diff --git a/debug_gym/gym/envs/swe_smith.py b/debug_gym/gym/envs/swe_smith.py index 120caf54..3be5a9c9 100644 --- a/debug_gym/gym/envs/swe_smith.py +++ b/debug_gym/gym/envs/swe_smith.py @@ -105,11 +105,37 @@ def setup_task(self): def setup_terminal(self): super().setup_terminal() - # Apply bug patch. - self.terminal.run(f"git apply - <<'EOF'\n{self.bug_patch}\nEOF", raises=True) - self.terminal.run( - f"git commit -am 'Applying bug patch for {self.task_name}' --no-verify" + # Apply bug patch, also to the index so files it adds are tracked. The + # index is refreshed first since `--index` compares the files' stat info. + # Note: `raises=True` doesn't cover timeouts, hence the extra checks. + success, output = self.terminal.run( + "git update-index -q --refresh && " + f"git apply --index - <<'EOF'\n{self.bug_patch}\nEOF", + raises=True, ) + if not success: + raise RuntimeError(f"Failed to apply the bug patch:\n{output}") + + # Start a new git history whose only commit is the buggy code. Committing + # the bug on top of the original code would let the agent restore the + # original code (i.e. the gold patch) with `git checkout HEAD~1 -- .`. + success, output = self.terminal.run( + [ + "tracked_files=$(mktemp)", + 'git ls-files -z > "$tracked_files"', + "rm -rf .git", + "git init -q -b main", + "git config user.name 'debug-gym'", + "git config user.email '<>'", + "git --literal-pathspecs add -f" + ' --pathspec-from-file="$tracked_files" --pathspec-file-nul', + 'rm "$tracked_files"', + "git commit -qm 'Initial commit' --no-verify", + ], + raises=True, + ) + if not success: + raise RuntimeError(f"Failed to reset the git history:\n{output}") def calculate_score(self, eval_output: EvalOutput) -> int: test_status_map = self.log_parser(eval_output.output) diff --git a/tests/gym/envs/test_swe_smith.py b/tests/gym/envs/test_swe_smith.py index 3305ceb5..73244ed7 100644 --- a/tests/gym/envs/test_swe_smith.py +++ b/tests/gym/envs/test_swe_smith.py @@ -98,16 +98,54 @@ def test_setup_task(get_swe_smith_env): @pytest.if_docker_running def test_setup_terminal(get_swe_smith_env): env = get_swe_smith_env() - task_name = "john-kurkowski__tldextract.3d1bf184.combine_file__1vnuqpt4" env.reset() - _, git_logs = env.terminal.run("git log -n 4") - # For SWE-Smith the base commit is found in the branch associated to the - # instance id and is different from the one in the main branch. - assert f"Applying bug patch for {task_name}" in git_logs - - _, git_diff = env.terminal.run("git show HEAD", strip_output=False) - git_diff = git_diff[git_diff.index("diff --git") :] - assert git_diff == env.bug_patch + + # The bug patch is applied and committed. + success, _ = env.terminal.run( + f"git apply --reverse --check - <<'EOF'\n{env.bug_patch}\nEOF" + ) + assert success + _, git_status = env.terminal.run("git status --porcelain") + assert git_status == "" + + # The git history doesn't contain the original code, e.g. the agent can't + # undo the bug with `git checkout HEAD~1 -- .`. + _, nb_commits = env.terminal.run("git rev-list --all --count") + assert nb_commits == "1" + _, nb_commit_objects = env.terminal.run( + "git cat-file --batch-all-objects --batch-check | grep -c ' commit '" + ) + assert nb_commit_objects == "1" + success, _ = env.terminal.run("git checkout HEAD~1 -- .") + assert not success + + +@pytest.if_docker_running +def test_setup_terminal_with_bug_patch_adding_a_file(get_swe_smith_env): + env = get_swe_smith_env() + new_file_patch = ( + "diff --git a/new_file.py b/new_file.py\n" + "new file mode 100644\n" + "--- /dev/null\n" + "+++ b/new_file.py\n" + "@@ -0,0 +1 @@\n" + "+print('hello')\n" + ) + env.task_data = dict( + env.task_data, patch=env.task_data["patch"].rstrip("\n") + "\n" + new_file_patch + ) + env.reset() + + # The added file is part of the initial commit. + _, tracked = env.terminal.run("git ls-files new_file.py") + assert tracked == "new_file.py" + _, git_status = env.terminal.run("git status --porcelain") + assert git_status == "" + + # The gold patch removes it. + env.apply_gold_patch() + success, _ = env.terminal.run("test -e new_file.py") + assert not success @pytest.if_docker_running