From a8d71702b50b4f0babf64aa919ec907d2a8b4b1f Mon Sep 17 00:00:00 2001 From: jimmy-holdout <346065422+jimmy-holdout@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:58:13 +0000 Subject: [PATCH] Do not keep the original code in the git history of SWE-smith tasks SWESmithEnv committed the bug patch on top of the code from before the bug. For bug patches that only modify or delete tracked files, `git checkout HEAD~1 -- .` gave back exactly the code the gold patch produces, and submitting it scored like the gold patch without any debugging. The image's .git also kept the commit objects of the SWE-smith task branches after the remote was removed. After applying the bug patch, replace .git with a new repository whose only commit is the buggy code. It tracks the files the old index tracked after the bug patch, including files the patch adds (`git commit -am` left those untracked). --- debug_gym/gym/envs/swe_smith.py | 34 ++++++++++++++++--- tests/gym/envs/test_swe_smith.py | 56 +++++++++++++++++++++++++++----- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/debug_gym/gym/envs/swe_smith.py b/debug_gym/gym/envs/swe_smith.py index 120caf54d..3be5a9c92 100644 --- a/debug_gym/gym/envs/swe_smith.py +++ b/debug_gym/gym/envs/swe_smith.py @@ -105,11 +105,37 @@ def setup_task(self): def setup_terminal(self): super().setup_terminal() - # Apply bug patch. - self.terminal.run(f"git apply - <<'EOF'\n{self.bug_patch}\nEOF", raises=True) - self.terminal.run( - f"git commit -am 'Applying bug patch for {self.task_name}' --no-verify" + # Apply bug patch, also to the index so files it adds are tracked. The + # index is refreshed first since `--index` compares the files' stat info. + # Note: `raises=True` doesn't cover timeouts, hence the extra checks. + success, output = self.terminal.run( + "git update-index -q --refresh && " + f"git apply --index - <<'EOF'\n{self.bug_patch}\nEOF", + raises=True, ) + if not success: + raise RuntimeError(f"Failed to apply the bug patch:\n{output}") + + # Start a new git history whose only commit is the buggy code. Committing + # the bug on top of the original code would let the agent restore the + # original code (i.e. the gold patch) with `git checkout HEAD~1 -- .`. + success, output = self.terminal.run( + [ + "tracked_files=$(mktemp)", + 'git ls-files -z > "$tracked_files"', + "rm -rf .git", + "git init -q -b main", + "git config user.name 'debug-gym'", + "git config user.email '<>'", + "git --literal-pathspecs add -f" + ' --pathspec-from-file="$tracked_files" --pathspec-file-nul', + 'rm "$tracked_files"', + "git commit -qm 'Initial commit' --no-verify", + ], + raises=True, + ) + if not success: + raise RuntimeError(f"Failed to reset the git history:\n{output}") def calculate_score(self, eval_output: EvalOutput) -> int: test_status_map = self.log_parser(eval_output.output) diff --git a/tests/gym/envs/test_swe_smith.py b/tests/gym/envs/test_swe_smith.py index 3305ceb5a..73244ed76 100644 --- a/tests/gym/envs/test_swe_smith.py +++ b/tests/gym/envs/test_swe_smith.py @@ -98,16 +98,54 @@ def test_setup_task(get_swe_smith_env): @pytest.if_docker_running def test_setup_terminal(get_swe_smith_env): env = get_swe_smith_env() - task_name = "john-kurkowski__tldextract.3d1bf184.combine_file__1vnuqpt4" env.reset() - _, git_logs = env.terminal.run("git log -n 4") - # For SWE-Smith the base commit is found in the branch associated to the - # instance id and is different from the one in the main branch. - assert f"Applying bug patch for {task_name}" in git_logs - - _, git_diff = env.terminal.run("git show HEAD", strip_output=False) - git_diff = git_diff[git_diff.index("diff --git") :] - assert git_diff == env.bug_patch + + # The bug patch is applied and committed. + success, _ = env.terminal.run( + f"git apply --reverse --check - <<'EOF'\n{env.bug_patch}\nEOF" + ) + assert success + _, git_status = env.terminal.run("git status --porcelain") + assert git_status == "" + + # The git history doesn't contain the original code, e.g. the agent can't + # undo the bug with `git checkout HEAD~1 -- .`. + _, nb_commits = env.terminal.run("git rev-list --all --count") + assert nb_commits == "1" + _, nb_commit_objects = env.terminal.run( + "git cat-file --batch-all-objects --batch-check | grep -c ' commit '" + ) + assert nb_commit_objects == "1" + success, _ = env.terminal.run("git checkout HEAD~1 -- .") + assert not success + + +@pytest.if_docker_running +def test_setup_terminal_with_bug_patch_adding_a_file(get_swe_smith_env): + env = get_swe_smith_env() + new_file_patch = ( + "diff --git a/new_file.py b/new_file.py\n" + "new file mode 100644\n" + "--- /dev/null\n" + "+++ b/new_file.py\n" + "@@ -0,0 +1 @@\n" + "+print('hello')\n" + ) + env.task_data = dict( + env.task_data, patch=env.task_data["patch"].rstrip("\n") + "\n" + new_file_patch + ) + env.reset() + + # The added file is part of the initial commit. + _, tracked = env.terminal.run("git ls-files new_file.py") + assert tracked == "new_file.py" + _, git_status = env.terminal.run("git status --porcelain") + assert git_status == "" + + # The gold patch removes it. + env.apply_gold_patch() + success, _ = env.terminal.run("test -e new_file.py") + assert not success @pytest.if_docker_running