Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 30 additions & 4 deletions debug_gym/gym/envs/swe_smith.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,37 @@ def setup_task(self):
def setup_terminal(self):
super().setup_terminal()

# Apply bug patch.
self.terminal.run(f"git apply - <<'EOF'\n{self.bug_patch}\nEOF", raises=True)
self.terminal.run(
f"git commit -am 'Applying bug patch for {self.task_name}' --no-verify"
# Apply bug patch, also to the index so files it adds are tracked. The
# index is refreshed first since `--index` compares the files' stat info.
# Note: `raises=True` doesn't cover timeouts, hence the extra checks.
success, output = self.terminal.run(
"git update-index -q --refresh && "
f"git apply --index - <<'EOF'\n{self.bug_patch}\nEOF",
raises=True,
)
if not success:
raise RuntimeError(f"Failed to apply the bug patch:\n{output}")

# Start a new git history whose only commit is the buggy code. Committing
# the bug on top of the original code would let the agent restore the
# original code (i.e. the gold patch) with `git checkout HEAD~1 -- .`.
success, output = self.terminal.run(
[
"tracked_files=$(mktemp)",
'git ls-files -z > "$tracked_files"',
"rm -rf .git",
"git init -q -b main",
"git config user.name 'debug-gym'",
"git config user.email '<>'",
"git --literal-pathspecs add -f"
' --pathspec-from-file="$tracked_files" --pathspec-file-nul',
'rm "$tracked_files"',
"git commit -qm 'Initial commit' --no-verify",
],
raises=True,
)
if not success:
raise RuntimeError(f"Failed to reset the git history:\n{output}")

def calculate_score(self, eval_output: EvalOutput) -> int:
test_status_map = self.log_parser(eval_output.output)
Expand Down
56 changes: 47 additions & 9 deletions tests/gym/envs/test_swe_smith.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,16 +98,54 @@ def test_setup_task(get_swe_smith_env):
@pytest.if_docker_running
def test_setup_terminal(get_swe_smith_env):
env = get_swe_smith_env()
task_name = "john-kurkowski__tldextract.3d1bf184.combine_file__1vnuqpt4"
env.reset()
_, git_logs = env.terminal.run("git log -n 4")
# For SWE-Smith the base commit is found in the branch associated to the
# instance id and is different from the one in the main branch.
assert f"Applying bug patch for {task_name}" in git_logs

_, git_diff = env.terminal.run("git show HEAD", strip_output=False)
git_diff = git_diff[git_diff.index("diff --git") :]
assert git_diff == env.bug_patch

# The bug patch is applied and committed.
success, _ = env.terminal.run(
f"git apply --reverse --check - <<'EOF'\n{env.bug_patch}\nEOF"
)
assert success
_, git_status = env.terminal.run("git status --porcelain")
assert git_status == ""

# The git history doesn't contain the original code, e.g. the agent can't
# undo the bug with `git checkout HEAD~1 -- .`.
_, nb_commits = env.terminal.run("git rev-list --all --count")
assert nb_commits == "1"
_, nb_commit_objects = env.terminal.run(
"git cat-file --batch-all-objects --batch-check | grep -c ' commit '"
)
assert nb_commit_objects == "1"
success, _ = env.terminal.run("git checkout HEAD~1 -- .")
assert not success


@pytest.if_docker_running
def test_setup_terminal_with_bug_patch_adding_a_file(get_swe_smith_env):
env = get_swe_smith_env()
new_file_patch = (
"diff --git a/new_file.py b/new_file.py\n"
"new file mode 100644\n"
"--- /dev/null\n"
"+++ b/new_file.py\n"
"@@ -0,0 +1 @@\n"
"+print('hello')\n"
)
env.task_data = dict(
env.task_data, patch=env.task_data["patch"].rstrip("\n") + "\n" + new_file_patch
)
env.reset()

# The added file is part of the initial commit.
_, tracked = env.terminal.run("git ls-files new_file.py")
assert tracked == "new_file.py"
_, git_status = env.terminal.run("git status --porcelain")
assert git_status == ""

# The gold patch removes it.
env.apply_gold_patch()
success, _ = env.terminal.run("test -e new_file.py")
assert not success


@pytest.if_docker_running
Expand Down
Loading