Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .github/actions/test-if-changes/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,13 @@ runs:
if [[ -z "${{ inputs.version }}" ]]; then
pip install -e '.[dev]'
else
pip install "debug-gym[dev]==${{ inputs.version }}"
pip install -r requirements.txt
pip install pytest-asyncio pytest-cov pytest-xdist pytest-timeout pytest-env isort black Flask==3.1.3 Werkzeug==3.1.9
pip install --no-deps "debug-gym==${{ inputs.version }}"
fi
if [[ "${{ inputs.name }}" == "swesmith" ]]; then
pip install -r requirements-swesmith.txt
pip install --no-deps swesmith==0.0.4
fi
df -h
- name: Run tests
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/tests_pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,9 @@ jobs:
- name: Install dependencies
run: |
pip install --upgrade pip
pip install "debug-gym[dev]==${{ needs.get-latest-release.outputs.release-tag }}"
pip install -r requirements.txt
pip install pytest-asyncio pytest-cov pytest-xdist pytest-timeout pytest-env isort black Flask==3.1.3 Werkzeug==3.1.9
pip install --no-deps "debug-gym==${{ needs.get-latest-release.outputs.release-tag }}"
- name: Test - PR - Fast
env:
DEBUG_GYM_DEBUG: 1
Expand Down
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ To install development dependencies, run:

pip install -e '.[dev]'

SWE-Smith support is optional. From a repository checkout, install its dependencies and the package without its unused SGLang dependency:

pip install -r requirements-swesmith.txt
pip install --no-deps swesmith==0.0.4


**Set your API information in llm.yaml**

Expand Down
31 changes: 25 additions & 6 deletions debug_gym/gym/envs/swe_smith.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,18 @@
import docker
import yaml
from datasets import load_dataset, load_from_disk
from swesmith.build_repo.download_images import DOCKER_ORG, TAG
from swesmith.constants import MAP_REPO_TO_SPECS
from swesmith.harness.grading import TestStatus
from swesmith.harness.log_parsers import MAP_REPO_TO_PARSER, parse_log_pytest
from swesmith.harness.utils import get_test_command
from swesmith.utils import get_repo_commit_from_image_name

from debug_gym.constants import DEBUG_GYM_CACHE_DIR
from debug_gym.gym.entities import EvalOutput
from debug_gym.gym.envs.swe_bench import SWEBenchEnv
from debug_gym.gym.terminals.terminal import DebugGymLogger
from debug_gym.gym.utils import filter_problems

_SWESMITH_INSTALL_HELP = (
"SWE-Smith support requires installing requirements-swesmith.txt "
"and swesmith==0.0.4 without dependencies."
)


class SWESmithEnv(SWEBenchEnv):
CACHE = DEBUG_GYM_CACHE_DIR / "swe-smith"
Expand All @@ -25,6 +24,18 @@ class SWESmithEnv(SWEBenchEnv):
)

def setup_task(self):
try:
from swesmith.build_repo.download_images import DOCKER_ORG, TAG
from swesmith.constants import MAP_REPO_TO_SPECS
from swesmith.harness.log_parsers import (
MAP_REPO_TO_PARSER,
parse_log_pytest,
)
from swesmith.harness.utils import get_test_command
from swesmith.utils import get_repo_commit_from_image_name
except ImportError as exc:
raise ImportError(_SWESMITH_INSTALL_HELP) from exc

self.base_commit = (
self.task_data["base_commit"] if "base_commit" in self.task_data else "main"
)
Expand Down Expand Up @@ -138,6 +149,8 @@ def setup_terminal(self):
raise RuntimeError(f"Failed to reset the git history:\n{output}")

def calculate_score(self, eval_output: EvalOutput) -> int:
from swesmith.harness.grading import TestStatus

test_status_map = self.log_parser(eval_output.output)
f2p_score = sum(
1
Expand Down Expand Up @@ -192,6 +205,12 @@ def load_dataset(
**kwargs,
) -> dict:
logger = logger or DebugGymLogger("debug_gym")
if prepull_images:
try:
from swesmith.build_repo.download_images import DOCKER_ORG, TAG
except ImportError as exc:
raise ImportError(_SWESMITH_INSTALL_HELP) from exc

data_path = Path(dataset_id)
if data_path.is_file():
# Loading from local file.
Expand Down
17 changes: 17 additions & 0 deletions requirements-swesmith.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Install these in addition to requirements.txt. SWE-Smith's SGLang dependency
# is intentionally omitted because debug-gym does not use its inference server.
astor
ghapi
jupyter
libcst
litellm
matplotlib
modal
pre-commit
python-dotenv
sparklines
textual
tqdm
tree-sitter
tree-sitter-go
unidiff
4 changes: 0 additions & 4 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ tiktoken
docker==7.1.0
filelock==4.2.10
swebench==4.0.3
swesmith==0.0.4
# swesmith requires sglang, but sglang>0.5.2 pulls in heavy runtime
# dependencies (torch, flashinfer) that fail to build.
sglang==0.5.2
prompt_toolkit==3.0.51
anthropic==0.51.0
jinja2==3.1.6
Expand Down
Loading