From 3888a6303541e9c15492883cd499527118eb49e7 Mon Sep 17 00:00:00 2001 From: lostmartian Date: Tue, 1 Sep 2026 23:41:58 +0530 Subject: [PATCH 1/4] fix(init): generated workflows invoke the diff CLI correctly --- CHANGELOG.md | 9 +++++++++ src/agentdiff/init_wizard.py | 6 ++---- tests/test_init_wizard.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb2afebb..3a152768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `kerrshift/agentdiff/.github/actions/agentdiff-check`; old `lostmartian/agentdiff` URLs continue to work via GitHub redirects. +### Fixed +- **`agentdiff init` generated workflows used an invalid CLI form**: the gate + and PR-report steps invoked `agentdiff --baseline `, + which the E1 diff-default patch parses as `diff` with a missing positional + (`Missing argument 'candidate_path'`) — `--baseline` is the baseline-*store* + option, not the first positional argument. Generated workflows now invoke + `agentdiff diff --scenario ` explicitly, with a + regression test pinning the valid form. + ## [0.4.0] - 2026-08-28 AgentDiff v0.4.0 — trace capture, Goodhart-resistant gating, and a full docs diff --git a/src/agentdiff/init_wizard.py b/src/agentdiff/init_wizard.py index 571ba646..4e88c087 100644 --- a/src/agentdiff/init_wizard.py +++ b/src/agentdiff/init_wizard.py @@ -144,8 +144,7 @@ def detect_framework() -> FrameworkInfo: - name: Gate against baseline envelope run: | - agentdiff {candidate} \\ - --baseline {baseline} \\ + agentdiff diff {baseline} {candidate} \\ --fail-on-regression \\ --scenario {scenario} @@ -160,8 +159,7 @@ def detect_framework() -> FrameworkInfo: - name: Post PR report if: always() run: | - agentdiff {candidate} \\ - --baseline {baseline} \\ + agentdiff diff {baseline} {candidate} \\ --format pr --pr ${{{{ github.event.number }}}} \\ --scenario {scenario} """ diff --git a/tests/test_init_wizard.py b/tests/test_init_wizard.py index 79b1b57d..c34f2e86 100644 --- a/tests/test_init_wizard.py +++ b/tests/test_init_wizard.py @@ -62,6 +62,20 @@ def test_workflow_is_valid_action_yaml(self, tmp_path): # record target placeholder is explicitly marked assert "EDIT ME" in text + def test_workflow_invokes_diff_with_valid_cli_form(self, tmp_path): + """Regression: generated workflows must call `agentdiff diff BASELINE + CANDIDATE`. The bare form (`agentdiff candidate --baseline …`) hits + the E1 diff-default patch and fails with a missing positional, since + `--baseline` is the baseline-*store* option, not the first positional. + """ + write_config(tmp_path, framework=GENERIC, scenario="refund") + text = (tmp_path / ".github/workflows/agentdiff.yml").read_text() + assert ( + "agentdiff diff baselines/refund.envelope.json traces/candidate.json" + in text + ) + assert "--baseline " not in text + def test_refuses_to_clobber_without_force(self, tmp_path): write_config(tmp_path, framework=GENERIC) with pytest.raises(FileExistsError): From d48d80cd398f85be5c997a978609bd49eeeff50b Mon Sep 17 00:00:00 2001 From: lostmartian Date: Wed, 2 Sep 2026 00:03:13 +0530 Subject: [PATCH 2/4] fix(init): approve workflow repo resolution + bot self-trigger guard --- CHANGELOG.md | 7 +++++++ src/agentdiff/init_wizard.py | 9 +++++---- tests/test_init_wizard.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a152768..4f31abcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 option, not the first positional argument. Generated workflows now invoke `agentdiff diff --scenario ` explicitly, with a regression test pinning the valid form. +- **`agentdiff init --with-approve` runtime fixes** (found by running the + generated bot end-to-end on a live repo): pre-checkout `gh` calls + (`pr view`, `run list`, `pr comment`) now pass `-R ${{ github.repository }}` + so they resolve outside a git checkout; the trigger filter uses native `if` + expressions (no `${{ }}` wrapper, which GitHub re-evaluates and mis-handles) + plus a `[bot]`-login-suffix guard so the bot's own approval comment can + never re-trigger itself. ## [0.4.0] - 2026-08-28 diff --git a/src/agentdiff/init_wizard.py b/src/agentdiff/init_wizard.py index 4e88c087..80e247c2 100644 --- a/src/agentdiff/init_wizard.py +++ b/src/agentdiff/init_wizard.py @@ -200,9 +200,10 @@ def detect_framework() -> FrameworkInfo: jobs: approve: if: >- - ${{{{ github.event.issue.pull_request != null }}}} + github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/agentdiff approve') && github.event.comment.user.type != 'Bot' + && !endsWith(github.event.comment.user.login, '[bot]') runs-on: ubuntu-latest steps: - name: Verify commenter has write access @@ -212,7 +213,7 @@ def detect_framework() -> FrameworkInfo: PERM=$(gh api repos/${{{{ github.repository }}}}/collaborators/${{{{ github.event.comment.user.login }}}}/permission --jq .permission) if [ "$PERM" != "write" ] && [ "$PERM" != "admin" ]; then echo "Commenter ${{{{ github.event.comment.user.login }}}} lacks write access ($PERM). Refusing." - gh pr comment ${{{{ github.event.issue.number }}}} --body "⛔ /agentdiff approve requires write access." + gh pr comment ${{{{ github.event.issue.number }}}} -R ${{{{ github.repository }}}} --body "⛔ /agentdiff approve requires write access." exit 1 fi @@ -265,7 +266,7 @@ def detect_framework() -> FrameworkInfo: env: GH_TOKEN: ${{{{ steps.token.outputs.token }}}} run: | - REF=$(gh pr view ${{{{ github.event.issue.number }}}} --json headRefName --jq .headRefName) + REF=$(gh pr view ${{{{ github.event.issue.number }}}} -R ${{{{ github.repository }}}} --json headRefName --jq .headRefName) echo "ref=$REF" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v4 @@ -279,7 +280,7 @@ def detect_framework() -> FrameworkInfo: env: GH_TOKEN: ${{{{ steps.token.outputs.token }}}} run: | - RUN_ID=$(gh run list --workflow "AgentDiff Check" --branch "${{{{ steps.pr.outputs.ref }}}}" --limit 1 --json databaseId --jq '.[0].databaseId') + RUN_ID=$(gh run list --workflow "AgentDiff Check" --branch "${{{{ steps.pr.outputs.ref }}}}" -R ${{{{ github.repository }}}} --limit 1 --json databaseId --jq '.[0].databaseId') echo "id=$RUN_ID" >> "$GITHUB_OUTPUT" - name: Download candidate trace diff --git a/tests/test_init_wizard.py b/tests/test_init_wizard.py index c34f2e86..a6fe4121 100644 --- a/tests/test_init_wizard.py +++ b/tests/test_init_wizard.py @@ -76,6 +76,23 @@ def test_workflow_invokes_diff_with_valid_cli_form(self, tmp_path): ) assert "--baseline " not in text + def test_approve_workflow_filters_bots_without_template_if(self, tmp_path): + """The approve trigger must use native `if` expressions (no `${{ }}` + wrapper — GitHub re-evaluates the substituted string, which lets + `!= null`-style interpolations misbehave) and must exclude bot + identities both by type and by the `[bot]` login suffix, so the + bot's own approval comment can never re-trigger itself. + """ + write_config(tmp_path, framework=GENERIC, with_approve=True) + text = (tmp_path / ".github/workflows/agentdiff-approve.yml").read_text() + assert "github.event.issue.pull_request != null" in text + assert "${{ github.event.issue.pull_request" not in text + assert "github.event.comment.user.type != 'Bot'" in text + assert "!endsWith(github.event.comment.user.login, '[bot]')" in text + # pre-checkout gh calls must resolve the repo explicitly + assert "gh pr view ${{ github.event.issue.number }} -R ${{ github.repository }}" in text + assert 'gh run list --workflow "AgentDiff Check" --branch "${{ steps.pr.outputs.ref }}" -R ${{ github.repository }}' in text + def test_refuses_to_clobber_without_force(self, tmp_path): write_config(tmp_path, framework=GENERIC) with pytest.raises(FileExistsError): From 2bb094eacf31623f6640366c42725757366d1b4a Mon Sep 17 00:00:00 2001 From: lostmartian Date: Wed, 2 Sep 2026 00:04:08 +0530 Subject: [PATCH 3/4] style: ruff format test file --- tests/test_init_wizard.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_init_wizard.py b/tests/test_init_wizard.py index a6fe4121..8584fea6 100644 --- a/tests/test_init_wizard.py +++ b/tests/test_init_wizard.py @@ -90,8 +90,14 @@ def test_approve_workflow_filters_bots_without_template_if(self, tmp_path): assert "github.event.comment.user.type != 'Bot'" in text assert "!endsWith(github.event.comment.user.login, '[bot]')" in text # pre-checkout gh calls must resolve the repo explicitly - assert "gh pr view ${{ github.event.issue.number }} -R ${{ github.repository }}" in text - assert 'gh run list --workflow "AgentDiff Check" --branch "${{ steps.pr.outputs.ref }}" -R ${{ github.repository }}' in text + assert ( + "gh pr view ${{ github.event.issue.number }} -R ${{ github.repository }}" + in text + ) + assert ( + 'gh run list --workflow "AgentDiff Check" --branch "${{ steps.pr.outputs.ref }}" -R ${{ github.repository }}' + in text + ) def test_refuses_to_clobber_without_force(self, tmp_path): write_config(tmp_path, framework=GENERIC) From 86743fc099b9f30ef7d60bed02653ae8ef49a426 Mon Sep 17 00:00:00 2001 From: lostmartian Date: Wed, 2 Sep 2026 00:17:48 +0530 Subject: [PATCH 4/4] feat(init): generated gate workflows brand PR reports via hosted identity --- CHANGELOG.md | 5 +++++ src/agentdiff/init_wizard.py | 29 ++++++++++++++++++++++++++++- tests/test_init_wizard.py | 11 +++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f31abcc..c5861a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`fail_on_identical_loops`, `max_tool_repeats`), and `[scenario.x.tolerances]` (`step_count_std_dev`, `divergence_ceiling`). A repo with a single scenario needs no `--scenario` flag. +- **Branded PR reports from generated gate workflows** (`agentdiff init`): + the gate's PR-report step mints an `agentdiff[bot]` token from the hosted + identity service (`token.agentdiff.app`) when the AgentDiff App is + installed on the repo, and silently falls back to the workflow's own + `GITHUB_TOKEN` otherwise. Verified live on `agentdiff-demo`. - **CLI statistical mode**: envelope baselines gate via variance bands (step-count band, envelope-relative cost ceiling = max of the relative cap and the k·sigma band, divergence ceiling); strict single-run baselines are diff --git a/src/agentdiff/init_wizard.py b/src/agentdiff/init_wizard.py index 80e247c2..fb65ebc7 100644 --- a/src/agentdiff/init_wizard.py +++ b/src/agentdiff/init_wizard.py @@ -156,8 +156,31 @@ def detect_framework() -> FrameworkInfo: path: {candidate} retention-days: 7 + - name: AgentDiff bot token (hosted service, optional) + id: bot_token + if: always() + env: + SERVICE_URL: {token_service} + GH_TOKEN: ${{{{ github.token }}}} + REPO: ${{{{ github.repository }}}} + run: | + # Branded agentdiff[bot] identity when the AgentDiff App is + # installed on this repo; silently falls back to GITHUB_TOKEN + # otherwise. Stateless mint, 1h expiry, nothing stored. + RESP=$(curl -sS -X POST "$SERVICE_URL/token" \\ + -H "Content-Type: application/json" \\ + -d "{{\\"repository\\": \\"$REPO\\", \\"token\\": \\"$GH_TOKEN\\"}}" || true) + BOT_TOKEN=$(echo "$RESP" | python3 -c 'import json,sys + try: print(json.load(sys.stdin).get("token","")) + except Exception: print("")' 2>/dev/null || true) + if [ -n "$BOT_TOKEN" ]; then + echo "token=$BOT_TOKEN" >> "$GITHUB_OUTPUT" + fi + - name: Post PR report if: always() + env: + GITHUB_TOKEN: ${{{{ steps.bot_token.outputs.token || secrets.GITHUB_TOKEN }}}} run: | agentdiff diff {baseline} {candidate} \\ --format pr --pr ${{{{ github.event.number }}}} \\ @@ -330,7 +353,11 @@ def detect_framework() -> FrameworkInfo: def _render_workflow(runs: int, scenario: str, baseline: str, candidate: str) -> str: return WORKFLOW_TEMPLATE.format( - runs=runs, scenario=scenario, baseline=baseline, candidate=candidate + runs=runs, + scenario=scenario, + baseline=baseline, + candidate=candidate, + token_service=TOKEN_SERVICE_URL, ) diff --git a/tests/test_init_wizard.py b/tests/test_init_wizard.py index 8584fea6..2bcee3f2 100644 --- a/tests/test_init_wizard.py +++ b/tests/test_init_wizard.py @@ -99,6 +99,17 @@ def test_approve_workflow_filters_bots_without_template_if(self, tmp_path): in text ) + def test_gate_workflow_brands_pr_report_via_hosted_identity(self, tmp_path): + """The generated gate workflow mints an agentdiff[bot] token from the + hosted identity service for the PR report, falling back to the + workflow's own GITHUB_TOKEN when the App isn't installed. + """ + write_config(tmp_path, framework=GENERIC, scenario="refund") + text = (tmp_path / ".github/workflows/agentdiff.yml").read_text() + assert "https://token.agentdiff.app" in text + assert "${{ steps.bot_token.outputs.token || secrets.GITHUB_TOKEN }}" in text + assert "AgentDiff bot token (hosted service, optional)" in text + def test_refuses_to_clobber_without_force(self, tmp_path): write_config(tmp_path, framework=GENERIC) with pytest.raises(FileExistsError):