diff --git a/CHANGELOG.md b/CHANGELOG.md index c5861a60..e0044624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 expressions (no `${{ }}` wrapper, which GitHub re-evaluates and mis-handles) plus a `[bot]`-login-suffix guard so the bot's own approval comment can never re-trigger itself. +- **Gate provenance misdescribed statistical runs (G7)**: in envelope mode the + `Gate: …` line showed the legacy single-run knobs (`max_divergence=0.3`, + `max_cost_delta=10.0%`) while `compare_envelope` actually judged with the + scenario tolerances. The line now renders + `Gate [statistical envelope: , N=]: divergence_ceiling=…, + max_cost_increase_pct=…, step_count_std_dev=…` for envelope runs; strict/v1 + baselines keep the legacy form. ## [0.4.0] - 2026-08-28 diff --git a/src/agentdiff/cli.py b/src/agentdiff/cli.py index fbe97ff3..2d751864 100644 --- a/src/agentdiff/cli.py +++ b/src/agentdiff/cli.py @@ -425,7 +425,6 @@ def diff( typer.echo(f"Baseline config not found: {e}", err=True) sys.exit(2) threshold_changes = diff_gate_thresholds(baseline_cfg, cfg) - gate_provenance = provenance_line(cfg, config_source) stale_days = ( stale_days if stale_days is not None @@ -508,6 +507,12 @@ def diff( typer.echo(f"Ingestion error: {e}", err=True) sys.exit(2) + # G7 provenance must describe the gate that actually judged this run — + # in envelope mode that is the scenario tolerances, not the legacy knobs. + gate_provenance = provenance_line( + cfg, config_source, scenario_cfg=scenario_cfg, envelope=baseline_env + ) + try: recovery_kwargs = {} if max_recovery_ratio is not None: diff --git a/src/agentdiff/governance.py b/src/agentdiff/governance.py index 7f039e9f..bc5f5143 100644 --- a/src/agentdiff/governance.py +++ b/src/agentdiff/governance.py @@ -9,7 +9,8 @@ from dataclasses import dataclass -from agentdiff.config import AgentDiffConfig +from agentdiff.config import AgentDiffConfig, ScenarioConfig +from agentdiff.models.envelope import BaselineEnvelope # The gate knobs that decide pass/fail in the CLI diff path, in display order. _GATED_KEYS: tuple[str, ...] = ( @@ -78,18 +79,51 @@ def diff_gate_thresholds( ] -def provenance_line(cfg: AgentDiffConfig, config_path: str | None) -> str: +def provenance_line( + cfg: AgentDiffConfig, + config_path: str | None, + *, + scenario_cfg: ScenarioConfig | None = None, + envelope: BaselineEnvelope | None = None, +) -> str: """G7 — one-line, self-describing gate summary for any report. Names the active thresholds and where they came from, so every diff answers "what rules judged me?" without opening the config. + + When a statistical envelope is being judged (mode ``statistical`` with + N >= 2 runs), the line must describe the *actual* statistical gate — + the scenario tolerances that ``compare_envelope`` applied — not the + legacy single-run knobs. """ - gates = effective_gates(cfg) + statistical = ( + envelope is not None and envelope.mode == "statistical" and envelope.n_runs >= 2 + ) source = ( f"agentdiff.toml ({config_path})" if config_path else "defaults (no agentdiff.toml found)" ) + gates = effective_gates(cfg) + invariant_parts = [ + f"fail_on_identical_loops={str(gates['fail_on_identical_loops']).lower()}" + ] + if gates["max_tool_repeats"] is not None: + invariant_parts.append(f"max_tool_repeats={gates['max_tool_repeats']}") + if statistical: + tol = getattr(scenario_cfg, "tolerances", None) + parts = [ + f"divergence_ceiling={tol.divergence_ceiling if tol else 0.35}", + "max_cost_increase_pct=" + f"{scenario_cfg.max_cost_increase_pct if scenario_cfg else 20.0}%", + f"step_count_std_dev={tol.step_count_std_dev if tol else 2.0}", + *invariant_parts, + ] + prefix = ( + f"Gate [statistical envelope: {envelope.scenario}, N={envelope.n_runs}]" + ) + return f"{prefix}: {', '.join(parts)} — source: {source}" + parts = [ f"max_divergence={gates['max_divergence']}", f"max_loops={gates['max_loops']}", @@ -97,9 +131,5 @@ def provenance_line(cfg: AgentDiffConfig, config_path: str | None) -> str: ] if gates["max_recovery_ratio"] is not None: parts.append(f"max_recovery_ratio={gates['max_recovery_ratio']}") - parts.append( - f"fail_on_identical_loops={str(gates['fail_on_identical_loops']).lower()}" - ) - if gates["max_tool_repeats"] is not None: - parts.append(f"max_tool_repeats={gates['max_tool_repeats']}") + parts.extend(invariant_parts) return f"Gate: {', '.join(parts)} — source: {source}" diff --git a/tests/test_governance.py b/tests/test_governance.py index 2140a967..a1d35665 100644 --- a/tests/test_governance.py +++ b/tests/test_governance.py @@ -231,6 +231,65 @@ def test_provenance_line_with_path(self, tmp_path): assert "max_recovery_ratio=1.5" in line # opt-in gate appears when set assert f"source: agentdiff.toml ({cfg_file})" in line + @staticmethod + def _statistical_envelope(n_runs: int = 3): + from agentdiff.models.envelope import BaselineEnvelope + + runs = [ + record_run("json:loads", task_input={"s": json.dumps({"a": i})}) + for i in range(n_runs) + ] + return BaselineEnvelope.from_runs(runs, scenario="default") + + def test_provenance_line_statistical_envelope_reports_scenario_tolerances(self): + """In envelope mode the provenance must describe the gate that actually + judged the run — scenario tolerances + envelope identity — not the + legacy single-run knobs (which never apply in this path). + """ + from agentdiff.config import ScenarioConfig + from agentdiff.governance import provenance_line + + scenario = ScenarioConfig( + name="default", + max_cost_increase_pct=20.0, + ) # tolerances default to 0.35 / 2.0 + line = provenance_line( + AgentDiffConfig(), + None, + scenario_cfg=scenario, + envelope=self._statistical_envelope(3), + ) + assert "Gate [statistical envelope: default, N=3]" in line + assert "divergence_ceiling=0.35" in line + assert "max_cost_increase_pct=20.0%" in line + assert "step_count_std_dev=2.0" in line + assert "fail_on_identical_loops=true" in line + assert "max_divergence=" not in line # legacy knob must not leak + assert "max_cost_delta=" not in line + + def test_provenance_line_statistical_without_scenario_uses_defaults(self): + from agentdiff.governance import provenance_line + + line = provenance_line( + AgentDiffConfig(), None, envelope=self._statistical_envelope(2) + ) + assert "Gate [statistical envelope: default, N=2]" in line + assert "divergence_ceiling=0.35" in line + assert "max_cost_increase_pct=20.0%" in line + assert "step_count_std_dev=2.0" in line + + def test_provenance_line_strict_envelope_keeps_legacy_form(self): + """A strict single-run baseline (v1) keeps the legacy provenance.""" + from agentdiff.governance import provenance_line + from agentdiff.models.envelope import BaselineEnvelope + + env = BaselineEnvelope.from_runs( + [record_run("json:loads", task_input={"s": "{}"})], mode="strict" + ) + line = provenance_line(AgentDiffConfig(), None, envelope=env) + assert "max_divergence=0.3" in line + assert "statistical envelope" not in line + def test_terminal_format_includes_provenance(self, tmp_path): base, cand = TestCLIBaselineConfig()._traces(tmp_path) result = runner.invoke(