Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
expressions (no `${{ }}` wrapper, which GitHub re-evaluates and mis-handles)
plus a `[bot]`-login-suffix guard so the bot's own approval comment can
never re-trigger itself.
- **Gate provenance misdescribed statistical runs (G7)**: in envelope mode the
`Gate: …` line showed the legacy single-run knobs (`max_divergence=0.3`,
`max_cost_delta=10.0%`) while `compare_envelope` actually judged with the
scenario tolerances. The line now renders
`Gate [statistical envelope: <scenario>, N=<n>]: divergence_ceiling=…,
max_cost_increase_pct=…, step_count_std_dev=…` for envelope runs; strict/v1
baselines keep the legacy form.

## [0.4.0] - 2026-08-28

Expand Down
7 changes: 6 additions & 1 deletion src/agentdiff/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,6 @@ def diff(
typer.echo(f"Baseline config not found: {e}", err=True)
sys.exit(2)
threshold_changes = diff_gate_thresholds(baseline_cfg, cfg)
gate_provenance = provenance_line(cfg, config_source)
stale_days = (
stale_days
if stale_days is not None
Expand Down Expand Up @@ -508,6 +507,12 @@ def diff(
typer.echo(f"Ingestion error: {e}", err=True)
sys.exit(2)

# G7 provenance must describe the gate that actually judged this run —
# in envelope mode that is the scenario tolerances, not the legacy knobs.
gate_provenance = provenance_line(
cfg, config_source, scenario_cfg=scenario_cfg, envelope=baseline_env
)

try:
recovery_kwargs = {}
if max_recovery_ratio is not None:
Expand Down
46 changes: 38 additions & 8 deletions src/agentdiff/governance.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@

from dataclasses import dataclass

from agentdiff.config import AgentDiffConfig
from agentdiff.config import AgentDiffConfig, ScenarioConfig
from agentdiff.models.envelope import BaselineEnvelope

# The gate knobs that decide pass/fail in the CLI diff path, in display order.
_GATED_KEYS: tuple[str, ...] = (
Expand Down Expand Up @@ -78,28 +79,57 @@ def diff_gate_thresholds(
]


def provenance_line(cfg: AgentDiffConfig, config_path: str | None) -> str:
def provenance_line(
cfg: AgentDiffConfig,
config_path: str | None,
*,
scenario_cfg: ScenarioConfig | None = None,
envelope: BaselineEnvelope | None = None,
) -> str:
"""G7 — one-line, self-describing gate summary for any report.

Names the active thresholds and where they came from, so every diff
answers "what rules judged me?" without opening the config.

When a statistical envelope is being judged (mode ``statistical`` with
N >= 2 runs), the line must describe the *actual* statistical gate —
the scenario tolerances that ``compare_envelope`` applied — not the
legacy single-run knobs.
"""
gates = effective_gates(cfg)
statistical = (
envelope is not None and envelope.mode == "statistical" and envelope.n_runs >= 2
)
source = (
f"agentdiff.toml ({config_path})"
if config_path
else "defaults (no agentdiff.toml found)"
)
gates = effective_gates(cfg)
invariant_parts = [
f"fail_on_identical_loops={str(gates['fail_on_identical_loops']).lower()}"
]
if gates["max_tool_repeats"] is not None:
invariant_parts.append(f"max_tool_repeats={gates['max_tool_repeats']}")
if statistical:
tol = getattr(scenario_cfg, "tolerances", None)
parts = [
f"divergence_ceiling={tol.divergence_ceiling if tol else 0.35}",
"max_cost_increase_pct="
f"{scenario_cfg.max_cost_increase_pct if scenario_cfg else 20.0}%",
f"step_count_std_dev={tol.step_count_std_dev if tol else 2.0}",
*invariant_parts,
]
prefix = (
f"Gate [statistical envelope: {envelope.scenario}, N={envelope.n_runs}]"
)
return f"{prefix}: {', '.join(parts)} — source: {source}"

parts = [
f"max_divergence={gates['max_divergence']}",
f"max_loops={gates['max_loops']}",
f"max_cost_delta={gates['max_cost_delta']}%",
]
if gates["max_recovery_ratio"] is not None:
parts.append(f"max_recovery_ratio={gates['max_recovery_ratio']}")
parts.append(
f"fail_on_identical_loops={str(gates['fail_on_identical_loops']).lower()}"
)
if gates["max_tool_repeats"] is not None:
parts.append(f"max_tool_repeats={gates['max_tool_repeats']}")
parts.extend(invariant_parts)
return f"Gate: {', '.join(parts)} — source: {source}"
59 changes: 59 additions & 0 deletions tests/test_governance.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,65 @@ def test_provenance_line_with_path(self, tmp_path):
assert "max_recovery_ratio=1.5" in line # opt-in gate appears when set
assert f"source: agentdiff.toml ({cfg_file})" in line

@staticmethod
def _statistical_envelope(n_runs: int = 3):
from agentdiff.models.envelope import BaselineEnvelope

runs = [
record_run("json:loads", task_input={"s": json.dumps({"a": i})})
for i in range(n_runs)
]
return BaselineEnvelope.from_runs(runs, scenario="default")

def test_provenance_line_statistical_envelope_reports_scenario_tolerances(self):
"""In envelope mode the provenance must describe the gate that actually
judged the run — scenario tolerances + envelope identity — not the
legacy single-run knobs (which never apply in this path).
"""
from agentdiff.config import ScenarioConfig
from agentdiff.governance import provenance_line

scenario = ScenarioConfig(
name="default",
max_cost_increase_pct=20.0,
) # tolerances default to 0.35 / 2.0
line = provenance_line(
AgentDiffConfig(),
None,
scenario_cfg=scenario,
envelope=self._statistical_envelope(3),
)
assert "Gate [statistical envelope: default, N=3]" in line
assert "divergence_ceiling=0.35" in line
assert "max_cost_increase_pct=20.0%" in line
assert "step_count_std_dev=2.0" in line
assert "fail_on_identical_loops=true" in line
assert "max_divergence=" not in line # legacy knob must not leak
assert "max_cost_delta=" not in line

def test_provenance_line_statistical_without_scenario_uses_defaults(self):
from agentdiff.governance import provenance_line

line = provenance_line(
AgentDiffConfig(), None, envelope=self._statistical_envelope(2)
)
assert "Gate [statistical envelope: default, N=2]" in line
assert "divergence_ceiling=0.35" in line
assert "max_cost_increase_pct=20.0%" in line
assert "step_count_std_dev=2.0" in line

def test_provenance_line_strict_envelope_keeps_legacy_form(self):
"""A strict single-run baseline (v1) keeps the legacy provenance."""
from agentdiff.governance import provenance_line
from agentdiff.models.envelope import BaselineEnvelope

env = BaselineEnvelope.from_runs(
[record_run("json:loads", task_input={"s": "{}"})], mode="strict"
)
line = provenance_line(AgentDiffConfig(), None, envelope=env)
assert "max_divergence=0.3" in line
assert "statistical envelope" not in line

def test_terminal_format_includes_provenance(self, tmp_path):
base, cand = TestCLIBaselineConfig()._traces(tmp_path)
result = runner.invoke(
Expand Down
Loading