From 10496010f45d32852d305a035fc67214972e5162 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Thu, 19 Nov 2026 02:00:46 -0500 Subject: [PATCH] Extend evidence-strength notes to the cost objective The noisy-baseline note only measured latency. For the cost objective it now compares estimated cost between repetitions of the same task, using the same report the cost gates use, and separately reports completed baseline attempts whose cost cannot be measured because of unknown pricing or estimated token counts. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++ agentloop/improvement.py | 49 ++++++++++++++++++++++++++++--------- docs/GUIDED_OPTIMIZATION.md | 8 ++++-- tests/test_improvement.py | 38 +++++++++++++++++++++++++--- 4 files changed, 87 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae4f360..008aab9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- Evidence-strength notes cover the cost objective. `improve start` and + `improve compare` flag a noisy baseline when estimated cost of the same task + varies between repetitions by at least 10% and more than the required + improvement, and report baseline attempts whose cost cannot be measured + because of unknown pricing or estimated token counts. + ### Documentation - The README has an Install section covering the PyPI package, the npm diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 32bb854..2074191 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -24,6 +24,7 @@ from agentloop.replay import ReplayGates, build_replay_report from agentloop.replay import _summary as replay_summary from agentloop.schema import TraceValidationError, validate_elapsed_ms +from agentloop.tokens import is_token_basis_evaluable from agentloop.tracer import AgentTrace VERSION = "1.0" @@ -433,7 +434,9 @@ def planning_guidance(workload, manifest, traces, objective, minimum_improvement The sign test needs at least five pairs before even a unanimous result can fall below p = 0.05. When repeated baseline attempts of the same task vary - by more than the required improvement, smaller differences may be noise. + in the objective (latency, or estimated cost) by more than the required + improvement, smaller differences may be noise. For the cost objective, + attempts whose cost cannot be measured are reported separately. """ notes = [] pairs = len(workload["tasks"]) * workload["repetitions"] @@ -455,26 +458,50 @@ def planning_guidance(workload, manifest, traces, objective, minimum_improvement "measured. Use at least 2 repetitions to see how noisy the agent is.", } ) - elif objective == "latency": - completed = _completed(manifest, traces) + else: + completed = { + key: trace + for key, trace in _completed(manifest, traces).items() + if _trace_usable(trace) + } + if objective == "latency": + values = {key: trace.elapsed_ms for key, trace in completed.items()} + else: + values, unmeasurable = {}, 0 + for key, trace in completed.items(): + report = trace.report() + if report.get("has_unknown_cost") or not is_token_basis_evaluable( + report.get("token_status") + ): + unmeasurable += 1 + else: + values[key] = float(report.get("estimated_cost_usd") or 0.0) + if unmeasurable: + notes.append( + { + "kind": "cost_unmeasurable", + "attempts": unmeasurable, + "message": f"{unmeasurable} of {len(completed)} completed baseline " + "attempts have unknown model pricing or estimated token counts, so " + "their cost cannot be compared and cost gates will be indeterminate. " + "Record provider token usage and use models with known pricing.", + } + ) spreads = [] for task in workload["tasks"]: - values = [ - trace.elapsed_ms - for (task_id, _), trace in completed.items() - if task_id == task["id"] and _trace_usable(trace) - ] - middle = statistics.median(values) if len(values) >= 2 else 0 + task_values = [value for (task_id, _), value in values.items() if task_id == task["id"]] + middle = statistics.median(task_values) if len(task_values) >= 2 else 0 if middle > 0: - spreads.append((max(values) - min(values)) / middle * 100) + spreads.append((max(task_values) - min(task_values)) / middle * 100) if spreads: spread = statistics.median(spreads) if spread >= NOISY_SPREAD_PCT and spread > minimum_improvement: notes.append( { "kind": "noisy_baseline", + "objective": objective, "spread_pct": round(spread, 2), - "message": f"Baseline latency of the same task varied by a median " + "message": f"Baseline {objective} of the same task varied by a median " f"{spread:.1f}% between repetitions, more than the " f"{minimum_improvement:g}% required improvement. Smaller differences " "may be noise: require a larger improvement, add repetitions, or " diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 487e9a6..fef0c30 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -286,8 +286,12 @@ print these notes, record them in the frozen session, and list them under - fewer than five pairs: even if every pair improves, the sign test cannot fall below p = 0.05; - one repetition per task: run-to-run variation cannot be measured; -- noisy baseline latency: repetitions of the same task varied by a median of at - least 10%, more than `--minimum-improvement`. +- noisy baseline: repetitions of the same task varied by a median of at least + 10% in the selected objective, more than `--minimum-improvement`. Latency uses + trace time; from AgentLoop 0.2.7 the cost objective uses estimated cost; +- unmeasurable cost (0.2.7 and newer, cost objective only): some completed + baseline attempts have unknown model pricing or estimated token counts, so + cost gates for them will be indeterminate. The notes do not change the decision. Respond by adding tasks or repetitions, requiring a larger improvement, or comparing against an interleaved control. diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 8ce9660..4ba5028 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -909,17 +909,31 @@ def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload): assert (tmp_path / "compare/results/report.json").is_file() -def _guidance(repetitions, elapsed, *, objective="latency", minimum=0, tasks=3): +def _guidance( + repetitions, elapsed, *, objective="latency", minimum=0, tasks=3, costs=None, report=None +): workload = { "tasks": [{"id": f"t{index}"} for index in range(tasks)], "repetitions": repetitions, } + costs = costs or [0.0] * repetitions runs, traces = [], {} for index in range(tasks): for repetition in range(repetitions): key = (f"t{index}", repetition) runs.append({"task_id": key[0], "repetition": repetition, "status": "completed"}) - traces[key] = SimpleNamespace(elapsed_ms=elapsed[repetition], metadata={}, events=[]) + summary = { + "estimated_cost_usd": costs[repetition], + "has_unknown_cost": False, + "token_status": "exact", + **(report or {}), + } + traces[key] = SimpleNamespace( + elapsed_ms=elapsed[repetition], + metadata={}, + events=[], + report=lambda summary=summary: summary, + ) return planning_guidance(workload, {"runs": runs}, traces, objective, minimum) @@ -940,7 +954,25 @@ def test_planning_guidance_flags_baseline_noise_above_the_required_gain(): def test_planning_guidance_is_quiet_for_stable_or_well_bounded_baselines(): assert _guidance(2, [100, 104]) == [] assert _guidance(2, [100, 140], minimum=50) == [] - assert _guidance(2, [100, 140], objective="cost") == [] + assert _guidance(2, [100, 140], objective="cost", costs=[0.010, 0.0104]) == [] + assert _guidance(2, [100, 104], objective="cost", costs=[0.0, 0.0]) == [] + + +def test_planning_guidance_flags_noisy_baseline_cost(): + notes = _guidance(2, [100, 100], objective="cost", costs=[0.010, 0.016], minimum=10) + assert [note["kind"] for note in notes] == ["noisy_baseline"] + assert notes[0]["objective"] == "cost" + assert notes[0]["spread_pct"] == pytest.approx(46.15, abs=0.01) + assert "Baseline cost of the same task" in notes[0]["message"] + + +@pytest.mark.parametrize("report", [{"has_unknown_cost": True}, {"token_status": "estimated"}]) +def test_planning_guidance_reports_unmeasurable_cost(report): + notes = _guidance(2, [100, 100], objective="cost", costs=[0.01, 0.02], report=report) + assert [note["kind"] for note in notes] == ["cost_unmeasurable"] + assert notes[0]["attempts"] == 6 + assert "6 of 6 completed baseline attempts" in notes[0]["message"] + assert _guidance(2, [100, 100], report=report) == [] def test_start_records_planning_notes_in_the_session_and_plan(tmp_path, workload):