diff --git a/CHANGELOG.md b/CHANGELOG.md index 211c328..07ebddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ trace, study, receipt, API and database schema versions. with a TypeScript SDK trace, and writes a failed trace when the agent throws. Without `--agent` it creates an offline example and a `package.json`. CI runs the complete guided workflow against the packed SDK on Node.js 22 and 24. +- Evidence-strength notes cover the cost objective. `improve start` and + `improve compare` flag a noisy baseline when estimated cost of the same task + varies between repetitions by at least 10% and more than the required + improvement, and report baseline attempts whose cost cannot be measured + because of unknown pricing or estimated token counts. ### Documentation diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 32bb854..2074191 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -24,6 +24,7 @@ from agentloop.replay import ReplayGates, build_replay_report from agentloop.replay import _summary as replay_summary from agentloop.schema import TraceValidationError, validate_elapsed_ms +from agentloop.tokens import is_token_basis_evaluable from agentloop.tracer import AgentTrace VERSION = "1.0" @@ -433,7 +434,9 @@ def planning_guidance(workload, manifest, traces, objective, minimum_improvement The sign test needs at least five pairs before even a unanimous result can fall below p = 0.05. When repeated baseline attempts of the same task vary - by more than the required improvement, smaller differences may be noise. + in the objective (latency, or estimated cost) by more than the required + improvement, smaller differences may be noise. For the cost objective, + attempts whose cost cannot be measured are reported separately. """ notes = [] pairs = len(workload["tasks"]) * workload["repetitions"] @@ -455,26 +458,50 @@ def planning_guidance(workload, manifest, traces, objective, minimum_improvement "measured. Use at least 2 repetitions to see how noisy the agent is.", } ) - elif objective == "latency": - completed = _completed(manifest, traces) + else: + completed = { + key: trace + for key, trace in _completed(manifest, traces).items() + if _trace_usable(trace) + } + if objective == "latency": + values = {key: trace.elapsed_ms for key, trace in completed.items()} + else: + values, unmeasurable = {}, 0 + for key, trace in completed.items(): + report = trace.report() + if report.get("has_unknown_cost") or not is_token_basis_evaluable( + report.get("token_status") + ): + unmeasurable += 1 + else: + values[key] = float(report.get("estimated_cost_usd") or 0.0) + if unmeasurable: + notes.append( + { + "kind": "cost_unmeasurable", + "attempts": unmeasurable, + "message": f"{unmeasurable} of {len(completed)} completed baseline " + "attempts have unknown model pricing or estimated token counts, so " + "their cost cannot be compared and cost gates will be indeterminate. " + "Record provider token usage and use models with known pricing.", + } + ) spreads = [] for task in workload["tasks"]: - values = [ - trace.elapsed_ms - for (task_id, _), trace in completed.items() - if task_id == task["id"] and _trace_usable(trace) - ] - middle = statistics.median(values) if len(values) >= 2 else 0 + task_values = [value for (task_id, _), value in values.items() if task_id == task["id"]] + middle = statistics.median(task_values) if len(task_values) >= 2 else 0 if middle > 0: - spreads.append((max(values) - min(values)) / middle * 100) + spreads.append((max(task_values) - min(task_values)) / middle * 100) if spreads: spread = statistics.median(spreads) if spread >= NOISY_SPREAD_PCT and spread > minimum_improvement: notes.append( { "kind": "noisy_baseline", + "objective": objective, "spread_pct": round(spread, 2), - "message": f"Baseline latency of the same task varied by a median " + "message": f"Baseline {objective} of the same task varied by a median " f"{spread:.1f}% between repetitions, more than the " f"{minimum_improvement:g}% required improvement. Smaller differences " "may be noise: require a larger improvement, add repetitions, or " diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 173541a..8de1391 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -315,8 +315,12 @@ print these notes, record them in the frozen session, and list them under - fewer than five pairs: even if every pair improves, the sign test cannot fall below p = 0.05; - one repetition per task: run-to-run variation cannot be measured; -- noisy baseline latency: repetitions of the same task varied by a median of at - least 10%, more than `--minimum-improvement`. +- noisy baseline: repetitions of the same task varied by a median of at least + 10% in the selected objective, more than `--minimum-improvement`. Latency uses + trace time; from AgentLoop 0.2.7 the cost objective uses estimated cost; +- unmeasurable cost (0.2.7 and newer, cost objective only): some completed + baseline attempts have unknown model pricing or estimated token counts, so + cost gates for them will be indeterminate. The notes do not change the decision. Respond by adding tasks or repetitions, requiring a larger improvement, or comparing against an interleaved control. diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 8ce9660..4ba5028 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -909,17 +909,31 @@ def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload): assert (tmp_path / "compare/results/report.json").is_file() -def _guidance(repetitions, elapsed, *, objective="latency", minimum=0, tasks=3): +def _guidance( + repetitions, elapsed, *, objective="latency", minimum=0, tasks=3, costs=None, report=None +): workload = { "tasks": [{"id": f"t{index}"} for index in range(tasks)], "repetitions": repetitions, } + costs = costs or [0.0] * repetitions runs, traces = [], {} for index in range(tasks): for repetition in range(repetitions): key = (f"t{index}", repetition) runs.append({"task_id": key[0], "repetition": repetition, "status": "completed"}) - traces[key] = SimpleNamespace(elapsed_ms=elapsed[repetition], metadata={}, events=[]) + summary = { + "estimated_cost_usd": costs[repetition], + "has_unknown_cost": False, + "token_status": "exact", + **(report or {}), + } + traces[key] = SimpleNamespace( + elapsed_ms=elapsed[repetition], + metadata={}, + events=[], + report=lambda summary=summary: summary, + ) return planning_guidance(workload, {"runs": runs}, traces, objective, minimum) @@ -940,7 +954,25 @@ def test_planning_guidance_flags_baseline_noise_above_the_required_gain(): def test_planning_guidance_is_quiet_for_stable_or_well_bounded_baselines(): assert _guidance(2, [100, 104]) == [] assert _guidance(2, [100, 140], minimum=50) == [] - assert _guidance(2, [100, 140], objective="cost") == [] + assert _guidance(2, [100, 140], objective="cost", costs=[0.010, 0.0104]) == [] + assert _guidance(2, [100, 104], objective="cost", costs=[0.0, 0.0]) == [] + + +def test_planning_guidance_flags_noisy_baseline_cost(): + notes = _guidance(2, [100, 100], objective="cost", costs=[0.010, 0.016], minimum=10) + assert [note["kind"] for note in notes] == ["noisy_baseline"] + assert notes[0]["objective"] == "cost" + assert notes[0]["spread_pct"] == pytest.approx(46.15, abs=0.01) + assert "Baseline cost of the same task" in notes[0]["message"] + + +@pytest.mark.parametrize("report", [{"has_unknown_cost": True}, {"token_status": "estimated"}]) +def test_planning_guidance_reports_unmeasurable_cost(report): + notes = _guidance(2, [100, 100], objective="cost", costs=[0.01, 0.02], report=report) + assert [note["kind"] for note in notes] == ["cost_unmeasurable"] + assert notes[0]["attempts"] == 6 + assert "6 of 6 completed baseline attempts" in notes[0]["message"] + assert _guidance(2, [100, 100], report=report) == [] def test_start_records_planning_notes_in_the_session_and_plan(tmp_path, workload):