Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ trace, study, receipt, API and database schema versions.
with a TypeScript SDK trace, and writes a failed trace when the agent throws.
Without `--agent` it creates an offline example and a `package.json`. CI runs
the complete guided workflow against the packed SDK on Node.js 22 and 24.
- Evidence-strength notes cover the cost objective. `improve start` and
`improve compare` flag a noisy baseline when estimated cost of the same task
varies between repetitions by at least 10% and more than the required
improvement, and report baseline attempts whose cost cannot be measured
because of unknown pricing or estimated token counts.

### Documentation

Expand Down
49 changes: 38 additions & 11 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from agentloop.replay import ReplayGates, build_replay_report
from agentloop.replay import _summary as replay_summary
from agentloop.schema import TraceValidationError, validate_elapsed_ms
from agentloop.tokens import is_token_basis_evaluable
from agentloop.tracer import AgentTrace

VERSION = "1.0"
Expand Down Expand Up @@ -433,7 +434,9 @@ def planning_guidance(workload, manifest, traces, objective, minimum_improvement

The sign test needs at least five pairs before even a unanimous result can
fall below p = 0.05. When repeated baseline attempts of the same task vary
by more than the required improvement, smaller differences may be noise.
in the objective (latency, or estimated cost) by more than the required
improvement, smaller differences may be noise. For the cost objective,
attempts whose cost cannot be measured are reported separately.
"""
notes = []
pairs = len(workload["tasks"]) * workload["repetitions"]
Expand All @@ -455,26 +458,50 @@ def planning_guidance(workload, manifest, traces, objective, minimum_improvement
"measured. Use at least 2 repetitions to see how noisy the agent is.",
}
)
elif objective == "latency":
completed = _completed(manifest, traces)
else:
completed = {
key: trace
for key, trace in _completed(manifest, traces).items()
if _trace_usable(trace)
}
if objective == "latency":
values = {key: trace.elapsed_ms for key, trace in completed.items()}
else:
values, unmeasurable = {}, 0
for key, trace in completed.items():
report = trace.report()
if report.get("has_unknown_cost") or not is_token_basis_evaluable(
report.get("token_status")
):
unmeasurable += 1
else:
values[key] = float(report.get("estimated_cost_usd") or 0.0)
if unmeasurable:
notes.append(
{
"kind": "cost_unmeasurable",
"attempts": unmeasurable,
"message": f"{unmeasurable} of {len(completed)} completed baseline "
"attempts have unknown model pricing or estimated token counts, so "
"their cost cannot be compared and cost gates will be indeterminate. "
"Record provider token usage and use models with known pricing.",
}
)
spreads = []
for task in workload["tasks"]:
values = [
trace.elapsed_ms
for (task_id, _), trace in completed.items()
if task_id == task["id"] and _trace_usable(trace)
]
middle = statistics.median(values) if len(values) >= 2 else 0
task_values = [value for (task_id, _), value in values.items() if task_id == task["id"]]
middle = statistics.median(task_values) if len(task_values) >= 2 else 0
if middle > 0:
spreads.append((max(values) - min(values)) / middle * 100)
spreads.append((max(task_values) - min(task_values)) / middle * 100)
if spreads:
spread = statistics.median(spreads)
if spread >= NOISY_SPREAD_PCT and spread > minimum_improvement:
notes.append(
{
"kind": "noisy_baseline",
"objective": objective,
"spread_pct": round(spread, 2),
"message": f"Baseline latency of the same task varied by a median "
"message": f"Baseline {objective} of the same task varied by a median "
f"{spread:.1f}% between repetitions, more than the "
f"{minimum_improvement:g}% required improvement. Smaller differences "
"may be noise: require a larger improvement, add repetitions, or "
Expand Down
8 changes: 6 additions & 2 deletions docs/GUIDED_OPTIMIZATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,12 @@ print these notes, record them in the frozen session, and list them under
- fewer than five pairs: even if every pair improves, the sign test cannot fall
below p = 0.05;
- one repetition per task: run-to-run variation cannot be measured;
- noisy baseline latency: repetitions of the same task varied by a median of at
least 10%, more than `--minimum-improvement`.
- noisy baseline: repetitions of the same task varied by a median of at least
10% in the selected objective, more than `--minimum-improvement`. Latency uses
trace time; from AgentLoop 0.2.7 the cost objective uses estimated cost;
- unmeasurable cost (0.2.7 and newer, cost objective only): some completed
baseline attempts have unknown model pricing or estimated token counts, so
cost gates for them will be indeterminate.

The notes do not change the decision. Respond by adding tasks or repetitions,
requiring a larger improvement, or comparing against an interleaved control.
Expand Down
38 changes: 35 additions & 3 deletions tests/test_improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -909,17 +909,31 @@ def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload):
assert (tmp_path / "compare/results/report.json").is_file()


def _guidance(repetitions, elapsed, *, objective="latency", minimum=0, tasks=3):
def _guidance(
repetitions, elapsed, *, objective="latency", minimum=0, tasks=3, costs=None, report=None
):
workload = {
"tasks": [{"id": f"t{index}"} for index in range(tasks)],
"repetitions": repetitions,
}
costs = costs or [0.0] * repetitions
runs, traces = [], {}
for index in range(tasks):
for repetition in range(repetitions):
key = (f"t{index}", repetition)
runs.append({"task_id": key[0], "repetition": repetition, "status": "completed"})
traces[key] = SimpleNamespace(elapsed_ms=elapsed[repetition], metadata={}, events=[])
summary = {
"estimated_cost_usd": costs[repetition],
"has_unknown_cost": False,
"token_status": "exact",
**(report or {}),
}
traces[key] = SimpleNamespace(
elapsed_ms=elapsed[repetition],
metadata={},
events=[],
report=lambda summary=summary: summary,
)
return planning_guidance(workload, {"runs": runs}, traces, objective, minimum)


Expand All @@ -940,7 +954,25 @@ def test_planning_guidance_flags_baseline_noise_above_the_required_gain():
def test_planning_guidance_is_quiet_for_stable_or_well_bounded_baselines():
assert _guidance(2, [100, 104]) == []
assert _guidance(2, [100, 140], minimum=50) == []
assert _guidance(2, [100, 140], objective="cost") == []
assert _guidance(2, [100, 140], objective="cost", costs=[0.010, 0.0104]) == []
assert _guidance(2, [100, 104], objective="cost", costs=[0.0, 0.0]) == []


def test_planning_guidance_flags_noisy_baseline_cost():
notes = _guidance(2, [100, 100], objective="cost", costs=[0.010, 0.016], minimum=10)
assert [note["kind"] for note in notes] == ["noisy_baseline"]
assert notes[0]["objective"] == "cost"
assert notes[0]["spread_pct"] == pytest.approx(46.15, abs=0.01)
assert "Baseline cost of the same task" in notes[0]["message"]


@pytest.mark.parametrize("report", [{"has_unknown_cost": True}, {"token_status": "estimated"}])
def test_planning_guidance_reports_unmeasurable_cost(report):
notes = _guidance(2, [100, 100], objective="cost", costs=[0.01, 0.02], report=report)
assert [note["kind"] for note in notes] == ["cost_unmeasurable"]
assert notes[0]["attempts"] == 6
assert "6 of 6 completed baseline attempts" in notes[0]["message"]
assert _guidance(2, [100, 100], report=report) == []


def test_start_records_planning_notes_in_the_session_and_plan(tmp_path, workload):
Expand Down
Loading