diff --git a/README.md b/README.md index d9c9690..77fa7ac 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,10 @@ stored original by address instead of reading and re-hashing it. > session's cost follows its turns (each about 18,000 input tokens, cache > included), the model narrows most commands itself, and outputs over about > 30 KB are saved to a file by Claude Code, which shows its own preview, so -> views reached the model on six of the sixteen tasks. The reports are in +> views reached the model on six of the sixteen tasks. Counted by the API's +> tokenizer, the views of the 26 benchmark outputs take 92.6% fewer tokens +> than the raw outputs; replayed steps show the agent sometimes makes one +> verifying call where a view leaves the answer out. The reports are in > `bench/results/`. Whole-session cost depends on your host, model and task. > Measure your own setup before you tell anyone a percentage. > [What is verified](docs/verification.md) · diff --git a/bench/README.md b/bench/README.md index 637dba1..a21723c 100644 --- a/bench/README.md +++ b/bench/README.md @@ -36,7 +36,12 @@ The current live campaign is `live-20261105-121-haiku.json`: 1.2.1 against no Scopelet, in sessions that search like a default Claude Code session. A session with Scopelet cost about 9% less on average, the 95% interval below zero, with 47 of 48 tasks right against 48; see -[1.2.1 against no Scopelet](#121-against-no-scopelet-november-2026). +[1.2.1 against no Scopelet](#121-against-no-scopelet-november-2026). The +current token count (`tokens-20261105-122.json`, the 1.2.2 release) finds +views 92.6% smaller in tokens than the raw outputs on 26 cases, exactly as in +1.0, and the current replay (`nap-20261105-122.json`) finds the view keeps the +agent's next action on 59% of draws, against 92% for raw output: where a view +leaves the answer out, the agent makes one verifying call first. The 1.0 campaign (`*-20261016-single*.json`) measured the single format (binary `ba3eb754e660…`) against the 0.11.1 release (`c4432301dec7…`, compact-v4 @@ -157,6 +162,15 @@ Flags: `--live` (required), `--scopelet BINARY` (default default `claude-opus-5-5` and `claude-haiku-5-5`), `--out PATH` (default `bench/results/tokens.json`, which must not exist). +The current report, `results/tokens-20261105-122.json`, counts the 1.2.2 +release binary (`59d26d8cee4a…`) on the 26 cases for `claude-opus-5-5` and +`claude-haiku-5-5`: 521,703 tokens raw, 38,471 in views, 92.6% fewer, and no +case costs more than its raw output (`rust_small`, under the file-read +threshold, passes through at 1,239). Every case counts exactly what 1.0 did: +the views of these outputs have not changed since. `giant_unicode_line`'s +fact lies past the 1.1 KB prefix the view keeps of that one line: recoverable +with `expand`, not visible, as in the 1.0 report. + The kept `results/tokens.json` measured the 0.10.0 grammar parts, before the format was unified: on its 26 cases compact-v5 cost 11.8% fewer view tokens than v4 for both models, and no case cost more. Most of that gap was the @@ -502,6 +516,30 @@ BINARY` (default `target/release/scopelet`), `--baseline BINARY`, `--arm NAME` `--out DIR` (required, must not exist). The report names each condition's binary by hash. +`results/nap-20261105-122.json` is the current replay: the 48 `off` sessions +of the 1.2.1 campaign (`live-20261105-121-haiku.json`), with raw output and +the 1.2.2 release binary's view (`single`, `59d26d8cee4a…`), `claude-haiku-5-5`, +3 draws each. Of their large results, 13 were saved by Claude Code (persisted; +every arm shows the same preview); 17 steps remain, no failed call: + +| Form | View bytes | Mean input tokens | NAP | Coarse NAP | +|---|---:|---:|---:|---:| +| raw | 281,361 | 11,202 | 0.92 | 1.00 | +| single | 73,774 | 4,637 | 0.59 | 0.63 | + +The view kept the next action on every step of `fix_test`, +`build_warnings_medium` and `test_report_medium` (9 steps), where the failure +or the error's location is in the view. On `grep_context` (3 steps) the raw +output let the model answer at once; with the view, which says +`gaps are not absence`, it ran one more search for `refund` first. The prompt +names that word, but prompt anchors are specific terms only (identifiers, +paths, quoted text, error codes), so the two `refund` lines among 200 calls +were not ranked in. On `diff_medium` (3 steps) it read `quota.py` or diffed it +again before answering on six of nine draws. The other two steps +(`build_warnings`, `stack_trace`) are noise: raw's own NAP there is 0.33. A view +costs a verifying call where it leaves the answer out, and the live campaign +shows that call is cheaper than reading everything. + `results/nap-20261016.json` (before 1.0: four forms, raw, compact-v4, v5 with its grammar only and v5 whole) replays the 36 `off` sessions of the haiku campaign with `claude-haiku-5-5`. Of their large results, 8 were saved by diff --git a/bench/results/nap-20261105-122.json b/bench/results/nap-20261105-122.json new file mode 100644 index 0000000..b0f12ec --- /dev/null +++ b/bench/results/nap-20261105-122.json @@ -0,0 +1,2312 @@ +{ + "kind": "next-action preservation; tokens from the API usage, not estimates", + "model": "claude-haiku-5-5", + "draws": 3, + "conditions": [ + { + "name": "raw", + "binary_sha256": null + }, + { + "name": "single", + "binary_sha256": "59d26d8cee4a6445d4fc4861386d967af2b0318ebee2ca3c67db321efce7e116" + } + ], + "steps": [ + { + "transcript": "0-build_warnings_medium-off.jsonl.gz", + "round": 0, + "task": "build_warnings_medium", + "step": 0, + "tool": "Bash", + "command": "python3 build.py", + "view_bytes": { + "raw": 17584, + "single": 4077 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 9637, + "cache_creation_input_tokens": 1164, + "cache_read_input_tokens": 0, + "output_tokens": 164, + "input_total": 10801, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 9637, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 124, + "input_total": 10801, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 9637, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 134, + "input_total": 10801, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 401, + "input_total": 3171, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 361, + "input_total": 3171, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 264, + "input_total": 3171, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ] + } + }, + { + "transcript": "0-diff_medium-off.jsonl.gz", + "round": 0, + "task": "diff_medium", + "step": 0, + "tool": "Bash", + "command": "git diff", + "view_bytes": { + "raw": 15650, + "single": 8071 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 0.3333333333333333 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.3333333333333333 + }, + "draws": { + "raw": [ + { + "input_tokens": 7777, + "cache_creation_input_tokens": 1130, + "cache_read_input_tokens": 0, + "output_tokens": 133, + "input_total": 8907, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 153, + "input_total": 8907, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 125, + "input_total": 8907, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 562, + "input_total": 5117, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 607, + "input_total": 5117, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "git diff", + "paths": [], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 235, + "input_total": 5117, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "/home/user/src/quota.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "0-fix_test-off.jsonl.gz", + "round": 0, + "task": "fix_test", + "step": 0, + "tool": "Bash", + "command": "python3 -m unittest -v test_inventory 2>&1; echo \"---\"; cat inventory.py", + "view_bytes": { + "raw": 21854, + "single": 1418 + }, + "raw_action": { + "kind": "tool", + "paths": [ + "inventory.py" + ], + "pattern": null, + "program": null, + "range": null, + "tool": "Edit" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 9770, + "cache_creation_input_tokens": 1164, + "cache_read_input_tokens": 0, + "output_tokens": 243, + "input_total": 10934, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 9770, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 236, + "input_total": 10934, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 9770, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 236, + "input_total": 10934, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 619, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 285, + "input_total": 1783, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 619, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 347, + "input_total": 1783, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 619, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1164, + "output_tokens": 318, + "input_total": 1783, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "0-grep_context-off.jsonl.gz", + "round": 0, + "task": "grep_context", + "step": 0, + "tool": "Bash", + "command": "grep -rn -C2 charge_card src", + "view_bytes": { + "raw": 23157, + "single": 8179 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 0.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 15218, + "cache_creation_input_tokens": 1162, + "cache_read_input_tokens": 0, + "output_tokens": 141, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 140, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 190, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 5175, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 495, + "input_total": 6337, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Grep", + "program": null, + "paths": [ + "src" + ], + "pattern": "refund", + "range": null + }, + "error": null + }, + { + "input_tokens": 5175, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 525, + "input_total": 6337, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Grep", + "program": null, + "paths": [ + "src" + ], + "pattern": "refund", + "range": null + }, + "error": null + }, + { + "input_tokens": 5175, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 421, + "input_total": 6337, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [], + "pattern": "refund", + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "0-test_report_medium-off.jsonl.gz", + "round": 0, + "task": "test_report_medium", + "step": 1, + "tool": "Bash", + "command": "python3 -m unittest -v test_shipping", + "view_bytes": { + "raw": 10449, + "single": 274 + }, + "raw_action": { + "correct": null, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 5285, + "cache_creation_input_tokens": 7539, + "cache_read_input_tokens": 0, + "output_tokens": 133, + "input_total": 12824, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 5285, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7539, + "output_tokens": 115, + "input_total": 12824, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 5285, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7539, + "output_tokens": 79, + "input_total": 12824, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 131, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7539, + "output_tokens": 116, + "input_total": 7670, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 131, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7539, + "output_tokens": 101, + "input_total": 7670, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 131, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7539, + "output_tokens": 128, + "input_total": 7670, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + } + ] + } + }, + { + "transcript": "1-build_warnings-off.jsonl.gz", + "round": 1, + "task": "build_warnings", + "step": 0, + "tool": "Bash", + "command": "python3 build.py 2>&1 | tail -n 200", + "view_bytes": { + "raw": 6040, + "single": 4047 + }, + "raw_action": { + "kind": "tool", + "paths": [ + "/tmp/build.log", + "build.py" + ], + "pattern": "^error", + "program": "python3", + "range": "head:60", + "tool": "Bash" + }, + "nap": { + "raw": 0.3333333333333333, + "single": 0.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.6666666666666666 + }, + "draws": { + "raw": [ + { + "input_tokens": 3349, + "cache_creation_input_tokens": 1130, + "cache_read_input_tokens": 0, + "output_tokens": 213, + "input_total": 4479, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "python3", + "paths": [ + "/tmp/build.log", + "build.py" + ], + "pattern": "^error", + "range": "head:60" + }, + "error": null + }, + { + "input_tokens": 3349, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 198, + "input_total": 4479, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "python3", + "paths": [ + "/tmp/build.log", + "build.py" + ], + "pattern": "^error", + "range": "head:40" + }, + "error": null + }, + { + "input_tokens": 3349, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 193, + "input_total": 4479, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "python3", + "paths": [ + "/tmp/build.log", + "build.py" + ], + "pattern": "^error", + "range": "head:80" + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 2056, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 244, + "input_total": 3186, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "python3", + "paths": [ + "/tmp/build.log", + "build.py" + ], + "pattern": "\"^(error", + "range": "head:20" + }, + "error": null + }, + { + "input_tokens": 2056, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 247, + "input_total": 3186, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "python3", + "paths": [ + "/tmp/build_out.txt", + "build.py" + ], + "pattern": "\"^(error", + "range": "head:80" + }, + "error": null + }, + { + "input_tokens": 2056, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 240, + "input_total": 3186, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "python3", + "paths": [ + "/tmp/build.log", + "build.py" + ], + "pattern": "\"^(error", + "range": "head:20" + }, + "error": null + } + ] + } + }, + { + "transcript": "1-build_warnings_medium-off.jsonl.gz", + "round": 1, + "task": "build_warnings_medium", + "step": 0, + "tool": "Bash", + "command": "python3 build.py", + "view_bytes": { + "raw": 17584, + "single": 4077 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 9637, + "cache_creation_input_tokens": 1128, + "cache_read_input_tokens": 0, + "output_tokens": 170, + "input_total": 10765, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 9637, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1128, + "output_tokens": 101, + "input_total": 10765, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 9637, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1128, + "output_tokens": 140, + "input_total": 10765, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1128, + "output_tokens": 283, + "input_total": 3135, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1128, + "output_tokens": 331, + "input_total": 3135, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1128, + "output_tokens": 235, + "input_total": 3135, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ] + } + }, + { + "transcript": "1-diff_medium-off.jsonl.gz", + "round": 1, + "task": "diff_medium", + "step": 0, + "tool": "Bash", + "command": "git diff", + "view_bytes": { + "raw": 15650, + "single": 8071 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 0.6666666666666666 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.6666666666666666 + }, + "draws": { + "raw": [ + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 145, + "input_total": 8907, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 126, + "input_total": 8907, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 126, + "input_total": 8907, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 664, + "input_total": 5117, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 305, + "input_total": 5117, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1130, + "output_tokens": 676, + "input_total": 5117, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "git diff", + "paths": [], + "pattern": null, + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "1-fix_test-off.jsonl.gz", + "round": 1, + "task": "fix_test", + "step": 0, + "tool": "Bash", + "command": "python3 -m unittest -v test_inventory 2>&1; echo \"EXIT=$?\"; ls -la", + "view_bytes": { + "raw": 21579, + "single": 1091 + }, + "raw_action": { + "kind": "tool", + "paths": [ + "inventory.py" + ], + "pattern": null, + "program": null, + "range": null, + "tool": "Read" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 9679, + "cache_creation_input_tokens": 1168, + "cache_read_input_tokens": 0, + "output_tokens": 52, + "input_total": 10847, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 9679, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1168, + "output_tokens": 52, + "input_total": 10847, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 9679, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1168, + "output_tokens": 52, + "input_total": 10847, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1168, + "output_tokens": 130, + "input_total": 1668, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1168, + "output_tokens": 180, + "input_total": 1668, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 500, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1168, + "output_tokens": 319, + "input_total": 1668, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "1-grep_context-off.jsonl.gz", + "round": 1, + "task": "grep_context", + "step": 0, + "tool": "Bash", + "command": "grep -rn -C2 charge_card src", + "view_bytes": { + "raw": 23157, + "single": 8179 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 0.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 153, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 158, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 146, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 5184, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 342, + "input_total": 6346, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [], + "pattern": "refund", + "range": null + }, + "error": null + }, + { + "input_tokens": 5184, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 291, + "input_total": 6346, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [], + "pattern": "refund", + "range": null + }, + "error": null + }, + { + "input_tokens": 5184, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 462, + "input_total": 6346, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [], + "pattern": "refund", + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "1-stack_trace-off.jsonl.gz", + "round": 1, + "task": "stack_trace", + "step": 0, + "tool": "Bash", + "command": "ls -la logs && grep -n -i \"settlement\\|Caused by\\|exception\" logs/service.log | head -100", + "view_bytes": { + "raw": 9507, + "single": 3993 + }, + "raw_action": { + "kind": "tool", + "paths": [ + "service.log" + ], + "pattern": "\"ERROR\\", + "program": "grep", + "range": "head:150", + "tool": "Bash" + }, + "nap": { + "raw": 0.3333333333333333, + "single": 0.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 5194, + "cache_creation_input_tokens": 1171, + "cache_read_input_tokens": 0, + "output_tokens": 150, + "input_total": 6365, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [ + "service.log" + ], + "pattern": "\"ERROR\\", + "range": "head:150" + }, + "error": null + }, + { + "input_tokens": 5194, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1171, + "output_tokens": 197, + "input_total": 6365, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [ + "service.log" + ], + "pattern": "\"Caused", + "range": "head:100" + }, + "error": null + }, + { + "input_tokens": 5194, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1171, + "output_tokens": 166, + "input_total": 6365, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [ + "service.log" + ], + "pattern": "\"ERROR", + "range": "head:50" + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 2194, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1171, + "output_tokens": 250, + "input_total": 3365, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [ + "logs/service.log" + ], + "pattern": "\"Caused", + "range": "head:50" + }, + "error": null + }, + { + "input_tokens": 2194, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1171, + "output_tokens": 188, + "input_total": 3365, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [ + "logs/service.log" + ], + "pattern": "\"Caused", + "range": "head:100" + }, + "error": null + }, + { + "input_tokens": 2194, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1171, + "output_tokens": 240, + "input_total": 3365, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [ + "logs/service.log" + ], + "pattern": "\"caused", + "range": "head:150" + }, + "error": null + } + ] + } + }, + { + "transcript": "1-test_report_medium-off.jsonl.gz", + "round": 1, + "task": "test_report_medium", + "step": 1, + "tool": "Bash", + "command": "python3 -m unittest -v test_shipping", + "view_bytes": { + "raw": 10449, + "single": 274 + }, + "raw_action": { + "correct": null, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 5285, + "cache_creation_input_tokens": 7556, + "cache_read_input_tokens": 0, + "output_tokens": 103, + "input_total": 12841, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 5285, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7556, + "output_tokens": 103, + "input_total": 12841, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 5285, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7556, + "output_tokens": 109, + "input_total": 12841, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 131, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7556, + "output_tokens": 116, + "input_total": 7687, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 131, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7556, + "output_tokens": 120, + "input_total": 7687, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 131, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7556, + "output_tokens": 79, + "input_total": 7687, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + } + ] + } + }, + { + "transcript": "2-build_warnings_medium-off.jsonl.gz", + "round": 2, + "task": "build_warnings_medium", + "step": 0, + "tool": "Bash", + "command": "python3 build.py", + "view_bytes": { + "raw": 17584, + "single": 4077 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 9637, + "cache_creation_input_tokens": 1131, + "cache_read_input_tokens": 0, + "output_tokens": 142, + "input_total": 10768, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 9637, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1131, + "output_tokens": 137, + "input_total": 10768, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 9637, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1131, + "output_tokens": 135, + "input_total": 10768, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1131, + "output_tokens": 314, + "input_total": 3138, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1131, + "output_tokens": 231, + "input_total": 3138, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 2007, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1131, + "output_tokens": 363, + "input_total": 3138, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ] + } + }, + { + "transcript": "2-diff_medium-off.jsonl.gz", + "round": 2, + "task": "diff_medium", + "step": 0, + "tool": "Bash", + "command": "git diff", + "view_bytes": { + "raw": 15650, + "single": 8071 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 0.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 7777, + "cache_creation_input_tokens": 1129, + "cache_read_input_tokens": 0, + "output_tokens": 145, + "input_total": 8906, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1129, + "output_tokens": 143, + "input_total": 8906, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 7777, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1129, + "output_tokens": 145, + "input_total": 8906, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1129, + "output_tokens": 378, + "input_total": 5116, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Read", + "program": null, + "paths": [ + "src/quota.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1129, + "output_tokens": 711, + "input_total": 5116, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "git diff", + "paths": [ + "src/area_bh.py", + "src/quota.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 3987, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1129, + "output_tokens": 666, + "input_total": 5116, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "git diff", + "paths": [ + "src/quota.py" + ], + "pattern": "^[+-]", + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "2-fix_test-off.jsonl.gz", + "round": 2, + "task": "fix_test", + "step": 0, + "tool": "Bash", + "command": "python3 -m unittest -v test_inventory; echo \"EXIT=$?\"; echo ---; cat inventory.py", + "view_bytes": { + "raw": 21861, + "single": 1426 + }, + "raw_action": { + "kind": "tool", + "paths": [ + "inventory.py" + ], + "pattern": null, + "program": null, + "range": null, + "tool": "Edit" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 9778, + "cache_creation_input_tokens": 1170, + "cache_read_input_tokens": 0, + "output_tokens": 331, + "input_total": 10948, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 9778, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1170, + "output_tokens": 339, + "input_total": 10948, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 9778, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1170, + "output_tokens": 307, + "input_total": 10948, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 629, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1170, + "output_tokens": 341, + "input_total": 1799, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 629, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1170, + "output_tokens": 364, + "input_total": 1799, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + }, + { + "input_tokens": 629, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1170, + "output_tokens": 317, + "input_total": 1799, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Edit", + "program": null, + "paths": [ + "inventory.py" + ], + "pattern": null, + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "2-grep_context-off.jsonl.gz", + "round": 2, + "task": "grep_context", + "step": 0, + "tool": "Bash", + "command": "grep -rn -C2 charge_card src", + "view_bytes": { + "raw": 23157, + "single": 8175 + }, + "raw_action": { + "correct": true, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 0.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 0.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 118, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 97, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + }, + { + "input_tokens": 15218, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 141, + "input_total": 16380, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": true + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 5190, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 341, + "input_total": 6352, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Grep", + "program": null, + "paths": [ + "src" + ], + "pattern": "refund", + "range": null + }, + "error": null + }, + { + "input_tokens": 5190, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 323, + "input_total": 6352, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [], + "pattern": "refund", + "range": null + }, + "error": null + }, + { + "input_tokens": 5190, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1162, + "output_tokens": 294, + "input_total": 6352, + "stop_reason": "tool_use", + "action": { + "kind": "tool", + "tool": "Bash", + "program": "grep", + "paths": [], + "pattern": "refund", + "range": null + }, + "error": null + } + ] + } + }, + { + "transcript": "2-test_report_medium-off.jsonl.gz", + "round": 2, + "task": "test_report_medium", + "step": 0, + "tool": "Bash", + "command": "python3 -m unittest -v test_shipping", + "view_bytes": { + "raw": 10449, + "single": 274 + }, + "raw_action": { + "correct": null, + "kind": "final" + }, + "nap": { + "raw": 1.0, + "single": 1.0 + }, + "nap_coarse": { + "raw": 1.0, + "single": 1.0 + }, + "draws": { + "raw": [ + { + "input_tokens": 5285, + "cache_creation_input_tokens": 7720, + "cache_read_input_tokens": 0, + "output_tokens": 109, + "input_total": 13005, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 5285, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7720, + "output_tokens": 113, + "input_total": 13005, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 5285, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7720, + "output_tokens": 82, + "input_total": 13005, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + } + ], + "single": [ + { + "input_tokens": 130, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7720, + "output_tokens": 108, + "input_total": 7850, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 130, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7720, + "output_tokens": 124, + "input_total": 7850, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + }, + { + "input_tokens": 130, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7720, + "output_tokens": 120, + "input_total": 7850, + "stop_reason": "end_turn", + "action": { + "kind": "final", + "correct": null + }, + "error": null + } + ] + } + } + ], + "skipped": [ + { + "transcript": "0-listing_artifacts-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "0-pytest_many-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "0-review_diff-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "0-search_symbol-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "1-listing_artifacts-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "1-pytest_many-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "1-review_diff-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "1-search_symbol-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "2-build_warnings-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "2-listing_artifacts-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "2-pytest_many-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "2-review_diff-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + }, + { + "transcript": "2-search_symbol-off.jsonl.gz", + "step": 0, + "tool": "Bash", + "reason": "persisted by the host: every arm shows its preview" + } + ], + "summary": { + "steps": 17, + "steps_scored": 17, + "conditions": { + "raw": { + "mean_nap": 0.9215686274509803, + "mean_nap_coarse": 1.0, + "view_bytes": 281361, + "mean_input_total": 11202.176470588236, + "draws": 51, + "errors": 0 + }, + "single": { + "mean_nap": 0.5882352941176471, + "mean_nap_coarse": 0.6274509803921569, + "view_bytes": 73774, + "mean_input_total": 4637.470588235294, + "draws": 51, + "errors": 0 + } + }, + "by_task": { + "build_warnings": { + "steps": 1, + "raw": 0.3333333333333333, + "single": 0.0, + "coarse": { + "raw": 1.0, + "single": 0.6666666666666666 + } + }, + "build_warnings_medium": { + "steps": 3, + "raw": 1.0, + "single": 1.0, + "coarse": { + "raw": 1.0, + "single": 1.0 + } + }, + "diff_medium": { + "steps": 3, + "raw": 1.0, + "single": 0.3333333333333333, + "coarse": { + "raw": 1.0, + "single": 0.3333333333333333 + } + }, + "fix_test": { + "steps": 3, + "raw": 1.0, + "single": 1.0, + "coarse": { + "raw": 1.0, + "single": 1.0 + } + }, + "grep_context": { + "steps": 3, + "raw": 1.0, + "single": 0.0, + "coarse": { + "raw": 1.0, + "single": 0.0 + } + }, + "stack_trace": { + "steps": 1, + "raw": 0.3333333333333333, + "single": 0.0, + "coarse": { + "raw": 1.0, + "single": 0.0 + } + }, + "test_report_medium": { + "steps": 3, + "raw": 1.0, + "single": 1.0, + "coarse": { + "raw": 1.0, + "single": 1.0 + } + } + } + } +} diff --git a/bench/results/tokens-20261105-122.json b/bench/results/tokens-20261105-122.json new file mode 100644 index 0000000..0f419e1 --- /dev/null +++ b/bench/results/tokens-20261105-122.json @@ -0,0 +1,541 @@ +{ + "kind": "exact token counts from POST /v1/messages/count_tokens, net of the message frame", + "binary_sha256": { + "single": "59d26d8cee4a6445d4fc4861386d967af2b0318ebee2ca3c67db321efce7e116" + }, + "models": [ + "claude-opus-5-5", + "claude-haiku-5-5" + ], + "arms": [ + "single" + ], + "fixtures": { + "small": { + "bytes": { + "raw": 32, + "single": 32 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 8, + "single": 8 + }, + "claude-haiku-5-5": { + "raw": 8, + "single": 8 + } + } + }, + "diagnostics": { + "bytes": { + "raw": 136063, + "single": 304 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 56024, + "single": 137 + }, + "claude-haiku-5-5": { + "raw": 56024, + "single": 137 + } + } + }, + "repeated_errors": { + "bytes": { + "raw": 28039, + "single": 163 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 9014, + "single": 81 + }, + "claude-haiku-5-5": { + "raw": 9014, + "single": 81 + } + } + }, + "json": { + "bytes": { + "raw": 171834, + "single": 3209 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 59989, + "single": 1088 + }, + "claude-haiku-5-5": { + "raw": 59989, + "single": 1088 + } + } + }, + "jsonl": { + "bytes": { + "raw": 171832, + "single": 3694 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 60986, + "single": 1247 + }, + "claude-haiku-5-5": { + "raw": 60986, + "single": 1247 + } + } + }, + "hidden_receipt": { + "bytes": { + "raw": 136193, + "single": 428 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 56056, + "single": 166 + }, + "claude-haiku-5-5": { + "raw": 56056, + "single": 166 + } + } + }, + "giant_unicode_line": { + "bytes": { + "raw": 12000, + "single": 1094 + }, + "facts_visible": { + "single": false + }, + "tokens": { + "claude-opus-5-5": { + "raw": 3001, + "single": 302 + }, + "claude-haiku-5-5": { + "raw": 3001, + "single": 302 + } + } + }, + "crlf": { + "bytes": { + "raw": 137031, + "single": 266 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 56009, + "single": 119 + }, + "claude-haiku-5-5": { + "raw": 56009, + "single": 119 + } + } + }, + "cargo_test": { + "bytes": { + "raw": 9481, + "single": 2654 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 4251, + "single": 1263 + }, + "claude-haiku-5-5": { + "raw": 4251, + "single": 1263 + } + } + }, + "tsc": { + "bytes": { + "raw": 14716, + "single": 4080 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 6929, + "single": 1933 + }, + "claude-haiku-5-5": { + "raw": 6929, + "single": 1933 + } + } + }, + "pytest_traceback": { + "bytes": { + "raw": 27092, + "single": 3454 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 10169, + "single": 1409 + }, + "claude-haiku-5-5": { + "raw": 10169, + "single": 1409 + } + } + }, + "go_test": { + "bytes": { + "raw": 7242, + "single": 1423 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 4269, + "single": 758 + }, + "claude-haiku-5-5": { + "raw": 4269, + "single": 758 + } + } + }, + "java_npe": { + "bytes": { + "raw": 34113, + "single": 1509 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 16103, + "single": 693 + }, + "claude-haiku-5-5": { + "raw": 16103, + "single": 693 + } + } + }, + "jest": { + "bytes": { + "raw": 14580, + "single": 851 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 7758, + "single": 397 + }, + "claude-haiku-5-5": { + "raw": 7758, + "single": 397 + } + } + }, + "eslint": { + "bytes": { + "raw": 13977, + "single": 839 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 5899, + "single": 387 + }, + "claude-haiku-5-5": { + "raw": 5899, + "single": 387 + } + } + }, + "git_diff": { + "bytes": { + "raw": 12022, + "single": 3849 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 7213, + "single": 2111 + }, + "claude-haiku-5-5": { + "raw": 7213, + "single": 2111 + } + } + }, + "git_log_p": { + "bytes": { + "raw": 21353, + "single": 7083 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 12685, + "single": 3937 + }, + "claude-haiku-5-5": { + "raw": 12685, + "single": 3937 + } + } + }, + "kubectl_json": { + "bytes": { + "raw": 60370, + "single": 3465 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 17730, + "single": 1312 + }, + "claude-haiku-5-5": { + "raw": 17730, + "single": 1312 + } + } + }, + "gh_api": { + "bytes": { + "raw": 9033, + "single": 1094 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 4726, + "single": 517 + }, + "claude-haiku-5-5": { + "raw": 4726, + "single": 517 + } + } + }, + "service_log": { + "bytes": { + "raw": 106651, + "single": 783 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 67214, + "single": 448 + }, + "claude-haiku-5-5": { + "raw": 67214, + "single": 448 + } + } + }, + "rg_context": { + "bytes": { + "raw": 19902, + "single": 1131 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 10063, + "single": 575 + }, + "claude-haiku-5-5": { + "raw": 10063, + "single": 575 + } + } + }, + "rust_small": { + "bytes": { + "raw": 3569, + "single": 3569 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 1239, + "single": 1239 + }, + "claude-haiku-5-5": { + "raw": 1239, + "single": 1239 + } + } + }, + "rust_large": { + "bytes": { + "raw": 21207, + "single": 16378 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 8466, + "single": 6688 + }, + "claude-haiku-5-5": { + "raw": 8466, + "single": 6688 + } + } + }, + "cargo_workspace": { + "bytes": { + "raw": 10874, + "single": 1699 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 5787, + "single": 878 + }, + "claude-haiku-5-5": { + "raw": 5787, + "single": 878 + } + } + }, + "listing": { + "bytes": { + "raw": 23153, + "single": 8147 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 11984, + "single": 4401 + }, + "claude-haiku-5-5": { + "raw": 11984, + "single": 4401 + } + } + }, + "large_source": { + "bytes": { + "raw": 48591, + "single": 16107 + }, + "facts_visible": { + "single": true + }, + "tokens": { + "claude-opus-5-5": { + "raw": 18131, + "single": 6377 + }, + "claude-haiku-5-5": { + "raw": 18131, + "single": 6377 + } + } + } + }, + "frame_tokens": { + "claude-opus-5-5": 8, + "claude-haiku-5-5": 8 + }, + "summary": { + "claude-opus-5-5": { + "fixtures_counted": 26, + "fixtures_missing": 0, + "tokens": { + "raw": 521703, + "single": 38471 + }, + "single_vs_raw": { + "saved_tokens": 483232, + "saved_percent": 92.63, + "fixtures_worse": [] + } + }, + "claude-haiku-5-5": { + "fixtures_counted": 26, + "fixtures_missing": 0, + "tokens": { + "raw": 521703, + "single": 38471 + }, + "single_vs_raw": { + "saved_tokens": 483232, + "saved_percent": 92.63, + "fixtures_worse": [] + } + } + } +}