Skip to content

claude-opus-5 on nine splits: the annapolis displacement does not generalise (#139) #146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonfroehlich merged 17 commits into from
Sep 17, 2026
Merged

claude-opus-5 on nine splits: the annapolis displacement does not generalise (#139) #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
e2f66fc
Publish the claude-opus-5 (effort low) detections for eight more spli…
jonfroehlich Aug 19, 2026
db7d0de
Recover the #139 leg's spend from Cloud Monitoring: $70.41, and what …
jonfroehlich Aug 19, 2026
e392d74
Score the nine-split Opus leg: the annapolis displacement does not ge…
jonfroehlich Aug 19, 2026
8bd4c43
Review fixes: recover the #122 spend, un-stale the coverage claims (#…
jonfroehlich Aug 20, 2026
ed9f34f
Split the #122 legs by effort: Opus recovers to 0.1%, Sonnet does not…
jonfroehlich Aug 20, 2026
4dbc11f
Commit the Cloud Monitoring minute series, so the recovered costs out…
jonfroehlich Sep 3, 2026
c7cee65
Pin the published effort split to the committed series, not to a copi…
jonfroehlich Sep 3, 2026
247df90
Date the telemetry, point the cost sections at the committed series (…
jonfroehlich Sep 3, 2026
29681d9
Merge origin/main: re-pool the opus/low leg over the eight-split boar…
jonfroehlich Sep 17, 2026
b8b1c95
Stop the effort-split tests from loading the repo-root .env into pytest
jonfroehlich Sep 17, 2026
b0248d6
Keep output-only minutes in the minute series, and search the last fu…
jonfroehlich Sep 17, 2026
26075e2
Attribute the 251st Opus pano honestly, and open the daily snapshot i…
jonfroehlich Sep 17, 2026
3d291b3
Share one Cloud Monitoring fetch and one .env reader between the two …
jonfroehlich Sep 17, 2026
383f2b4
Scope "the largest gap" to the eight pooled splits; six of the other …
jonfroehlich Sep 17, 2026
f2b3af6
Say which Claude files have a recorded cache check, and date the ledg…
jonfroehlich Sep 17, 2026
278d6f2
Refuse a per-pano rate that leaves zero input per leg instead of divi…
jonfroehlich Sep 17, 2026
e1ea5bb
Merge main: keep both halves of vertex_usage.py (#147's --reconcile, …
jonfroehlich Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
328 changes: 231 additions & 97 deletions analysis_out/scoreboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,59 @@
"manual_gold_recall": null,
"manual_gold_ap": null
},
{
"model": "claude-opus-5-effort-low",
"label": "claude-opus-5",
"spec": "claude:claude-opus-5",
"provider": "claude",
"standing": false,
"display": "Claude Opus 5 (low)",
"class": "chat-vlm",
"operating_point": 0.0,
"operating_point_note": "no score",
"coverage": "8/8",
"complete": true,
"pooled_splits": [
"richmond",
"bend",
"clovis",
"morgantown",
"annapolis",
"paterson",
"gainesville",
"laurens_mapillary"
],
"precision": 0.56161,
"recall": 0.585547,
"f1": 0.5678,
"ap": null,
"ap_bundle": null,
"ap_is_substituted": false,
"fp_per_pano": 1.123888,
"micro_precision": 0.560082,
"micro_recall": 0.589433,
"f1_min": 0.42953,
"f1_max": 0.648551,
"f1_min_split": "laurens_mapillary",
"f1_max_split": "morgantown",
"n_splits_run": 11,
"laurens_gsv_f1": 0.437186,
"laurens_gsv_precision": 0.488764,
"laurens_gsv_recall": 0.395455,
"laurens_gsv_ap": null,
"budapest_district5_f1": 0.377778,
"budapest_district5_precision": 0.425,
"budapest_district5_recall": 0.34,
"budapest_district5_ap": null,
"sao_paulo_f1": 0.468227,
"sao_paulo_precision": 0.44164,
"sao_paulo_recall": 0.498221,
"sao_paulo_ap": null,
"manual_gold_f1": null,
"manual_gold_precision": null,
"manual_gold_recall": null,
"manual_gold_ap": null
},
{
"model": "gemini-3.7-flash",
"label": "gemini-3.7-flash",
Expand Down Expand Up @@ -924,53 +977,6 @@
"manual_gold_recall": null,
"manual_gold_ap": null
},
{
"model": "claude-opus-5-effort-low",
"label": "claude-opus-5",
"spec": "claude:claude-opus-5",
"provider": "claude",
"standing": false,
"display": "Claude Opus 5 (low)",
"class": "chat-vlm",
"operating_point": 0.0,
"operating_point_note": "no score",
"coverage": "2/8",
"complete": false,
"pooled_splits": [
"annapolis",
"laurens_mapillary"
],
"precision": 0.528598,
"recall": 0.495492,
"f1": 0.50898,
"ap": null,
"ap_bundle": null,
"ap_is_substituted": false,
"fp_per_pano": 1.074553,
"micro_precision": 0.53831,
"micro_recall": 0.504604,
"f1_min": 0.42953,
"f1_max": 0.58843,
"f1_min_split": "laurens_mapillary",
"f1_max_split": "annapolis",
"n_splits_run": 3,
"laurens_gsv_f1": 0.437186,
"laurens_gsv_precision": 0.488764,
"laurens_gsv_recall": 0.395455,
"laurens_gsv_ap": null,
"budapest_district5_f1": null,
"budapest_district5_precision": null,
"budapest_district5_recall": null,
"budapest_district5_ap": null,
"sao_paulo_f1": null,
"sao_paulo_precision": null,
"sao_paulo_recall": null,
"sao_paulo_ap": null,
"manual_gold_f1": null,
"manual_gold_precision": null,
"manual_gold_recall": null,
"manual_gold_ap": null
},
{
"model": "claude-sonnet-5-effort-low",
"label": "claude-sonnet-5",
Expand Down Expand Up @@ -2019,6 +2025,184 @@
"fp_per_pano": 1.16
}
},
"claude-opus-5-effort-low": {
"richmond": {
"split": "richmond",
"precision": 0.522673,
"recall": 0.706452,
"f1": 0.600823,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 219,
"fp": 200,
"fn": 91,
"n_panos": 124,
"n_gt_recall": 310,
"fp_per_pano": 1.612903
},
"bend": {
"split": "bend",
"precision": 0.586207,
"recall": 0.623853,
"f1": 0.604444,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 204,
"fp": 144,
"fn": 123,
"n_panos": 110,
"n_gt_recall": 327,
"fp_per_pano": 1.309091
},
"clovis": {
"split": "clovis",
"precision": 0.528302,
"recall": 0.574359,
"f1": 0.550369,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 112,
"fp": 100,
"fn": 83,
"n_panos": 125,
"n_gt_recall": 195,
"fp_per_pano": 0.8
},
"morgantown": {
"split": "morgantown",
"precision": 0.62807,
"recall": 0.670412,
"f1": 0.648551,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 179,
"fp": 106,
"fn": 88,
"n_panos": 125,
"n_gt_recall": 267,
"fp_per_pano": 0.848
},
"annapolis": {
"split": "annapolis",
"precision": 0.572347,
"recall": 0.605442,
"f1": 0.58843,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 178,
"fp": 133,
"fn": 116,
"n_panos": 125,
"n_gt_recall": 294,
"fp_per_pano": 1.064
},
"paterson": {
"split": "paterson",
"precision": 0.751701,
"recall": 0.559494,
"f1": 0.641509,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 221,
"fp": 73,
"fn": 174,
"n_panos": 125,
"n_gt_recall": 395,
"fp_per_pano": 0.584
},
"gainesville": {
"split": "gainesville",
"precision": 0.418733,
"recall": 0.558824,
"f1": 0.47874,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 152,
"fp": 211,
"fn": 120,
"n_panos": 125,
"n_gt_recall": 272,
"fp_per_pano": 1.688
},
"laurens_mapillary": {
"split": "laurens_mapillary",
"precision": 0.484848,
"recall": 0.385542,
"f1": 0.42953,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 96,
"fp": 102,
"fn": 153,
"n_panos": 94,
"n_gt_recall": 249,
"fp_per_pano": 1.085106
},
"laurens_gsv": {
"split": "laurens_gsv",
"precision": 0.488764,
"recall": 0.395455,
"f1": 0.437186,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 87,
"fp": 91,
"fn": 133,
"n_panos": 86,
"n_gt_recall": 220,
"fp_per_pano": 1.05814
},
"budapest_district5": {
"split": "budapest_district5",
"precision": 0.425,
"recall": 0.34,
"f1": 0.377778,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 102,
"fp": 138,
"fn": 198,
"n_panos": 125,
"n_gt_recall": 300,
"fp_per_pano": 1.104
},
"sao_paulo": {
"split": "sao_paulo",
"precision": 0.44164,
"recall": 0.498221,
"f1": 0.468227,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 140,
"fp": 177,
"fn": 141,
"n_panos": 125,
"n_gt_recall": 281,
"fp_per_pano": 1.416
}
},
"gemini-3.7-flash": {
"richmond": {
"split": "richmond",
Expand Down Expand Up @@ -3415,56 +3599,6 @@
"fp_per_pano": 2.048
}
},
"claude-opus-5-effort-low": {
"annapolis": {
"split": "annapolis",
"precision": 0.572347,
"recall": 0.605442,
"f1": 0.58843,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 178,
"fp": 133,
"fn": 116,
"n_panos": 125,
"n_gt_recall": 294,
"fp_per_pano": 1.064
},
"laurens_mapillary": {
"split": "laurens_mapillary",
"precision": 0.484848,
"recall": 0.385542,
"f1": 0.42953,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 96,
"fp": 102,
"fn": 153,
"n_panos": 94,
"n_gt_recall": 249,
"fp_per_pano": 1.085106
},
"laurens_gsv": {
"split": "laurens_gsv",
"precision": 0.488764,
"recall": 0.395455,
"f1": 0.437186,
"ap": null,
"ap_source": "bundle",
"ap_bundle": null,
"bundle_floor": null,
"tp": 87,
"fp": 91,
"fn": 133,
"n_panos": 86,
"n_gt_recall": 220,
"fp_per_pano": 1.05814
}
},
"claude-sonnet-5-effort-low": {
"annapolis": {
"split": "annapolis",
Expand Down
1 change: 1 addition & 0 deletions benchmark/model_detections/claude-opus-5-effort-low__bend.json
View file
Open in desktop

Large diffs are not rendered by default.

Loading
Loading