diff --git a/analysis_out/scoreboard.json b/analysis_out/scoreboard.json index a951944d..c6077005 100644 --- a/analysis_out/scoreboard.json +++ b/analysis_out/scoreboard.json @@ -931,6 +931,98 @@ "manual_gold_recall": null, "manual_gold_ap": null }, + { + "model": "claude-fable-5-effort-low-anthropic", + "label": "claude-fable-5", + "spec": "claude:claude-fable-5", + "provider": "claude", + "standing": false, + "display": "Claude Fable 5 (low, anthropic)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "1/8", + "complete": false, + "pooled_splits": [ + "annapolis" + ], + "precision": 0.579268, + "recall": 0.646259, + "f1": 0.610932, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 1.104, + "micro_precision": 0.579268, + "micro_recall": 0.646259, + "f1_min": 0.610932, + "f1_max": 0.610932, + "f1_min_split": "annapolis", + "f1_max_split": "annapolis", + "n_splits_run": 1, + "laurens_gsv_f1": null, + "laurens_gsv_precision": null, + "laurens_gsv_recall": null, + "laurens_gsv_ap": null, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "claude-fable-5-1-effort-low-anthropic", + "label": "claude-fable-5-1", + "spec": "claude:claude-fable-5-1", + "provider": "claude", + "standing": false, + "display": "Claude Fable 5.1 (low, anthropic)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "1/8", + "complete": false, + "pooled_splits": [ + "annapolis" + ], + "precision": 0.637037, + "recall": 0.585034, + "f1": 0.609929, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.784, + "micro_precision": 0.637037, + "micro_recall": 0.585034, + "f1_min": 0.609929, + "f1_max": 0.609929, + "f1_min_split": "annapolis", + "f1_max_split": "annapolis", + "n_splits_run": 1, + "laurens_gsv_f1": null, + "laurens_gsv_precision": null, + "laurens_gsv_recall": null, + "laurens_gsv_ap": null, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, { "model": "claude-opus-5-effort-high", "label": "claude-opus-5", @@ -3581,6 +3673,42 @@ "fp_per_pano": 11.282258 } }, + "claude-fable-5-effort-low-anthropic": { + "annapolis": { + "split": "annapolis", + "precision": 0.579268, + "recall": 0.646259, + "f1": 0.610932, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 190, + "fp": 138, + "fn": 104, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 1.104 + } + }, + "claude-fable-5-1-effort-low-anthropic": { + "annapolis": { + "split": "annapolis", + "precision": 0.637037, + "recall": 0.585034, + "f1": 0.609929, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 172, + "fp": 98, + "fn": 122, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.784 + } + }, "claude-opus-5-effort-high": { "annapolis": { "split": "annapolis", diff --git a/analysis_out/usage_log.jsonl b/analysis_out/usage_log.jsonl index 2576c64b..0741e2f4 100644 --- a/analysis_out/usage_log.jsonl +++ b/analysis_out/usage_log.jsonl @@ -5,3 +5,9 @@ {"ts": "2026-09-04T17:21:26+00:00", "bundle": "laurens_mapillary", "label": "claude-opus-5", "provider": "claude", "model_id": "claude-opus-5", "model_versions": {"claude-opus-5": 552}, "panos_scored": 92, "signature": {"provider": "claude", "model_id": "claude-opus-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "stop_reasons": {"tool_use": 552}, "calls": 552, "input_tokens": 1121112, "output_tokens": 37238, "thoughts_tokens": 506, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 6.5365, "pricing": {"input_per_m": 5.0, "output_per_m": 25.0, "cache_read_per_m": 0.5, "cache_write_per_m": 6.25, "as_of": "2026-08-15", "note": "Vertex `global`; regional +10%."}} {"ts": "2026-09-04T18:03:21+00:00", "bundle": "laurens_gsv", "label": "claude-opus-5", "provider": "claude", "model_id": "claude-opus-5", "model_versions": {"claude-opus-5": 516}, "panos_scored": 86, "signature": {"provider": "claude", "model_id": "claude-opus-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "stop_reasons": {"tool_use": 516}, "calls": 516, "input_tokens": 1047996, "output_tokens": 34611, "thoughts_tokens": 306, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 6.1053, "pricing": {"input_per_m": 5.0, "output_per_m": 25.0, "cache_read_per_m": 0.5, "cache_write_per_m": 6.25, "as_of": "2026-08-15", "note": "Vertex `global`; regional +10%."}} {"ts": "2026-09-04T18:04:47+00:00", "bundle": "laurens_mapillary", "label": "claude-opus-5", "provider": "claude", "model_id": "claude-opus-5", "model_versions": {"claude-opus-5": 12}, "panos_scored": 94, "signature": {"provider": "claude", "model_id": "claude-opus-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "stop_reasons": {"tool_use": 12}, "calls": 12, "input_tokens": 24372, "output_tokens": 685, "thoughts_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.139, "pricing": {"input_per_m": 5.0, "output_per_m": 25.0, "cache_read_per_m": 0.5, "cache_write_per_m": 6.25, "as_of": "2026-08-15", "note": "Vertex `global`; regional +10%."}} +{"ts": "2026-09-05T14:21:06+00:00", "bundle": "annapolis", "label": "claude-fable-5-1", "provider": "claude", "model_id": "claude-fable-5-1", "model_versions": {"claude-fable-5-1": 30}, "panos_scored": 5, "signature": {"provider": "claude", "model_id": "claude-fable-5-1", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "serving_path": "anthropic", "stop_reasons": {"tool_use": 30}, "calls": 30, "input_tokens": 60990, "output_tokens": 4107, "thoughts_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.8153, "pricing": {"input_per_m": 10.0, "output_per_m": 50.0, "cache_read_per_m": 0.25, "cache_write_per_m": 12.5, "as_of": "2026-09-05", "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier. Cache read $0.25/MTok is a rate of its own, not 0.1x input."}} +{"ts": "2026-09-05T14:22:15+00:00", "bundle": "annapolis", "label": "claude-fable-5-1", "provider": "claude", "model_id": "claude-fable-5-1", "model_versions": {"claude-fable-5-1": 24}, "panos_scored": 5, "signature": {"provider": "claude", "model_id": "claude-fable-5-1", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "serving_path": "anthropic", "stop_reasons": {"tool_use": 24}, "calls": 24, "input_tokens": 48792, "output_tokens": 3252, "thoughts_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.6505, "pricing": {"input_per_m": 10.0, "output_per_m": 50.0, "cache_read_per_m": 0.25, "cache_write_per_m": 12.5, "as_of": "2026-09-05", "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier. Cache read $0.25/MTok is a rate of its own, not 0.1x input."}} +{"ts": "2026-09-05T14:23:20+00:00", "bundle": "annapolis", "label": "claude-fable-5", "provider": "claude", "model_id": "claude-fable-5", "model_versions": {"claude-fable-5": 30}, "panos_scored": 5, "signature": {"provider": "claude", "model_id": "claude-fable-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "serving_path": "anthropic", "stop_reasons": {"tool_use": 30}, "calls": 30, "input_tokens": 61050, "output_tokens": 2955, "thoughts_tokens": 961, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.7582, "pricing": {"input_per_m": 10.0, "output_per_m": 50.0, "cache_read_per_m": 1.0, "cache_write_per_m": 12.5, "as_of": "2026-09-05", "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier."}} +{"ts": "2026-09-05T14:23:45+00:00", "bundle": "annapolis", "label": "claude-fable-5", "provider": "claude", "model_id": "claude-fable-5", "model_versions": {"claude-fable-5": 18}, "panos_scored": 5, "signature": {"provider": "claude", "model_id": "claude-fable-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "serving_path": "anthropic", "stop_reasons": {"tool_use": 18}, "calls": 18, "input_tokens": 36630, "output_tokens": 1776, "thoughts_tokens": 657, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.4551, "pricing": {"input_per_m": 10.0, "output_per_m": 50.0, "cache_read_per_m": 1.0, "cache_write_per_m": 12.5, "as_of": "2026-09-05", "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier."}} +{"ts": "2026-09-05T15:52:23+00:00", "bundle": "annapolis", "label": "claude-fable-5-1", "provider": "claude", "model_id": "claude-fable-5-1", "model_versions": {"claude-fable-5-1": 720}, "panos_scored": 125, "signature": {"provider": "claude", "model_id": "claude-fable-5-1", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "serving_path": "anthropic", "stop_reasons": {"tool_use": 720}, "calls": 720, "input_tokens": 1463760, "output_tokens": 104430, "thoughts_tokens": 254, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 19.8591, "pricing": {"input_per_m": 10.0, "output_per_m": 50.0, "cache_read_per_m": 0.25, "cache_write_per_m": 12.5, "as_of": "2026-09-05", "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier. Cache read $0.25/MTok is a rate of its own, not 0.1x input."}} +{"ts": "2026-09-05T16:47:23+00:00", "bundle": "annapolis", "label": "claude-fable-5", "provider": "claude", "model_id": "claude-fable-5", "model_versions": {"claude-fable-5": 720}, "panos_scored": 125, "signature": {"provider": "claude", "model_id": "claude-fable-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "serving_path": "anthropic", "stop_reasons": {"tool_use": 720}, "calls": 720, "input_tokens": 1465200, "output_tokens": 76418, "thoughts_tokens": 23699, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 18.4729, "pricing": {"input_per_m": 10.0, "output_per_m": 50.0, "cache_read_per_m": 1.0, "cache_write_per_m": 12.5, "as_of": "2026-09-05", "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier."}} diff --git a/benchmark/model_detections/claude-fable-5-1-effort-low-anthropic__annapolis.json b/benchmark/model_detections/claude-fable-5-1-effort-low-anthropic__annapolis.json new file mode 100644 index 00000000..a73d3e87 --- /dev/null +++ b/benchmark/model_detections/claude-fable-5-1-effort-low-anthropic__annapolis.json @@ -0,0 +1 @@ +{"city":"annapolis","detections":{"1005430537402761":[[0.4345408485743551,0.5557636087370782,null],[0.6655601355079884,0.5933833663209835,null],[0.7106337704344909,0.5888939185329167,null],[0.907571162570173,0.5743476729439813,null],[0.10500922427317755,0.6021657194491776,null],[0.2147591834907387,0.5677113836632993,null],[0.24272508873169196,0.5399301534999916,null],[0.3429601604001834,0.5464297372977205,null]],"1018302322713012":[[0.7320510584684159,0.5645756164990132,null],[0.0659102401153484,0.5634410203537521,null],[0.9045237364136239,0.558288432395561,null],[0.2900134844263801,0.6105481258371527,null]],"1023104848740281":[[0.03315768565300292,0.5270549105998589,null]],"1027083038026430":[[0.4494458110047701,0.5120493876407682,null],[0.5960624524120239,0.5253226828856696,null],[0.029908883487156646,0.5327287673478861,null],[0.965251898779896,0.5264985372930353,null]],"1031592527886900":[[0.96137639443635,0.536396873041274,null],[0.024451527848509624,0.5417469999308371,null],[0.09455413513326061,0.5567358444116826,null]],"1034575561078277":[[0.37475470013352713,0.6310400944776104,null],[0.6642826273711921,0.5230841548133994,null],[0.9518422843890715,0.5428314737923294,null],[0.1383999484363636,0.641119925883727,null]],"1040516234061456":[[0.45231359018554435,0.5340773077805947,null],[0.5879415658840079,0.5132990015952157,null],[0.7127387751607439,0.5456370411323572,null],[0.873002191084102,0.5479736946473215,null],[0.09096021524536613,0.5686662200430981,null],[0.06811866582579151,0.5545959173910229,null],[0.12098671828157692,0.5582922760912995,null],[0.3453767442581098,0.5803290463596125,null]],"1054705589196732":[[0.39961933761305046,0.5906369657721152,null],[0.6207991622909468,0.595509576739715,null],[0.5811116254474141,0.558147705617108,null]],"1066275427712945":[[0.953896673202315,0.5755289638840222,null],[0.13769468001371954,0.6652021278267506,null],[0.1946997554400267,0.7053895806619046,null],[0.21441221569974694,0.7268246968145029,null]],"1069711381045172":[[0.6613986116060431,0.5816628074708824,null],[0.9379888498846651,0.5573009378555114,null]],"1074557820562424":[[0.4554866600797433,0.6002959684906961,null],[0.7277945766148539,0.548705594577364,null],[0.7436769219953383,0.6321744129423592,null],[0.2776103025153244,0.7531231028940956,null],[0.3524420067554897,0.7115430102713372,null]],"1130786224993585":[[0.918871442538319,0.5523248726247377,null],[0.031188281926668282,0.5683966070711614,null],[0.3325877165042449,0.6871527826829221,null]],"1179546065891425":[[0.37627904455169486,0.6034891835097256,null],[0.7721576536496553,0.5658778042146406,null],[0.7295550974598626,0.5716434908212074,null],[0.904923362231387,0.540788818712543,null],[0.0421958075373946,0.5485622517887974,null],[0.25166793545378163,0.6357029021225082,null]],"1186577468484043":[[0.8250754275777836,0.5741390898262303,null]],"1254255151701056":[[0.7357991551529823,0.6226503455912396,null],[0.6704945647526055,0.5513631126364825,null],[0.11553083738724479,0.6339425830453544,null],[0.3537608790653267,0.6581037231637568,null]],"1255568348226573":[],"1265191387234427":[[0.4623167700869688,0.5323881991716146,null],[0.575228922794874,0.5315385578851392,null],[0.8823188642886937,0.5488910394109013,null]],"1319468485116956":[[0.09215914763526156,0.594795316411272,null]],"1348745456021634":[[0.6484350976047134,0.5388098694837896,null],[0.7182160681463096,0.5516014551769685,null],[0.05723748940693535,0.5445860116122637,null]],"1349860502075738":[[0.6147091904344868,0.56041519444563,null],[0.9884894233111701,0.5611192404644959,null],[0.3766568138562805,0.729874409232011,null]],"1378841722703885":[],"1405036700051880":[[0.4225790475181545,0.5343361436610575,null],[0.14875742891081206,0.5916564997627152,null]],"1417173041993097":[[0.384346201402954,0.5686320729563016,null],[0.7110461291026409,0.5594998509416115,null],[0.7559885597601008,0.5629249287182697,null],[0.8499196320782525,0.5565470965868032,null],[0.07780753358516851,0.5444819831034295,null],[0.9077844503608661,0.5481875563064084,null],[0.24407881741746074,0.5715386298728016,null],[0.04979120722094993,0.5436455622857675,null],[0.3342706215482345,0.586349933858582,null]],"1424988461692696":[[0.4544818399080716,0.5472397514277609,null],[0.6039997571711029,0.5415092676350238,null],[0.9454732666345759,0.6146864577913264,null],[0.08726096279517997,0.6097387215002352,null],[0.8807908266518011,0.5751962484139599,null],[0.2264880925736561,0.5751404624046953,null],[0.36032242085439203,0.5477145542438496,null]],"1435162300398313":[[0.46994484672595027,0.5072404069943028,null]],"1475792212785589":[[0.3966807368656998,0.522249649919642,null],[0.7543039368606848,0.5467071168210764,null]],"1528518111324684":[[0.4292570777963481,0.5730081837034763,null],[0.6229246457303261,0.6062483122671357,null],[0.839154602290108,0.6070018752926034,null],[0.057906617801614435,0.5765676303412088,null],[0.35232718321387346,0.5403617399670325,null],[0.26157306034194966,0.5092325440240343,null]],"1554822665323668":[[0.8773833879969182,0.590021282144852,null]],"1577720189426545":[[0.581831434394586,0.5225110346766269,null],[0.6971092245227217,0.5596384269289,null]],"1635539566853832":[],"1746146992466005":[[0.7079723069978157,0.593296605367577,null],[0.09893551351058205,0.6030771404621158,null],[0.15938059476778493,0.6885471321013783,null]],"1796129530825058":[[0.7344763717288247,0.5518605178106075,null],[0.22417853302260143,0.604901098507961,null]],"1808761949309259":[[0.3860431184016318,0.5362235652530992,null]],"2005223349857247":[],"2007477026275162":[[0.38947007741979034,0.5896259086237859,null],[0.6513529630009938,0.5643326275513457,null],[0.9112282919221202,0.5714273766756267,null],[0.10616906066902021,0.578987549037389,null],[0.35842325705645395,0.5888018464289153,null],[0.4264258083470478,0.5715411583798148,null]],"2033244330365072":[],"2048474495485493":[[0.3795887611052578,0.590467056883257,null],[0.6605771960869673,0.5862912128096985,null]],"2050450181762428":[[0.4657691933282389,0.5201057570090146,null],[0.3741175529057157,0.6094290547076706,null],[0.6349872772234262,0.5890400110499208,null]],"2087242684945790":[[0.1303646327483831,0.6679824659893457,null],[0.315718762542872,0.523446865196433,null]],"221474716428145":[[0.4164282777878587,0.5546441374160574,null],[0.6217946498851888,0.538804877334323,null],[0.26643772792344844,0.6750575717624772,null],[0.38901656737185775,0.5488268423500617,null]],"2358385930959873":[[0.07011810630108006,0.5368026292861278,null]],"24092441563734589":[[0.9450034310422606,0.5507309820281178,null]],"260599643174895":[[0.39038576446962203,0.6025042277966891,null],[0.05802954918219205,0.579277895449465,null]],"263227562888871":[],"263456786691932":[[0.5668881584571072,0.5492245261659569,null],[0.40368184047526057,0.5547852844997526,null],[0.5405041340155486,0.5027683870039679,null],[0.5028206242687863,0.498884938355975,null]],"266139323080373":[[0.9892747497313342,0.558925877427857,null]],"269324986100409":[[0.8761512198007932,0.572087390643336,null]],"270044664458604":[[0.9094782016618819,0.6023813536318082,null],[0.835908405231987,0.6040601875614561,null],[0.8026793228316107,0.5417277023242054,null],[0.23321129276663616,0.57520302759599,null]],"270895215940194":[],"272909839049970":[[0.7525278434235418,0.5411433568834639,null]],"2901245953423830":[[0.42217998533687406,0.5537479665815832,null],[0.583311900524055,0.5527481159319598,null],[0.3997049191456622,0.5760409083795696,null]],"2925949640956963":[[0.9292975948935223,0.5593551741672927,null],[0.01974124905928676,0.5456283190816591,null]],"294097413515834":[[0.8363514519274231,0.5963221666928301,null],[0.289566825076058,0.7340926635471439,null]],"297886629678677":[[0.4251102221023241,0.5325066105759748,null],[0.5974134261462356,0.526445711222184,null]],"299861069631715":[[0.5676928036835801,0.5356133337537946,null],[0.6682568750447365,0.5969259928595155,null]],"310957541571911":[[0.3826817231449102,0.5602167853630526,null]],"317716534218426":[[0.40115113767145016,0.5742247715949544,null],[0.6087575969380562,0.5132232479078987,null],[0.1154530095566832,0.6057519621896243,null]],"324732793509915":[[0.5263566448755567,0.521780026098545,null],[0.8007106612337356,0.6404757868723416,null],[0.12108053897185755,0.6458119898271075,null]],"328410443119195":[[0.3906819344401979,0.6039179771623607,null]],"329577912221811":[[0.44461665412752527,0.5160767055337183,null],[0.8345472947144824,0.569121958252781,null],[0.43390046895211437,0.5938697134966853,null]],"334237679155324":[[0.37498345709116343,0.6099828904669402,null],[0.16014558642590104,0.6266816988870334,null]],"335077195781777":[[0.8334872831803836,0.5776706944986146,null],[0.1137095789331064,0.5377150702089065,null],[0.18302888482432472,0.5698963115648346,null]],"340722771792916":[[0.37318540670856104,0.5961854659160578,null]],"342904991620622":[],"344613277972089":[[0.5623835459775617,0.546031357656428,null],[0.7713412295254907,0.569918739149181,null],[0.9510529516621486,0.5560804072919404,null]],"3450660371930065":[[0.6012275795098219,0.5297809151689363,null]],"3474716149511064":[[0.41028613329815966,0.5640557908987711,null],[0.6439738412017659,0.5628771114555748,null],[0.390767039261151,0.593560245577619,null]],"348332217713751":[[0.5305382958138132,0.526393931569887,null],[0.575184524200866,0.5174888160583219,null],[0.3369375483891699,0.6145755113026167,null]],"351023274068767":[[0.5770706370590086,0.5369580859793737,null],[0.4474824018809501,0.5399845511285859,null],[0.7313423473381779,0.5980805657556053,null],[0.7873641511270861,0.5767897684118666,null],[0.9539165682735058,0.5371997547252051,null],[0.12444708459025206,0.6115717235447607,null],[0.16684198405285505,0.6536182220801569,null],[0.1449277465898558,0.5899380854642595,null],[0.19404918506721947,0.7031337959152422,null]],"351046994156490":[[0.4554638961842969,0.5124246508780501,null],[0.5436779950587096,0.514563735671997,null],[0.05701057382868541,0.5600965275615688,null]],"352023870627096":[[0.12553839643889875,0.6106254274885543,null],[0.19707057792939936,0.6731865077578476,null]],"354530760362063":[[0.4454894790536624,0.5418237266336714,null]],"360016932424336":[[0.11940232844000115,0.6047866839765679,null],[0.04553820635450745,0.5564413724139884,null]],"360772096383255":[[0.5577139100894638,0.5272964050740621,null],[0.4666408634477342,0.5305623193455754,null],[0.8177100160135371,0.6119886774181488,null]],"371650204319915":[[0.12289085144529044,0.603739384763903,null],[0.04007642799993244,0.5540252244453764,null]],"373863014117157":[[0.2896376978577324,0.5693484789056124,null]],"381873453353748":[[0.7068696671660527,0.6267440898192993,null],[0.32257082939596593,0.5861635090687168,null]],"3946684388792708":[[0.44455012698793506,0.5193453865675939,null],[0.5728237438532581,0.5327170213967045,null],[0.5272598287448028,0.506792397090869,null]],"4146719218776864":[],"4445476159010798":[[0.08869002430858652,0.5793794228071406,null]],"4542231422476340":[[0.5401206841302498,0.5588448623080408,null],[0.6542992476513112,0.5523109294550058,null],[0.8437982429553663,0.5958473801208721,null]],"4578453255540160":[[0.7644826586014295,0.5760349036639267,null],[0.8883260593733713,0.552774908715735,null],[0.25166793545378163,0.6357029021225082,null]],"4807548019274072":[[0.46064587983874355,0.5156236186254205,null]],"4843511522330195":[[0.8911872000553263,0.5390104970671961,null],[0.05644801941409944,0.5336964261324789,null]],"4918993041463207":[[0.6146242421612964,0.5771031322151624,null]],"552683216160777":[],"555650998901334":[[0.7544432878763796,0.5829983332306943,null],[0.9510251221848431,0.5547578946165782,null]],"567965147905936":[[0.3903260666813069,0.6152768418470471,null],[0.07335716176556684,0.5884234916455955,null]],"573014264148691":[[0.3493515015513899,0.6893072734457031,null],[0.8597944586090741,0.5681998248700748,null],[0.3018628483164012,0.6822158051998394,null]],"591884792217872":[[0.5883720876421035,0.5687623327969349,null],[0.4194801302036395,0.5631855522290077,null]],"621448865493882":[[0.21730936913120535,0.6433692580407055,null]],"634400838879405":[[0.4121861127635815,0.6471636548194014,null],[0.7656583631925209,0.6251370985847474,null],[0.9307292790940725,0.5602784140801307,null]],"653812816884043":[],"6647525068693964":[[0.14185176166432922,0.6777056362308251,null],[0.24595016033508316,0.6819271846014267,null]],"686286580131737":[[0.42156790775952796,0.5439099545504777,null],[0.8970437925918506,0.5549949462724931,null],[0.057989115014257575,0.5647143163550368,null]],"6927560153933821":[[0.5698000234757676,0.5353790474661829,null],[0.8279793067694172,0.5912771268403592,null],[0.9692714263500963,0.5378274336771244,null]],"6965548880171391":[],"705263804809661":[[0.5781202845288592,0.5441956567759866,null],[0.09043057216763523,0.5896628874287843,null]],"712773504232349":[[0.8814117569052731,0.5537332271888318,null]],"716590860313634":[],"733299878608344":[],"738497988323553":[[0.6978209649836341,0.5790746512782114,null],[0.857146990662502,0.5762680339756421,null],[0.09049387541248338,0.573281375856109,null]],"812237619662282":[[0.4460370664344798,0.5444081295696699,null]],"819578196836418":[[0.15485227325130568,0.6408912422337452,null]],"832744555517667":[[0.4301622479046715,0.5426494832457806,null],[0.7712350304074281,0.5884575500664071,null],[0.19959415668625946,0.6349375999658409,null]],"843279940788401":[[0.7330677300404618,0.5738969548041561,null],[0.1672952312088487,0.5898582003798314,null]],"846876249550191":[[0.39747629792441763,0.5398966833469612,null],[0.36221453323612796,0.5942657170697057,null]],"863337525341813":[],"863365891868341":[],"863961825178792":[[0.558742412617275,0.5301148367532612,null],[0.46295697606945707,0.5224663233240903,null],[0.9404385833001287,0.555511105042491,null],[0.03477799342400173,0.526995566908308,null]],"865533918497811":[[0.44841214482286246,0.5124857186117383,null],[0.5838969385656458,0.5310724301593436,null],[0.5540726808423005,0.5087611196812493,null]],"867977875056387":[[0.6100637568921164,0.536641071969133,null],[0.38237048306359234,0.6116758468862431,null]],"869773754548856":[],"874414166619589":[[0.40182086338345113,0.5734577775002081,null]],"876143044170826":[[0.5260138906538709,0.5262771686308029,null]],"877430540067191":[[0.4528540133838872,0.5771803003536592,null],[0.5724475841521143,0.544265881534373,null]],"891735058502193":[[0.32692991770981206,0.6160307736183801,null]],"900958664931957":[[0.4167226155878158,0.5508183845625114,null],[0.7665815237739466,0.5394946903055439,null],[0.5678605605335802,0.5095267284357479,null]],"924877929312275":[[0.6109729493823276,0.599821504275255,null],[0.34483120221434704,0.7353962335294995,null]],"928118321383655":[[0.6852915738107487,0.5377532403795582,null],[0.8427476687355833,0.5606162003941512,null],[0.8856861830347691,0.5440523465231014,null]],"955762218327225":[],"967427853833872":[[0.4179399229922534,0.5209726638032304,null],[0.780686416052023,0.5418107423832234,null],[0.8460975436242373,0.5925146918474955,null],[0.8739013987793283,0.5926134425384934,null],[0.12139499865303671,0.5765298021798452,null],[0.19303138223570943,0.5829254286256962,null],[0.22126031339026786,0.5838582885158919,null]],"972140606663998":[[0.7939185448031024,0.6034470935514056,null],[0.03251456468005676,0.5450334328747372,null]],"991242088827423":[[0.7584699649050333,0.5667143788157423,null],[0.028523861529420047,0.5619637620760995,null]],"997534874804455":[[0.5711690807235126,0.5446812722162655,null]]},"model":"claude-fable-5-1","n_panos":125,"n_uncached":0,"pins":{"claude_effort":"low","claude_serving_path":"anthropic"},"published_as":"claude-fable-5-1-effort-low-anthropic","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-fable-5-1","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-fable-5-effort-low-anthropic__annapolis.json b/benchmark/model_detections/claude-fable-5-effort-low-anthropic__annapolis.json new file mode 100644 index 00000000..680be388 --- /dev/null +++ b/benchmark/model_detections/claude-fable-5-effort-low-anthropic__annapolis.json @@ -0,0 +1 @@ +{"city":"annapolis","detections":{"1005430537402761":[[0.4341884911097198,0.5531616576527615,null],[0.669196964989205,0.5936690559128929,null],[0.7075432609567642,0.5870584440722129,null],[0.8944107958463763,0.5768066461557494,null],[0.10829739010243516,0.6006156933772258,null],[0.20405229378257422,0.5672823795052235,null],[0.25513195162226043,0.5366198081284475,null],[0.33693738963959435,0.539053435562282,null]],"1018302322713012":[[0.7187468668798673,0.5655615535702132,null],[0.8353871911391113,0.5492047409969231,null],[0.7472054838783639,0.5308174675461053,null],[0.06365375598276307,0.5630340856936744,null],[0.2920307087338069,0.6097610717052668,null]],"1023104848740281":[[0.8370798031828488,0.5387892854815958,null]],"1027083038026430":[[0.4394315587766505,0.5088512824968233,null],[0.6008306082504111,0.5200449803786785,null],[0.5653136350205332,0.5125263380143914,null],[0.07882516075975327,0.5523402924404621,null],[0.029546455840147368,0.525675044917047,null]],"1031592527886900":[[0.9565582474999839,0.537630883422683,null],[0.0349799618830689,0.5512262338051968,null],[0.12487966876646794,0.5525490697855213,null],[0.1731961279150664,0.59803874778831,null]],"1034575561078277":[[0.37720234392810625,0.6138608813792228,null],[0.6542100838547342,0.5230163068526856,null],[0.9507373525324598,0.5414477334549271,null],[0.13628605103465113,0.6398982737952275,null]],"1040516234061456":[[0.45475379242303654,0.5349869080735457,null],[0.6071981084934454,0.5129840641508004,null],[0.7224199329013415,0.5467731562971558,null],[0.8860867286546437,0.547082409272142,null],[0.06527323791392847,0.5635491935825702,null],[0.09948420709071304,0.5634126933957877,null],[0.3551152664556557,0.5826980808477155,null]],"1054705589196732":[[0.618687631636981,0.5942164794461156,null],[0.4385911240106467,0.5668270882544747,null],[0.3610425525553289,0.6296027668298967,null]],"1066275427712945":[[0.9499708560850619,0.5832856527957959,null],[0.9868791591207365,0.573423595993148,null],[0.13446891063186517,0.6605808926412137,null],[0.1929410933638922,0.7081151669797451,null],[0.21482602017617108,0.729966596342176,null]],"1069711381045172":[[0.6614215814685716,0.5790672568682267,null],[0.9324969101296671,0.5649870628113614,null]],"1074557820562424":[[0.3786858918581951,0.6887705486036689,null],[0.7062654698551029,0.5469298510495767,null],[0.5987397510260051,0.5441148756304875,null],[0.7253597700023131,0.591891869912325,null],[0.07162814243467641,0.5794436551437923,null],[0.27119011031130147,0.7472725672491729,null],[0.3372391025092486,0.7110908761336329,null]],"1130786224993585":[[0.029716180769369882,0.5713075688677856,null],[0.9295547063946256,0.5457607286470919,null],[0.325591337797757,0.6559451847447737,null]],"1179546065891425":[[0.3710113911623513,0.6245010473528435,null],[0.7637971115314914,0.5688948395042509,null],[0.6839529478892523,0.5603705926211859,null],[0.7861285065341885,0.568914878419474,null],[0.7297440999456384,0.5705096133081121,null],[0.043352106591016015,0.54559983061681,null],[0.8895026201946309,0.5425967865506975,null],[0.253157118996979,0.636445434877067,null]],"1186577468484043":[[0.5734003339710511,0.5326588519177006,null],[0.7829451299570507,0.5260394565448285,null],[0.9582456532697585,0.5423544004249129,null]],"1254255151701056":[[0.419334454917206,0.5917482379863134,null],[0.5680314301923589,0.542871207812863,null],[0.641853057450344,0.5572638638981549,null],[0.7457044814786207,0.6164168832360344,null],[0.8424522828464459,0.611412684221448,null],[0.11725782662369616,0.6486493469685228,null],[0.08418822967873851,0.6033204183772496,null]],"1255568348226573":[],"1265191387234427":[[0.4612248737929384,0.5386894822221239,null],[0.5849053776635702,0.5382688156785043,null],[0.5546822783301544,0.5104179270296719,null]],"1319468485116956":[[0.4062740530776629,0.5217368270558281,null],[0.09691012143382927,0.5889081355847815,null]],"1348745456021634":[[0.6328783341903941,0.5356263221931788,null],[0.6754057232809009,0.5358608689092814,null],[0.7186812874578092,0.5555756021662729,null],[0.05522197425228409,0.5481080335692362,null],[0.015703801806464412,0.5298098785210777,null]],"1349860502075738":[[0.9896914582616708,0.5605955816199486,null]],"1378841722703885":[],"1405036700051880":[[0.42587991537440273,0.5356477888256279,null],[0.18850125269572415,0.584134833780799,null]],"1417173041993097":[[0.38016243125935417,0.5760701657647083,null],[0.7125401782541443,0.5572051723263769,null],[0.753876526994917,0.5585227356998278,null],[0.8501512649801194,0.5543532627556516,null],[0.9057929488333532,0.5432072366315284,null],[0.06061944252669049,0.5437232056746986,null],[0.08902236186956364,0.5441188294375293,null],[0.25219104448423035,0.5670698923868641,null],[0.3373782072599528,0.583998096973669,null]],"1424988461692696":[[0.4563810505201191,0.5492294650946652,null],[0.39403083639665076,0.5603912610785236,null],[0.6083732308854936,0.5430606186512625,null],[0.7799920968046677,0.5521853383687987,null],[0.9418563958799746,0.610719668304855,null],[0.09359815151211487,0.605856540974075,null],[0.8923223455477785,0.5822864007440771,null],[0.22320533015898564,0.5781742721349549,null],[0.3596101962303025,0.5477500977314191,null]],"1435162300398313":[[0.4735348051584368,0.5067981767443137,null],[0.03438336311991069,0.5340665615583697,null]],"1475792212785589":[[0.4125292648893968,0.5240307740213316,null],[0.7584131318973404,0.5468467796221526,null],[0.3197958001695686,0.5717059828512615,null]],"1528518111324684":[[0.43109117325524404,0.5694336184724809,null],[0.621830772656127,0.6045971723112553,null],[0.8383594340684731,0.6085199003902074,null],[0.05444627449788175,0.5766110374009907,null],[0.2680348635151816,0.6236613795176832,null],[0.35431009185272094,0.5400366780413177,null]],"1554822665323668":[[0.36957017714285745,0.6551869480813772,null],[0.8885960348136852,0.5917495147082383,null]],"1577720189426545":[[0.5510287582132861,0.5159954612551745,null],[0.6984759001440828,0.5581788917968534,null],[0.5728429463535646,0.5223636043463316,null]],"1635539566853832":[],"1746146992466005":[[0.703281796777791,0.6029800616467862,null],[0.10388378728975095,0.557524984446047,null],[0.135053667202499,0.61701635005596,null]],"1796129530825058":[[0.7450262587126706,0.5487527217003604,null],[0.8890929647907615,0.5610598255985183,null],[0.11607341044270819,0.548692865805316,null],[0.22285695598123656,0.6095225380572006,null],[0.24618234020055624,0.6152518869734216,null]],"1808761949309259":[[0.3855371358282509,0.5545377638068262,null]],"2005223349857247":[],"2007477026275162":[[0.3829765321647778,0.5890451489381614,null],[0.40588755088323253,0.5835262502543602,null],[0.5884652849555939,0.5366198081284475,null],[0.6482172593519417,0.5644797822121441,null],[0.902301628724977,0.5733779599879713,null],[0.11427829548480456,0.5784171333117516,null],[0.09198102231035565,0.5832415671853263,null]],"2033244330365072":[],"2048474495485493":[[0.3785997772590425,0.5916173914908283,null],[0.6614215814685716,0.5790672568682267,null],[0.7509472620729485,0.5729164556173312,null],[0.06557317800929213,0.5544690971562005,null]],"2050450181762428":[[0.39457869543441015,0.5782964211249227,null],[0.6160913777374353,0.5882916685232127,null],[0.43901754871194265,0.5254170213063895,null],[0.5495713784074989,0.528521355325874,null],[0.6467119075382096,0.6062890651386525,null],[0.3740470688596882,0.6067587983382605,null]],"2087242684945790":[[0.8843061127564589,0.5792315588693022,null],[0.16051224134486836,0.6715349860410403,null],[0.31573415830838625,0.5229462203564298,null]],"221474716428145":[[0.4073257383609425,0.5628262636799444,null],[0.6210111404245819,0.540276615965003,null],[0.3847985800510013,0.5616252627500478,null],[0.2412739518787959,0.6563504398190172,null]],"2358385930959873":[],"24092441563734589":[[0.945554542673221,0.5512385828108735,null]],"260599643174895":[[0.3894713878497046,0.6045762054719369,null],[0.0646034732623903,0.5845341194092112,null]],"263227562888871":[],"263456786691932":[[0.5612803623464514,0.5489406319069043,null],[0.39732850715096474,0.5532584290904041,null],[0.5340769533987737,0.5023352563897157,null]],"266139323080373":[[0.6097983410140967,0.523729744352394,null],[0.8115348247165517,0.5546935958542132,null],[0.9893246328021026,0.556174831098459,null]],"269324986100409":[[0.8862125674503467,0.531010878534472,null]],"270044664458604":[[0.9048487349737236,0.6068549449871435,null],[0.8174675500115085,0.6029965864273884,null],[0.7506812507078382,0.5355666085992454,null],[0.2429052212158268,0.5478688870360643,null]],"270895215940194":[],"272909839049970":[[0.7847154465203124,0.5779909321944007,null],[0.6216128731113127,0.5324933197908683,null],[0.2629486503982136,0.5600854323527881,null]],"2901245953423830":[[0.4248199870125017,0.5567054255088398,null],[0.57156322019862,0.5573353952956366,null],[0.3990707219593999,0.575097554494765,null]],"2925949640956963":[[0.42671950522853297,0.5685539779190982,null],[0.9282944687501251,0.5598443706498535,null],[0.018970810007188865,0.5440595773854725,null]],"294097413515834":[[0.8323881596543832,0.5913244869768606,null],[0.27983326944097986,0.742990161375608,null]],"297886629678677":[[0.42061789890700274,0.5449678544997595,null],[0.6016621544458222,0.5335525143866863,null],[0.12020604871223517,0.6142432077226989,null]],"299861069631715":[[0.5674803989678154,0.5358755328161281,null],[0.4031391370600471,0.5813324377752668,null],[0.6660324665747435,0.5951550500582974,null]],"310957541571911":[[0.38210199184598376,0.5633284045336131,null],[0.9164970063441749,0.571358465491029,null]],"317716534218426":[[0.39854300258409425,0.5736180892939743,null],[0.11485945705984746,0.6048244486925491,null]],"324732793509915":[[0.5264471475315935,0.5207891703378077,null],[0.4945034374826202,0.5261064236611359,null],[0.782997410892261,0.6439805598278734,null],[0.8082825475485975,0.632974474639669,null],[0.12515689672225305,0.6387959192133376,null],[0.8749834570911634,0.6099828904669402,null]],"328410443119195":[[0.38502692699985686,0.5997941755147602,null]],"329577912221811":[[0.4421407930632142,0.49982719246719215,null],[0.8372629954158441,0.5668487328906705,null],[0.8955953303622144,0.54730381229109,null],[0.4317094336701504,0.5909080256990776,null]],"334237679155324":[[0.37225519462260265,0.599872384416362,null],[0.15664189618574093,0.6265514945613908,null]],"335077195781777":[[0.83425257653224,0.5748052540321718,null],[0.11027578958039497,0.5352672144130708,null],[0.1841156786959257,0.5659194982407323,null],[0.07988032023343994,0.5492184502984249,null]],"340722771792916":[[0.3733068827152404,0.5969617246355541,null],[0.06366008177313898,0.5782028594230652,null]],"342904991620622":[],"344613277972089":[[0.5271592760074352,0.5129797606071084,null],[0.9569691719526527,0.5505929375259623,null]],"3450660371930065":[[0.5992839045758535,0.5296186877970525,null]],"3474716149511064":[[0.4143549787240211,0.5571512543374777,null],[0.6478709231331796,0.5521074836894798,null],[0.9633619164068548,0.5484579885442968,null]],"348332217713751":[[0.5284926110576542,0.5264573625359662,null],[0.5772225950665337,0.4978804057127477,null],[0.3346509337293778,0.6176294837651979,null]],"351023274068767":[[0.4500066023326714,0.5412084723387011,null],[0.5752445448866352,0.5371767111757041,null],[0.7270134855969201,0.599970398397535,null],[0.9496228835423883,0.5333993645815354,null],[0.12319657276501517,0.6148463294599786,null],[0.14315748394995864,0.5877702160180688,null],[0.17035376689435855,0.6545141366410677,null],[0.19514745639229597,0.6990165659691397,null]],"351046994156490":[[0.45634003615224317,0.5143278747768177,null],[0.558853827041082,0.5262661129962632,null]],"352023870627096":[[0.9102517852179273,0.5448532017447901,null],[0.12201720208913991,0.6094010828092052,null],[0.03939008597790966,0.5438138703291849,null],[0.1426324361804247,0.6285192794637288,null]],"354530760362063":[[0.4420617368240466,0.5424991750448207,null]],"360016932424336":[[0.1178970820250933,0.6016909239206303,null],[0.046469506869185384,0.5542265108863622,null]],"360772096383255":[[0.5278455502852074,0.535885952966422,null],[0.785334233155502,0.6052044583127799,null],[0.8172332666121734,0.6116571333142901,null]],"371650204319915":[[0.03645444419723465,0.5545896372711324,null],[0.11497465332743118,0.6060129919347162,null],[0.07101875563397353,0.5506351831706671,null]],"373863014117157":[],"381873453353748":[[0.7055648048284073,0.6263800638506141,null],[0.32107678667128603,0.58320736297879,null]],"3946684388792708":[[0.43630383443396015,0.5148415417832352,null],[0.5750417988882384,0.5364897580753428,null],[0.4998638316728783,0.5064158372317414,null],[0.9141217762148028,0.5588128260817926,null]],"4146719218776864":[[0.7363379594410245,0.5800343649449026,null]],"4445476159010798":[[0.09761335819770667,0.5861087567812906,null],[0.9284286097880423,0.57020959157251,null],[0.05270055753141151,0.55380854604526,null]],"4542231422476340":[[0.5557014855572956,0.5677297229992568,null],[0.6535360285985234,0.5536515794909674,null],[0.8535050843569691,0.6032956687489957,null]],"4578453255540160":[[0.7640070804167881,0.5737216382339286,null],[0.8945896291579114,0.5499598549652476,null],[0.04034667948901827,0.5580163257265145,null],[0.25166793545378163,0.6357029021225082,null]],"4807548019274072":[[0.7177258374756088,0.5414496114803049,null]],"4843511522330195":[[0.8920751989113423,0.5376899690064677,null],[0.056329996670711646,0.5391493884632288,null]],"4918993041463207":[[0.6156739552999093,0.5801781848483019,null]],"552683216160777":[[0.598651510863062,0.5484844392276348,null]],"555650998901334":[[0.7550070091906935,0.5802523747622741,null],[0.8605683316218425,0.5530595674581638,null],[0.9093950489785751,0.5489683395472208,null],[0.9359368840010678,0.554688134149387,null],[0.4090924472858777,0.5078316107866695,null],[0.3550114741519964,0.5140175357281501,null]],"567965147905936":[[0.3890071106060413,0.6132649269944025,null],[0.6794751020451819,0.5591459761486993,null],[0.06978332792137387,0.5920756596676072,null]],"573014264148691":[[0.36731461857251024,0.6416452648047178,null],[0.8530633693371903,0.5672088139472201,null],[0.3023330015655459,0.6804010773236796,null]],"591884792217872":[[0.42985857188093174,0.5462948228732569,null],[0.5748437749292143,0.5239784190172184,null],[0.1670394771268039,0.687153991791504,null]],"621448865493882":[],"634400838879405":[[0.8013459757430752,0.594803328376732,null]],"653812816884043":[[0.6205344425225219,0.5464084875534841,null]],"6647525068693964":[[0.9486862014922567,0.5363559884829445,null],[0.1385139371399985,0.6774096254847248,null],[0.2458487477533381,0.6920168134997935,null]],"686286580131737":[[0.4245609659903946,0.5443400101786912,null],[0.8973000378623546,0.5513868116419923,null],[0.061193895404214604,0.5621140053280883,null]],"6927560153933821":[[0.5684839839230944,0.5373691146068524,null],[0.8201764821132347,0.5889837304544702,null],[0.9695128480188404,0.5412140612932197,null]],"6965548880171391":[],"705263804809661":[[0.572681784974356,0.5315562788872897,null],[0.7571549851272825,0.6026467380132037,null]],"712773504232349":[[0.9010522142851106,0.5484209326018142,null]],"716590860313634":[],"733299878608344":[[0.7085433817515616,0.6288293505160594,null]],"738497988323553":[[0.6964869178842253,0.5763633474997205,null],[0.8289274609440287,0.5699463488213963,null],[0.8637524240327197,0.573492578226481,null]],"812237619662282":[[0.44415017504893045,0.5506074148504397,null]],"819578196836418":[[0.006340358080083541,0.5524255368648373,null],[0.1316086687575837,0.6225391716647359,null],[0.1538105447805087,0.6340604918525956,null]],"832744555517667":[[0.5570476318445624,0.519274047774422,null],[0.7708180419060597,0.5898764243097059,null],[0.20067448851777642,0.6393262259248633,null]],"843279940788401":[[0.7258211463538613,0.5739055213803428,null],[0.1719760575020291,0.5863056348194722,null]],"846876249550191":[[0.3891059954889123,0.5372356124655142,null]],"863337525341813":[],"863365891868341":[[0.0012935518777387744,0.5372220718479644,null]],"863961825178792":[[0.5581983900670586,0.529670866764602,null],[0.45910109020413975,0.5208730047702463,null],[0.9396067517031452,0.5414995238026427,null]],"865533918497811":[[0.45045497883378827,0.5162794178522773,null],[0.5904225077926387,0.5285466866302809,null],[0.5579401167412852,0.5295365965004134,null]],"867977875056387":[[0.5671892047842131,0.5375973246259093,null],[0.6125140334340128,0.5393416485066281,null],[0.3833929425216194,0.6094224405316914,null]],"869773754548856":[],"874414166619589":[[0.40507127617213895,0.5739725219654545,null],[0.6132131999709644,0.5593339384302712,null]],"876143044170826":[[0.5189832040736351,0.5229232769712996,null],[0.3823479598723223,0.5619281826982638,null]],"877430540067191":[[0.45784132872333017,0.5480414176162433,null],[0.5781654707032854,0.5427507660961528,null]],"891735058502193":[],"900958664931957":[[0.4172659443348091,0.5552960192074229,null],[0.5696842226097489,0.5068931029872327,null],[0.766336110500696,0.5404219655764151,null]],"924877929312275":[[0.602085742175894,0.5996334281341118,null]],"928118321383655":[[0.6846750434550039,0.536206487026391,null],[0.8792355871393897,0.5400035078548928,null]],"955762218327225":[],"967427853833872":[[0.4124251222316741,0.521197091076949,null],[0.7630874082894263,0.6059987026642593,null],[0.8429930571134223,0.5949907777100092,null],[0.08448309180776975,0.5723820029848796,null],[0.8727171045305531,0.5907932248828492,null],[0.1754490111187727,0.5891440507450911,null],[0.21337550843340075,0.5767039349789381,null]],"972140606663998":[[0.7612363607808036,0.6121299538604708,null],[0.7942991553005552,0.5953549474832156,null],[0.8508443366446752,0.5433094735258386,null],[0.028476639477429844,0.5402663362176003,null],[0.9518757931436924,0.5298329553498436,null]],"991242088827423":[[0.760263751671719,0.5674575350585507,null],[0.032599766084921045,0.5594801721240297,null]],"997534874804455":[[0.5680759285825986,0.5443429349607075,null]]},"model":"claude-fable-5","n_panos":125,"n_uncached":0,"pins":{"claude_effort":"low","claude_serving_path":"anthropic"},"published_as":"claude-fable-5-effort-low-anthropic","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-fable-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/docs/figures/scoreboard_by_split.png b/docs/figures/scoreboard_by_split.png index 45e6a623..5ce325ee 100644 Binary files a/docs/figures/scoreboard_by_split.png and b/docs/figures/scoreboard_by_split.png differ diff --git a/docs/figures/scoreboard_f1.png b/docs/figures/scoreboard_f1.png index 72fb3ccc..9ce5dbd8 100644 Binary files a/docs/figures/scoreboard_f1.png and b/docs/figures/scoreboard_f1.png differ diff --git a/docs/model_comparison.md b/docs/model_comparison.md index 373207ca..5ae89e2c 100644 --- a/docs/model_comparison.md +++ b/docs/model_comparison.md @@ -59,6 +59,8 @@ and `tests/test_roster.py` fails if the two stop agreeing. Regenerate it with | `claude-opus-5-effort-high` | claude | sparse | 2026-08-15 | — published, not in these tables | | `claude-sonnet-5-effort-low` | claude | sparse | 2026-08-15 | — published, not in these tables | | `claude-sonnet-5-effort-high` | claude | sparse | 2026-08-15 | — published, not in these tables | +| `claude-fable-5-1-effort-low-anthropic` | claude | sparse | 2026-09-05 | — published, not in these tables | +| `claude-fable-5-effort-low-anthropic` | claude | sparse | 2026-09-05 | — published, not in these tables | **"Not in these tables" covers three different situations, and the difference matters.** The tables below are the zero-shot comparison; a leg can sit outside them because its @@ -77,33 +79,38 @@ the matrix above exists to make, stated here because the artifact is already in would otherwise read as a withheld result. Promoting it is one field in the registry plus a re-run; since #122 froze the #46 witness pool, doing so no longer disturbs that human pass. -**Two more off-roster models, `claude-sonnet-5` and `claude-opus-5`, have been run** — four -legs, both models × effort `low`/`high` (#122). They are absent from the roster tables -below, so no number in this document moves; `claude-opus-5` at `low` **is** scored in -[`model_scoreboard.md`](model_scoreboard.md), which keys off coverage rather than -`standing`. Detections are published -(`benchmark/model_detections/claude-*-effort-*__*.json`, 14 files) and 12 of the 14 are -verified identical to the desktop cache with `export_model_cache.py --verify` (the four -annapolis legs and the eight #139 splits); the two Laurens files were exported with -`n_uncached` 0 on the machine that ran #151, which never had a recorded `--verify`, and -are re-scored from the committed detections by `tests/test_claude_published_legs.py` — a -different guarantee, stated as such. The write-up is the "Claude on Vertex" section -further down, where the -annapolis result table is re-derived from those files by +**Six Claude legs are off-roster.** Four are `claude-sonnet-5` and `claude-opus-5` at +effort `low`/`high` (#122), served on Vertex; two are `claude-fable-5` and `claude-fable-5-1` +at effort `low` (#156), served on Anthropic's first-party API because Vertex gates that +family — which is why their published names carry `-anthropic`. All six are absent from the +roster tables below, so no number in this document moves; `claude-opus-5` at `low` **is** +scored in [`model_scoreboard.md`](model_scoreboard.md), which keys off coverage rather than +`standing`. Detections are published (`benchmark/model_detections/claude-*__*.json`, 16 +files) and 14 of the 16 are verified identical to the cache that produced them with +`export_model_cache.py --verify` (the four Vertex annapolis legs, the eight #139 splits, and +the two Fable annapolis legs); the two Laurens files were exported with `n_uncached` 0 on the +machine that ran #151, which never had a recorded `--verify`, and are re-scored from the +committed detections by `tests/test_claude_published_legs.py` — a different guarantee, +stated as such. The write-ups are the "Claude on Vertex" section and the "Claude Fable on +annapolis" section further down, where every result table is re-derived from those files by `tests/test_claude_published_legs.py`. -Coverage is no longer uniform across the four, so it is stated per leg: +Coverage is not uniform across the six, so it is stated per leg: - **`claude-opus-5` at `low` has run eleven splits** — everything but `manual_gold`: the nine of #139 plus both Laurens arms (#151). The `manual_gold` absence is a decision, not a pending run: `gemini-3.1-pro-preview` has no `manual_gold` row either, so a Claude-only run there would have no peer to compare against (#144). -- **The other three legs have run annapolis only.** A gap in coverage, not a withheld - result. It was estimated at "about $57 at `low` effort" before anyone ran it; the nine-split - Opus leg then **measured $70.41 for eight splits** (§"Cost accounting" below), so budget - roughly **$8.80 per split** for Opus and re-derive Sonnet from its own $3.60 annapolis leg - rather than from that average. Re-running `high` comprehensively would roughly double the - bill to re-measure a result we already have, which is why it stays at one split. +- **The other five legs have run annapolis only.** A gap in coverage, not a withheld + result. For the Opus/Sonnet legs it was estimated at "about $57 at `low` effort" before + anyone ran it; the nine-split Opus leg then **measured $70.41 for eight splits** (§"Cost + accounting" below), so budget roughly **$8.80 per split** for Opus and re-derive Sonnet + from its own $3.60 annapolis leg rather than from that average. Re-running `high` + comprehensively would roughly double the bill to re-measure a result we already have, + which is why it stays at one split. The two Fable legs are annapolis-only for a different + reason: both clear the pre-registered expansion gate, and the expansion has not been + taken — `manual_gold` alone is ~$154–165 per id at their measured rates (§"Claude Fable + on annapolis" below). **The three `y*_pano` rows are the supervised YOLO baseline** (#51), the one part of the registry that is not zero-shot. They have run on all ten splits and are scored, but under the @@ -2100,7 +2107,7 @@ pair has no such check — that day's telemetry does not separate. **Every number in this table is re-derivable from committed files**, with no `.model_cache`, no API key and no GPU: the per-panorama detections are published under `benchmark/model_detections/claude-*-effort-*__annapolis.json`, and -`tests/test_claude_annapolis_leg.py` recomputes the whole table from them on every CI run. +`tests/test_claude_published_legs.py` recomputes the whole table from them on every CI run. A number edited here without re-running anything fails the suite. **Effort is an operating-point dial, never a quality lever.** Both models move the same @@ -2113,11 +2120,12 @@ Same shape as this benchmark's Qwen 8B→32B finding, where scaling flipped the instead of fixing it. Note that the expensive setting is the worse one — 127k thinking tokens to lose 0.068 F1. -**`claude-opus-5` at `low` is the strongest general model measured on annapolis**, at -0.588 — the first to displace `gemini-3.1-pro-preview` (0.567) from that slot. Against +**`claude-opus-5` at `low` was the strongest general model measured on annapolis**, at +0.588 — the first to displace `gemini-3.1-pro-preview` (0.567) from that slot — **until the +two Fable legs (0.611 and 0.610, next section) displaced it in turn.** Against `claude-sonnet-5` at the same effort it gains **+0.224 recall at essentially unchanged precision** (0.589 → 0.572), which is a capability difference rather than a threshold -shift. RampNet still leads it by **0.251** (0.839 vs 0.588). +shift. RampNet leads it by **0.251** (0.839 vs 0.588). > **Superseded on the pooled board, 2026-08-19 (#139); re-pooled over eight splits > 2026-09-17 after #151 added `laurens_mapillary`.** That displacement is an annapolis @@ -2183,13 +2191,106 @@ prefix. That puts a 125-pano leg at **≈$3.60** and all ten splits at **≈$61* batch). Effort is the dominant lever: thinking bills as output at $10/MTok, and `low` spends none of it. +### Claude Fable on annapolis (#156): the first legs served off Vertex + +**Serving path caveat, and it travels with every number below.** Vertex gates the whole +Fable family behind a project-level publisher data-sharing setting +(`PublisherModelConfig.data_sharing_enabled_provider`), so these two legs did **not** run +on the Vertex path the four legs above used. They ran on Anthropic's first-party API under +`--claude-serving-path anthropic`, a different account and a different rate card for the +same weights. Same rig, same prompt, same tool definition, same `effort=low`, same JPEG +q90 encoding. The path is deliberately **not** part of the detection cache key — it +changes who bills, not what was asked, and putting it in the key would have orphaned the +$28.82 of paid detections above — so it is recorded in `analysis_out/usage_log.jsonl` and +in each published file's `pins` instead. Whether the two paths return bit-identical +detections for one model id is **untested**; nothing here depends on it, because no model +was run on both. + +| model | P | R | F1 | tp/fp/fn | boxes/pano | thinking tok | cost, 720 calls (120 of 125 panos) | +| :--- | ---: | ---: | ---: | :--- | ---: | ---: | ---: | +| `claude-fable-5` | 0.579 | 0.646 | **0.611** | 190/138/104 | 2.72 | 23,699 | $18.47 | +| `claude-fable-5-1` | 0.637 | 0.585 | **0.610** | 172/98/122 | 2.28 | 254 | $19.86 | +| *`claude-opus-5` (low), for reference* | *0.572* | *0.605* | *0.588* | *178/133/116* | *2.56* | *523* | *$8.94* | + +The cost and thinking-token columns are the two 720-call full-leg rows in +`analysis_out/usage_log.jsonl` (2026-09-05 15:52 and 16:47), which cover 120 of the 125 +panos: the other 5 were served from the cache the calibration pass had already written. +Counting that pass's first 30 calls on those 5 panos ($0.76 for `claude-fable-5`, $0.82 for +`claude-fable-5-1`), each id's whole split cost **$19.23** and **$20.67**. The calibration's +second pass (42 calls, $1.11, re-issued after a cache-write gap) is in the ledger too but +belongs to neither leg's number. The Opus row is the whole 125-pano leg, from console +output rather than the ledger (see the gap stated below). + +**Both displace `claude-opus-5` at the top of this split**, which had itself displaced +`gemini-3.1-pro-preview` (0.567). This is the first time a general-purpose model has beaten +Opus here. RampNet still leads by **0.228** (0.839 vs 0.611). + +**Within the family, the version is an operating-point dial — not a quality lever.** The +two are separated by **0.001 F1**, which is nothing, while sitting at visibly different +operating points: 5.1 trades 0.061 recall for 0.058 precision against 5, and emits 0.44 +fewer boxes per pano. That is the same shape as the effort finding above and the same shape +as the Qwen 8B→32B inversion: the knob moves *where* on the P/R curve the model sits, and +the ceiling does not move. Which one to prefer is therefore a decision about the +objective, not about the models — and under this project's recall-first framing, where a +false negative is permanent and a false positive is cheap, that argues for `claude-fable-5` +despite it being the older id. + +**The always-on-thinking cost premise was wrong, and this is where it was measured.** +Because the Fable family cannot disable thinking (`{"type": "disabled"}` is a 400 and +`budget_tokens` was removed), #156 predicted a cost band "wider than a flat 2x" — no +near-zero-thinking floor to make an `effort=low` leg cheap. It is a flat 2x. Fable is +$10/$50 per MTok against Opus's $5/$25, and per call the legs cost 2.15x and 2.31x the +Opus leg ($0.0257 and $0.0276 against $0.0119) — the whole-split totals above, $19.23 and +$20.67 against $8.94, give the same ratios. +`claude-fable-5` spent ~33 thinking tokens/call and `claude-fable-5-1` ~0.35, against +Opus-low's ~0.7 — always-on thinking is *adaptive*, and on a localization task at low +effort it costs essentially nothing. A 5-pano calibration predicted the full-leg cost to +within 1.5% on both ids. + +**These legs are `standing=False` and cover annapolis only (1 of 8 pooled splits).** Both +clear the pre-registered 0.567 gate for expanding to the full split set, so that expansion +is now a live decision rather than a hypothetical — `manual_gold` is 1,000 panos × 6 +views = 6,000 calls, so at the measured full-leg rates ($0.0257/call for `claude-fable-5`, +$0.0276/call for `claude-fable-5-1`) it is **~$154 and ~$165 per id** for that split alone. +It has **not** been taken, and no other split has been run. + +**Reproducing them** (the detections are committed; nothing below needs an API key): + +```bash +pytest -q tests/test_claude_published_legs.py # recompute both rows from committed files +``` + +Re-exporting from a `.model_cache` that produced them needs the serving path as well as the +effort — not because it changes the cache lookup (it does not), but because the registry +uses it to resolve the leg's published filename: + +```bash +for m in claude-fable-5-1 claude-fable-5; do + python scripts/analysis/export_model_cache.py --splits annapolis --models claude:$m --claude-effort low --claude-serving-path anthropic + python scripts/analysis/export_model_cache.py --verify --splits annapolis --models claude:$m --claude-effort low --claude-serving-path anthropic +done +``` + +Re-running them from scratch needs `ANTHROPIC_API_KEY` (the repo-root `.env` is the +gitignored home for it) and costs ~$40 (the whole-split totals above): + +```bash +for m in claude-fable-5-1 claude-fable-5; do + python scripts/model_comparison/compare.py benchmark/annapolis --models claude:$m --claude-serving-path anthropic --claude-effort low +done +``` + +Check reachability first — `python scripts/model_comparison/probe_claude_models.py +--serving-path anthropic` — because a key with no credit balance authenticates and then +fails every call with a 400 that says so. + ### Reproducing these four legs, and one gap in the record The detections are committed, so the table above can be re-derived by anyone with a clone and nothing else: ```bash -pytest -q tests/test_claude_annapolis_leg.py # recompute the table from committed files +pytest -q tests/test_claude_published_legs.py # recompute the table from committed files ``` Re-exporting them from a `.model_cache` that produced them needs the leg's settings, because diff --git a/docs/model_scoreboard.md b/docs/model_scoreboard.md index 9890e16d..15d22d1d 100644 --- a/docs/model_scoreboard.md +++ b/docs/model_scoreboard.md @@ -123,7 +123,7 @@ and F1 cannot tell you. ## Legs that have not run every pooled split -Five legs have run one split each, so they have no pooled mean to put in the table above — +Seven legs have run one split each, so they have no pooled mean to put in the table above — a one-city average printed beside an eight-city one is exactly the comparison the coverage column exists to prevent. They are reported per split instead, at the split they ran on: @@ -133,6 +133,8 @@ column exists to prevent. They are reported per split instead, at the split they |---|---|---|--:|--:|--:|--:|--:|--:| | Mask2Former Vistas (curb cut) | supervised transfer | `richmond` | 0.411 | 0.697 | 0.517 | 0.513 | 2.5 | 216/309/94 | | Mask2Former Vistas (+curb) | supervised transfer | `richmond` | 0.126 | 0.648 | 0.210 | 0.089 | 11.3 | 201/1399/109 | +| Claude Fable 5 (low, anthropic) | chat VLM | `annapolis` | 0.579 | 0.646 | 0.611 | – | 1.1 | 190/138/104 | +| Claude Fable 5.1 (low, anthropic) | chat VLM | `annapolis` | 0.637 | 0.585 | 0.610 | – | 0.8 | 172/98/122 | | Claude Opus 5 (high) | chat VLM | `annapolis` | 0.430 | 0.656 | 0.520 | – | 2.0 | 193/256/101 | | Claude Sonnet 5 (low) | chat VLM | `annapolis` | 0.589 | 0.381 | 0.463 | – | 0.6 | 112/78/182 | | Claude Sonnet 5 (high) | chat VLM | `annapolis` | 0.506 | 0.415 | 0.456 | – | 1.0 | 122/119/172 | @@ -142,14 +144,22 @@ column exists to prevent. They are reported per split instead, at the split they Two things worth carrying out of that table, both from splits where the roster's own numbers are directly above them in `model_comparison.md`: +- **Claude Fable 5 at low effort is the strongest challenger measured on annapolis** (F1 + 0.611), with Fable 5.1 at 0.610 — both displace Claude Opus 5 at low effort (0.588), + which had itself displaced gemini-3.1-pro (0.567) there. The two Fable legs ran on + Anthropic's first-party API rather than the Vertex path every other Claude leg used, + because Vertex gates that family; the path is pinned in each published file and the + caveat travels with the numbers in `model_comparison.md` (#156). They are annapolis-only: + both clear the pre-registered expansion gate, and the expansion has not been taken. - **More thinking makes it worse.** Claude Opus 5 drops from 0.588 at low effort to 0.520 at high on annapolis, and the same direction holds for Sonnet 5 (0.463 → 0.456). Effort moves - the operating point; it does not raise the ceiling (#122). The low-effort leg has since run - eleven splits (#139, #151) and is in the headline table above — **and its annapolis lead - over `gemini-3.1-pro` did not survive the other seven pooled splits**; the two tie within - 0.01, see the note under that table. The high-effort leg stays here, at one split, because - re-running it comprehensively would roughly double the bill to re-measure a result we - already have. + the operating point; it does not raise the ceiling (#122) — and the Fable version is the + same kind of dial: 5 and 5.1 sit 0.001 F1 apart at visibly different P/R points (#156). + The low-effort Opus leg has since run eleven splits (#139, #151) and is in the headline + table above — **and its annapolis lead over `gemini-3.1-pro` did not survive the other + seven pooled splits**; the two tie within 0.01, see the note under that table. The + high-effort leg stays here, at one split, because re-running it comprehensively would + roughly double the bill to re-measure a result we already have. - **Supervised transfer fixes most of the precision problem and still loses.** Mask2Former reading Vistas' `Curb Cut` class scores 0.517 on richmond with **12.4× OWLv2's precision** and no training at all — but RampNet's 0.855 on that split is 0.337 clear of @@ -180,6 +190,8 @@ numbers are directly above them in `model_comparison.md`: | Grounding DINO | 0.053 | 0.073 | 0.035 | 0.042 | 0.055 | 0.068 | 0.055 | 0.045 | 0.053 | 0.054 | 0.042 | 0.049 | 0.082 | | Mask2Former Vistas (curb cut) | 0.517 | – | – | – | – | – | – | – | – | – | – | – | – | | Mask2Former Vistas (+curb) | 0.210 | – | – | – | – | – | – | – | – | – | – | – | – | +| Claude Fable 5 (low, anthropic) | – | – | – | – | 0.611 | – | – | – | – | – | – | – | – | +| Claude Fable 5.1 (low, anthropic) | – | – | – | – | 0.610 | – | – | – | – | – | – | – | – | | Claude Opus 5 (high) | – | – | – | – | 0.520 | – | – | – | – | – | – | – | – | | Claude Sonnet 5 (low) | – | – | – | – | 0.463 | – | – | – | – | – | – | – | – | | Claude Sonnet 5 (high) | – | – | – | – | 0.456 | – | – | – | – | – | – | – | – | @@ -413,9 +425,10 @@ Omissions are content, so they are named rather than left as blanks: because it is untested. - **`manual_gold` has no null-recall pass** (O(n²) in panos), so the open detectors' recall discount is unmeasured on that split. -- **Five legs have one split each**, so they are in the partial table rather than the - headline: the two Vistas arms (richmond), and Claude Opus 5 (high) plus both Sonnet 5 legs - (annapolis). Extending either to the full pool is a run, not a code change. +- **Seven legs have run one split each**, so they are in the partial table rather than the + headline: the two Vistas arms (richmond), and five Claude legs on annapolis — Claude Opus 5 + (high), both Sonnet 5 legs, and both Fable legs. Extending any of them to the full pool is + a run, not a code change. - **`claude-opus-5-effort-low` is scored here but is not a standing roster entry.** It has full 8/8 coverage, so it appears in every table above; `standing` stays `False` because `roster.py` forbids a *pinned* leg from being standing — a scored entry has to be what a diff --git a/docs/replication.md b/docs/replication.md index 3e99b7f4..6ab5959f 100644 --- a/docs/replication.md +++ b/docs/replication.md @@ -21,7 +21,7 @@ lives on one machine. | `benchmark/miss_taxonomy_46/*.json` (human verdicts) | small | **committed** | ✅ | | RampNet model weights | — | HF `projectsidewalk/rampnet-model` | ✅ | | Stage 1 dataset | **463 GB** (test split ~44 GB) | HF `projectsidewalk/rampnet-dataset` | ✅ | -| `benchmark/model_detections/` (challenger detections) | 25.2 MB (146 files) | **committed** ✅ | ✅ | +| `benchmark/model_detections/` (challenger detections) | 25.3 MB (148 files) | **committed** ✅ | ✅ | | **`location_data/` (the paper's government inventories)** | 71.8 MB | **committed** ✅ | ✅ | | **`street_data/` derivative (what the pipeline actually reads)** | 18.7 MB | **committed** ✅ | ✅ | | `street_data/` raw downloads (NY file alone is 669 MB) | 801 MB | git-ignored; HF #21 pending | ⚠️ superseded by the derivative | @@ -42,7 +42,7 @@ in this sentence — the list here was one of the things that drifted. single-panorama shards keyed by an opaque SHA-1 of (label, signature, city, pano), unreadable without reconstructing detector signatures. `scripts/analysis/export_model_cache.py` consolidates it into human-readable files, one per (model, split), keyed by panorama id with the detector -signature recorded inside. As of 2026-09-17 that is **146 files, 25.2 MB**, and every one of +signature recorded inside. As of 2026-09-17 that is **148 files, 25.3 MB**, and every one of them belongs to a registered leg: | what | files | where it is written up | @@ -51,7 +51,8 @@ them belongs to a registered leg: | `gemini-3.7-flash`, twelve splits, published ahead of its write-up (#120) | 12 | §below | | the supervised YOLO pano trio, twelve splits each (#51) | 36 | [`model_comparison.md` §supervised baseline](model_comparison.md), and the [training record](../scripts/model_comparison/yolo_baseline/README.md) | | `claude-opus-5` at `low` effort, eleven splits (#122; the pool by #139, both Laurens arms by #151) | 11 | [`model_comparison.md` §Claude](model_comparison.md) | -| the other three Claude legs, annapolis only (#122) | 3 | [`model_comparison.md` §Claude](model_comparison.md) | +| the other three Vertex Claude legs, annapolis only (#122) | 3 | [`model_comparison.md` §Claude](model_comparison.md) | +| the two Fable legs, annapolis only, served on Anthropic's first-party API (#156) | 2 | [`model_comparison.md` §Claude Fable on annapolis](model_comparison.md) | | the two Mapillary Vistas class-set arms, richmond only (#126) | 2 | [`model_comparison.md` §Vistas](model_comparison.md) | `rampnet` is a row in every results table and has no file here: it is read from each bundle's diff --git a/rampnet/roster.py b/rampnet/roster.py index 10372bfc..f121d1af 100644 --- a/rampnet/roster.py +++ b/rampnet/roster.py @@ -34,6 +34,15 @@ ``claude-sonnet-5`` at effort ``low`` and at effort ``high`` are different runs with different cache keys and different results. Each is its own entry, and ``pins`` names the knob it holds, as ``(("claude_effort", "high"),)``. + + **A pin is what the leg needs to REPRODUCE, which is a superset of what enters the + signature.** ``claude_serving_path`` (#156) is the first pin that is not a + signature key: a Fable leg must run against ``anthropic`` because Vertex gates that + family, but the path does not change the detections and deliberately stays out of the + cache key (see ``ClaudeDetector.signature``). Pinning it anyway is what keeps a + bare ``claude:claude-fable-5-1`` from resolving to a Vertex run that 403s. Note the + consequence for naming: ``published_as`` must spell out every pin's value, so such a + leg is ``claude-fable-5-1-effort-low-anthropic``. * ``published_as`` is the filename stem under ``benchmark/model_detections/``, and it defaults to ``label``. It exists because ``label`` cannot carry a pin: the label is baked into cache keys that were already paid for, so renaming it orphans the @@ -232,6 +241,31 @@ def slug(label): published_as="claude-sonnet-5-effort-high", note="1.98 boxes/pano. Loses F1 to effort in the same direction as Opus, " "which is what makes that a pattern rather than one model's quirk."), + # #156. The first legs served OFF Vertex -- that family is gated there behind a + # publisher data-sharing setting, so these ran on Anthropic's first-party API. + # The serving path is NOT in the detection signature (it does not change the + # answer; see ClaudeDetector.signature), but it IS pinned, because reproducing + # these legs requires it -- which is why `published_as` spells it out. + Challenger( + spec="claude:claude-fable-5-1", label="claude-fable-5-1", provider="claude", + density="sparse", standing=False, added="2026-09-05", + pins=(("claude_effort", "low"), ("claude_serving_path", "anthropic")), + published_as="claude-fable-5-1-effort-low-anthropic", + note="F1 0.610 on annapolis (P 0.637 / R 0.585), 2.28 boxes/pano. Clears " + "the 0.567 gate and displaces claude-opus-5 effort-low (0.588) -- the " + "first general-purpose model to do so on this split. Tied with " + "claude-fable-5 (0.611) at a MORE PRECISE operating point, which is " + "the whole difference between them."), + Challenger( + spec="claude:claude-fable-5", label="claude-fable-5", provider="claude", + density="sparse", standing=False, added="2026-09-05", + pins=(("claude_effort", "low"), ("claude_serving_path", "anthropic")), + published_as="claude-fable-5-effort-low-anthropic", + note="F1 0.611 on annapolis (P 0.579 / R 0.646), 2.72 boxes/pano. " + "Indistinguishable from claude-fable-5-1 on F1 while trading 0.058 " + "precision for 0.061 recall: within this family the model version is " + "an operating-point dial, the same shape as effort in #123. Also " + "spends ~33 thinking tokens/call against 5.1's ~0.35, for no F1."), ) #: Specs whose label cannot be derived from the spec, because the ``model_id`` slot @@ -291,6 +325,14 @@ def slug(label): # cache key and every lookup misses. "claude_image_format": None, "claude_temperature": None, + "claude_max_tokens": None, + # Which account serves the calls. `vertex` is what all four published legs ran + # on, so it stays the default; the Fable legs pin `anthropic` because Vertex + # gates that family. Unlike the three settings above, this one does NOT enter + # the detection signature -- see ClaudeDetector.signature -- so changing it + # does not orphan the cache. It is still a pin, because reproducing a Fable + # leg requires it. + "claude_serving_path": "vertex", "qwen_model": "Qwen/Qwen3-VL-8B-Instruct", "qwen_coord_space": "auto", "owlv2_model": "google/owlv2-large-patch14-ensemble", @@ -439,6 +481,17 @@ def label_for(spec, cargs=None): return PROVIDER_DEFAULTS.get(key, provider) +def legs_of(spec, cargs=None): + """Every registered leg a spec could name, in roster order. + + ``leg_for`` picks one of these by matching pins against ``cargs``. Callers that + need to know *why* nothing matched -- the exporter, which must not fall back to + a bare filename when every candidate is pinned -- read the whole list. + """ + label = label_for(spec, cargs) + return [c for c in ROSTER if c.spec == spec or c.label == label] + + def leg_for(spec, cargs=None): """The registered leg a run resolves to, or ``None`` if it is not registered. @@ -448,8 +501,7 @@ def leg_for(spec, cargs=None): name a file without being told (see ``published_name``) instead of relying on whoever ran it to remember ``--publish-as``. """ - label = label_for(spec, cargs) - candidates = [c for c in ROSTER if c.spec == spec or c.label == label] + candidates = legs_of(spec, cargs) for c in candidates: # a pinned leg wins when its pins match if c.pins and all(getattr(cargs, k, None) == v for k, v in c.pins): return c diff --git a/scripts/analysis/export_model_cache.py b/scripts/analysis/export_model_cache.py index ed244af8..9f830312 100644 --- a/scripts/analysis/export_model_cache.py +++ b/scripts/analysis/export_model_cache.py @@ -119,11 +119,35 @@ def publication_name(spec, cargs, publish_as=None): nothing, so the overwrite guard stays quiet, and surfaces only later as a file that belongs to no registered leg. The registry knows the answer; this is the one place that writes the filename, so this is where it should ask. + + And when the registry knows the spec but none of its legs match the run's + settings, this REFUSES rather than falling back to the plain label. Every leg + of that model is pinned, so a run that matches none of them is either a new + leg (name it with ``--publish-as``) or, far more likely, the right leg exported + at the wrong flags: both Fable legs pin ``claude_serving_path=anthropic`` while + the default is ``vertex``, so a plain ``--models claude:claude-fable-5`` used + to write ``claude-fable-5__annapolis.json`` with ``pins: {}`` and no error. The + serving path is not in the cache key, so the lookup even succeeded. """ if publish_as: return publish_as leg = roster.leg_for(spec, cargs) - return roster.published_name(leg) if leg is not None else spec_label(spec, cargs) + if leg is not None: + return roster.published_name(leg) + candidates = roster.legs_of(spec, cargs) + if candidates: + known = "; ".join( + f"{roster.published_name(c)}: " + + ", ".join(f"{k}={v}" for k, v in c.pins) for c in candidates) + actual = ", ".join( + f"{k}={getattr(cargs, k, None)}" + for k in sorted({k for c in candidates for k, _ in c.pins})) + raise ValueError( + f"{spec!r} is registered, but every leg of it is pinned and none matches " + f"this run ({actual}). Refusing to publish it as the bare label " + f"{spec_label(spec, cargs)!r}. Registered legs: {known}. Pass the " + f"matching flags, or --publish-as to name a genuinely new leg.") + return spec_label(spec, cargs) def export(cache_dir, out_dir, splits, specs, allow_partial=False, overrides=None, @@ -159,6 +183,8 @@ def export(cache_dir, out_dir, splits, specs, allow_partial=False, overrides=Non "would write to the same file.") for spec in specs: name = publication_name(spec, cargs, publish_as) + leg = roster.leg_for(spec, cargs) + pins = dict(leg.pins) if leg is not None else {} for city in splits: bundle = os.path.join(REPO, "benchmark", city) if not os.path.exists(os.path.join(bundle, "records.jsonl")): @@ -197,8 +223,16 @@ def export(cache_dir, out_dir, splits, specs, allow_partial=False, overrides=Non collisions.append((label, city, path)) continue with open(path, "w", encoding="utf-8") as fh: + # `pins` is the leg's identity as the registry states it. Every pin + # used to be a signature field, so `signature` recorded them all by + # construction; claude_serving_path (#156) is the first that is not + # -- it changes who bills, not what was asked, so it is kept out of + # the cache key on purpose. Recording the pins here keeps the + # published file self-describing anyway, which is what lets a reader + # tell a Vertex-served leg from a first-party one. Absent on files + # published before this field existed; readers must tolerate that. json.dump({"model": label, "published_as": name, - "city": city, "signature": sig, + "city": city, "signature": sig, "pins": pins, "n_panos": len(dets), "n_uncached": missing, "detections": dets}, fh, **DUMP_KW) written.append((label, city, len(dets), missing, os.path.getsize(path))) @@ -345,6 +379,15 @@ def main(argv=None): "must match the run or the export finds no cache.") p.add_argument("--claude-tool-choice", default="auto", choices=["auto", "forced"], help="Claude tool choice of the producing run (signature field).") + p.add_argument("--claude-serving-path", default="vertex", + choices=["vertex", "anthropic"], + help="Which account served the producing run. UNLIKE the two flags " + "above this is NOT a signature field -- the serving path does " + "not change the detections, so it plays no part in finding the " + "cache. It is here only so the registry lookup can match a leg " + "pinned on it (#156's Fable legs are), which is what lets the " + "filename come from the roster instead of a --publish-as typed " + "from memory.") p.add_argument("--publish-as", help="Filename stem for this leg, when the model id alone does not " "identify it — e.g. claude-sonnet-5 at two effort levels are two " @@ -372,7 +415,10 @@ def main(argv=None): specs = [s.strip() for s in args.models.split(",") if s.strip()] overrides = {"tiling": args.tiling, "yolo_imgsz": args.yolo_imgsz, "claude_effort": args.claude_effort, - "claude_tool_choice": args.claude_tool_choice} + "claude_tool_choice": args.claude_tool_choice, + # Not a signature field, so it does not affect the cache lookup -- + # it is here purely so leg_for() can match a leg pinned on it. + "claude_serving_path": args.claude_serving_path} if args.verify: compared, problems, vacuous, unpublished = verify( diff --git a/scripts/analysis/scoreboard.py b/scripts/analysis/scoreboard.py index a8b2350e..40f1fc69 100644 --- a/scripts/analysis/scoreboard.py +++ b/scripts/analysis/scoreboard.py @@ -195,6 +195,10 @@ "claude-opus-5-effort-high": "Claude Opus 5 (high)", "claude-sonnet-5-effort-low": "Claude Sonnet 5 (low)", "claude-sonnet-5-effort-high": "Claude Sonnet 5 (high)", + # #156: served on Anthropic's first-party API, not Vertex -- the pin is part of + # the published name, so it is part of the display name too. + "claude-fable-5-effort-low-anthropic": "Claude Fable 5 (low, anthropic)", + "claude-fable-5-1-effort-low-anthropic": "Claude Fable 5.1 (low, anthropic)", "mask2former-vistas-curb-cut": "Mask2Former Vistas (curb cut)", "mask2former-vistas-curb-cut+curb": "Mask2Former Vistas (+curb)", } diff --git a/scripts/model_comparison/compare.py b/scripts/model_comparison/compare.py index b0440d60..03a0636a 100644 --- a/scripts/model_comparison/compare.py +++ b/scripts/model_comparison/compare.py @@ -508,6 +508,15 @@ def report_usage(detector, label, city, panos_scored, usage_log_path, timing=Non # ~6x more tiled than whole-pano, and without this two such runs log # identically. Same reasoning as the detections export (9e87290). "signature": detector.signature() if hasattr(detector, "signature") else None, + # Which account served this run (#156). Deliberately NOT in `signature` + # above -- that is the cache key, and the path does not change the answer + # -- so this is the only place it is written down, and the only thing + # that can distinguish a Vertex leg from a first-party one afterwards + # (`model_versions` cannot; both report the bare model id). It also says + # which reconciliation is available: vertex_usage.py can recover + # server-side spend for `vertex` and has no equivalent for `anthropic`. + # None for providers that have no such notion. + "serving_path": getattr(detector, "serving_path", None), # How each call terminated. Empty for providers that report nothing. "stop_reasons": stop_reasons or None, **usage, @@ -719,6 +728,25 @@ def build_parser(): "SUPPRESSES THINKING, which makes --claude-effort inert. " "'auto' (default) lets effort actually do something. Also " "part of the cache key.") + ap.add_argument("--claude-serving-path", default=_D["claude_serving_path"], + choices=["vertex", "anthropic"], + help="Which account serves the calls. 'vertex' (default) is " + "ADC + GOOGLE_CLOUD_PROJECT, what all four published " + "Claude legs ran on. 'anthropic' is the first-party API " + "via ANTHROPIC_API_KEY, needed for models Vertex gates " + "(the Fable family). NOT part of the cache key -- the " + "same request should return the same detections whoever " + "bills for it -- but it IS recorded in --usage-log, and " + "a number produced on one path carries that as a caveat " + "when compared with numbers from the other.") + ap.add_argument("--claude-max-tokens", type=int, default=_D["claude_max_tokens"], + help="Ceiling on one call's answer. Thinking bills against it, so " + "a call that thinks too long is CUT OFF mid-thought and the " + "run aborts rather than scoring it as 'no ramps'. Default " + "4096 is what the published legs ran and is ample at " + "effort=low there; the Fable family cannot disable thinking, " + "so it may need raising. A non-default value is a cache-key " + "change.") ap.add_argument("--claude-image-format", default=_D["claude_image_format"], choices=["jpeg", "png"], help="How each view is encoded before it is sent. Default 'jpeg' " "(q90) is what the published annapolis legs ran; 'png' is " diff --git a/scripts/model_comparison/detectors.py b/scripts/model_comparison/detectors.py index 17fcec55..9dda19a7 100644 --- a/scripts/model_comparison/detectors.py +++ b/scripts/model_comparison/detectors.py @@ -924,16 +924,91 @@ def boxes_from_claude_response(resp): # closes that gap at the price of re-running the legs; see docs/model_comparison.md. CLAUDE_AS_RUN_IMAGE_FORMAT = "jpeg" CLAUDE_AS_RUN_TEMPERATURE = None # i.e. the provider default, NOT greedy +# What the four published annapolis legs sent. Thinking bills against this, so it +# is the ceiling on how long a call may think before answering -- see +# ClaudeDetector._TRUNCATED_STOP_REASON for why hitting it is raised rather than +# scored. Held at the as-run value so the paid detection cache keeps its key. +CLAUDE_AS_RUN_MAX_TOKENS = 4096 CLAUDE_IMAGE_FORMATS = {"jpeg": ("JPEG", "image/jpeg"), "png": ("PNG", "image/png")} +# Which account serves the call (#156). NOT one of the as-run constants above, +# because it is deliberately absent from the signature -- see +# ClaudeDetector.signature. `vertex` is what all four published legs ran on; +# `anthropic` is the first-party API, reached when Vertex gates a model the way +# it gates the Fable family behind the publisher data-sharing setting. +CLAUDE_SERVING_PATHS = ("vertex", "anthropic") +CLAUDE_AS_RUN_SERVING_PATH = "vertex" + + +# Model ids on which `tool_choice` of `tool`/`any` returns a 400, per Anthropic's API +# docs (the copy bundled with Claude Code's claude-api reference, cached 2026-06-24, +# read 2026-09-17): the restriction is listed as one of the breaking changes Fable 5.1 +# introduced relative to Fable 5, and Mythos 5.1 shares it. Nothing in this repo has +# measured it -- every published Fable leg ran `auto`, and the probe never sent a +# forced tool_choice -- so this is a statement from the docs, not a result. +CLAUDE_FORCED_TOOLS_REJECTED = frozenset({"claude-fable-5-1", "claude-mythos-5-1"}) + +# Ids this guard refuses WITHOUT a source either way. `claude-fable-5` was refused +# from the start (commit 1213fbb, "forced tool use is gone in that family") on the +# assumption that the whole family rejects it; the docs above say the restriction +# arrived with 5.1, which would make Fable 5 accept it. No probe on record settles +# this. It stays refused because a wrong refusal costs one flag, while a wrong +# acceptance costs a leg that 400s on every view. To settle it, one 16-token call: +# +# python scripts/model_comparison/probe_claude_models.py \ +# --serving-path anthropic --models claude-fable-5 --tool-choice forced +# +# A 200 means the id belongs in neither set (move it, and move its case from +# test_forced_tool_choice_is_refused_before_the_run_starts to +# test_forced_tool_choice_still_works_on_the_ids_it_was_measured_on); +# a 400 means it belongs in CLAUDE_FORCED_TOOLS_REJECTED. Record the status code in +# #156 either way. +CLAUDE_FORCED_TOOLS_UNVERIFIED = frozenset({"claude-fable-5"}) + + +def claude_forced_tools_status(model_id): + """``"rejected"``, ``"unverified"`` or ``None`` for ``model_id``. + + Matched on the EXACT id, not a substring: the docs say the restriction is + per-version (5.1, not the family), so a family match would refuse an id nobody + has looked at. An id in neither set is assumed to allow forcing, which is the + behaviour every leg published so far relies on. Checked at construction rather + than probed because the point is to fail before the first paid call.""" + if model_id in CLAUDE_FORCED_TOOLS_REJECTED: + return "rejected" + if model_id in CLAUDE_FORCED_TOOLS_UNVERIFIED: + return "unverified" + return None -class ClaudeDetector(_VLMDetector): - """Claude via Google Cloud Vertex AI (#122). - Runs on the SAME credential path as the Gemini legs -- Vertex + ADC, same - project, same ``global`` location -- so it needs no new secret. Vertex's - Claude rates match Anthropic's first-party rates on ``global`` (regional - endpoints cost 10% more), verified 2026-08-15; see pricing.py. +def claude_forbids_forced_tools(model_id): + """Whether the detector refuses ``tool_choice="forced"`` on ``model_id``.""" + return claude_forced_tools_status(model_id) is not None + + +class ClaudeDetector(_VLMDetector): + """Claude, over either of two serving paths (#122, #156). + + ``serving_path="vertex"`` (the default, and what all four published legs ran + on) uses Google Cloud Vertex AI: the SAME credential path as the Gemini legs + -- Vertex + ADC, same project, same ``global`` location -- so it needs no new + secret. Vertex's Claude rates match Anthropic's first-party rates on + ``global`` (regional endpoints cost 10% more), verified 2026-08-15. + + ``serving_path="anthropic"`` uses Anthropic's first-party API and an + ``ANTHROPIC_API_KEY``. It exists because Vertex gates the Fable family behind + a project-level publisher data-sharing setting; first-party serves both Fable + ids today (probed 2026-09-05, see probe_claude_models.py). Same weights, and + the rate card is separate but currently identical for the models we price. + + **The path does not enter the cache key** -- see ``signature`` -- so a leg run + on one path and re-scored on the other reuses detections rather than re-paying. + That is deliberate, and it makes recording which path ran a job for the + usage log (compare.report_usage), not the signature. Two consequences worth + knowing before you switch a leg: a number produced here carries its serving + path as a caveat that must travel with it into docs/model_comparison.md, and + ``scripts/analysis/vertex_usage.py`` can reconcile spend server-side only for + the vertex path -- on first-party, ``--usage-log`` is the only record. """ name = "claude" @@ -945,8 +1020,14 @@ class ClaudeDetector(_VLMDetector): def __init__(self, model_id="claude-sonnet-5", max_edge=None, tile=True, project=None, location=None, effort="low", tool_choice="auto", views=None, image_format=CLAUDE_AS_RUN_IMAGE_FORMAT, - temperature=CLAUDE_AS_RUN_TEMPERATURE): + temperature=CLAUDE_AS_RUN_TEMPERATURE, + max_tokens=CLAUDE_AS_RUN_MAX_TOKENS, + serving_path=CLAUDE_AS_RUN_SERVING_PATH): super().__init__(model_id, max_edge, tile=tile, views=views) + if serving_path not in CLAUDE_SERVING_PATHS: + raise ValueError(f"unknown serving_path {serving_path!r} " + f"(choose from: {', '.join(CLAUDE_SERVING_PATHS)})") + self.serving_path = serving_path if image_format not in CLAUDE_IMAGE_FORMATS: raise ValueError(f"unknown image_format {image_format!r} " f"(choose from: {', '.join(sorted(CLAUDE_IMAGE_FORMATS))})") @@ -974,6 +1055,33 @@ def __init__(self, model_id="claude-sonnet-5", max_edge=None, tile=True, # back as a tool call (no text fallback needed) and is the better choice # at effort=low, where there is no thinking to lose. self.tool_choice = tool_choice + # Some ids reject `tool_choice` of `tool` or `any` with a 400 -- see + # CLAUDE_FORCED_TOOLS_REJECTED / _UNVERIFIED for which, and on what + # evidence. Caught at construction rather than at call time because the + # provider's 400 does not name --claude-tool-choice, and a leg that fails + # on every one of ~750 views is an afternoon lost to an error message + # about a knob nobody set deliberately. + status = claude_forced_tools_status(self.model_id) + if self.tool_choice == "forced" and status is not None: + if status == "rejected": + why = (f"{self.model_id} rejects tool_choice `tool`/`any` with a 400 " + f"(per Anthropic's API docs; see CLAUDE_FORCED_TOOLS_REJECTED " + f"in detectors.py).") + else: + why = (f"whether {self.model_id} accepts tool_choice `tool`/`any` has " + f"not been measured, so it is refused until it is. Settle it " + f"with `python scripts/model_comparison/probe_claude_models.py " + f"--serving-path anthropic --models {self.model_id} " + f"--tool-choice forced` and update " + f"CLAUDE_FORCED_TOOLS_UNVERIFIED in detectors.py.") + raise ValueError( + f"--claude-tool-choice forced is not supported on {self.model_id}: " + f"{why} Use `auto` (the default). Note the tradeoff the other way is " + f"real too: `auto` is what lets --claude-effort do anything at all.") + # Thinking bills against max_tokens. On the Opus/Sonnet legs effort=low + # thinks near zero, so 4096 was ample; the Fable family cannot disable + # thinking at all, so its floor is higher and this may need raising. + self.max_tokens = max_tokens self._client = None self.init_usage() self._usage_warned = False @@ -997,7 +1105,16 @@ def signature(self): Image encoding and temperature are inputs too, but they appear ONLY when they deviate from what the published legs ran (see CLAUDE_AS_RUN_IMAGE_FORMAT). Recording an unchanged default would change - the hash without changing the run and orphan the paid annapolis cache.""" + the hash without changing the run and orphan the paid annapolis cache. + + ``serving_path`` is NOT here, and not under the deviation rule either. It + names which account answered, not what was asked -- the same model id on + the same views should return the same detections whoever bills for them, + so adding it would fragment the cache along an axis that does not change + the answer, and would orphan $28.82 of paid annapolis detections the + moment any leg moved. It is recorded in the usage log instead, where + provenance belongs. If the two paths ever DO diverge in output, that is a + finding to measure and write down, not a cache key to add quietly.""" sig = super().signature() sig["effort"] = self.effort sig["tool_choice"] = self.tool_choice @@ -1006,6 +1123,11 @@ def signature(self): sig["image_format"] = self.image_format if self.temperature != CLAUDE_AS_RUN_TEMPERATURE: sig["temperature"] = self.temperature + # Same deviation-only rule: max_tokens truncates the answer when it binds, + # so it is an input to the detections -- but recording the as-run value + # would rewrite the key of every paid detection already in the cache. + if self.max_tokens != CLAUDE_AS_RUN_MAX_TOKENS: + sig["max_tokens"] = self.max_tokens return sig def location_warning(self): @@ -1014,7 +1136,13 @@ def location_warning(self): pricing.py's Claude rates are the ``global`` ones; Vertex bills regional endpoints 10% above them. GOOGLE_CLOUD_LOCATION is shared with the Gemini legs, so a region set for Gemini's sake silently makes every Claude cost - figure ~9% low. Returns None when there is nothing to say.""" + figure ~9% low. Returns None when there is nothing to say. + + Vertex-only. First-party billing has no endpoint dimension, so a + GOOGLE_CLOUD_LOCATION left set for Gemini's sake must not raise a pricing + alarm about a path it does not price.""" + if self.serving_path != "vertex": + return None if self.location == "global": return None return (f"[{self.model_id}] WARNING: {self.location!r} is a REGIONAL Vertex " @@ -1023,30 +1151,56 @@ def location_warning(self): f"Unset GOOGLE_CLOUD_LOCATION (or set it to 'global') to match the " f"table.") + # max_retries above the SDK default of 2: a full-city run is ~750 calls and a + # transient 429/5xx mid-leg is expensive to redo. Matches the Gemini rig's 5 + # attempts, and applies to both serving paths -- a rate limit is a rate limit. + _MAX_RETRIES = 5 + def _ensure_ready(self): + if self._client is not None: + return + warning = self.location_warning() + if warning: + print(warning) + self._client = (self._anthropic_client() if self.serving_path == "anthropic" + else self._vertex_client()) + + def _vertex_client(self): try: from anthropic import AnthropicVertex except ImportError as e: raise ImportError( "ClaudeDetector needs the `anthropic[vertex]` package " "(pip install -r requirements-vlm.txt)") from e - if self._client is not None: - return if not self.project: raise RuntimeError( "Claude on Vertex needs GOOGLE_CLOUD_PROJECT (and ADC via " "`gcloud auth application-default login`) -- the same credentials " "the Gemini legs use. Each Claude model must also be enabled " "separately in Vertex Model Garden; an un-enabled model 404s with " - "'was not found or your project does not have access to it'.") - # max_retries above the SDK default of 2: a full-city run is ~750 calls and - # a transient 429/5xx mid-leg is expensive to redo. Matches the Gemini rig's - # 5 attempts. - warning = self.location_warning() - if warning: - print(warning) - self._client = AnthropicVertex(project_id=self.project, region=self.location, - max_retries=5) + "'was not found or your project does not have access to it'. If " + "the model is gated rather than un-enabled (the Fable family is, " + "behind the publisher data-sharing setting), " + "--claude-serving-path anthropic is the other way in.") + return AnthropicVertex(project_id=self.project, region=self.location, + max_retries=self._MAX_RETRIES) + + def _anthropic_client(self): + try: + from anthropic import Anthropic + except ImportError as e: + raise ImportError( + "ClaudeDetector needs the `anthropic` package " + "(pip install -r requirements-vlm.txt)") from e + if not os.environ.get("ANTHROPIC_API_KEY"): + raise RuntimeError( + "Claude on the first-party API needs ANTHROPIC_API_KEY. It reads " + "from the environment or the gitignored repo-root .env, the same " + "file GOOGLE_CLOUD_PROJECT lives in. Note the API is PREPAID: a " + "valid key with no credit balance authenticates and then fails " + "every call with a 400 saying so. Check reachability first with " + "`probe_claude_models.py --serving-path anthropic`.") + return Anthropic(max_retries=self._MAX_RETRIES) # Vertex intermittently answers a perfectly good request with 404 "Publisher # model ... was not found or your project does not have access to it", most @@ -1085,12 +1239,19 @@ def _raw_detect(self, image): b64, media_type = self._encode_image(image) + # Vertex-only. The transient 404 below is an artifact of entitlement + # propagating unevenly across Vertex serving backends; on the first-party + # API a 404 means the model id does not exist for this account, which is + # permanent. Retrying it there would turn a typo into 30 seconds of + # backoff before the same error, which is how a wrong --claude-model + # reads as a network problem. + attempts = self._NOT_FOUND_RETRIES if self.serving_path == "vertex" else 1 delay = self._NOT_FOUND_BACKOFF - for attempt in range(self._NOT_FOUND_RETRIES): + for attempt in range(attempts): try: return self._call(b64, media_type) except NotFoundError: - if attempt == self._NOT_FOUND_RETRIES - 1: + if attempt == attempts - 1: raise if not self._not_found_warned: self._not_found_warned = True @@ -1115,7 +1276,7 @@ def _call(self, b64, media_type): kwargs["temperature"] = self.temperature resp = self._client.messages.create( model=self.model_id, - max_tokens=4096, + max_tokens=self.max_tokens, # effort is allowed by the org policy; structured_outputs is not. output_config={"effort": self.effort}, tools=[CLAUDE_BOX_TOOL], @@ -1148,7 +1309,10 @@ def _check_stop_reason(self, resp): f"(effort={self.effort}). Thinking bills against max_tokens, so a " f"high-effort call can be cut off mid-thought; scoring that as " f"'no curb ramps' would be a silent recall loss AND would be " - f"cached. Raise max_tokens or lower --claude-effort.") + f"cached. Raise --claude-max-tokens (currently {self.max_tokens}) " + f"or lower --claude-effort. On the Fable family only the first " + f"works: thinking cannot be turned off there, so effort has a " + f"floor.") if reason == "refusal" and not self._refusal_warned: self._refusal_warned = True print(f"[{self.model_id}] WARNING: a call ended in `refusal`, which " @@ -1865,7 +2029,11 @@ def build_detector(provider, model_id, records, args): tool_choice=getattr(args, "claude_tool_choice", None) or _D("claude_tool_choice"), image_format=(getattr(args, "claude_image_format", None) or CLAUDE_AS_RUN_IMAGE_FORMAT), - temperature=getattr(args, "claude_temperature", CLAUDE_AS_RUN_TEMPERATURE)) + temperature=getattr(args, "claude_temperature", CLAUDE_AS_RUN_TEMPERATURE), + max_tokens=(getattr(args, "claude_max_tokens", None) + or CLAUDE_AS_RUN_MAX_TOKENS), + serving_path=(getattr(args, "claude_serving_path", None) + or _D("claude_serving_path"))) if provider == "qwen": mid = model_id or args.qwen_model coord_space = getattr(args, "qwen_coord_space", "auto") diff --git a/scripts/model_comparison/pricing.py b/scripts/model_comparison/pricing.py index 5f12d5d0..2ea86cf8 100644 --- a/scripts/model_comparison/pricing.py +++ b/scripts/model_comparison/pricing.py @@ -70,6 +70,32 @@ "cache_read_per_m": 0.50, "cache_write_per_m": 6.25, "as_of": "2026-08-15", "note": "Vertex `global`; regional +10%.", }, + # Claude Fable via Anthropic's FIRST-PARTY API (#156), not Vertex: that family + # is gated on Vertex behind the publisher data-sharing setting, so these legs + # run on a different account and a different rate card. Source: the Anthropic + # pricing page (claude.com/pricing), read 2026-09-05 -- i.e. the page that + # actually governs this billing path. No endpoint dimension here, so unlike + # the Vertex rows above there is no regional premium to get wrong. + # + # Both ids are $10/$50, exactly 2x claude-opus-5. The cache rows are why this + # had to be READ: Fable 5 reads cache at the standard 0.1x of input ($1.00), + # but Fable 5.1 reads at $0.25/MTok, which is NOT any multiplier of its input + # rate. Deriving these from the usual multipliers would have overstated 5.1's + # cache reads 4x. Both write at 1.25x input, the standard 5-minute-TTL rate. + # Zero on every run so far -- nothing sets cache_control, and the tool + # definition renders below the minimum cacheable prefix -- but recorded so the + # first run that does is priced whole rather than silently low. + "claude-fable-5": { + "input_per_m": 10.00, "output_per_m": 50.00, + "cache_read_per_m": 1.00, "cache_write_per_m": 12.50, "as_of": "2026-09-05", + "note": "Anthropic first-party API, NOT Vertex (gated there). No regional tier.", + }, + "claude-fable-5-1": { + "input_per_m": 10.00, "output_per_m": 50.00, + "cache_read_per_m": 0.25, "cache_write_per_m": 12.50, "as_of": "2026-09-05", + "note": ("Anthropic first-party API, NOT Vertex (gated there). No regional " + "tier. Cache read $0.25/MTok is a rate of its own, not 0.1x input."), + }, } diff --git a/scripts/model_comparison/probe_claude_models.py b/scripts/model_comparison/probe_claude_models.py new file mode 100644 index 00000000..f026f270 --- /dev/null +++ b/scripts/model_comparison/probe_claude_models.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +"""Ask a serving path whether a Claude model id is reachable on our account. + +Answers one question before any leg is planned: *is this model id real, and is +it enabled for us?* Both serving paths distinguish those two cases, and the +distinction is the whole point of running this. + +Two paths, because #156 forced the question. Every paid Claude leg published so +far ran on **Vertex** (`--serving-path vertex`, the default), which is why that +is what this script was originally written against. Vertex gates the Fable +family behind a project setting we do not control, so the alternative is +Anthropic's **first-party API** (`--serving-path anthropic`), a different +credential and a different rate card for the same weights. + +Status semantics, which differ by path: + + 200 reachable now -- a leg can run today. Prints stop_reason and output + tokens too; see PROBE_MAX_TOKENS for why those are worth reading. + + vertex: + 403 the id RESOLVED but the project is not entitled. The message says why; + for the Fable family it is the publisher data-sharing setting, which is + a project-level configuration change, not something a script should + make. + 404 Vertex did not resolve the id at all -- almost always a spelling that + does not exist (`claude-fable-5.1` vs `claude-fable-5-1`), NOT a + permission problem. + + anthropic: + 404 the id does not exist for this account -- the same "wrong spelling" + signal Vertex sends as a 404. + 401 the key is bad or revoked. Distinct from every model-level answer, so a + wall of 401s is a credential problem and nothing else. + 400 the request was understood and refused. The two that matter here are an + empty credit balance and the Fable family's data-retention requirement; + both name themselves in the message, which is why it is printed + verbatim. + +That last row of the vertex block is what makes a 403 informative. A 403 on +every `anthropic` id would be consistent with a blanket publisher block; a 404 +on a misspelling alongside 403s on real ids proves the gate is per-model and +that the ids behind it exist. Probing a deliberately-wrong spelling is therefore +part of the measurement, not a typo -- see PROBE_CONTROL. The same control earns +its place on the first-party path, where a 404 otherwise reads as "not enabled +for us" rather than "not a model". + +`--tool-choice forced` asks a second question: *does this id accept a forced +`tool_choice` (`{"type": "tool", ...}`)?* Anthropic's docs list that as a 400 on +`claude-fable-5-1` and say the restriction arrived with 5.1; whether +`claude-fable-5` accepts it has never been measured here, and +`detectors.CLAUDE_FORCED_TOOLS_UNVERIFIED` refuses it until someone runs this. +The request is validated before any generation, so a 400 whose message names +`tool_choice` is the answer, and a 200 -- even one that stops at `max_tokens` +before the tool call -- means the id accepts it. The forced request also carries +`output_config={"effort": "low"}`, because that is what `ClaudeDetector._call` +sends beside `tool_choice` on every call, and effort is the one axis forcing is +known to interact with (forcing suppresses thinking; the Fable family cannot +disable it). A 200 on the bare shape would not answer for the detector's +request. Record the status code in #156 and update the two sets in detectors.py. + +Cost: each 200 is a real generate call, capped at --max-tokens, so a full run +costs a fraction of a cent. Every non-200 costs nothing. Nothing here writes to +the detection cache or the usage log -- this is a reachability probe, not a leg, +and its spend is too small to be worth a ledger row. + +Usage: + + # Vertex (ADC, the path the four published Claude legs ran on) + python scripts/model_comparison/probe_claude_models.py + + # Anthropic first-party (ANTHROPIC_API_KEY, from the environment or .env) + python scripts/model_comparison/probe_claude_models.py --serving-path anthropic + + python scripts/model_comparison/probe_claude_models.py --models claude-opus-5 + + # Does claude-fable-5 accept a forced tool_choice? (see detectors.py) + python scripts/model_comparison/probe_claude_models.py --serving-path anthropic \ + --models claude-fable-5 --tool-choice forced +""" +import argparse +import os +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] + +# The ids #156 is about, plus claude-opus-5 as a POSITIVE CONTROL: without a +# known-good id in the same run, a wall of 403s (or 401s) is indistinguishable +# from broken credentials. +PROBE_MODELS = ("claude-fable-5", "claude-fable-5-1", "claude-opus-5") + +# A spelling Anthropic does not publish. Expected to 404 -- see the module +# docstring for why a deliberate miss is load-bearing here. +PROBE_CONTROL = "claude-fable-5.1" + +# Deliberately tiny: this asks whether the door opens, not how the model +# performs. It is low enough that the Fable family -- which cannot disable +# thinking -- may stop at `max_tokens` before emitting any text. That is still a +# 200 and still proves reachability, which is why the stop_reason is printed +# rather than treated as a failure. Bracketing Fable's actual thinking floor is +# a separate measurement; --max-tokens is here so it needs no edit to this file. +PROBE_MAX_TOKENS = 16 +PROBE_PROMPT = "Reply with the single word: ok" + +SERVING_PATHS = ("vertex", "anthropic") +TOOL_CHOICES = ("auto", "forced") + +# Sent with the forced probe, as the detector sends it on every call. `low` is +# what every published Claude leg ran at, and the setting under which the +# detector's own comment calls `forced` the better choice. +PROBE_EFFORT = "low" + +# The smallest tool that can be forced. Not the detector's box tool: this asks +# whether the REQUEST SHAPE is accepted, and a schema this small keeps the +# answer from depending on anything else. +PROBE_TOOL = { + "name": "ok", + "description": "Acknowledge the request.", + "input_schema": {"type": "object", "properties": {"ok": {"type": "boolean"}}, + "required": ["ok"], "additionalProperties": False}, +} + + +def _load_dotenv(): + """Reuse compare.py's .env loader so the probe reads the same credentials + file the legs do -- GOOGLE_CLOUD_PROJECT for the vertex path, + ANTHROPIC_API_KEY for the first-party one. Imported inside the function (the + vertex_usage.py idiom) so this module still imports without the detector + stack on the path.""" + try: + from compare import load_dotenv + except ImportError: + return + load_dotenv(str(REPO)) + + +def _sdk(): + """The ``anthropic`` package, or the same ImportError ``ClaudeDetector`` raises. + + This is the script the docs tell a new user to run first, so a bare + ``ModuleNotFoundError`` is the wrong first thing for it to say: the detector + it fronts names the requirements file, and so does this.""" + try: + import anthropic + except ImportError as e: + raise ImportError( + "probe_claude_models.py needs the `anthropic[vertex]` package " + "(pip install -r requirements-vlm.txt)") from e + return anthropic + + +def make_client(args, ap): + """The SDK client for one serving path, or ``ap.error`` with what is missing. + + ``max_retries=0`` on both: a 403/404/401 is the answer here, not a blip, and + retrying it would only slow the probe down and blur the reading.""" + anthropic = _sdk() + if args.serving_path == "vertex": + if not args.project: + ap.error("no project: pass --project or set GOOGLE_CLOUD_PROJECT " + "(and authenticate with `gcloud auth application-default " + "login`)") + return anthropic.AnthropicVertex(project_id=args.project, region=args.location, + max_retries=0) + + if not os.environ.get("ANTHROPIC_API_KEY"): + ap.error("no ANTHROPIC_API_KEY: put it in the repo-root .env (which is " + "gitignored) or the environment. Keys are minted at " + "console.anthropic.com; a key with no credit balance " + "authenticates and then fails each call with a 400 that says " + "so.") + return anthropic.Anthropic(max_retries=0) + + +def probe(client, model_id, max_tokens=PROBE_MAX_TOKENS, tool_choice="auto"): + """``(status, detail)`` for one model id. Never raises. + + ``tool_choice="forced"`` sends ``PROBE_TOOL`` with ``tool_choice`` of type + ``tool`` and ``output_config.effort``, the shape ``ClaudeDetector._call`` + sends under ``--claude-tool-choice forced``.""" + anthropic = _sdk() + + request = dict(model=model_id, max_tokens=max_tokens, + messages=[{"role": "user", "content": PROBE_PROMPT}]) + if tool_choice == "forced": + request["tools"] = [PROBE_TOOL] + request["tool_choice"] = {"type": "tool", "name": PROBE_TOOL["name"]} + request["output_config"] = {"effort": PROBE_EFFORT} + try: + resp = client.messages.create(**request) + except anthropic.APIStatusError as e: + # e.message is the provider's own text; it carries the actionable part + # (which setting, which API) and is the reason this prints it verbatim + # rather than a summary of it. + return e.status_code, (e.message or "").strip() + except anthropic.APIConnectionError as e: + return "conn", f"connection error: {e}" + + # A 200 is the answer, but not the whole of it. `stop_reason` distinguishes a + # model that answered from one that spent the whole budget thinking, and the + # output count is the first (very coarse) datapoint on where the Fable + # family's thinking floor sits -- the unknown that --claude-max-tokens + # exists for on the detector. + detail = f"served by {getattr(resp, 'model', model_id)}" + usage = getattr(resp, "usage", None) + if usage is not None: + detail += (f"; {getattr(usage, 'input_tokens', 0) or 0} in / " + f"{getattr(usage, 'output_tokens', 0) or 0} out tokens") + reason = getattr(resp, "stop_reason", None) + if reason: + detail += f"; stop_reason={reason}" + if reason == "max_tokens": + detail += (f" (spent the whole {max_tokens}-token budget before " + f"answering -- still reachable; raise --max-tokens to " + f"see it reply)") + return 200, detail + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--serving-path", choices=SERVING_PATHS, default="vertex", + help="Which account answers. `vertex` (default) is what the " + "four published Claude legs ran on; `anthropic` is the " + "first-party API, a different credential and a " + "different rate card for the same weights " + "(default: %(default)s)") + ap.add_argument("--models", nargs="+", default=list(PROBE_MODELS), + help="model ids to probe (default: %(default)s)") + ap.add_argument("--no-control", action="store_true", + help=f"skip the {PROBE_CONTROL} 404 control") + ap.add_argument("--max-tokens", type=int, default=PROBE_MAX_TOKENS, + help="ceiling on each probe answer. Thinking bills against " + "it, so on the Fable family the default may be spent " + "before any text is emitted (default: %(default)s)") + ap.add_argument("--tool-choice", choices=TOOL_CHOICES, default="auto", + help="`forced` also sends a one-field tool with tool_choice " + "type `tool`, to measure whether the id accepts forced " + "tool use (a 400 naming tool_choice) -- the question " + "detectors.CLAUDE_FORCED_TOOLS_UNVERIFIED is waiting " + "on (default: %(default)s)") + ap.add_argument("--project", default=None, + help="vertex only; defaults to GOOGLE_CLOUD_PROJECT") + ap.add_argument("--location", default=None, + help="vertex only; the endpoint. `global` is what the legs " + "run on and is priced 10%% below regional " + "(default: global)") + args = ap.parse_args(argv) + + # After parsing, so an explicit flag still wins over the file. + _load_dotenv() + if args.project is None: + args.project = os.environ.get("GOOGLE_CLOUD_PROJECT") + if args.location is None: + args.location = os.environ.get("GOOGLE_CLOUD_LOCATION") or "global" + + client = make_client(args, ap) + + ids = list(args.models) + if not args.no_control and PROBE_CONTROL not in ids: + ids.append(PROBE_CONTROL) + + if args.serving_path == "vertex": + print(f"serving_path=vertex project={args.project} " + f"location={args.location}", end="") + else: + print("serving_path=anthropic (first-party API, ANTHROPIC_API_KEY)", end="") + print(f" tool_choice={args.tool_choice}\n") + + worst = 0 + for model_id in ids: + status, detail = probe(client, model_id, max_tokens=args.max_tokens, + tool_choice=args.tool_choice) + print(f"{model_id:<24} {status}") + for line in detail.splitlines(): + print(f"{'':<24} {line}") + print() + if model_id in args.models and status != 200: + worst = 1 + return worst + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_claude_published_legs.py b/tests/test_claude_published_legs.py index e2dd01bc..c906f030 100644 --- a/tests/test_claude_published_legs.py +++ b/tests/test_claude_published_legs.py @@ -57,17 +57,36 @@ "laurens_gsv": (86, 220), } -# (model, effort, split) -> (panos, n_gt, tp, fp, fn, P, R, F1) exactly as published. +# (model, effort, serving_path, split) -> (panos, n_gt, tp, fp, fn, P, R, F1) exactly +# as published. The serving path is part of the key because it is part of the leg's +# published NAME (#156): it does not enter the detection signature -- it changes who +# bills, not what was asked -- but a Fable leg cannot run on Vertex at all, so two +# rows of this table now come from a different account than the other six, and a +# fixture that could not say which would be hiding that. PUBLISHED_LEGS = { - ("claude-sonnet-5", "low", "annapolis"): (125, 294, 112, 78, 182, 0.589, 0.381, 0.463), - ("claude-sonnet-5", "high", "annapolis"): (125, 294, 122, 119, 172, 0.506, 0.415, 0.456), - ("claude-opus-5", "low", "annapolis"): (125, 294, 178, 133, 116, 0.572, 0.605, 0.588), - ("claude-opus-5", "high", "annapolis"): (125, 294, 193, 256, 101, 0.430, 0.656, 0.520), - ("claude-opus-5", "low", "laurens_mapillary"): (94, 249, 96, 102, 153, 0.485, 0.386, 0.430), - ("claude-opus-5", "low", "laurens_gsv"): (86, 220, 87, 91, 133, 0.489, 0.395, 0.437), + ("claude-sonnet-5", "low", "vertex", "annapolis"): (125, 294, 112, 78, 182, 0.589, 0.381, 0.463), + ("claude-sonnet-5", "high", "vertex", "annapolis"): (125, 294, 122, 119, 172, 0.506, 0.415, 0.456), + ("claude-opus-5", "low", "vertex", "annapolis"): (125, 294, 178, 133, 116, 0.572, 0.605, 0.588), + ("claude-opus-5", "high", "vertex", "annapolis"): (125, 294, 193, 256, 101, 0.430, 0.656, 0.520), + ("claude-opus-5", "low", "vertex", "laurens_mapillary"): (94, 249, 96, 102, 153, 0.485, 0.386, 0.430), + ("claude-opus-5", "low", "vertex", "laurens_gsv"): (86, 220, 87, 91, 133, 0.489, 0.395, 0.437), + ("claude-fable-5-1", "low", "anthropic", "annapolis"): (125, 294, 172, 98, 122, 0.637, 0.585, 0.610), + ("claude-fable-5", "low", "anthropic", "annapolis"): (125, 294, 190, 138, 104, 0.579, 0.646, 0.611), } +def _publish_as(model, effort, serving): + """The filename stem this leg publishes under. + + Mirrors ``roster.published_name`` without importing the registry, so this file + stays an independent check on the published numbers rather than a second reading + of the same source. `vertex` is elided because it is the default and the six + Vertex legs were published before the path was a pin. + """ + stem = f"{model}-effort-{effort}" + return stem if serving == "vertex" else f"{stem}-{serving}" + + def _bundle(split): return os.path.join(REPO, "benchmark", split) @@ -77,9 +96,9 @@ def _ground_truths(split): return C.ground_truths_from_verdicts(records, verdicts) -def _score(model, effort, split, gts): +def _score(model, effort, serving, split, gts): pub = load_detections(model, split, PUBLISHED, - publish_as=f"{model}-effort-{effort}") + publish_as=_publish_as(model, effort, serving)) assert pub is not None, f"no published export for {model}/{effort}/{split}" tp = fp = n_gt = panos = 0 for pid, gt in gts.items(): @@ -105,10 +124,11 @@ def test_the_bundles_are_the_ones_the_numbers_were_read_against(split): assert sum(len(g.gt_points) for g in gts.values()) == want_gt -@pytest.mark.parametrize("model,effort,split", sorted(PUBLISHED_LEGS)) -def test_published_claude_numbers_reproduce_from_the_committed_detections(model, effort, split): - want = PUBLISHED_LEGS[(model, effort, split)] - got = _score(model, effort, split, _ground_truths(split)) +@pytest.mark.parametrize("model,effort,serving,split", sorted(PUBLISHED_LEGS)) +def test_published_claude_numbers_reproduce_from_the_committed_detections( + model, effort, serving, split): + want = PUBLISHED_LEGS[(model, effort, serving, split)] + got = _score(model, effort, serving, split, _ground_truths(split)) assert got[:5] == want[:5], ( f"{model}/{effort}/{split}: (panos, n_gt, tp, fp, fn) = {got[:5]}, " f"published {want[:5]}") @@ -125,11 +145,11 @@ def test_every_leg_covers_the_whole_split(split): printed a plausible table. This is the guard that makes it impossible to reintroduce quietly.""" gts = _ground_truths(split) - legs = [k for k in PUBLISHED_LEGS if k[2] == split] + legs = [k for k in PUBLISHED_LEGS if k[3] == split] assert legs, f"no published legs for {split}" - for model, effort, _ in legs: + for model, effort, serving, _ in legs: pub = load_detections(model, split, PUBLISHED, - publish_as=f"{model}-effort-{effort}") + publish_as=_publish_as(model, effort, serving)) missing = set(gts) - set(pub) assert not missing, f"{model}/{effort}/{split} is missing {len(missing)} pano(s)" for pano in RECOVERED.get(split, ()): @@ -142,23 +162,30 @@ def test_every_leg_covers_the_whole_split(split): def test_each_leg_is_published_under_its_own_name(): """Two effort levels of one model id are two legs. Publishing them under the bare id would put both on one filename and silently keep only the last.""" - paths = {published_path(m, split, PUBLISHED, publish_as=f"{m}-effort-{e}") - for m, e, split in PUBLISHED_LEGS} + paths = {published_path(m, split, PUBLISHED, publish_as=_publish_as(m, e, sp)) + for m, e, sp, split in PUBLISHED_LEGS} assert len(paths) == len(PUBLISHED_LEGS) for path in paths: assert os.path.exists(path), f"missing published export: {path}" -@pytest.mark.parametrize("model,effort,split", sorted(PUBLISHED_LEGS)) -def test_every_export_records_the_signature_that_produced_it(model, effort, split): +@pytest.mark.parametrize("model,effort,serving,split", sorted(PUBLISHED_LEGS)) +def test_every_export_records_the_signature_that_produced_it( + model, effort, serving, split): """Provenance has to survive in the file, because the cache it came from is git-ignored and the effort level is not visible in the detections.""" - path = published_path(model, split, PUBLISHED, publish_as=f"{model}-effort-{effort}") + path = published_path(model, split, PUBLISHED, + publish_as=_publish_as(model, effort, serving)) with open(path, encoding="utf-8") as fh: blob = json.load(fh) sig = blob["signature"] assert blob["model"] == model - assert blob["published_as"] == f"{model}-effort-{effort}" + assert blob["published_as"] == _publish_as(model, effort, serving) + # The serving path is NOT in the signature, by design. On the legs where it is + # not the default it must therefore be legible from `pins`, or the published + # file cannot say which account produced it. + if serving != "vertex": + assert blob["pins"]["claude_serving_path"] == serving assert sig["provider"] == "claude" assert sig["model_id"] == model assert sig["effort"] == effort @@ -175,11 +202,123 @@ def test_the_laurens_arms_are_the_pair_the_rig_comparison_rests_on(): A comparison whose two halves came from different effort levels (or different model ids) would attribute to the imagery a difference that is really a configuration change -- the exact confound the second arm exists to remove.""" - arms = [k for k in PUBLISHED_LEGS if k[2].startswith("laurens_")] - assert {k[2] for k in arms} == {"laurens_mapillary", "laurens_gsv"} - assert {(k[0], k[1]) for k in arms} == {("claude-opus-5", "low")} - f1 = {k[2]: PUBLISHED_LEGS[k][7] for k in arms} + arms = [k for k in PUBLISHED_LEGS if k[3].startswith("laurens_")] + assert {k[3] for k in arms} == {"laurens_mapillary", "laurens_gsv"} + # Serving path is pinned here too: both arms must come from the same account, + # for the same reason they must share an effort level. + assert {(k[0], k[1], k[2]) for k in arms} == {("claude-opus-5", "low", "vertex")} + f1 = {k[3]: PUBLISHED_LEGS[k][7] for k in arms} # The published reading: the strongest zero-shot model is FLAT across the rigs # (+0.007) where RampNet gains +0.115. If a re-run ever moves this materially, # the write-up's argument changes and should be re-read, not silently updated. assert abs(f1["laurens_gsv"] - f1["laurens_mapillary"]) < 0.01 + + +# --------------------------------------------------------------------------- # +# the prose that ranks the annapolis legs (#122, #156) +# --------------------------------------------------------------------------- # +def test_the_strongest_annapolis_claim_tracks_the_published_legs(): + """When the Fable legs landed, docs/model_scoreboard.md and this doc both kept + saying claude-opus-5 was the strongest general model on annapolis, in the + present tense, above a table that showed otherwise. The scoreboard got a pin + in review round 1; this is the same pin for model_comparison.md, read off the + published detections rather than off any number typed into the doc. + + The sentence is allowed either shape -- "X is the strongest" when X is the best + published annapolis leg, or "X was the strongest ... until Y" when Y is -- but + the leg it names has to agree with the files.""" + import re + with open(os.path.join(REPO, "docs", "model_comparison.md"), encoding="utf-8") as fh: + doc = fh.read() + scored = {k: _score(*k, _ground_truths("annapolis")) + for k in PUBLISHED_LEGS if k[3] == "annapolis"} + best = max(scored, key=lambda k: scored[k][7]) + m = re.search(r"\*\*`([\w.-]+)` at `(\w+)` (is|was) the strongest general model " + r"measured on annapolis\*\*", doc) + assert m, "the strongest-general-model sentence is gone from model_comparison.md" + named, effort, verb = m.groups() + if verb == "is": + assert (named, effort) == best[:2], ( + f"the doc says {named}/{effort} is the strongest on annapolis; the published " + f"files say {best[0]}/{best[1]} (F1 {scored[best][7]:.3f})") + else: + assert (named, effort) != best[:2], f"{named}/{effort} IS still the best; say so" + # The supersession has to name the leg that did it, close by. + after = doc[m.end():m.end() + 400] + assert "until" in after and f"`{best[0]}`" in doc, after + # ...and the Fable section's own claim has to hold: both Fable legs above Opus. + opus = scored[("claude-opus-5", "low", "vertex", "annapolis")][7] + for k in scored: + if k[0].startswith("claude-fable"): + assert scored[k][7] > opus, (k, scored[k][7], opus) + assert "**Both displace `claude-opus-5` at the top of this split**" in doc + # The coverage paragraph at the top of the doc names every off-roster Claude leg + # (not "annapolis only" any more: Opus-low has eleven splits since #139/#151). + assert "**Six Claude legs are off-roster.**" in doc + assert len({k[:3] for k in scored}) == 6 + + +# --------------------------------------------------------------------------- # +# the Fable cost figures, re-derived from the ledger (#156) +# --------------------------------------------------------------------------- # +USAGE_LOG = os.path.join(REPO, "analysis_out", "usage_log.jsonl") +DOC = os.path.join(REPO, "docs", "model_comparison.md") + + +def _fable_ledger(): + """``{model_id: [rows]}`` for the two Fable ids, in ledger order.""" + rows = {} + with open(USAGE_LOG, encoding="utf-8") as fh: + for line in fh: + if not line.strip(): + continue + r = json.loads(line) + if r.get("provider") == "claude" and "fable" in r["model_id"]: + rows.setdefault(r["model_id"], []).append(r) + return rows + + +def test_the_fable_cost_figures_are_what_the_ledger_says(): + """The doc's cost column was the 720-call row (120 panos), labelled as if it were + the leg; the manual_gold estimate quoted one id's calibration rate and did not + follow from it (6,000 calls at $0.0272 is $163, not $155). Every figure the doc + now states is recomputed here from the committed ledger. (S3 on PR #157.)""" + with open(DOC, encoding="utf-8") as fh: + doc = fh.read() + ledger = _fable_ledger() + assert set(ledger) == {"claude-fable-5", "claude-fable-5-1"} + # Both legs: one 720-call full row, and a calibration pass of 30 calls on the + # 5 panos the full row then read from cache (a second, partial calibration pass + # follows it -- that one belongs to neither number). + want = {} + for model, rows in ledger.items(): + full = [r for r in rows if r["calls"] == 720] + cal = [r for r in rows if r["panos_scored"] == 5] + assert len(full) == 1 and full[0]["panos_scored"] == 125 + assert cal[0]["calls"] == 30 and 720 + 30 == 125 * 6 + per_call = full[0]["est_cost_usd"] / 720 + want[model] = { + "row": full[0]["est_cost_usd"], + "whole": full[0]["est_cost_usd"] + cal[0]["est_cost_usd"], + "per_call": per_call, + "manual_gold": per_call * 1000 * 6, + "thinking": full[0]["thoughts_tokens"], + } + f5, f51 = want["claude-fable-5"], want["claude-fable-5-1"] + assert f"| `claude-fable-5` | 0.579 | 0.646 | **0.611** | 190/138/104 | 2.72 | " \ + f"{f5['thinking']:,} | ${f5['row']:.2f} |" in doc + assert f"| `claude-fable-5-1` | 0.637 | 0.585 | **0.610** | 172/98/122 | 2.28 | " \ + f"{f51['thinking']:,} | ${f51['row']:.2f} |" in doc + assert "cost, 720 calls (120 of 125 panos)" in doc # the column says what it is + assert f"each id's whole split cost **${f5['whole']:.2f}** and **${f51['whole']:.2f}**" in doc + assert f"(${f5['per_call']:.4f}/call for `claude-fable-5`," in doc + assert f"${f51['per_call']:.4f}/call for `claude-fable-5-1`)" in doc + assert f"**~${f5['manual_gold']:.0f} and ~${f51['manual_gold']:.0f} per id**" in doc + # The Opus leg has no ledger row (its tokens were never logged; see the doc), so + # $8.94 for 750 calls is a stated number, and the ratio is checked against it. + opus_per_call = 8.94 / 750 + assert f"per call the legs cost {f5['per_call'] / opus_per_call:.2f}x and " \ + f"{f51['per_call'] / opus_per_call:.2f}x the" in doc + # The thing the old sentence got wrong: an estimate must follow from the rate + # beside it. $155 at $0.0272/call would have needed 5,699 calls. + assert "$0.0272/call it is" not in doc diff --git a/tests/test_export_model_cache.py b/tests/test_export_model_cache.py index 34caf061..9f2593db 100644 --- a/tests/test_export_model_cache.py +++ b/tests/test_export_model_cache.py @@ -275,6 +275,40 @@ def test_a_pinned_leg_publishes_under_its_registry_name_without_being_told(): assert em.publication_name("gemini:gemini-9-turbo", cargs) == "gemini-9-turbo" +@pytest.mark.parametrize("spec", ["claude:claude-fable-5", "claude:claude-fable-5-1"]) +def test_a_registered_spec_whose_pins_all_miss_refuses_the_bare_label(spec): + """Both Fable legs pin claude_serving_path=anthropic; the default is vertex. The + path is not in the cache key, so an export at the defaults found every pano, + named the file `claude-fable-5__annapolis.json`, wrote `pins: {}`, collided with + nothing, and reported success. The effort pin never had this hole because its + default (`low`) matches the published legs; the serving-path default matches + neither. (S1 on PR #157.)""" + cargs = em._compare_args(".model_cache") + assert cargs.claude_serving_path == "vertex" # the trap, as shipped + with pytest.raises(ValueError) as err: + em.publication_name(spec, cargs) + msg = str(err.value) + # The message names what the registry knows and what the run had, so the fix is + # readable off the error rather than off the roster. + assert "claude_serving_path=anthropic" in msg + assert "claude_serving_path=vertex" in msg + assert "--publish-as" in msg + cargs.claude_serving_path = "anthropic" + assert em.publication_name(spec, cargs).endswith("-effort-low-anthropic") + + +def test_the_same_hole_on_the_effort_pin_is_closed_too(): + """Pre-existing shape of the same bug: `claude:claude-sonnet-5` has legs at + effort low and high only, so `medium` used to publish as the bare + `claude-sonnet-5`.""" + cargs = em._compare_args(".model_cache") + cargs.claude_effort = "medium" + with pytest.raises(ValueError, match="claude_effort=medium"): + em.publication_name("claude:claude-sonnet-5", cargs) + # --publish-as remains the way to name a leg that is genuinely new. + assert em.publication_name("claude:claude-sonnet-5", cargs, "sonnet-medium") == "sonnet-medium" + + def test_publish_as_refuses_more_than_one_spec(tmp_path): with pytest.raises(ValueError, match="ONE leg"): em.export("/nope", str(tmp_path), ["annapolis"], diff --git a/tests/test_model_comparison.py b/tests/test_model_comparison.py index 1359cb23..1d7622e3 100644 --- a/tests/test_model_comparison.py +++ b/tests/test_model_comparison.py @@ -1135,11 +1135,14 @@ def test_claude_as_run_defaults_stay_out_of_the_signature(): det = ClaudeDetector(model_id="claude-sonnet-5") assert det.image_format == detectors.CLAUDE_AS_RUN_IMAGE_FORMAT == "jpeg" assert det.temperature is detectors.CLAUDE_AS_RUN_TEMPERATURE is None + assert det.max_tokens == detectors.CLAUDE_AS_RUN_MAX_TOKENS == 4096 sig = det.signature() assert "image_format" not in sig and "temperature" not in sig + assert "max_tokens" not in sig -@pytest.mark.parametrize("kwargs", [{"image_format": "png"}, {"temperature": 0.0}]) +@pytest.mark.parametrize("kwargs", [{"image_format": "png"}, {"temperature": 0.0}, + {"max_tokens": 8192}]) def test_claude_deviating_from_the_as_run_settings_invalidates_the_cache(kwargs): base = ClaudeDetector(model_id="claude-sonnet-5") other = ClaudeDetector(model_id="claude-sonnet-5", **kwargs) @@ -1147,6 +1150,75 @@ def test_claude_deviating_from_the_as_run_settings_invalidates_the_cache(kwargs) cache_key("claude-sonnet-5", other.signature(), "annapolis", "p1") +def test_claude_serving_path_stays_out_of_the_cache_key(): + """Which account billed the call is not part of what was asked (#156). + + The same model id, prompt and views should return the same detections whoever + serves them, so putting the path in the key would fragment the cache along an + axis that does not change the answer — and would orphan $28.82 of paid + annapolis detections the instant a leg moved. Provenance goes to the usage + log instead. This is the pair to test_claude_cache_key_is_frozen: that one + pins the four published keys, this one says a path switch cannot move them.""" + vertex = ClaudeDetector(model_id="claude-sonnet-5", serving_path="vertex") + first_party = ClaudeDetector(model_id="claude-sonnet-5", serving_path="anthropic") + assert "serving_path" not in vertex.signature() + assert vertex.signature() == first_party.signature() + assert cache_key("claude-sonnet-5", vertex.signature(), "annapolis", "p1") == \ + cache_key("claude-sonnet-5", first_party.signature(), "annapolis", "p1") + + +def test_claude_rejects_an_unknown_serving_path(): + """Refused at construction, like the Fable forced-tool guard: a typo that + reaches the client constructor fails with the SDK's error, not ours.""" + with pytest.raises(ValueError, match="serving_path"): + ClaudeDetector(model_id="claude-sonnet-5", serving_path="bedrock") + + +def test_usage_record_carries_the_serving_path(tmp_path): + """The one place the path is written down, so it has to actually be written. + + `model_versions` cannot stand in — both paths report the bare model id — and + the signature deliberately omits it, so without this field nothing + distinguishes a Vertex leg from a first-party one after the fact. It also + says which reconciliation exists: vertex_usage.py recovers server-side spend + for `vertex` and has no first-party equivalent.""" + det = ClaudeDetector(model_id="claude-fable-5-1", serving_path="anthropic") + det.accumulate_usage(10, 5, 0) + log = tmp_path / "usage_log.jsonl" + report_usage(det, "claude-fable-5-1", "annapolis", 125, str(log)) + rec = json.loads(log.read_text(encoding="utf-8").splitlines()[-1]) + assert rec["serving_path"] == "anthropic" + assert "serving_path" not in (rec["signature"] or {}) + + +def test_the_regional_pricing_warning_is_vertex_only(): + """GOOGLE_CLOUD_LOCATION is shared with the Gemini legs, so it can be set to a + region for reasons that have nothing to do with Claude. On Vertex that really + does make the cost figures ~9% low and is worth shouting about; on the + first-party path there is no endpoint dimension, and warning anyway would + train the reader to ignore a real alarm.""" + regional = ClaudeDetector(model_id="claude-sonnet-5", location="us-east5") + assert "REGIONAL" in (regional.location_warning() or "") + first_party = ClaudeDetector(model_id="claude-fable-5-1", location="us-east5", + serving_path="anthropic") + assert first_party.location_warning() is None + + +@pytest.mark.parametrize("model_id", ["claude-fable-5", "claude-fable-5-1"]) +def test_both_fable_ids_are_priced_from_the_first_party_card(model_id): + """#156 runs both ids, so both need a price — and the cache rows had to be + READ, not derived. Fable 5 reads cache at the standard 0.1x of input; Fable + 5.1 reads at $0.25/MTok, which is no multiple of its $10 input rate. Deriving + it would have overstated 5.1's cache reads 4x.""" + from pricing import price_for + p = price_for(model_id) + assert (p["input_per_m"], p["output_per_m"]) == (10.00, 50.00) + assert p["as_of"] == "2026-09-05" + assert "first-party" in p["note"] + assert price_for("claude-fable-5")["cache_read_per_m"] == 1.00 + assert price_for("claude-fable-5-1")["cache_read_per_m"] == 0.25 + + def test_claude_tool_choice_is_in_the_cache_key(): auto = ClaudeDetector(model_id="claude-sonnet-5", tool_choice="auto") forced = ClaudeDetector(model_id="claude-sonnet-5", tool_choice="forced") @@ -1154,6 +1226,110 @@ def test_claude_tool_choice_is_in_the_cache_key(): cache_key("claude-sonnet-5", forced.signature(), "bend", "p1") +@pytest.mark.parametrize("model_id, status", [ + ("claude-fable-5-1", "rejected"), # per Anthropic's docs + ("claude-mythos-5-1", "rejected"), # same docs, same restriction + ("claude-fable-5", "unverified"), # refused until someone measures it +]) +def test_forced_tool_choice_is_refused_before_the_run_starts(model_id, status): + """Some ids reject `tool_choice` `tool`/`any` with a 400 -- and one is refused + without a measurement either way (see CLAUDE_FORCED_TOOLS_UNVERIFIED). + + A leg discovers a 400 on its first call and then repeats it ~750 times, so the + refusal has to happen at construction, where it costs nothing and can name the + flag the provider's own error does not. The message also has to say WHICH + kind of refusal it is: a documented 400 and an unmeasured id call for + different responses (change the flag vs. run the probe).""" + assert detectors.claude_forced_tools_status(model_id) == status + with pytest.raises(ValueError, match="tool-choice") as err: + ClaudeDetector(model_id=model_id, tool_choice="forced") + msg = str(err.value) + if status == "unverified": + assert "not been measured" in msg + assert f"probe_claude_models.py --serving-path anthropic --models {model_id} " \ + f"--tool-choice forced" in msg + else: + assert "400" in msg and "not been measured" not in msg + # `auto` -- the default, and what a Fable leg must run -- is unaffected. + assert ClaudeDetector(model_id=model_id, tool_choice="auto").tool_choice == "auto" + + +def test_the_forced_tool_guard_matches_exact_ids_not_a_family_substring(): + """The guard used to be `"fable" in model_id`. Anthropic's docs say the 400 + arrived with 5.1, not with the family, so a substring match refused ids nobody + had looked at and could not say why. Now every refused id is listed, with its + evidence, and anything else is allowed -- which is the assumption every + published leg relies on.""" + assert detectors.CLAUDE_FORCED_TOOLS_REJECTED.isdisjoint( + detectors.CLAUDE_FORCED_TOOLS_UNVERIFIED) + for mid in detectors.CLAUDE_FORCED_TOOLS_REJECTED | detectors.CLAUDE_FORCED_TOOLS_UNVERIFIED: + assert detectors.claude_forbids_forced_tools(mid) + # A spelling that is not a listed id is not refused, whatever it contains. + for mid in ("claude-fable-6", "claude-fable-5-1-fast", "fable", None, ""): + assert detectors.claude_forced_tools_status(mid) is None + assert not detectors.claude_forbids_forced_tools(mid) + + +def test_forced_tool_choice_still_works_on_the_ids_it_was_measured_on(): + """The guard keys on the id, so it must not spread to the existing legs. + + "Measured", not "published with": every published Claude leg ran `auto` (all + six annapolis signatures and all twelve ledger rows say so). The only forced + measurement on record is the one-view Vertex check in docs/model_comparison.md + ("The tool is offered, not forced"), on these two ids.""" + for mid in ("claude-opus-5", "claude-sonnet-5"): + assert not detectors.claude_forbids_forced_tools(mid) + assert ClaudeDetector(model_id=mid, tool_choice="forced").tool_choice == "forced" + + +def test_the_probe_can_send_the_forced_tool_choice_the_guard_is_waiting_on(monkeypatch): + """The unverified set points at `probe_claude_models.py --tool-choice forced`; + this checks that flag builds the request the detector would send -- tool_choice + type `tool` naming a declared tool, AND output_config.effort beside it, since + effort is the axis forcing is known to interact with and the detector never + sends one without the other -- and that `auto` sends none of it, without a + network. Measuring the answer is deliberately NOT done here. + + Runs without the `anthropic` package: it is not in requirements-dev.txt, so + CI does not have it, and the first version of this test failed there on the + probe's import guard. The probe only needs the SDK for its two exception + classes, so a stub stands in for it and the request-shape check still runs.""" + import types + sys.path.insert(0, os.path.join(REPO_ROOT, "scripts", "model_comparison")) + import probe_claude_models as probe + + class _StatusError(Exception): + pass + + class _ConnError(Exception): + pass + + monkeypatch.setattr(probe, "_sdk", lambda: types.SimpleNamespace( + APIStatusError=_StatusError, APIConnectionError=_ConnError)) + + seen = {} + + class _Messages: + def create(self, **kw): + seen.clear() + seen.update(kw) + return types.SimpleNamespace(model=kw["model"], usage=None, + stop_reason="tool_use") + + client = types.SimpleNamespace(messages=_Messages()) + status, _ = probe.probe(client, "claude-fable-5", tool_choice="forced") + assert status == 200 + assert seen["tool_choice"] == {"type": "tool", "name": probe.PROBE_TOOL["name"]} + assert [t["name"] for t in seen["tools"]] == [probe.PROBE_TOOL["name"]] + assert seen["output_config"] == {"effort": "low"} + # ...the same keys ClaudeDetector._call sends, so the probe answers for it. + assert {"model", "max_tokens", "output_config", "tools", "tool_choice", + "messages"} <= set(seen) + probe.probe(client, "claude-fable-5") + assert "tool_choice" not in seen and "tools" not in seen + assert "output_config" not in seen # the reachability probe is unchanged + + # --- Claude: what pixels the model actually sees ---------------------------- def test_claude_image_encoding_is_explicit_and_round_trips(): diff --git a/tests/test_roster.py b/tests/test_roster.py index abd664fe..00d5f009 100644 --- a/tests/test_roster.py +++ b/tests/test_roster.py @@ -233,7 +233,17 @@ def test_each_published_file_names_the_leg_it_says_it_is(): assert payload["signature"]["model_id"] == entry.label, name for key, value in entry.pins: sig_key = key.split("_", 1)[1] # claude_effort -> effort - assert payload["signature"][sig_key] == value, (name, key) + if sig_key in payload["signature"]: + assert payload["signature"][sig_key] == value, (name, key) + else: + # A pin that is NOT a signature field (claude_serving_path, #156: + # it changes who bills, not what was asked, so it is deliberately + # kept out of the cache key). The file still has to name it, or + # nothing in the published artifact distinguishes a Vertex-served + # leg from a first-party one. `pins` is absent on files published + # before that field existed — those legs pin only signature + # fields, so they never reach this branch. + assert payload.get("pins", {}).get(key) == value, (name, key) # --------------------------------------------------------------------------- # diff --git a/tests/test_scoreboard.py b/tests/test_scoreboard.py index 4df5332f..cdb9e144 100644 --- a/tests/test_scoreboard.py +++ b/tests/test_scoreboard.py @@ -532,6 +532,10 @@ def test_single_split_legs_stay_out_of_the_pooled_tables(board): "claude-opus-5-effort-high": "1/8", "claude-sonnet-5-effort-low": "1/8", "claude-sonnet-5-effort-high": "1/8", + # #156, annapolis only. Both clear the expansion gate, so these may well + # grow past 1/8 later; until they do, partial coverage means excluded. + "claude-fable-5-1-effort-low-anthropic": "1/8", + "claude-fable-5-effort-low-anthropic": "1/8", } single = [m for m in board["models"] if not m["complete"]] assert {m["model"] for m in single} == set(want_coverage) @@ -561,6 +565,54 @@ def test_partial_table_names_the_split_every_number_came_from(board): assert "Claude Opus 5 (low)" not in table +_WORDS = {2: "Two", 3: "Three", 4: "Four", 5: "Five", 6: "Six", 7: "Seven", 8: "Eight", + 9: "Nine", 10: "Ten", 11: "Eleven", 12: "Twelve"} + + +def test_the_prose_beside_the_partial_table_agrees_with_it(board): + """The generator rewrites the tables and nothing else. The sentences around them + are hand-written, and when the Fable legs landed (#156) the table gained two rows + that beat the leg the prose still called the strongest challenger, while the leg + count it quoted stayed at six. (S2 on PR #157.) + + Two checks, both against the board rather than against any number typed here: + the count of legs outside the pooled tables, and which leg is the best challenger + on annapolis -- the split every Claude leg has run, and the one the prose singles + out. + """ + with open(sb.DEFAULT_DOC, encoding="utf-8", newline="") as fh: + doc = fh.read() + partial = [m for m in board["models"] if not m["complete"]] + one_split = [m for m in partial if m["n_splits_run"] == 1] + more = [m for m in partial if m["n_splits_run"] > 1] + if not more: + # Every partial leg is one-split (the state since #139 completed Opus-low), + # and the sentence says exactly that. + want = f"{_WORDS[len(partial)]} legs have run one split each" + else: + # A breakdown, which has to add up. The first version of this test pinned + # the total only, and the sentence said "six of them one split each" + # beside a three-split leg: six plus one is seven. + want = (f"{_WORDS[len(partial)]} legs have not run the pooled splits — " + f"{_WORDS[len(one_split)].lower()} of them one split each") + assert doc.count(want) == 2, ( + f"expected {want!r} twice in docs/model_scoreboard.md (the section opener and " + f"the 'What is missing' bullet); the board has {len(partial)} partial legs, " + f"{len(one_split)} of them one-split, {[m['model'] for m in more]} with more") + + best = max((m for m in board["models"] if m["model"] != "rampnet" + and "annapolis" in board["per_split"][m["model"]]), + key=lambda m: board["per_split"][m["model"]]["annapolis"]["f1"]) + claim = re.search(r"\*\*(.+?) is the strongest challenger measured on annapolis\*\*", doc) + assert claim, "the strongest-challenger sentence is gone" + # Whole model name, not a prefix: "Claude Fable 5" is a prefix of "Claude Fable + # 5.1 (low, anthropic)", and the two are 0.001 F1 apart, so a prefix match + # would stay green through the one flip that is actually plausible. + assert best["display"].split(" (")[0] == claim.group(1).split(" at ")[0], ( + f"the doc names {claim.group(1)!r}; the board says {best['display']!r} " + f"(F1 {board['per_split'][best['model']]['annapolis']['f1']:.3f})") + + def test_a_leg_from_an_unmapped_provider_is_classified_not_dropped(): """A roster entry whose provider predates this module still has to reach the board.""" leg = roster.Challenger(spec="newthing:x", label="x", provider="newthing",