diff --git a/analysis_out/scoreboard.json b/analysis_out/scoreboard.json index 90797721..a951944d 100644 --- a/analysis_out/scoreboard.json +++ b/analysis_out/scoreboard.json @@ -415,6 +415,59 @@ "manual_gold_recall": null, "manual_gold_ap": null }, + { + "model": "claude-opus-5-effort-low", + "label": "claude-opus-5", + "spec": "claude:claude-opus-5", + "provider": "claude", + "standing": false, + "display": "Claude Opus 5 (low)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "8/8", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville", + "laurens_mapillary" + ], + "precision": 0.56161, + "recall": 0.585547, + "f1": 0.5678, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 1.123888, + "micro_precision": 0.560082, + "micro_recall": 0.589433, + "f1_min": 0.42953, + "f1_max": 0.648551, + "f1_min_split": "laurens_mapillary", + "f1_max_split": "morgantown", + "n_splits_run": 11, + "laurens_gsv_f1": 0.437186, + "laurens_gsv_precision": 0.488764, + "laurens_gsv_recall": 0.395455, + "laurens_gsv_ap": null, + "budapest_district5_f1": 0.377778, + "budapest_district5_precision": 0.425, + "budapest_district5_recall": 0.34, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.468227, + "sao_paulo_precision": 0.44164, + "sao_paulo_recall": 0.498221, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, { "model": "gemini-3.7-flash", "label": "gemini-3.7-flash", @@ -924,53 +977,6 @@ "manual_gold_recall": null, "manual_gold_ap": null }, - { - "model": "claude-opus-5-effort-low", - "label": "claude-opus-5", - "spec": "claude:claude-opus-5", - "provider": "claude", - "standing": false, - "display": "Claude Opus 5 (low)", - "class": "chat-vlm", - "operating_point": 0.0, - "operating_point_note": "no score", - "coverage": "2/8", - "complete": false, - "pooled_splits": [ - "annapolis", - "laurens_mapillary" - ], - "precision": 0.528598, - "recall": 0.495492, - "f1": 0.50898, - "ap": null, - "ap_bundle": null, - "ap_is_substituted": false, - "fp_per_pano": 1.074553, - "micro_precision": 0.53831, - "micro_recall": 0.504604, - "f1_min": 0.42953, - "f1_max": 0.58843, - "f1_min_split": "laurens_mapillary", - "f1_max_split": "annapolis", - "n_splits_run": 3, - "laurens_gsv_f1": 0.437186, - "laurens_gsv_precision": 0.488764, - "laurens_gsv_recall": 0.395455, - "laurens_gsv_ap": null, - "budapest_district5_f1": null, - "budapest_district5_precision": null, - "budapest_district5_recall": null, - "budapest_district5_ap": null, - "sao_paulo_f1": null, - "sao_paulo_precision": null, - "sao_paulo_recall": null, - "sao_paulo_ap": null, - "manual_gold_f1": null, - "manual_gold_precision": null, - "manual_gold_recall": null, - "manual_gold_ap": null - }, { "model": "claude-sonnet-5-effort-low", "label": "claude-sonnet-5", @@ -2019,6 +2025,184 @@ "fp_per_pano": 1.16 } }, + "claude-opus-5-effort-low": { + "richmond": { + "split": "richmond", + "precision": 0.522673, + "recall": 0.706452, + "f1": 0.600823, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 219, + "fp": 200, + "fn": 91, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 1.612903 + }, + "bend": { + "split": "bend", + "precision": 0.586207, + "recall": 0.623853, + "f1": 0.604444, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 204, + "fp": 144, + "fn": 123, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 1.309091 + }, + "clovis": { + "split": "clovis", + "precision": 0.528302, + "recall": 0.574359, + "f1": 0.550369, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 112, + "fp": 100, + "fn": 83, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.8 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.62807, + "recall": 0.670412, + "f1": 0.648551, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 179, + "fp": 106, + "fn": 88, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.848 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.572347, + "recall": 0.605442, + "f1": 0.58843, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 178, + "fp": 133, + "fn": 116, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 1.064 + }, + "paterson": { + "split": "paterson", + "precision": 0.751701, + "recall": 0.559494, + "f1": 0.641509, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 221, + "fp": 73, + "fn": 174, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.584 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.418733, + "recall": 0.558824, + "f1": 0.47874, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 152, + "fp": 211, + "fn": 120, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 1.688 + }, + "laurens_mapillary": { + "split": "laurens_mapillary", + "precision": 0.484848, + "recall": 0.385542, + "f1": 0.42953, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 96, + "fp": 102, + "fn": 153, + "n_panos": 94, + "n_gt_recall": 249, + "fp_per_pano": 1.085106 + }, + "laurens_gsv": { + "split": "laurens_gsv", + "precision": 0.488764, + "recall": 0.395455, + "f1": 0.437186, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 87, + "fp": 91, + "fn": 133, + "n_panos": 86, + "n_gt_recall": 220, + "fp_per_pano": 1.05814 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.425, + "recall": 0.34, + "f1": 0.377778, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 102, + "fp": 138, + "fn": 198, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 1.104 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.44164, + "recall": 0.498221, + "f1": 0.468227, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "bundle_floor": null, + "tp": 140, + "fp": 177, + "fn": 141, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 1.416 + } + }, "gemini-3.7-flash": { "richmond": { "split": "richmond", @@ -3415,56 +3599,6 @@ "fp_per_pano": 2.048 } }, - "claude-opus-5-effort-low": { - "annapolis": { - "split": "annapolis", - "precision": 0.572347, - "recall": 0.605442, - "f1": 0.58843, - "ap": null, - "ap_source": "bundle", - "ap_bundle": null, - "bundle_floor": null, - "tp": 178, - "fp": 133, - "fn": 116, - "n_panos": 125, - "n_gt_recall": 294, - "fp_per_pano": 1.064 - }, - "laurens_mapillary": { - "split": "laurens_mapillary", - "precision": 0.484848, - "recall": 0.385542, - "f1": 0.42953, - "ap": null, - "ap_source": "bundle", - "ap_bundle": null, - "bundle_floor": null, - "tp": 96, - "fp": 102, - "fn": 153, - "n_panos": 94, - "n_gt_recall": 249, - "fp_per_pano": 1.085106 - }, - "laurens_gsv": { - "split": "laurens_gsv", - "precision": 0.488764, - "recall": 0.395455, - "f1": 0.437186, - "ap": null, - "ap_source": "bundle", - "ap_bundle": null, - "bundle_floor": null, - "tp": 87, - "fp": 91, - "fn": 133, - "n_panos": 86, - "n_gt_recall": 220, - "fp_per_pano": 1.05814 - } - }, "claude-sonnet-5-effort-low": { "annapolis": { "split": "annapolis", diff --git a/benchmark/model_detections/claude-opus-5-effort-low__bend.json b/benchmark/model_detections/claude-opus-5-effort-low__bend.json new file mode 100644 index 00000000..eeb7928e --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__bend.json @@ -0,0 +1 @@ +{"city":"bend","detections":{"-6wL7HzdK7t8DTC_zDPOLw":[[0.32799777673054553,0.5892253591220026,null]],"0CvMs02xHlg3mCMHJdc6Xg":[[0.5959468519241533,0.5704631702702925,null],[0.38834020395069974,0.6676965025988052,null],[0.447591097105487,0.5709223934782521,null],[0.8547787921714165,0.6166601909378416,null],[0.881876373229711,0.6056549601347094,null],[0.09092414009912664,0.6352060755338139,null],[0.9295818814006289,0.6057780010380298,null],[0.20492627416133685,0.5563202483617455,null],[0.23836765984862174,0.5532797804999599,null]],"0FMaNGiSOoTGqLgh8jHfYQ":[[0.36550737654366067,0.6657729185030118,null],[0.775522688063903,0.7271514886473585,null],[0.7544137299046776,0.7098127208596313,null],[0.8460236481570486,0.720629133194443,null],[0.3010802945166566,0.6580390018664326,null],[0.23675538195080198,0.5718269005051095,null],[0.1514759565022834,0.5481814166708925,null],[0.11898579821003469,0.548866034713048,null],[0.32771622296595804,0.6544675653367396,null]],"0SgaioiGd5Yi_5wc6m-nJg":[[0.47403819705370226,0.5375108689014578,null],[0.5672675450920432,0.5361381737501236,null]],"1k8PFgEWlZnrY-j007WMAQ":[],"2S_Fj0bq7qkdKUBw9-g1XA":[[0.35023200853288516,0.7168539361806682,null],[0.7954799650628053,0.644906635289489,null],[0.8249762168349353,0.6382857803261213,null],[0.12200935922939399,0.6963846349336753,null],[0.8827834097351445,0.61734237122372,null],[0.11389153352031078,0.5387124765548233,null],[0.19829211412787628,0.5655899886836925,null],[0.1910921168463069,0.6733704249435569,null]],"3EUb8DGIsQCcw8z9LOBnAA":[[0.6022238899393468,0.5868940123451235,null],[0.5189512176644572,0.539051281269813,null]],"4ijauQusmQIJgw4YLTkYsA":[],"5NVgLhChZXnD_D-z3SLZyQ":[[0.6272541725715064,0.6446350227048991,null],[0.7234048815935554,0.6137332785701748,null],[0.9322435802313439,0.560489846305573,null],[0.11682279800463058,0.5551351158304245,null],[0.26290929683448006,0.5730770008668176,null],[0.3202726648055951,0.5846233060812986,null]],"5U3KS0M2Y8w3qsDGGV610g":[[0.3181892628706924,0.5813519371795517,null]],"62TiKmlmxt_xn_68w1lgAw":[[0.39658506042670355,0.6284878075561907,null],[0.6189056885260928,0.6249632873323792,null],[0.5706368935524697,0.5291664180434676,null]],"6WC0hdAYRsSAcluKSs5iRg":[],"7B5Vphda5PDmZJ5q1RQWdg":[],"87nWq0hrTZLm2_UXRgPhZQ":[[0.9504958722123227,0.5762865913114302,null],[0.04995202625795053,0.5599455171686102,null]],"8H6LgNn56hay95HiBYrhHw":[[0.7466842837356821,0.6166188155710137,null],[0.7959143829774951,0.5303876222638921,null],[0.8234930980216071,0.57752812585064,null],[0.8920834511451968,0.5209574397008692,null],[0.27542179615040374,0.5842821224612811,null],[0.3183190736824453,0.5702351459498343,null]],"8KtYwvo9b34f3zm-afipNg":[[0.7719909320495255,0.5642925270794937,null]],"8VPbrd8XBzDOIVoo5Xwz8g":[[0.9455747982608225,0.5494786602859728,null],[0.09115908240242282,0.5505040142789305,null]],"8cphEWdm85w5CwOw4rm0_A":[[0.5638988279456746,0.6147440132766498,null],[0.38863342993204575,0.5856079063856963,null],[0.9329145409626413,0.541020066615672,null],[0.27992675777976844,0.6076975831233316,null],[0.2841074021287496,0.5349272467807941,null],[0.41706845658390124,0.5713001506425825,null]],"91IlnKR3b8rCc1Zx5XGy7w":[[0.46083230910388967,0.5446097979950112,null],[0.8961122320408171,0.6179392172181498,null],[0.061619811829026094,0.5411240246936339,null]],"9DA1aapwjQcdUP2HEfrhfg":[[0.5741536533888719,0.5425806866885635,null],[0.7798930674749915,0.6016880647847591,null],[0.7567568166640726,0.6047307899759966,null],[0.9363418413537261,0.5733429093076879,null],[0.08912086034484834,0.6268467484519484,null],[0.07256184073097527,0.5861956714147835,null],[0.04931141894606711,0.5702565978123786,null]],"9kW9cxpuj7q8DMzf-ClrQQ":[[0.4272058224729009,0.5506499760611074,null]],"A5AVKFhm7unr9Rs55aQ1Tw":[[0.4314332578236816,0.5317066398297352,null]],"BfLStXTTg1-0FjTrZcvYWg":[[0.5826365237667792,0.53626187071391,null],[0.6911809366579817,0.5641558913008484,null],[0.2916335354695234,0.5415849545093562,null]],"CIdYXefgwcZw4L9PqIpuHQ":[],"DFx_w62mJn4Qm4upFTXytQ":[[0.012487749339946896,0.5442356327852285,null]],"DJ8Zp111zu6KnMZz-0PHgQ":[[0.921320772186279,0.6281883817251093,null],[0.8948910878223932,0.6256784998877788,null],[0.2698717033033449,0.5948079151692693,null]],"DaCRMtjRrkxBB468GFSAvQ":[[0.6117137863944151,0.5535704326579046,null],[0.6428246202453125,0.5694934808118604,null],[0.6849674027925832,0.564486578161938,null],[0.8607492341978151,0.5476928024868281,null],[0.16540111478005082,0.5246000963083473,null],[0.2586462974539481,0.525327987618796,null],[0.33998617578883983,0.5411417978341592,null]],"E15SCY6RTiDuNUu_trZk4Q":[[0.44738255274569927,0.5813287928967235,null],[0.5853115855708654,0.5731839857380572,null],[0.6988090975792148,0.6091928231295196,null],[0.8896363861246616,0.5697889215442501,null],[0.9409957652377992,0.5564870623801562,null],[0.1134507325508074,0.5410997697933938,null],[0.15071391454345873,0.5495042575799084,null],[0.1870703666808608,0.544010911025386,null],[0.281934208668537,0.5510537867547118,null],[0.3619758469143036,0.5524398256035331,null],[0.4081883630015622,0.6227146151345772,null]],"E1wxX3pnvZdWrWnUkfkNkA":[[0.5694756186449264,0.6317783719507244,null],[0.5896446516301787,0.5635819342724707,null]],"HraXxn_y7_j4o8_4iS_JDQ":[[0.7747281977331235,0.5773127129513796,null],[0.2036031753477967,0.5623905113488765,null],[0.23476903386770337,0.5563511182840398,null]],"I8Y9zDjNKL1k5HUv6BSyFA":[[0.6666666666666666,0.7932144477677419,null],[0.7364295405252012,0.536840921758121,null],[0.7743471731111254,0.6889912955047864,null],[0.8356380744695793,0.5385326908191094,null],[0.8732610721529115,0.6088776216662487,null],[0.0797914929033195,0.5242612953861951,null]],"IQT-DUkA06hxOus4rPb47A":[[0.8745517661505833,0.6072757971810752,null],[0.04945397778564997,0.5317266440476136,null]],"I_t9Bp1OzV2m-ZKFHssOdg":[[0.9355668951426652,0.5616952437220972,null],[0.9609530269315887,0.5386731610940511,null],[0.07391852465788445,0.5368566332299674,null]],"JliGuAWGKCyOS1LBoywZlA":[],"Jv7Zl2erbO_g10pRN-IS_Q":[[0.8023120502129717,0.7207680415403119,null],[0.8503652606672794,0.713299805711611,null],[0.20223748742645548,0.6512640386341828,null],[0.08476971059256339,0.5508240327911957,null]],"Kv2haiYlkJCiaO61RVWwQA":[[0.6466426701151408,0.6566004519829254,null],[0.7752050766142351,0.6171966274927831,null],[0.9109889902615808,0.5530737538078814,null],[0.07711249835121597,0.543620088895182,null],[0.11112676869519028,0.5581854088386149,null],[0.28057996216349,0.6008366140362063,null],[0.23079824639333124,0.5887754977246734,null]],"LMwVmdxU9opG4yzbotuw6g":[[0.5262176157805591,0.546423307040242,null]],"LN819vhVLLeYwzRsBh2zAg":[[0.43837776567555387,0.54236639175945,null],[0.5363595654545592,0.5484774833817354,null],[0.7038827888677824,0.6251692654976287,null],[0.2732736709367869,0.639613099882039,null],[0.31622741119804787,0.5695813118824254,null]],"LhpYKYMPIDXskyHjHW1nOQ":[[0.47455155815069794,0.545133300143911,null],[0.5802928486385298,0.5231024857356621,null],[0.8895739973525654,0.5505625329564018,null],[0.946929265444577,0.5374306225170964,null]],"NS7lT-itv0JiN6UHVvBxsg":[[0.6099682739181795,0.586451008783911,null],[0.9247498721483569,0.5559032407358537,null],[0.058457941741179964,0.5808443083311955,null]],"Noi7DhIT3xbCjA2Ip4QOyg":[[0.6280531269745757,0.6538610266410699,null],[0.41305965792509486,0.5327462735034407,null],[0.44464325984396164,0.5315739105319759,null],[0.30033764697482424,0.6433295674495959,null]],"NzRL8weHSGkfduBHnrgd5g":[[0.5608878018523057,0.5722537023211696,null],[0.024635242723297757,0.5393831094258504,null],[0.9553977176993635,0.5335207493032714,null],[0.09093406900439999,0.5373555682514607,null],[0.3305753257271874,0.7153732642576285,null]],"OedGesfk606bGhg4OQESiQ":[[0.5485660941676737,0.5701010377850384,null],[0.0532498523055166,0.5394192198214129,null],[0.361375859969827,0.5672512535866097,null]],"PX5zU_2KkKO0Gr4FiSe9aw":[[0.5688733174340931,0.6056619187401849,null],[0.3077629434203375,0.5821298166444324,null],[0.39662145040938546,0.5367737346337915,null]],"PeJlsIXjn8VkEoryRRBBMg":[[0.5756220610496499,0.5553762865913638,null],[0.53304380994951,0.5298185182687171,null],[0.44395421785708017,0.5216745282591625,null],[0.6570360391940993,0.663619316298251,null],[0.723161042549856,0.6868725119897141,null],[0.9576055686301275,0.5361751243263234,null],[0.04819708026724534,0.532301643765132,null]],"PgOXmiCTtEV4ip7AJDqf2Q":[[0.38310877369246327,0.6659285588925399,null],[0.976504100268029,0.5391644611492274,null],[0.07863326049750952,0.534896523699889,null],[0.16666666666666657,0.699499772503862,null],[0.25268007099706546,0.720504490726903,null]],"RS0LXKWETF8PhWMbaexYlg":[[0.5354760828208612,0.523009096756976,null],[0.5066546832722911,0.5168381077232898,null]],"Ro8ZmvW9hyxIS4GVq1cpMQ":[[0.9054986006914683,0.6122796148310249,null],[0.12378824715796949,0.6107621791366188,null]],"SM4RUwTVtTzgSm3LWvzwqA":[[0.5594673718061126,0.5514601246321182,null],[0.41230246239434665,0.5365002372035234,null]],"ScnPNQ2cah0xzWlH28uoOw":[[0.6216464250147957,0.5933865583190457,null]],"ShaAxuhgD3rK5__gON66zA":[[0.5216690334911752,0.5713136557442804,null],[0.3403969435545656,0.5513279958193033,null]],"Tn5W3kvbdH7_Fpk29a_hrQ":[[0.422334795983747,0.5525412715839604,null],[0.5514623608332928,0.5552720268414709,null],[0.5921071694660698,0.549609635582722,null]],"TzKDX5ADk6lqRtZ-F8r6XQ":[[0.6678878115878479,0.6508226266005905,null],[0.7977033534798912,0.6045867135983525,null],[0.8907920408775425,0.5204253151830321,null],[0.2375238992003335,0.5556088644468178,null]],"UdLzRf1Yi67NOSMn6IZjHQ":[[0.5541008194703713,0.5459231088161401,null],[0.8564686439184908,0.6225419511975058,null],[0.9435087107134148,0.5518219241633906,null],[0.3623313844715744,0.5597384007823643,null]],"VIMFndX_oqm7vJJD8gDyEA":[[0.3628950272231796,0.6788999347410873,null],[0.636406258785607,0.7009931681817981,null],[0.6848563763083132,0.7674944221645381,null],[0.27470848766659794,0.6700270975706237,null],[0.2408701764773899,0.6675376397693511,null],[0.3411254071191959,0.6343061475940092,null]],"VMjtxZNqxl9E6oUXnsdI8Q":[[0.6340995244627228,0.6622347645665643,null],[0.6291413688330642,0.597887089666092,null],[0.6883725094234246,0.7043923484317114,null],[0.3214147263589577,0.6052944288701827,null],[0.3406680323254364,0.5735895359449752,null]],"VgWpqFkTwCIROvM0z-DkOw":[[0.6296530990512291,0.6429248075133459,null],[0.715564989272838,0.657662985307397,null],[0.11515082029236912,0.5293418352315059,null],[0.16489640078984258,0.552463228391052,null]],"WGdZz7jOkmmmyQYpbw3F8w":[[0.5039730424818935,0.5733579450085788,null],[0.9222346102276422,0.5542521899148527,null],[0.29861714950911306,0.6590309917286334,null],[0.2276965831827486,0.5527985610442786,null],[0.32288360664995164,0.6598498915902176,null]],"Wnu3AyufxG1XD9JvU5K-hg":[[0.6210768059551081,0.5889484446960527,null],[0.8556623519843225,0.5769370868728932,null],[0.9909218626254589,0.5301629275135962,null],[0.2100565824604595,0.5227224132411807,null],[0.24796215846151776,0.5277668281160288,null]],"X3HUl4G0JyP3fwNYcFAXVw":[[0.6217544043825494,0.6085753481096988,null],[0.7918939092461261,0.6868686681995005,null],[0.8666026190975257,0.6587332025504438,null],[0.11825202947214786,0.6629630228018003,null],[0.17237589287283572,0.6808632139700157,null],[0.21876414008870404,0.6069640961478202,null],[0.2242501223234753,0.6543868791874712,null]],"X769L3ombUWjLtZDlE2fqg":[[0.43993754133243246,0.5748265575453289,null],[0.41525945140041176,0.5394508288524082,null],[0.5576613878739213,0.5768834858313341,null],[0.4405434876819815,0.5254617534919795,null]],"ZO7V6RR9LtZT8l9-SjIQUg":[[0.4141552653266569,0.6339136275216741,null],[0.6350919288337388,0.8182817566875074,null],[0.8344785740492137,0.6994958577373557,null],[0.8734228428281237,0.6894117897122054,null],[0.09905026828520258,0.6767117923778843,null],[0.30547276119076383,0.6055500598669961,null]],"ZlXmdBYkhIpp-wNfPlSIFw":[[0.40169476309609875,0.5833396292555053,null],[0.6074424885353295,0.5821365202723594,null],[0.896583506572723,0.5755839770866509,null],[0.08930615816419624,0.5691073237152845,null],[0.1750764664659566,0.5634185225366369,null],[0.3451261766283241,0.5832248911875101,null]],"bCoStRGdrZB6z6QXF9w0yQ":[[0.5842420790559725,0.5856325326843616,null],[0.8927007153411185,0.626756627852068,null]],"bTkyJpLDQ7HCw4yOdqdNCg":[[0.6225577094003648,0.642072474340156,null],[0.3867654353024402,0.5683743851879477,null],[0.7634375861306121,0.7029696989727586,null],[0.7230712581305196,0.7008038209065721,null],[0.8265142126877227,0.5819229510315997,null],[0.9955635178608043,0.5348574620010667,null],[0.8626884657980983,0.6443582568198302,null],[0.23269577500639527,0.5574712470438065,null],[0.28423922058708395,0.5856925280650975,null]],"bVadNrPq_zXgbJjhhDSwDw":[[0.9067422167300063,0.5613270760944995,null],[0.8871740155547195,0.613020372344678,null]],"cqb7v_2KGHMeSy8VpvkcDQ":[[0.58700387926997,0.5437593098617293,null],[0.527350060282211,0.5254885669148204,null]],"dUy9p1NputrGH7SFDmbCpg":[[0.5774648480394347,0.5838520225278443,null],[0.4012419720071052,0.5556037501284381,null]],"f8KWeNBZLlwJXVSj5_0RAg":[[0.6019727348568629,0.5638857610468861,null],[0.8833043828034975,0.6146086328879111,null],[0.21910193847547288,0.6279901175976927,null]],"gTM14tSsKuUwbSxtTb4xoQ":[[0.46519194752587417,0.5389148959716392,null]],"hKMeDFlSe7fPsZpi4kS-aA":[[0.2614716384765049,0.6297043308926766,null]],"h_1bUHhTLKugR5XhzA1-nA":[[0.9564792148000971,0.5497622930590211,null],[0.042546551058541826,0.536419171499702,null],[0.15073183115822403,0.5821785892174864,null]],"hu3-VWXlGi42PkY38XPJzQ":[[0.6097636641454601,0.5850022603058364,null],[0.925346392197314,0.5688751956175356,null]],"kJr7eKpZcJO_W6V5XlLtBw":[[0.8811106142214014,0.6264049058865625,null],[0.2877159625944958,0.6070284534142749,null]],"kjNgQrRlUM8VkFMli6aIPQ":[[0.5859326817914534,0.5674708369046422,null],[0.8646968875828542,0.568938200582662,null]],"kkDJcaa4ZmOKSqhbnM1MvQ":[[0.7415947648933257,0.64136913223994,null],[0.9048811719765111,0.5545931345111758,null],[0.06517068079380534,0.54844896495931,null],[0.33150749969942717,0.622188676872321,null]],"kmEwK3ZUn80IJAPqjc4nnA":[[0.9025315111668952,0.5799964708109561,null],[0.07909942938622194,0.5965410107684135,null],[0.1249005807586005,0.5849471895308896,null],[0.2933009012065009,0.6036452593428304,null]],"lWmcJJwTxnbJ5OUloRKphA":[],"lijNEguHL6T106ODEBcylQ":[[0.603465844884701,0.6525159316791146,null],[0.3235566888389875,0.5645048734434828,null]],"m7R9z8VWIY94LLSiDgBdBQ":[[0.7070873716806363,0.6485751249250159,null]],"mVwc04XB9DxuIb05ExnrdA":[[0.8307513798920169,0.5696804662406517,null]],"mdQY8aMSdYrnHfHNXqj6GQ":[[0.5301711456874808,0.5324637730066231,null]],"nd-z43_liOX80bmhO15urg":[[0.9219466727092909,0.564326732203913,null],[0.1085579777975238,0.5383196989937752,null]],"o-cW-Lnr7Yd_VI9xiZ1DHw":[[0.08029802177553386,0.5718909335799155,null]],"oKUCusv3jdVCLmevp-P3zw":[[0.557976250137324,0.561804659595724,null]],"pIcQQrFa8_XDG7-VSRhAfQ":[[0.42122933764967124,0.5950157327643822,null],[0.5916214759376571,0.6058581232024108,null],[0.7811711348173851,0.7916766273826403,null],[0.0,0.8353757610230474,null],[0.16666666666666657,0.8099947234329781,null],[0.28064534699144034,0.6956587108795149,null],[0.3361065242971052,0.5462381443826246,null]],"qNYRurrxEhqZvO8GrwE9nA":[[0.43666029295872066,0.5372571307428887,null],[0.5244942719470929,0.539388481810176,null],[0.724468948788511,0.7148200692671925,null],[0.09311058072430689,0.5526866453897177,null]],"rB_gP9faDRSuiKVDu8h23A":[[0.5455501712981364,0.5529971391288936,null]],"rQtHh0Wai-D1FvbvqBlWdw":[[0.37629307770448644,0.6083614925092868,null],[0.7985059857078737,0.6391880400087795,null],[0.014948182090856854,0.5500738181332996,null],[0.14491728111309532,0.5455539362587979,null],[0.337925457624633,0.5945280809149317,null]],"rvwbAGgi-7mNlZHKSPC8WQ":[[0.4464942573616175,0.5336727510372316,null],[0.519115484633339,0.5353986198268846,null]],"rxF0qBiSbOEnMR9wwxpjEw":[[0.6342407777198305,0.6925685718899948,null],[0.6805665414496793,0.7059711758208765,null],[0.05112762112760422,0.5529509948565228,null],[0.10080737328734307,0.5548669610449962,null],[0.2946696901147582,0.6466112229177382,null]],"s4e-NVzTyqcyZ6_5npv6Ow":[[0.5040510414733481,0.5411641783307131,null],[0.5710386397018101,0.5355076522918394,null],[0.8622009731877076,0.7250848953489067,null],[0.34925219923813394,0.5758699860751705,null],[0.4382821177043614,0.5549549336993637,null]],"sLysKXt4_iSuR_gUCdxe5g":[[0.11229747818485591,0.5501294771088305,null],[0.2373151602855793,0.5562656803856265,null]],"sUMfdZmScVLBGMjvZ0MjCA":[[0.9599334491510558,0.5580421111978784,null],[0.04264372051718479,0.5490496759167587,null]],"tAnrbC_F_FXbg7kTiYywVA":[[0.6429982096507817,0.6934855802826985,null],[0.6700950542275639,0.6408938507016296,null],[0.7701836470673477,0.6000270704918477,null]],"tvNqBlxdm0rdkZgBOGVfog":[[0.06793191774464963,0.5355892587636435,null],[0.10376142618416345,0.5319527187417243,null],[0.18933954766142602,0.5701277964324869,null]],"ukV9s59lkyu-xEjUdK7CmQ":[[0.6269017521662337,0.5684162788697672,null],[0.8513127253476821,0.6800339401167264,null],[0.1308962195068023,0.6260416121558394,null]],"vX7HB8heH18fb_VkIUAagA":[[0.12158073169479477,0.5741840171539351,null]],"vwNiakvhRASLUkJZwdRB4A":[[0.7151967600196258,0.6506738929440502,null],[0.2704148851925335,0.6027480859427247,null]],"vxefPfMvnutF5i07I8rnuw":[[0.6029432044163187,0.582643001641681,null]],"woYJ5n0CfTGMuDWTBqcZAg":[[0.5797194814015053,0.5845655082012844,null],[0.4609360104488443,0.5389286168214669,null],[0.2730405368148173,0.584514933517648,null]],"wuFHDmC43ImDlZMS6-opJQ":[[0.4312147276686132,0.5244231576799688,null],[0.5319332548316005,0.5221319848300543,null]],"xKoseQiPNzFwaWfw9Rl6hg":[[0.8996730579711872,0.5487968696690035,null],[0.9283662882473402,0.547568732635947,null],[0.039742858456492,0.5388870845571107,null]],"xU7xT5kREy_Jnq_vYaXc8w":[[0.40520883994441853,0.5337064752517975,null],[0.6621865496720085,0.7493835638144737,null],[0.7238288830522035,0.7554335840904158,null],[0.7371388759303801,0.5864452554162067,null],[0.8444512235985435,0.5720713908899993,null],[0.2623290411777174,0.6246245317393168,null],[0.2672825211316058,0.5437950588572092,null],[0.3198779016143787,0.5550071352804763,null]],"yDV1n1O4f0QHWG869hAzbQ":[[0.389072440247589,0.572924721520888,null],[0.9448228231167892,0.5523123436764842,null],[0.07320696712750174,0.5515436716063099,null]],"yJmMJeV6PD3UZlAmm4cmWg":[[0.4959327773826545,0.5648829042094854,null],[0.8049914752510732,0.7534241680226563,null],[0.8364370049359502,0.7471255937016037,null],[0.15246626866989166,0.6202373187374309,null],[0.19182077910403744,0.608807336332151,null],[0.3646412011174595,0.5541564230516427,null]],"yPUv9rk8acshJKXa9_jMag":[[0.5193628194654675,0.5472429276106894,null],[0.44041130127452427,0.5410830865768895,null],[0.8931159802917518,0.5899797438147828,null],[0.1683951601834615,0.5898538422598084,null],[0.2375861399014505,0.6078785055625723,null]],"yuwkp3XCUwSbB8q7Og99GQ":[[0.8508411757565322,0.6100599537912926,null],[0.8814226334558232,0.5894832270155486,null],[0.11383209985125164,0.5658810003323899,null],[0.9798765458473577,0.5239037649683274,null],[0.15638083155019689,0.5855981443692064,null]],"zR6d6XkSFO-UYbgkD26kkw":[[0.8917822041984762,0.6014133076476306,null]],"zcVgeJb6FkDw1z__YhM1tg":[[0.6299173450922013,0.6177578849414762,null],[0.8532983125641445,0.5742398110179873,null],[0.8970396261229854,0.553186946215276,null],[0.9636307819865877,0.535243254353515,null]]},"model":"claude-opus-5","n_panos":110,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__budapest_district5.json b/benchmark/model_detections/claude-opus-5-effort-low__budapest_district5.json new file mode 100644 index 00000000..4106b5e4 --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__budapest_district5.json @@ -0,0 +1 @@ +{"city":"budapest_district5","detections":{"1003827842477344":[[0.5742269976813839,0.5999298359082126,null],[0.4107004130250646,0.5786869728330941,null],[0.36516487962070354,0.5694551589131329,null]],"1007938414942666":[],"1019095247256685":[[0.7750726875643079,0.6448252736444372,null]],"1027077646394579":[],"1037039761997906":[[0.6087557914667168,0.5517156809927186,null],[0.6469881155065597,0.5932981851552281,null],[0.7535915318382603,0.5662276159734605,null],[0.546625764283202,0.5845288160535176,null],[0.853729546200761,0.6098397509613208,null],[0.7766422080148085,0.5681155003273446,null],[0.9180553633381959,0.5784685611532521,null],[0.09038126637129273,0.5975530080881237,null],[0.2759968687306312,0.5375371548358998,null]],"1037329802140159":[[0.5334893989193116,0.7440162322623005,null],[0.5772712178005918,0.6793487437451945,null]],"1037999235592907":[],"1038280075426836":[[0.43451323548240334,0.6319410301488965,null],[0.6832683572098729,0.5570946630407565,null],[0.9430984176099326,0.511296032951607,null],[0.06856610128670476,0.5146427880195482,null],[0.27715216954092703,0.56819110380779,null],[0.3777502357184156,0.6466945655348411,null]],"1042570498111893":[],"1043138107118713":[[0.38805680263119674,0.5665361640300843,null],[0.17079653022304175,0.5524490707230293,null],[0.4065592154754943,0.5917164409495851,null]],"1046671274052950":[[0.5069428859599249,0.5662450056130357,null],[0.6097952989860662,0.5381054294114778,null]],"1137646927290075":[[0.552546050012966,0.5709021667418472,null],[0.5739606095021391,0.5450838521633857,null]],"1170370375008478":[[0.6201487379478684,0.6593099271530338,null],[0.6935551457299374,0.6801247732820827,null],[0.7303922006202997,0.6710055827227628,null]],"1188947286474652":[],"1216426636963136":[[0.7065105525706945,0.664171873801549,null]],"1234836825326131":[],"1263700272280191":[[0.8788863438528285,0.5287428725550661,null]],"1279768990703921":[[0.778202479251722,0.7184565320869576,null]],"1290858892857927":[[0.16434536227096386,0.6215747619891009,null],[0.20831613164817803,0.6273981628271763,null]],"1320252873424067":[[0.5207401231061982,0.7134682526074727,null],[0.5818980945255733,0.5984759322799804,null],[0.751517993027986,0.7324945463980341,null],[0.29593180446650913,0.6585634288048554,null]],"1321969539605547":[[0.1443376480156775,0.5769370868728932,null]],"1328862755794452":[],"1330282068787409":[[0.7349490991289548,0.6794149706525198,null],[0.9535539024498783,0.6656437766387143,null],[0.05315507646715234,0.6167315773753423,null]],"1338136654317995":[],"1358450323019655":[[0.41683740906976696,0.5636180050381212,null],[0.5430638531150506,0.5626036161866806,null]],"1382388957125864":[],"1402190408446458":[],"1402259001719568":[[0.5403151374821265,0.5373183615734582,null],[0.5912258296607058,0.5841166742700032,null],[0.3609248114924345,0.5364119887684106,null]],"1415048523727047":[[0.4408558487538713,0.5779807728733791,null],[0.6337135468495616,0.7053845765457611,null],[0.7836116170201533,0.6738676286535118,null],[0.9271421277175861,0.6483296012677375,null],[0.0721949082977108,0.6600222318651735,null]],"1479510223377234":[[0.9609120929577566,0.6949153212299073,null],[0.10266131912520338,0.6459562518310953,null],[0.3176032308271128,0.6406284635797401,null]],"1479649270603872":[[0.4611401911124019,0.5399690334721017,null],[0.15326841063952212,0.5927796157448825,null],[0.37535898925788574,0.5519990062680179,null]],"1485983376024324":[[0.7640011392183623,0.6333752342031841,null],[0.9600592057754171,0.5561743520064352,null],[0.09240854429779116,0.5621859273466467,null],[0.2412603546757766,0.5610381658093982,null]],"1493807495453961":[[0.7955857148979192,0.6673341328821621,null],[0.9389610864159987,0.5451821957474219,null],[0.06532921896175403,0.5168175388542663,null],[0.21556496471645836,0.5407901488439097,null]],"1503933797776035":[[0.9474904390618677,0.638822253842996,null],[0.9967499827585546,0.6396620240673246,null],[0.13928338167008086,0.5952919399064072,null],[0.04349136000698045,0.6312672308262521,null]],"1507586557510184":[[0.9035029966133328,0.5804522575532574,null]],"1526153925847560":[[0.686765549166064,0.5964992580228101,null],[0.07717540792068395,0.5628814722882229,null],[0.2717117793833739,0.6164342564303819,null],[0.32701139857002637,0.608490775978553,null]],"1526644485012786":[[0.3724667605751194,0.6179633021326153,null]],"1526903691607662":[],"1531415762060842":[],"1531940155140947":[],"1533099125112846":[[0.45759337269258277,0.6459866064784804,null]],"1547490957023264":[[0.3834214549869422,0.5504726715891237,null],[0.33690653791161773,0.5579373251530639,null]],"1548551026941868":[[0.8470588327627357,0.6995513744579558,null]],"1550619426690125":[[0.18957789927835472,0.6590546390956679,null]],"1554553479649915":[[0.49519707114342576,0.6010457010085672,null],[0.5434154582243259,0.6011105643880327,null],[0.8431717880930187,0.559222887374934,null],[0.9208127652051074,0.5426048563699949,null],[0.03272254989437756,0.514810912432857,null],[0.09805565502426916,0.5841774621036738,null],[0.2548844275538038,0.6031529773913399,null],[0.40534039994604915,0.5610750647790774,null]],"1560573605574046":[],"1579465080398346":[],"1621880408877798":[[0.48342809253834396,0.7169764011621245,null],[0.5540523298872733,0.7795003210578286,null],[0.8369001826278915,0.7703471819594145,null]],"1628684669262976":[[0.5343538628101515,0.552604918614321,null],[0.49466316433202734,0.5240818950817836,null],[0.36388439162247455,0.5451436214646295,null]],"1642079667427179":[],"1644929466791193":[[0.4131744860060321,0.5969641963146461,null]],"1665066454545157":[[0.9317098217621593,0.5474041039329313,null],[0.24943805091068327,0.7045411334615492,null],[0.32310154850609296,0.6961107122173928,null]],"1665856994688538":[],"1679632769964162":[[0.5049656254669639,0.5825355338033008,null],[0.39080635969816024,0.5619441993959753,null]],"1688417845615915":[[0.7192635825402531,0.6524715424067886,null],[0.801794950060922,0.639722693548157,null],[0.051915530054934045,0.6640237351908479,null],[0.28420579183536576,0.5500282685681472,null]],"1708279679842703":[[0.5352583806082227,0.5680382468236911,null],[0.5578955580486197,0.532829168586816,null],[0.9230025664819275,0.6199704980967582,null],[0.9936892316911672,0.7006103043340473,null],[0.023329915362342413,0.770941580469181,null]],"1711104372904021":[],"1726281301363646":[[0.4351551347958951,0.5644061103532426,null],[0.5425862051806892,0.561842001683546,null],[0.3689830539146166,0.5721661289108008,null]],"1736795824019170":[],"1743265983784166":[[0.4397800240094062,0.5373080365425891,null],[0.2521058360019597,0.6749898954300273,null]],"1743535723318388":[],"1743866716581524":[[0.827780487428281,0.49043433868761943,null]],"1762265868113165":[[0.5422743823607206,0.5271780388921269,null],[0.3894093258959951,0.5516127425788685,null]],"1769629404195835":[[0.4504943457529413,0.5885748569336801,null],[0.3787080085233414,0.5958235531351659,null]],"1779415216757575":[[0.8889911555314036,0.6375402799425116,null],[0.08469562685322024,0.5967148142536408,null]],"1872865880061015":[[0.6160487239980784,0.5859235233432316,null],[0.47715102601319553,0.5592857214345748,null],[0.7561698882288305,0.6318485168205459,null],[0.024683834683903283,0.6360725877045279,null],[0.2775156140299455,0.6262462908110367,null]],"1875441803044127":[[0.4738002515637401,0.5895762098721148,null],[0.5353126449659902,0.5568308053085894,null]],"1892190831489561":[],"1920855128875530":[[0.5526066247660986,0.5409856013492398,null],[0.2490302234338308,0.5846089368711436,null],[0.35064831498802723,0.5367445890907183,null]],"1933608350737216":[],"1980881726130460":[[0.49477998609407015,0.6121291824992946,null],[0.5779059375041468,0.609838809218977,null],[0.11256257813028647,0.7282694141532828,null],[0.27994159658412954,0.7234898900593408,null]],"2002394670457045":[[0.5010750733886085,0.49911707398246447,null],[0.5516756273503767,0.6652083596339855,null],[0.9587750149681098,0.693753681533754,null],[0.3005332539210417,0.7846726430342752,null]],"2039237576963899":[],"2055080831885811":[[0.715564154712506,0.6589243372825697,null],[0.8286697053080674,0.708630182570724,null],[0.8884238768142421,0.6103849530583744,null],[0.22155541733508094,0.5603841119253031,null],[0.2707306194140976,0.5719469578578844,null]],"2064297630816443":[[0.5650120701204007,0.5757778868098868,null],[0.20283423274287776,0.7492153507113681,null],[0.323822133958705,0.5955860354241935,null]],"2080767922502123":[[0.6775878777474826,0.5379221637847261,null]],"2088155352056225":[[0.45790723017120527,0.6667185159992901,null],[0.7271560733260981,0.5763737413315764,null],[0.7620865297040985,0.5795316199775402,null],[0.12510829776072535,0.6298685721049083,null],[0.3198101134737796,0.6773359294586215,null]],"2185625018892068":[[0.7596525941875192,0.5676940214660124,null],[0.9011848022528848,0.5244885777572663,null],[0.08469044400252934,0.5462644563900436,null],[0.14490301171820547,0.6121204089031874,null]],"2216048919129703":[[0.6891787488638792,0.49797796956910156,null]],"2221274008612179":[[0.5395793922926122,0.51323953126655,null]],"2243369796465669":[[0.445325764815991,0.5270053659899203,null]],"2392893427787498":[[0.7834409661477275,0.6206786706871694,null],[0.8153356459658156,0.594583247305788,null],[0.9061673380061344,0.5400977490386865,null],[0.21265095049737381,0.5960088988720489,null]],"2511246802645139":[],"2641788216158646":[[0.48619258550977523,0.5901230989032236,null],[0.6047405675459285,0.5687081954117874,null],[0.38141167653385477,0.5537988851552543,null]],"27177228765230843":[],"27260772950175386":[[0.09850065997267032,0.5706722377410157,null],[0.17056503964763192,0.6344130873740912,null],[0.3169299552210483,0.6523044175114121,null]],"28215663224697246":[[0.9928020919437219,0.5384976711184883,null]],"2962988900572350":[],"3015816262094291":[[0.7446402016508422,0.7533331540568791,null],[0.8524379370069202,0.6706287078930001,null],[0.8710366203051294,0.7252665694492424,null],[0.30802507810876995,0.5654932803246586,null]],"3140904756109373":[[0.46878855995992963,0.5859218831380522,null],[0.7278792050544401,0.6256480474694412,null],[0.8368208939759108,0.5951342527947542,null],[0.06293871857199979,0.5300382753518074,null],[0.09567637860189654,0.5282019774408444,null],[0.26287648031225286,0.5346146495080342,null]],"3296688647144785":[[0.8492290263657103,0.5693529678590443,null],[0.2268919716711849,0.6651291476275438,null],[0.29986154724314873,0.6501088204343647,null]],"335801701340862":[[0.6053707066869333,0.5237450852286505,null]],"3688080354666548":[[0.5832929394742143,0.5760218639150079,null],[0.6786216749081,0.5536822961016853,null],[0.04191535011202652,0.5891950770782051,null],[0.2996427444989207,0.6709501450700428,null]],"37046635288313572":[[0.45156465765515275,0.602488442215193,null],[0.24822860140725866,0.5731230441206889,null]],"37230045973275479":[[0.5346946437909734,0.6673782771425,null]],"3860523807571425":[[0.48459649582611797,0.6036195730298552,null],[0.5560041707798072,0.7450258022309295,null],[0.6699784231501155,0.7387337936404118,null],[0.7657002852889533,0.7252169997380573,null],[0.042246297183454684,0.5454259976916794,null],[0.09048737073211022,0.5608003482311682,null]],"3901990523265321":[[0.5398627651516628,0.5508574712312627,null],[0.5800601376543109,0.5730935992107349,null],[0.395131396449784,0.577212436479547,null]],"4122786137987976":[[0.5907507942030092,0.6587682020290948,null],[0.6158751098335633,0.6541773019186135,null],[0.8580533335994911,0.551576672178815,null],[0.34512520775454114,0.7064409618763525,null]],"427909170007735":[[0.9766294168100137,0.5360487066548394,null],[0.21672662575604706,0.5514975646633724,null]],"4426398164172326":[[0.4988087538927467,0.5579500217246514,null],[0.3240407316620959,0.6434907177785076,null]],"4478163572471700":[],"5026082804197254":[[0.5502732458478733,0.5572535575117903,null],[0.3898890275198988,0.5609501875153425,null]],"740730485632754":[[0.6302505415529857,0.648661145341414,null],[0.7631734672430615,0.6712115282552903,null]],"749536038146240":[[0.4715359335075884,0.7458790470687977,null],[0.6029005888295302,0.6162191328641167,null],[0.31598229288461166,0.6831904641273208,null]],"755126063369619":[[0.5118055878266459,0.5185350855359308,null],[0.5360714837817973,0.5176282134446321,null],[0.06071477763994765,0.5251972826347988,null]],"790116040739014":[[0.46604017446477863,0.5759761457599498,null]],"817455584410792":[[0.734862098341002,0.5153490612901228,null]],"820647700656490":[[0.9251356907775523,0.6679130916804104,null],[0.8881488668974816,0.6686588530788528,null],[0.12148678500841703,0.6626172618693436,null]],"833524132783867":[],"838276004852617":[],"841136762387293":[],"866540066042247":[[0.3960451254572748,0.588706545424956,null]],"873103842125428":[[0.0655750424962685,0.5384962639579302,null],[0.16290894630050112,0.5877871988590203,null]],"881246267480927":[[0.2828796065754418,0.556192663717176,null],[0.2906487531607851,0.6301685050542658,null]],"897403239302195":[],"917879701312494":[],"934293439634924":[],"945894851803282":[[0.40412316962147354,0.5553578765109717,null],[0.6235445067711347,0.5351191497584364,null],[0.7713090193437799,0.5554907532934449,null],[0.2142534984607996,0.6202148891805942,null],[0.23586907539120588,0.5997842644896701,null]],"986609046276593":[[0.5737450422619658,0.5354526808334586,null],[0.13843493874885515,0.5839113761472634,null],[0.3605996650305227,0.5794193516631776,null]],"986995307528492":[[0.550011419408243,0.7287563626051952,null],[0.7775327018325711,0.5053544621200763,null]],"991739220159007":[[0.7217102469614378,0.7404710786762386,null],[0.9991018566295922,0.7731659750701154,null],[0.043277953367748645,0.7033568304889698,null]],"992737779816905":[],"993081170303646":[[0.06926425105771311,0.5759714780759938,null],[0.10899788065345872,0.5698704625035363,null]],"994057612515361":[[0.5208837253914734,0.5618659781020356,null]],"997653559315299":[[0.5781429406399706,0.5952054661728243,null]]},"model":"claude-opus-5","n_panos":125,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__clovis.json b/benchmark/model_detections/claude-opus-5-effort-low__clovis.json new file mode 100644 index 00000000..2d2e0876 --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__clovis.json @@ -0,0 +1 @@ +{"city":"clovis","detections":{"1064896187252557":[[0.6848427353992609,0.5901788886854045,null]],"1110562892823067":[[0.6497335769670975,0.61250318517083,null]],"1115273772298326":[[0.5408145469919314,0.5566314646304157,null],[0.6166034519203876,0.5534710751906875,null],[0.38228702092751476,0.5518565457158899,null],[0.9137541210600257,0.5589778449906124,null],[0.9795346671663143,0.5668910034769629,null],[0.08003631907102715,0.5597868296714257,null],[0.3149933734497715,0.5562064300894689,null]],"1181210132352804":[[0.834318993510956,0.6161146005114868,null]],"1195656127552307":[[0.010617380962041567,0.5867506559988314,null],[0.3689662425057311,0.630860983209269,null]],"1197447770701938":[[0.045637639832767474,0.5914849217918281,null],[0.39948151409979943,0.5304932682815945,null]],"1198240240634034":[[0.3882482634888663,0.6487639717392835,null]],"1207873673000217":[],"1208670676234826":[[0.9915515022632759,0.5876929290788204,null]],"1243806399391315":[[0.38048202156054345,0.5918730280779313,null],[0.09850918062780673,0.5508041370365389,null]],"1281339068951215":[[0.5020234542821703,0.6313656932332069,null],[0.6801815706266786,0.5511957703505855,null]],"1362100907510446":[[0.5254203204099571,0.6528725218735334,null],[0.5644849965867894,0.6519205144707515,null],[0.039180275378923546,0.583868503842345,null]],"1370371966682122":[[0.13234775290279782,0.5569080750941187,null]],"139019651616598":[[0.6724628183866653,0.5278868132380613,null],[0.09841634210588956,0.5515151113394857,null],[0.3353892595853417,0.5936728505535405,null]],"1414692438885050":[[0.14097677379264573,0.5640825856134833,null],[0.30704446084557224,0.5643206466351989,null]],"142003227963196":[],"144203244404799":[[0.6878359014248492,0.5657371893057677,null],[0.8596711937356862,0.5654258999763462,null],[0.11738938835752138,0.5584284289974153,null]],"148678610517067":[],"150809456962552":[[0.38555928151034613,0.6228878738363539,null],[0.9985947775971763,0.5860567585365806,null],[0.35254910558758057,0.5983310752536715,null]],"1646577775732974":[[0.9194191977746264,0.6096461014391907,null]],"166346065432248":[[0.506779052632898,0.5521474808882856,null],[0.06497939563394406,0.5525175799780971,null],[0.3743808767244957,0.5682777076352666,null]],"167719748544354":[[0.4688173138068639,0.7011665353586005,null],[0.6498194289750852,0.6095057484000781,null],[0.012058907302049837,0.5887379193178576,null]],"168127948505506":[[0.6439341997507819,0.5957089025736325,null],[0.29491174886686394,0.5522985671073168,null]],"170392751596494":[[0.9928137339640154,0.6129847664635787,null],[0.4080998848156556,0.6677612217146173,null]],"1704575766415957":[[0.48443111325833493,0.5917911335714189,null],[0.008830198808771283,0.5818742489186333,null],[0.43848697031832906,0.606988270475284,null]],"171723391458583":[],"172282998233890":[[0.7133379301958747,0.558876981416263,null],[0.34141360881917004,0.6187086765668386,null]],"172922364659417":[[0.025717495934594048,0.5998570384157357,null],[0.3286349880345273,0.5685301448746415,null]],"177108551000367":[[0.2877030873933588,0.5838208490276449,null]],"183964400302616":[[0.861498857087942,0.5636410312379945,null],[0.9492706554624184,0.5696633407015682,null],[0.15969736057096562,0.5684953913286441,null],[0.3061355041658329,0.552253541744473,null]],"185678950103109":[[0.8351559689264592,0.5696849212349542,null],[0.05420197565333473,0.6187204722453478,null],[0.16278673616935097,0.5593163931484144,null]],"1878739745614992":[],"192621366051825":[],"195183319043473":[[0.7894393220397042,0.575692195693196,null],[0.0936775954150107,0.6192869481463709,null],[0.9751020471738563,0.5523761507484686,null]],"1978660535622283":[],"203883874892763":[],"204886314666549":[[0.927171256662102,0.550423916864257,null]],"217080690010555":[],"219131249781847":[[0.313577298173791,0.6003615948995058,null]],"221917589460448":[[0.6091334914124258,0.5755435141198194,null]],"222124426103957":[[0.3593881022149819,0.6882961415473249,null],[0.10137278738509209,0.6215123025840504,null]],"232786268184386":[[0.4625649983080479,0.6895663370205771,null]],"232891661506981":[[0.5872307309360124,0.5626855392470721,null],[0.6984952126219982,0.5861453565034166,null]],"233474051445151":[[0.4012031086124831,0.7268925362715599,null],[0.2758883453400827,0.5323401628884329,null],[0.3063665795460776,0.5695606394838806,null]],"2493443314123605":[[0.31796072869211195,0.6024307963374379,null]],"259170445960040":[[0.5661088783625724,0.5004711505439398,null]],"2652019851757090":[[0.681619301773745,0.5859888390558182,null],[0.9189932978022324,0.5680397540073139,null]],"285195270004030":[[0.6223495835373002,0.6448482568287156,null],[0.010511704144560485,0.5636488498981946,null]],"2904429339832545":[],"2908033546190932":[[0.6318573069411142,0.6553859232651671,null],[0.13763774886985403,0.5663452445770057,null]],"2909136316010153":[[0.14251254349314924,0.5553929938161599,null],[0.3314305497155612,0.5478555827745948,null]],"2938602763134225":[[0.8703155278830922,0.5494904143809617,null],[0.9645316026580819,0.5691259320499713,null],[0.041522436285592546,0.5835637953391014,null],[0.10286817859750291,0.5570388064032532,null]],"2977612369225705":[],"2979119369077618":[[0.5853446451415716,0.5926757302179492,null],[0.08837039093629401,0.5998827472757701,null],[0.2569415713000437,0.5615178773621284,null]],"298740268622763":[[0.09212506163412454,0.580363378860318,null]],"301397241499290":[[0.6481912581416096,0.5653145645408474,null],[0.898961039125932,0.5716515544744988,null],[0.33425257653224005,0.5748052540321718,null]],"301640868358262":[[0.6648672132444571,0.5621063919667535,null],[0.9416163697275757,0.5392097132086672,null]],"3043158182635438":[[0.44003189822030164,0.6976209928558801,null],[0.030545767602843832,0.587716219712397,null]],"308441850838672":[[0.6885363092317793,0.5645861182427699,null],[0.715454729167204,0.5518081472367286,null],[0.33808140023548977,0.5747743810660576,null]],"308559267339828":[[0.5560167892241102,0.5785395939266342,null],[0.6013340215115869,0.6015621490426082,null],[0.05892078137384604,0.6088681862594092,null],[0.056190496749846386,0.661663347630115,null],[0.30937519048052337,0.5470591946928922,null]],"310589334058179":[[0.7093023643169403,0.5451363637468649,null],[0.9923380979610736,0.5752939970014173,null]],"311175153796836":[],"314281570321010":[[0.30155751448632506,0.5739345331929866,null]],"323628349333170":[[0.08892302385181278,0.5434390669006428,null],[0.14057122014313694,0.5220337185317125,null],[0.3212313316861908,0.5832132798661934,null]],"3264932050400483":[[0.7080481254292004,0.5547101538446622,null],[0.053063396144135355,0.5661882780233483,null],[0.3253988425511981,0.5398057226085361,null],[0.4510390082334684,0.5403504933026702,null]],"327979002173875":[[0.9840764406347049,0.623162415245178,null]],"331704768384211":[[0.698034411174357,0.5216550239612695,null],[0.7214267245945967,0.5147064320647597,null]],"341385827421059":[[0.40349329320005456,0.5407645642354322,null]],"3706361862807633":[[0.5027369672712974,0.6396694473018947,null],[0.01354843321224014,0.5924790202222041,null],[0.41696469286753945,0.5866042088899324,null]],"373977847581697":[[0.4011246587148694,0.6637431663656563,null],[0.66165358556225,0.5353749346900869,null],[0.7134536162812499,0.5059124944494863,null],[0.9076387866739954,0.5486542331048472,null],[0.04435898515477649,0.5536391509359216,null],[0.3167385252722629,0.6005755887449303,null]],"376064363845819":[],"3893017870814523":[],"3906688372733138":[],"3912015842228191":[],"3939249736143912":[[0.3827395109758831,0.5707475732683702,null],[0.6118953115652783,0.5577309027320864,null],[0.6719732556142552,0.5521654846315079,null],[0.9055292522727028,0.5492174989347102,null],[0.11575757366681871,0.5537211968839105,null],[0.28515668281810014,0.5631181656048555,null]],"3975206075849203":[[0.7795372246963519,0.5528081432753502,null],[0.9155903455120857,0.6152282567356607,null]],"3975990335818804":[[0.6111304073050658,0.6755600015997358,null],[0.5832130994822757,0.6744454433786827,null],[0.862812961265454,0.6224100143155131,null]],"4029786067103641":[],"4088414157885402":[[0.6013764115150894,0.5817372182866551,null],[0.8112906776762336,0.5381712267030014,null]],"4134475506643495":[],"4142410022468972":[[0.6636242586306642,0.6010714441687676,null]],"4347306715281715":[[0.41283881788984644,0.7185150551001597,null],[0.323175445573714,0.6342063797496438,null]],"4648083425204950":[[0.6767178557210568,0.5542720550662037,null],[0.265391872978863,0.5191918645009234,null]],"476824956956024":[],"477834906826400":[[0.7082825181627144,0.5309350558098439,null]],"483031622925446":[[0.3966246350540138,0.6226100575041819,null],[0.6093786976236094,0.5709077845950006,null],[0.6646940421827223,0.5691187624018731,null],[0.3077465686177443,0.5896299111408306,null],[0.4222578895814164,0.5826443557336898,null]],"483328876252454":[[0.1420261229351562,0.5672057480757939,null]],"490105898988611":[[0.5395333773358126,0.7126774527244711,null],[0.6333308090669882,0.6593358726674721,null],[0.9205585971116194,0.5997616447197598,null],[0.9467154894980536,0.5986670929879028,null],[0.046761300323085564,0.5888269071883163,null]],"490344308689064":[[0.5318326386254996,0.5919157078494716,null]],"492435855406236":[[0.32296811710456963,0.5553594860193487,null]],"494271831919687":[[0.4000871650320226,0.6018724339605381,null]],"499704301465714":[[0.6122211360910457,0.5949912630999868,null],[0.647911565783178,0.6087709215775655,null],[0.992313620880483,0.58917420743687,null],[0.3178882650244531,0.5579597344133636,null]],"504758370672604":[[0.09436361162522089,0.579899577553343,null],[0.3060279655099624,0.5802715665077529,null]],"506669223705998":[[0.3830969926523068,0.7479396304876471,null],[0.2330467139330995,0.5127131748476523,null],[0.26004451720849836,0.5289326103672531,null]],"507337747134100":[[0.6857205233608965,0.5422039283052559,null],[0.8629255657089195,0.5457229717987195,null]],"507362710300684":[[0.48300840372986625,0.7289570624098209,null],[0.7071627620788666,0.5419252444693194,null],[0.6101572023211502,0.5898180248043474,null]],"517195446066836":[[0.6651741337684969,0.5593310689640263,null]],"525644228599460":[[0.5488036856988807,0.55267327600834,null]],"531478437866297":[],"542282330125531":[[0.14161776819872823,0.5491448338848026,null],[0.2777641095973909,0.5521988984870597,null]],"545146053319322":[[0.8807521859184503,0.5716232131762802,null]],"637757707623691":[[0.4441933310996041,0.7081022228043707,null],[0.009273303172055047,0.6006310533042729,null]],"744451279581577":[[0.6954064249436289,0.5850136273049376,null],[0.9001367889576777,0.5751505899045516,null]],"755820925099849":[[0.621280490894836,0.6018845980343274,null],[0.8911120170623704,0.597154976088329,null],[0.9523602455864474,0.5962980777147386,null],[0.09234505545943289,0.5519078579608515,null],[0.17729261197088736,0.553440823485352,null]],"773395216705340":[[0.37720234392810625,0.6138608813792228,null],[0.2738551905784049,0.5351908355977497,null]],"775225080310516":[[0.6154576191383648,0.5946395778146258,null],[0.8331927756240312,0.5243480241998102,null],[0.9801678882701237,0.5613642582350571,null]],"777059616337842":[],"777844299592863":[[0.6564688754633707,0.5628206666069964,null]],"778142686176383":[[0.8388053190427472,0.5385141390786785,null]],"796298687952468":[],"805303977081180":[[0.679285552536076,0.5572235358817655,null],[0.3076985212810087,0.5696547237974603,null]],"807482166820261":[[0.3821570009331998,0.6562591297236279,null],[0.008746660265717465,0.5659285484464522,null],[0.36150899786187224,0.6222742682394953,null]],"812831093002742":[],"825040278408836":[[0.458973109882725,0.5827808636150104,null]],"841498166782932":[[0.11612161224934847,0.5556390142300125,null],[0.05144398676113937,0.5586982890519718,null],[0.30281497646484856,0.5391316627928681,null]],"856069755259257":[],"858708244726347":[],"870854823644900":[[0.3906866761738934,0.6556020563974709,null],[0.00573392209436574,0.5980564663229407,null],[0.40893883279689586,0.5959574963120838,null]],"874947899752061":[],"931494460919519":[[0.04534608242235466,0.5804506565259936,null]],"963194777761975":[[0.6947550530340305,0.5468610304219497,null]],"972140163589048":[],"980307362774134":[],"987647778730799":[[0.5,0.6359879136347564,null],[0.3785444294060142,0.6722773541769859,null],[0.5547676142656426,0.6688993052368196,null],[0.26719328435578166,0.542033940086935,null],[0.30033369090370965,0.5589057301716777,null]],"995842280952828":[[0.6737493370922859,0.5291559995600488,null]]},"model":"claude-opus-5","n_panos":125,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__gainesville.json b/benchmark/model_detections/claude-opus-5-effort-low__gainesville.json new file mode 100644 index 00000000..e525b5d3 --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__gainesville.json @@ -0,0 +1 @@ +{"city":"gainesville","detections":{"--xM_B9BVXSNT9IcytVKug":[],"-FMeOe5iGG-5JFxSF0lxaQ":[[0.5363501081784026,0.539345313802034,null],[0.4341817801608394,0.5384707629489914,null],[0.10079781132559168,0.59250240228696,null]],"-Thw-Y56uiaGEg8I_zZwgw":[[0.538394478978532,0.5288426644911365,null],[0.4414099378888249,0.531093938753931,null]],"-zi5tN6Qyyeq8B1mczYfOg":[[0.8772921507704008,0.6407452938578787,null],[0.24021665974439843,0.5856751004617702,null]],"08ssXvyy311t_dOt8e3MVA":[[0.5671772312197307,0.5506917710804854,null],[0.7223085807277725,0.5745010404376184,null],[0.8411918037679771,0.5903409243585861,null],[0.8874809207092409,0.5779563122557785,null],[0.1825855325714672,0.5758699860751705,null],[0.44292093507394503,0.5364381016510082,null]],"1n-U7MQVuboJf00fwh9UBg":[[0.853291889030277,0.5520617491731554,null],[0.3123130293456597,0.5699440248098112,null]],"21qHYEKoNSGw9O3wF_dvBg":[[0.9362195761127565,0.5391756280351584,null],[0.06049266203618442,0.5500548011437527,null]],"2ZG41STPUADAWUJD-Rc4sA":[[0.9426050463843275,0.5395506865529647,null],[0.06024549215772046,0.5363228029400008,null]],"2xfTrO6eZbF0hKmvjGzN0A":[[0.12097178179728812,0.5407841123677681,null]],"3CbLaqDP3N3z64brBrC0eA":[[0.7654874267464107,0.6587064099708684,null],[0.7972427744074523,0.6594353605766501,null],[0.07293997213574654,0.5503440094511693,null]],"4ILOJcaHD7m47NGoY6aqRg":[[0.17646342703130036,0.5392532295144675,null],[0.345995967552429,0.526039666193216,null]],"4My9h4mmpjE82U1up3jWOA":[[0.6122058571380644,0.5637101137831312,null],[0.6638561057540773,0.5540964359513832,null],[0.9297525140678712,0.5428042270768707,null],[0.8674774526438938,0.5441505700754732,null],[0.8901192170354628,0.539116747241206,null],[0.049717458758245514,0.5379445766829666,null],[0.16666666666666657,0.5327926292801344,null],[0.35410280108897785,0.5382121729529624,null],[0.41817843379738046,0.5457646189813977,null]],"5vEoUj3H3aRkW-2h1B0JAQ":[[0.6325571594420107,0.6292054368552792,null]],"6-tNVth5ck0axWmPuKpiXg":[[0.958342455516588,0.5507100417461103,null]],"65kxFTBgmbOPUQbC612rdg":[[0.5659983881313441,0.5468395729409555,null],[0.5894217038823933,0.5468471798902866,null],[0.4246445913418412,0.5455649360504171,null],[0.7323857845890195,0.6474365932822616,null],[0.9417589637158139,0.5872411214051362,null],[0.28062374827174863,0.6244820076077144,null],[0.3215576796986915,0.58235798937465,null]],"6HfrzDBSveHUN1NpGsF31Q":[],"6KnaArlg4nGEGAGNkIw9NA":[[0.6244735522714312,0.6662022187574749,null],[0.596558020804276,0.5598774782486442,null],[0.6975051771506662,0.5673143687192692,null],[0.9492706554624184,0.5696633407015682,null]],"6esDvYaAUiUdZHtq1FCAfg":[[0.7442480118883724,0.7941604032983427,null],[0.6790626237518391,0.6127757562946681,null],[0.009130821708261239,0.815993721182787,null],[0.11692455403896646,0.6124507494842019,null],[0.1062192416663208,0.656669582362408,null]],"6mZXrVGoxQ2H1k8Jx2OZHA":[[0.7275556789241455,0.621075846687659,null],[0.8301690384918048,0.5939575648243626,null],[0.09345073627770895,0.6324756663891055,null]],"7U40JKd7HiOpFrTHa0-yQQ":[],"7Vk-zugISgNBVOV4g3r2jA":[[0.9478088113692582,0.6050309386365451,null],[0.15856530719866263,0.5738577468978358,null]],"7dme7qsPcOTglSlmpGp9Rw":[[0.6574014250552489,0.5606194565436105,null],[0.7477081744149923,0.561078613223411,null],[0.3373567719199204,0.5584856289435792,null]],"82y6sIvPm-_shG58klusgw":[[0.9398159083496661,0.590150380327582,null],[0.13346007938846144,0.5704478470115554,null]],"89lilp2BlhppxL-oQ0gLNg":[[0.5628270452080061,0.5626441014996457,null],[0.7376223537858194,0.5511436434794283,null],[0.7772292657033376,0.56585726932812,null],[0.8303762554164061,0.5538221107633747,null],[0.8839243789043307,0.554581858908435,null],[0.9669999296719202,0.5487010723199273,null],[0.023275857749602946,0.5407426115370906,null],[0.2021775650260807,0.5391345138001361,null],[0.3935533093239272,0.5373080365425891,null]],"8FPxXspDHn_FNxmQkr0j-g":[],"9D-oOS00uSg18nvfm8t9zQ":[[0.6016729693365561,0.6356909270261647,null],[0.3870818984929667,0.5885534785638025,null],[0.7075333985484329,0.654782853041282,null],[0.9775544930272616,0.5412982849758496,null],[0.04515522514855269,0.5372631743264812,null],[0.2732903404003555,0.5773692936094672,null],[0.29598169473935343,0.6180927040697306,null],[0.3469018641226781,0.5933591980166077,null]],"9QF3VpTsXpZei70YvA-H6w":[[0.6285581078473974,0.5517916436289914,null],[0.8457632205443032,0.5694836131465288,null],[0.22180382852881103,0.6077442318770249,null]],"9tS6faHm3jFdOth84XlbEQ":[[0.5577356075640434,0.5676851970279095,null],[0.3919673960652974,0.5563836310404404,null],[0.6515733512382657,0.5674178661433833,null],[0.8436969042289963,0.572377524637407,null],[0.9138873640164501,0.5756992681940147,null],[0.9472238627243983,0.5584545124100548,null],[0.09644615729825656,0.5715400950330216,null],[0.17828983526142245,0.5746142900076513,null],[0.3634307551105911,0.552889263341328,null]],"A44roYJy6tdzAK9yLNS_qQ":[[0.5218928489244614,0.5125635672773138,null],[0.8662889788567255,0.6153365656811653,null],[0.19797033148619736,0.5762303223763077,null],[0.2705250544621006,0.6251835144638676,null]],"AVRG2QwkPWmeX2iPSnKDcw":[[0.750937913236482,0.6827522799099968,null],[0.11505309456178947,0.6134662144197923,null]],"BtYzecFoq3-H5qaGzBh4iw":[[0.5347928075747286,0.5363451451405118,null],[0.2711162248430543,0.5438969421449182,null]],"BzXolOMX-XSEGwyXQpIsrA":[[0.5957196426289628,0.5668694264115878,null],[0.5379229886818312,0.5217032787109717,null],[0.5969894927700994,0.6150414044599665,null],[0.9834302534605202,0.5257284861307921,null],[0.3677026709839237,0.5528712992494151,null]],"CaClPAIWuWDFsoftxxmRig":[],"CabndIEXzzs5FAENnrsEug":[[0.7748477953004336,0.5680455599164989,null],[0.8995692973319492,0.5389142212607397,null],[0.26730196762709824,0.5551667153125803,null]],"D5NlzcW2s95DlOUPY0cVAw":[],"DSID5BsZ19qz5BuvC_ZDNA":[[0.7324883307593503,0.7160471447322839,null],[0.0590834203501478,0.5704150432996498,null],[0.930132113985837,0.5632626817150458,null],[0.26601598815415384,0.6790192725853431,null],[0.23221656374799615,0.6609493069583218,null]],"DXKg24FIKMrGUX6ilHn5Ew":[[0.9307851188303691,0.5894788848473553,null],[0.12085838918196673,0.5666603470153176,null]],"E2YAOGNgsJYuHjnUBvnjFg":[],"EhBhDcmWUDV5CaBKoAqi0A":[[0.6095171861778064,0.6125170199021162,null],[0.5631421852993154,0.5638995623691772,null],[0.9575869272216267,0.5420498761454365,null],[0.12501685646428107,0.6183649312145725,null],[0.053533885578323503,0.5374179216547359,null]],"FZOcjvm34N_9JGZyXQf7dg":[[0.5809340019972915,0.5206936243833871,null],[0.14928730814117486,0.5535166245773151,null]],"G7ttNgjW4KU9LJVeD1Ivdg":[[0.5525495950806806,0.5821734420088753,null],[0.25799915224460523,0.5750477705367335,null],[0.3316078479604507,0.5374834163485236,null]],"GJ9IMVsb4ULLwoTYEUCZtA":[],"GRTC3zOCFSc8QMIrAtwtzw":[[0.3851762062713038,0.5666647693138125,null],[0.5702377828957826,0.5731143532323726,null],[0.4339389252548498,0.5243784365511404,null],[0.6162864538866012,0.5767046944207939,null],[0.9420093037267852,0.5838282554152052,null]],"GhbzPi4bva0g3AlMDcioOg":[],"HLLOIKVLXvOkBCREUvcBzg":[[0.9388221166083928,0.5711299421639358,null],[0.11023228377359151,0.5661981724691373,null]],"HoOapW5g53qbEFjLA7yv3Q":[[0.44979231828361577,0.543737048701201,null],[0.6614694218331914,0.5736271030974932,null],[0.7024494077406817,0.5743879344909358,null],[0.7977010126207689,0.5943631178607622,null]],"ICUoYVV_dXvc9W3ge2PnWg":[[0.5848146361284279,0.5687518826553923,null],[0.6807273444333357,0.5495590509250365,null],[0.7101192188680574,0.5699268377471306,null],[0.3413111763191016,0.5429833711472082,null]],"Idfm_UDnRFcD78ZVeNnhVg":[[0.5454523765538513,0.5342206074455449,null],[0.5674613944678245,0.5325210988658685,null],[0.9036856059505618,0.5865767232536676,null]],"JOK56E_pGy9cwNPyiymZ2A":[[0.1187086702898455,0.5958945055773252,null],[0.36365147978205925,0.6043838043970402,null]],"JlYp0BtvGCVeIqBady7pGg":[[0.5822004440922629,0.5742472941497307,null]],"K732QTWoB9RZwrfSmw7RYA":[[0.4025389767611681,0.6042353022187618,null],[0.6393617118059979,0.6634016958717275,null],[0.8767890618458601,0.6106967443759238,null],[0.20460187889391657,0.5512740084095157,null],[0.2886296726281158,0.5790088229248259,null],[0.33525209910489173,0.6001888888526971,null]],"KEwb17BzIjKVur60MpItuA":[[0.549003088107228,0.5420539099868912,null],[0.5784766197492877,0.5759593858835411,null],[0.9136181465888585,0.611718552055515,null],[0.11640212524894256,0.6052309855535277,null],[0.15987819098264588,0.5529644868330601,null],[0.3396885784674759,0.540351382989571,null],[0.40099865908336607,0.5618425417370219,null]],"KH6m80K7GCy3eVrCdRMYZA":[],"KbZFzQai9wfXijoQDscWFw":[[0.6095021605772019,0.5648272251377396,null],[0.4534278915482347,0.5555389075268429,null],[0.700116143120608,0.6439947179183048,null],[0.16831230071878323,0.5607166789332584,null]],"KqZzM0v2PbaoxQmq6wk3Xw":[[0.7592908718061717,0.5899410164820132,null],[0.784608822926721,0.6378422427510427,null],[0.9393329610858523,0.5564925879102379,null],[0.10614278725670084,0.552163417594494,null],[0.339520662247798,0.5518831748926546,null]],"LDICnJgAhCSszYwolm8JRA":[[0.9085818317509508,0.582565436610476,null],[0.053949573789259964,0.5790582979633251,null]],"LEM6UenW5sPkdj5aaW4thw":[],"LLc383eAyneHduBTInH9nw":[[0.586879476535849,0.5221216894325957,null],[0.6535575303604798,0.5397210080357185,null]],"MTDJakccuk6hQfnZ3QEjxQ":[[0.6127706147850785,0.5532740886066917,null],[0.5577088856811623,0.5214411147836274,null]],"MVFGRGfew07C0mMkeXxhCA":[],"Mg3-OQnBkIqnfi4qnO5KbQ":[],"Mm5gyUYqmJmjeGopC1IgKg":[[0.4089536985530624,0.5519423553506488,null],[0.5410536054775428,0.5601046768693541,null],[0.38567804701677694,0.5847165841182781,null]],"PSS2iz32Sj0nE2ILJ6i8rA":[[0.622664713611297,0.5619624785874442,null],[0.9499538448007212,0.5441251777771466,null],[0.31573415830838625,0.5229462203564298,null],[0.3656072355207315,0.5162663194289748,null]],"P_bUUx5CIFcylFd9UAZRug":[[0.7627868627288228,0.6063919121084724,null],[0.22573306604157434,0.5384021055665397,null]],"SxMVl_DRQndQkQAvu9a01Q":[[0.6158932642619688,0.5980879065971922,null],[0.672499783257037,0.608202143969469,null],[0.9490936796304378,0.550368948093654,null],[0.0662981788341489,0.5495557114021468,null]],"TsU9TS4hpMHXVwv-YIrvCQ":[[0.6011036567020923,0.5773969005389564,null],[0.7717294606581535,0.6172149178566217,null],[0.2799208853068433,0.5272480470748784,null]],"UPTPPrW6iUFocod220aq8g":[[0.5043677683636544,0.5258582429288272,null],[0.6758324542195309,0.6144283899691895,null]],"UrwK5NxGtb1p1lGv2EIgdg":[[0.5050178097810658,0.6659766178137897,null],[0.6243872130615168,0.5695141552395313,null],[0.30317728600611427,0.5456926371532227,null]],"V1ze8IkgvM7tXueCGVfzbw":[[0.7715426722335322,0.6368501059462901,null],[0.1819229732597732,0.5918076929428141,null]],"VWMApFeYDoQmO9taDRV-lg":[[0.5838849441532027,0.6376173646217196,null],[0.4010508099213252,0.6330741053576832,null],[0.5468611524879075,0.6170066115593711,null],[0.8976841561318238,0.618933407498217,null],[0.25010773454686225,0.5962356638446059,null]],"VYsAOBkJ-7hdoDBIOhr-nQ":[],"Vk6u3cbrlksc6Cy5a1LVVA":[[0.4297674645285522,0.6344175755306025,null],[0.8498585158543406,0.7328776461180565,null],[0.04091732349295446,0.5560867763697815,null],[0.28518869061780916,0.5666187144225221,null]],"WVSHD61mDgZmv8fnzj3i6Q":[],"X45JWReulQB2gYvNrbUKGA":[[0.6255300096104337,0.5779816850559936,null],[0.15799980908345063,0.6138424718223185,null]],"XRcpuzY4BC5AypmNXaQuyQ":[[0.4461100508226351,0.5658475502603729,null]],"XWhHaUuheK2ypAkTt10B8A":[[0.42336948085006854,0.5344252041797688,null],[0.9125159039873173,0.611344611649857,null],[0.8782917534744901,0.6400965024160918,null],[0.12184706664554318,0.5718239638760279,null],[0.32566035828955475,0.5761514743535617,null]],"XuX3LCIaNmEI9FSBenjToA":[[0.7187752357605637,0.6295537484786148,null],[0.0726870052605077,0.572068595133475,null],[0.19457530905808662,0.5675392275653576,null],[0.21718791591084557,0.5637219044320126,null]],"YDTr8LXPDt5ieA8MA6ZXLA":[[0.7246639736163699,0.5301687187138882,null],[0.8736981564490042,0.5582836216606838,null],[0.942352303497573,0.5543151460256106,null],[0.4114795402419973,0.5303181414648476,null]],"YckCrqJ_I8sBF8wU5FpITw":[[0.11931868090664483,0.56041265852043,null],[0.10510868311947852,0.52162005684209,null]],"Z0pMjC8iJ6EDQzsBBrr_5Q":[[0.5981590057066443,0.6501227695502572,null],[0.6379717748254458,0.6733650495187885,null],[0.3760381032101044,0.5919402048071083,null]],"_2GSllZzac7Zfd02VoVgCA":[[0.9193684065184031,0.5757940042310729,null],[0.0454227284432282,0.5477682221682645,null],[0.14571062655574774,0.5474508176196177,null],[0.11704215271598012,0.5399710708711839,null],[0.3638857961843327,0.6684350442888017,null]],"amZHKZbJ7pOJ9kvr2VlK5A":[[0.5261973398938197,0.5242696499410106,null],[0.7959127483004786,0.7895551393934304,null],[0.7718295904631755,0.7935657102998224,null],[0.9618573485008441,0.8302035435068382,null],[0.2022033037413309,0.5851641315902845,null]],"b-azHLq5uJSlywVqKYbgHA":[[0.16714953330209642,0.6040724137898628,null],[0.23747213747918672,0.639157775829911,null]],"bcNsOc7RlDm9L9VmIGXYpg":[[0.4040849864682724,0.611269867380107,null],[0.6130551019359219,0.6416924141336939,null],[0.6607492474466543,0.6166493352843874,null],[0.9064612882432814,0.5672895400552412,null],[0.0693244813440475,0.5880862820182465,null],[0.3112236676443497,0.5552229422340721,null],[0.3751977319978328,0.5863531527373796,null]],"dVrNCxUTFAfeyVNMXQ1LTw":[[0.6249524792228561,0.625561728670146,null],[0.770183372934568,0.5545616731393099,null]],"dXNmCM3bRYfsvQ7xPivHaw":[[0.586033917977304,0.551029470182961,null],[0.5770714955369695,0.8164803750688928,null],[0.10294052970420992,0.5354917594453004,null],[0.9567682925639245,0.5276264288692483,null]],"dvMvyySAGHbGLfSDgYWHsw":[[0.5142761862022188,0.5123897794983464,null]],"eW6Mi_5E32tMqZTu_731bw":[],"fOc-SxUHjTYKCGN83egiYA":[[0.7912539366839262,0.6024900810923485,null],[0.8936874614261198,0.5804838515454874,null],[0.9193116987640828,0.5619878917829714,null],[0.36967624432977897,0.5597224285347417,null]],"g7ZWUQeOp1Kl0LrWi0qrWw":[[0.5753794392272644,0.543864398823354,null],[0.2213673406512937,0.5467543001382403,null]],"g8Jd1uL_FE_9eave1z2OpA":[[0.3875739344809217,0.6150394731130636,null],[0.6084337756123569,0.5712406897021037,null],[0.6578578813228069,0.5804317226807577,null],[0.8761933319419964,0.6039218283983397,null]],"gcAfPhmNaIDewoRc5O6yZQ":[],"h7I_Gn2mXFsjt09ESdkoOg":[],"hD1quvb9bv96mz7ElBTTgQ":[[0.7633284173923932,0.6736101886436889,null],[0.8341572713700002,0.6179352943375641,null],[0.35605460811961787,0.5954163124620385,null]],"hlYnMGBEa12Ajij0iL7yxg":[[0.7002225805990324,0.5662570440713925,null],[0.11568046060013287,0.5326485040457403,null],[0.14391868278270697,0.5720933005611297,null]],"ipoVm9spHhQGCMROwSdmmA":[[0.4195147157997469,0.5360704568205275,null],[0.5360968823203127,0.5286913519144312,null],[0.8780391561203023,0.663039234965935,null],[0.8902274316463765,0.5385049336801502,null]],"jE7jfpgpMn1zP3_DMdvMTg":[[0.7594366786517929,0.641848420306688,null],[0.942850131431032,0.5504569572381239,null],[0.3506543820111362,0.5565197725553964,null]],"jL3wxV95A0lO2xKUQukpIg":[[0.5317486272138718,0.5286091202060935,null],[0.8931731075905349,0.604889286153192,null]],"jqZrc8lSwNpLZbLrixSduQ":[],"kU1NFVXyUkQzaPpROkDVNg":[[0.5648566789221707,0.5221288959189226,null],[0.7387266696521524,0.5636291018652082,null],[0.7113915593647249,0.5457489116565846,null],[0.2657990253510788,0.5502065021575996,null],[0.29329047099143823,0.5453295709506775,null]],"ki1YlBoTB-ARFE17nF4PPg":[[0.6184256938576264,0.5907507975057176,null],[0.7227632046608103,0.6425254274186792,null],[0.9634558514964002,0.5380479764522668,null],[0.35679551078214433,0.5712028461235342,null],[0.3072523805323179,0.6010014277936329,null],[0.4255864039109619,0.5345090155223257,null]],"kjXjoHf2E-tZuI7RwMPgsw":[[0.7659697812197725,0.648327220515861,null],[0.8767627546455408,0.5937180015397799,null],[0.9564663576021117,0.5263810548905894,null],[0.19035641129783365,0.5619921414399335,null],[0.2810075235584416,0.5636509728805393,null]],"kozsgHrge4ofusqK2D_Hlw":[[0.5809967781658785,0.5390414229835233,null]],"ld_LKRyg3hte-YCeFY3avQ":[[0.9607178635120746,0.5676413553718423,null],[0.032108958420075495,0.545319880214894,null]],"mMSFxX8cKDjvaOmsrguJEA":[[0.5821183592056217,0.6270381845430093,null],[0.9635731308515019,0.5729240428979184,null],[0.08478051430852995,0.54530227230285,null],[0.3430038664184112,0.540309500926849,null]],"orMHYZcfpk4vTLLG6ciKUA":[[0.6148774703369309,0.5816996144954789,null],[0.6692495736058365,0.6058558211902725,null],[0.7380352502751208,0.6104720661576927,null],[0.9378305034586292,0.5500987469753915,null],[0.2808943134751856,0.5623608932477591,null],[0.38389119501985547,0.5447287754758832,null],[0.3426334649875742,0.5628418409609074,null]],"oyTDvou4dK-Hm3v-cIaToQ":[[0.3944999327658028,0.6096686278585306,null],[0.572001772578143,0.6109212777319186,null]],"pM_zjaOL4jt8_niIC0EU2Q":[[0.4390554167633421,0.5406973675911999,null],[0.5667599108442509,0.5359478525431365,null],[0.6868170227074712,0.6504075401001128,null],[0.11629573381487984,0.5893935035450031,null],[0.16882218466013027,0.6215763749873296,null],[0.3178520358507705,0.6065991600183415,null]],"q2yX7pwQF7_3SFGxEqlDHQ":[[0.7419772505939618,0.5607140997111182,null],[0.8939810480501911,0.5632497722435168,null],[0.2528793180774732,0.532158553699184,null]],"qAP6awcZVDK19stGxM31jQ":[[0.3744646075103375,0.6849531024952411,null]],"r2EDHivQkQoWWelu3IZOxg":[[0.5983271741554945,0.5561759291763207,null],[0.4167445513244286,0.5591250417936416,null],[0.7018927291008681,0.6135207620478338,null],[0.8001457332575784,0.633430597920657,null],[0.16190861576449966,0.6359342726911412,null],[0.2551859004712869,0.5700646926417844,null],[0.29540657128983117,0.5851630402412284,null]],"rFa5e9WO2AombOfKxw1GzA":[[0.5837149242358927,0.6462598356146723,null],[0.6975276312137898,0.6395209180548895,null],[0.7338522461775859,0.6316974104601766,null],[0.9517452671067109,0.5769116955597028,null],[0.2552858588429168,0.5788891506997661,null],[0.2780945919809304,0.5883851531556892,null]],"s7wxqo3wnK-LD6bUkCRGzA":[],"sIW4u-Mq7TORFHbsswcCzQ":[[0.4264146500931279,0.5763385783261016,null],[0.8857771900232654,0.6092087299722531,null],[0.8998464511114135,0.5603926423035165,null]],"scWBbnWIHdaaBRanRNRa4Q":[[0.40953989953642317,0.5385987054958932,null],[0.5426515633392002,0.539723059924255,null],[0.9451554819388134,0.5468685817246436,null],[0.37494722468808384,0.5600498059432054,null]],"smIcY196GPbaTpEbjkA7aA":[[0.5319285851177852,0.5318856151966516,null],[0.11408652734002867,0.6192251295979766,null],[0.14439488453588772,0.6256632535352287,null]],"tD8uJ0wHWy0RNPwb73S7Og":[[0.5863886227634577,0.5417596726235361,null],[0.6640705948436945,0.6088572159519385,null],[0.7341248380931681,0.6080382452092122,null],[0.7755579462379094,0.5376211345171662,null],[0.6294979219950648,0.5349475158002505,null],[0.7109574924107542,0.6091815827705426,null],[0.9881340895155375,0.542122815806185,null],[0.058475840048525185,0.5310594263022506,null]],"taAM_J38w0t7x9p1iOGptw":[[0.9496753485119909,0.5603220035180129,null]],"u0S_61wv1Q10xayttHdcbg":[],"vbW2W3-AE2tr1NJYHCuHgw":[[0.6836954823095713,0.6615385263892842,null],[0.7540280841412892,0.6692023683044845,null],[0.8842964280859077,0.5766602204464578,null],[0.17693743689475477,0.5847272312935565,null]],"wwbkgmoKv4MS5sMy3PGhqA":[[0.6595204346551551,0.5343158522734931,null]],"xExZyWexGcwGNf5LNhQBRg":[[0.6186152717760336,0.6629480365260765,null],[0.3903815928824572,0.5355956342431016,null],[0.07027493784810851,0.544308301358369,null]],"xMY6EkmbN9OZqwHeC_S7Zg":[[0.4136984419014396,0.6003592169606152,null],[0.7795953786457507,0.623805194230821,null],[0.8393691368973335,0.6282255011831607,null],[0.9103436802429475,0.6060986625249281,null],[0.3120974128195881,0.6425911534387159,null]],"yQOkeKGLo1dFuhOO7BFRFg":[[0.6442776256190222,0.5587607886883603,null],[0.8298422960176101,0.5957251879601749,null],[0.8843252122996712,0.5952628507239063,null],[0.19327399796851974,0.5883807244905144,null],[0.3221085690297805,0.5539727716377943,null]],"zTwcPkPUYZ7lRIowR3wXyg":[]},"model":"claude-opus-5","n_panos":125,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__morgantown.json b/benchmark/model_detections/claude-opus-5-effort-low__morgantown.json new file mode 100644 index 00000000..6d9ecfdf --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__morgantown.json @@ -0,0 +1 @@ +{"city":"morgantown","detections":{"1024920772140814":[],"1045092623940908":[[0.5376832299130312,0.5323881991716146,null],[0.43215416272238755,0.5275221341563071,null],[0.680275595187883,0.6373935138530206,null],[0.21923240080251494,0.5509725867093194,null],[0.2723825004454534,0.5494891158981043,null]],"1048998662992115":[],"1053392859080211":[[0.6266703165194868,0.5784023646620968,null],[0.7309975725298543,0.5399316701576179,null]],"1055892936046905":[[0.7251867295389572,0.6894589089897928,null],[0.9500149849259167,0.5762556647710698,null],[0.1007521401683637,0.5602307876583977,null],[0.22145962805931796,0.630057212265125,null]],"1061541095152683":[[0.5111374043892633,0.5342667826929471,null],[0.6692948884806725,0.6161021702283129,null],[0.7045532162183088,0.5868802695515326,null],[0.9837448565270304,0.5244731779945693,null],[0.09133696024635252,0.5909120700565786,null],[0.09979963275662096,0.6414387130547472,null]],"1070246644188505":[[0.35648371390478045,0.6024566577929111,null]],"1076796326927681":[[0.5923017138506224,0.522878595372408,null]],"1083964366388125":[[0.9123517472723331,0.5974753689511505,null],[0.1940213833673382,0.5403161351878585,null]],"1098428298746698":[[0.7279941969966565,0.6039890903840486,null],[0.9132643462032228,0.5154738532688391,null]],"1098721767937588":[[0.9192975384583074,0.5961938659730877,null],[0.18216401361951728,0.6195475989372565,null],[0.20657317308254647,0.6253851824248637,null]],"1108246054387150":[],"1113270700666667":[],"1116198312725600":[],"1128316922460832":[],"1159131791921689":[[0.0658388809314901,0.5639748774016714,null],[0.2476778108520909,0.6846222862956919,null]],"1174986150136618":[[0.7206825658632934,0.5253581287879469,null]],"1253759805583014":[[0.9334655959953951,0.6162519913254818,null],[0.15465707993452626,0.6097853341624784,null]],"1302688287512142":[[0.558333927737945,0.5452282746797185,null],[0.9533242300172677,0.6910341238957951,null]],"1314106923314560":[[0.4012998230703577,0.5732928154077997,null],[0.5727058640812672,0.5940426437898934,null],[0.6262119130264279,0.6097420909331193,null],[0.3379819931186524,0.5816722835195448,null]],"1318084892916238":[[0.21899160066156131,0.574756685145334,null]],"1320929785258220":[[0.662038932083721,0.640870350274698,null],[0.7639052182399497,0.6203338155099682,null],[0.2657658076243405,0.621853154181787,null]],"1323223598366351":[],"1329462397764594":[[0.5946056796135524,0.6107732311373417,null],[0.8992308688447861,0.6236714623663756,null],[0.9330593895067731,0.6137858230677711,null],[0.12546602419347974,0.585307821875741,null],[0.1826820573591153,0.596764740249085,null],[0.3470579442399007,0.6001440257213013,null]],"1361874984449863":[[0.8234382407447984,0.6189425705855613,null],[0.08687497908804559,0.605271938261206,null],[0.009086276253811365,0.5491298690198975,null]],"1379751865997896":[[0.9066723366294553,0.6105003086612439,null],[0.44655395135539583,0.5672348380094271,null]],"1380832652551536":[[0.6331978840787947,0.6412056432103036,null],[0.5396864093315059,0.5482350575485693,null],[0.8341409017999213,0.7271628475879864,null],[0.2494261971368007,0.6165383823594135,null]],"1412029892768315":[[0.3686482602129413,0.6402345130509344,null],[0.11219053437481269,0.62335016113472,null]],"1413501976231778":[[0.7017088025947386,0.5895241430281863,null]],"1413692656166618":[[0.38313230663588077,0.6926155164384542,null],[0.6744138755836186,0.6060518081655774,null],[0.903004580250937,0.6247757907739225,null],[0.07041503817230554,0.6334822929476938,null],[0.3437844517973965,0.6317363100520792,null]],"1415013822421025":[[0.40387673199880114,0.6540918715989499,null],[0.69172993740496,0.5955095734714855,null],[0.8064924090197886,0.5667805674318223,null],[0.0269061278486471,0.6696329177005461,null]],"1418204239105680":[[0.5351191287158796,0.548827121583859,null],[0.12215119511007128,0.6024736185009918,null]],"1433753780865996":[],"1443470343184880":[[0.2289324996518018,0.5911158563712575,null],[0.2515145252171439,0.5927335396907925,null]],"1455316282454101":[[0.5963062521537819,0.5287163294143961,null],[0.6473148215335869,0.6271544020954412,null],[0.7679243820309796,0.7092725897391234,null],[0.7301628968420115,0.6991086071699247,null],[0.24081002623503772,0.6215416442354953,null],[0.27912232218285976,0.5934030238162976,null],[0.38684009310436085,0.5495778482532546,null]],"1466487117546337":[],"1475722139747056":[[0.09316181074642216,0.575005161411721,null]],"1560632374758055":[[0.9942530064510231,0.6671982988416396,null],[0.17038959963425354,0.5822628383647983,null]],"1563664664398611":[[0.4281893591939754,0.5562815059571699,null]],"1599401410798071":[[0.7659639282104546,0.6961326339184579,null],[0.9379485001038341,0.5627672859074092,null],[0.058659554288304494,0.5619689190243212,null],[0.24581772592234263,0.6445499571465023,null]],"1700297240715914":[],"1756594321481394":[[0.21742741857344933,0.5662780640086151,null],[0.3247331112045102,0.5662122665444472,null]],"1763097830781857":[[0.6122868070498357,0.6112750072893898,null],[0.9545894818056241,0.609123267908101,null],[0.11962633973665487,0.5476254632771334,null],[0.053568638522669354,0.5571458037884376,null]],"1777363949394557":[[0.6081279472228609,0.6226696919669028,null],[0.6922152571634839,0.5409155205812952,null],[0.8840678149717356,0.6281051601420654,null],[0.7891481489281895,0.540383508357206,null],[0.950433205426099,0.696638490927345,null]],"1777706892706963":[[0.37882463966753804,0.6186541613928864,null],[0.6230084940861875,0.6068342622615139,null]],"1822937428270315":[[0.9755948488494135,0.5273270529054993,null],[0.17087450374196278,0.5634832195528373,null]],"1877033843193840":[[0.7380855279528142,0.6060433249821569,null],[0.18112893258460494,0.630892293593449,null],[0.31890172699783403,0.5773640778941773,null],[0.20809352941970022,0.6209544199936594,null]],"1921193534970603":[[0.9495207092546775,0.537634852160985,null]],"1940794952981900":[[0.5006434952589709,0.6037731425918366,null]],"2020608445007968":[[0.9014869010792483,0.6770881492568301,null],[0.7035175218286477,0.748904225621909,null],[0.02393213665115823,0.5535056762876491,null]],"2079289565741835":[[0.7389084049379573,0.6688560118739735,null],[0.9390814170237269,0.5392142485569786,null],[0.9717769627760875,0.5209955214735142,null],[0.13590958316809493,0.5253884623258706,null],[0.25102811845203954,0.5328346573497363,null],[0.28549192505457266,0.5408742614299529,null]],"2189963931355900":[[0.6205927781382227,0.564335894156323,null],[0.7761736681799005,0.6486404948823634,null]],"239851625849527":[],"24215058578133685":[[0.3820979411467005,0.64951207116105,null],[0.6258140087253539,0.619312441126944,null],[0.8836888989177187,0.6003282323485086,null]],"242318955595938":[[0.4066365479579177,0.6140038116005523,null],[0.8734221595248248,0.6240134301469046,null]],"24345562601705116":[[0.5402201196380374,0.5561495141886912,null],[0.7498698898286649,0.6516710327426273,null]],"2509132522616563":[[0.5641989478602478,0.533776830554976,null]],"254762570847217":[[0.06966613247546438,0.6081300794004042,null]],"2547728638744506":[[0.33977615884407264,0.6653108442620337,null]],"272487405737151":[[0.5205933562683887,0.5453316099284327,null],[0.6643485056539835,0.7056215787507938,null],[0.29942042142478387,0.5963219879889552,null]],"283488671383320":[[0.20300309620811358,0.5572881397346247,null],[0.34132351537882194,0.5656641827450876,null]],"300795532996657":[],"307295108505220":[[0.6516303754774467,0.6858802292631595,null],[0.7802699270217055,0.6286190697553797,null],[0.9208579686336766,0.5567530074310584,null],[0.9760868611205304,0.532169659113121,null],[0.04134058736979723,0.5327034996679351,null],[0.11825956500152662,0.5511427649105866,null],[0.17502182292554114,0.5390100725876932,null],[0.2506392589624223,0.6242726625690855,null]],"312712807925866":[[0.9134494573428469,0.6201868698442331,null],[0.09936576351115517,0.5956397778976862,null]],"345140178495984":[],"345988571673909":[],"346162854923531":[[0.5725174095648824,0.633886623435981,null],[0.34636385593385227,0.5975120677982223,null]],"348102994829328":[[0.17845457084977434,0.6067962358620186,null],[0.32225271097314134,0.5949386814985109,null]],"356422300574178":[[0.5615281405869768,0.5602314357083262,null],[0.7830025769313018,0.550113223201571,null],[0.8749576954006333,0.6022248264564012,null],[0.971461180564754,0.5503002352597433,null],[0.0756250871029937,0.5477184488908656,null],[0.1452374365146436,0.5368894118120114,null],[0.2822130916976252,0.6025442646506376,null]],"360514396681247":[[0.44032020482797857,0.5471025996925039,null],[0.9194801302036395,0.5631855522290078,null],[0.2633892138257261,0.5363413979019979,null]],"360752356915454":[[0.5764728807256936,0.5780238817615794,null]],"3625726214363805":[[0.5445894465415481,0.5834201463893591,null],[0.5904402517086288,0.5881002084166997,null],[0.6464202792314915,0.5605165414279142,null],[0.8652431464181705,0.5933580531387395,null],[0.9199100794196995,0.5981297075747695,null],[0.9770566060885546,0.5771808217505056,null],[0.0889568997907016,0.5511796759379026,null],[0.05286015010904127,0.5435421420499229,null],[0.29306412635376533,0.6090979027519258,null],[0.3792502962079207,0.5798621086979855,null],[0.26475802697349166,0.5745205177875083,null]],"362690176518230":[[0.0682422853687194,0.5635548878098108,null]],"375936148381288":[],"381456537940520":[],"3815491675442946":[],"382272334940513":[[0.5847999820742577,0.5652416144721967,null],[0.6127536194472668,0.570697745616811,null],[0.8003787957864936,0.558094483583896,null]],"390232193697883":[],"393398989743901":[[0.6767375845616095,0.6682539027355364,null],[0.5857291447818727,0.5889957681753386,null],[0.956325458807806,0.5518531805879039,null],[0.08432200754200708,0.5686207826933649,null]],"3940759156239014":[[0.9936933783558934,0.5358865851579735,null],[0.3500429281901301,0.5404263445385391,null]],"394696869697746":[[0.4299228025519432,0.5969590583460862,null],[0.37619333194199644,0.6039218283983397,null],[0.6161682771658098,0.6028749521104135,null],[0.6434889999300077,0.6072084447742739,null],[0.8357684179755491,0.6091595621906398,null],[0.8795909049804367,0.621084799926379,null],[0.11713707924794281,0.6328718489845624,null],[0.17536029480143744,0.6156556854551828,null]],"396936476205103":[[0.5411915425965108,0.5460268858952854,null]],"400080345817958":[[0.5999931964970051,0.6568766033532272,null]],"402410412144028":[[0.6539087587965435,0.5922214178990022,null]],"406227845237495":[],"407074681751773":[[0.7835591373344694,0.6409316943580778,null],[0.7317974472663318,0.6011634913175686,null],[0.2568041365104201,0.5992328913518761,null]],"408409558196279":[[0.6435217535569093,0.5519224328228506,null]],"413429987710546":[[0.8227159523712917,0.5867506559988314,null],[0.9823930928622168,0.5187195979624825,null]],"423595953348322":[],"426801413006387":[],"4607308572828610":[[0.5321839868966087,0.5597839942955414,null],[0.507678241942244,0.5648300704070738,null],[0.6425436879789095,0.5475850138062932,null],[0.7117179614567472,0.5446872474834531,null],[0.8487497210738635,0.5568623374217975,null],[0.02128030121274538,0.573030413888484,null],[0.19466998686041542,0.5540945384442081,null],[0.26772143516944485,0.5858410074132782,null],[0.32443826554280586,0.5862230335098525,null]],"533130179129190":[[0.4346724104121939,0.5522148624707944,null],[0.9132658652560071,0.5871589184017694,null],[0.05736332365217306,0.519038985918977,null],[0.4094725270227548,0.5387294104881297,null]],"559825463037840":[[0.388846363712231,0.5804084124793546,null],[0.05693048734555611,0.5522856889441077,null]],"612101157901284":[],"683976123635133":[[0.39190125594578634,0.6032483818670592,null],[0.6040399895807191,0.5911757378907773,null],[0.6519519640286069,0.5889112171175213,null]],"694851952818440":[[0.39265604971965673,0.5870679218963467,null],[0.5492937664046691,0.5948597619576659,null],[0.7991366782353712,0.6815230580032728,null],[0.8292595485714475,0.6836992903696351,null],[0.7536529314820613,0.5699838059102853,null],[0.8634497302203975,0.7043348896532569,null],[0.16916838046938548,0.5869243718202394,null]],"701464722124781":[[0.47265942953492845,0.6819031045148958,null],[0.4429967545644053,0.6041589034246071,null],[0.0996013957779891,0.6717676501639667,null],[0.0678634356747399,0.5870731901278176,null],[0.06653706323455594,0.6887360748195972,null],[0.3431717880930188,0.5592228873749339,null]],"708625001301257":[[0.6408420481479343,0.6466639015234684,null],[0.7089728395112719,0.6422581953270894,null],[0.9170647785088423,0.577450973613333,null]],"715907037015770":[[0.8445622353311681,0.6722218835294753,null],[0.06536037170019343,0.5890291608410008,null],[0.888826117675979,0.6367700759930164,null]],"716002773841007":[[0.6396797807996181,0.5866127743173815,null],[0.1219938759943417,0.6128796139197902,null]],"730248625779882":[[0.9162615234047121,0.5841829710050508,null],[0.9451423440269758,0.5262716044896583,null],[0.11713103539112807,0.5784929925493061,null]],"732241911917653":[[0.8679581267243364,0.7050384173186502,null],[0.8852731315880037,0.5983632869484462,null],[0.2133625552546548,0.6525176348199737,null]],"7393628424017132":[[0.38152822027078676,0.5967451559654331,null],[0.5356818441513393,0.5511761428374007,null],[0.6593965482204103,0.6466657931977953,null]],"747134177377694":[[0.4555659240344278,0.6238345375487325,null],[0.5559398155009109,0.5762973523823923,null],[0.38859190787914577,0.6011124458563303,null]],"749673027037288":[],"759399502183549":[[0.606381702295234,0.6122647123749029,null]],"760177245654837":[],"760811765935688":[[0.5541844835100621,0.6725292244304419,null],[0.007439966710827084,0.6072597339780987,null],[0.3998034746030354,0.6834324517959226,null]],"775966364379699":[[0.7754832609674307,0.7235474046023913,null],[0.9328910949253559,0.5904512023714077,null],[0.06603282119533732,0.5681361508609127,null],[0.3032298769708378,0.6559925284964268,null],[0.3475713737382129,0.6422784512499524,null]],"777238804420101":[[0.5097552586752309,0.5283633730970168,null],[0.5931985589568429,0.5166445155773087,null],[0.8432469060722465,0.537675570611502,null],[0.9748169630521379,0.529328986626196,null]],"778147450850347":[[0.5913806341582537,0.5895969193983466,null],[0.928261202441356,0.5837046336178613,null]],"781523387330174":[[0.637605634894924,0.6303366108080777,null],[0.8853591513260366,0.625129469796355,null],[0.16214051442210747,0.6067281774014448,null],[0.35219894940150254,0.6022046705675612,null]],"834261372050006":[[0.5894119996997191,0.5438199461939954,null],[0.6240604730451629,0.5485286184241851,null],[0.9097989522143326,0.5524989750593051,null],[0.29556987261426565,0.5421041735362218,null]],"8551050821585152":[],"863715805438268":[[0.38160056043091156,0.6016616351998801,null],[0.5657632135249314,0.618518184796977,null],[0.6090600145905991,0.6414468522387887,null],[0.9327821956423828,0.6157818436852761,null],[0.1009022990666959,0.5864653422099663,null]],"878431760506295":[],"914644690034345":[[0.11498111184171861,0.6015229788431612,null]],"916442743190808":[[0.6528263929776691,0.6896878090077952,null],[0.40852652336169604,0.5617513593305058,null]],"919312849636317":[[0.12007665184747729,0.5625502415663958,null],[0.14440157379389224,0.5957465398911193,null]],"919729533106221":[[0.42191437149815536,0.5614339025734233,null],[0.5332134515682021,0.5679465039027042,null],[0.951817689651459,0.5606996237355962,null],[0.9189451825370907,0.6024411413471743,null],[0.29680623064954603,0.6192868521653976,null]],"925860169582480":[],"927279915685911":[[0.1695102608228199,0.5609880966466365,null]],"931660678678586":[[0.5483428781366977,0.5638908661354218,null]],"941838830948431":[[0.9345012799703272,0.6305219852512993,null],[0.09635486381174385,0.6218413016621338,null]],"948824803432429":[[0.5117705140673874,0.7137568728189957,null],[0.5377684860980568,0.7406265261014269,null],[0.9505475443096087,0.5520019462540324,null],[0.47693950561468773,0.7130124229548278,null],[0.218338304770667,0.5500538834985994,null]]},"model":"claude-opus-5","n_panos":125,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__paterson.json b/benchmark/model_detections/claude-opus-5-effort-low__paterson.json new file mode 100644 index 00000000..d0ee08b3 --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__paterson.json @@ -0,0 +1 @@ +{"city":"paterson","detections":{"-2so4d79_XMQZkVRS5c0dw":[],"-csdipRzFYTMTtksUDLaHw":[],"0-zmUwLwl5ENJgqxheDklQ":[[0.7442076963369054,0.666512771442415,null],[0.8954827619831258,0.5981340271619429,null],[0.9384411878198117,0.5775291575619651,null],[0.12033304621802293,0.5733587773850926,null]],"0Drku25sOlOlWGiVf7uetw":[[0.40643819366652323,0.5577976709861937,null],[0.5561859927605102,0.5641830853673329,null],[0.8817710371339064,0.5970166539507875,null],[0.16712474871964872,0.5728102108024058,null]],"0fCPhcB0ZF3dHQ7nEIWABA":[[0.5118647182640741,0.5350347415567447,null],[0.9796484330227313,0.5550308114373059,null],[0.2880429329149401,0.5788192166148345,null],[0.31052842237054584,0.5814776313866328,null]],"0s94i47Wdggc9cHmBjEe8w":[[0.9300406074377806,0.573919447217982,null],[0.08704189116598066,0.5629764910270785,null]],"0uNeulmg68uHO_WKDIuqZw":[[0.2830075669648423,0.6281823848838668,null]],"1FJQJxlHdC_TzHOH9ynMFw":[[0.5421915136655853,0.5242335029981813,null],[0.41492484854283684,0.5227101639231356,null],[0.7002694475939458,0.5989873395629347,null],[0.26985411879977217,0.5600276914672624,null]],"1gJowvvmBQUnl6h2znJ8ow":[],"21Gw_VuYrSK42PCF32UGUw":[[0.5260892686948354,0.5403699238641942,null],[0.3856372882316173,0.5300687697759686,null],[0.7037993623453658,0.59648756144523,null],[0.24658900622618568,0.5625665520878073,null]],"2lovLRC7Pv6S94tMW3mDug":[[0.44216566639070215,0.5136836839827582,null],[0.5448256582739524,0.5112493810062034,null],[0.6697542380801857,0.5793821283252792,null],[0.28249203101515097,0.5836498149502793,null],[0.30851850398306657,0.5713945660572458,null]],"3BTAarqIRrxbpFHUFJhtSQ":[[0.4557192040480135,0.545266600510148,null],[0.7277265495268752,0.683134027314047,null],[0.15298803040039732,0.5582927687516632,null]],"3Tw1W-lv133l2efDeoEAYQ":[[0.3997570435632552,0.5307678822767006,null],[0.5472301785137031,0.5399041413061464,null]],"3dq9TqRXzopD0yvztHg8rg":[[0.9333657130699944,0.5940868865780191,null],[0.9949124397569107,0.5276358375493018,null],[0.16741575952008148,0.5615529305658111,null]],"4GUDoZXe6RgYNQX0m4b0Zg":[[0.725944355956603,0.6273684208851042,null],[0.09002695787751241,0.5363968076665593,null]],"4H5JBCKRb_RegGCUdvxVqQ":[[0.5656238140937099,0.5655815578217993,null]],"4cXVf74zyKLTQLiQtBObww":[[0.7879178495965736,0.6373037683201496,null],[0.04639428818586255,0.534411437109088,null],[0.1784987803073541,0.5265572680900398,null]],"5M-GzTlVe-lrOqHmypJqTA":[[0.5485246215041567,0.5362732238112647,null]],"5PJhqjhMTDTN_54DwP35Xw":[[0.4267816997542077,0.5690862254614854,null],[0.6823683508462826,0.6210507366450528,null],[0.8193893291110457,0.6232926701879399,null],[0.07397959478742572,0.5460064600118318,null],[0.44581048619556113,0.5425803390582035,null]],"60amgEGA1TEcbtRBTtxUTg":[[0.5671305965290631,0.5721572988242876,null],[0.5354967305006236,0.5277162086326728,null]],"75T9ZXImSNQMpkN794zWRQ":[[0.2446983423722512,0.5734665708276354,null]],"7ZSHYN4Na-esoxvqvTzjwQ":[[0.8346509337293778,0.6176294837651979,null],[0.9537749540555776,0.5510920390842119,null],[0.056062583321929926,0.5500684078316347,null],[0.18484632215288016,0.6214749389951881,null]],"80E3CRwZRxGtczZCQX3JNg":[[0.5483114097050087,0.5566792681951749,null]],"8NBnsxnKT5lNnv4ToKfO1Q":[],"8_b_fPkaKLKL1NpmMe3OMQ":[[0.11101789098992587,0.5700917229962931,null],[0.19936563529570378,0.599391518476377,null]],"9AZrydWz7yu5OFJmBXIzDg":[[0.5275654258756969,0.5300158593540955,null],[0.7953563286680857,0.5995312996003674,null],[0.35416073212936605,0.5561001279550531,null]],"ATosj02dIzTybtsR5ecUPA":[],"AYBFFp06z_3m70ETRK2Jtw":[[0.1115722867217282,0.5496666985218672,null],[0.2850708727665451,0.5660347063022367,null]],"AhV3GKVFnENcHwhG45WN2g":[[0.5748105061606632,0.5507938537759479,null],[0.7726970635889446,0.6081107735539057,null],[0.7462791406882816,0.5583752970697082,null],[0.23888457356129483,0.6058042264805595,null],[0.27008276872287906,0.5894249535380421,null]],"BjV06xLkvfHDCTJMA5WK6Q":[[0.10722174533134238,0.5417947846339911,null]],"Bv3GagtdyKXz2JVeZMpjrA":[[0.6917330387467144,0.6256695956236565,null]],"BvnpSlTjHxKyBiX5tQ02Mg":[[0.3990510132999179,0.6907373568614372,null],[0.5831456015740243,0.6522744590961105,null],[0.6789682617032267,0.6082702467356297,null],[0.9215998781009528,0.5764423876286819,null],[0.7083170400197898,0.6031945242396681,null],[0.28447115077461094,0.571669950418812,null]],"BwpJfUUUcWanSPtttY4KJw":[[0.4159424191442886,0.5337836075580803,null],[0.5329575022688076,0.5331127650387957,null],[0.7090361037078492,0.6041450256657529,null],[0.7685439768949827,0.6457347612481095,null],[0.25214954246107824,0.5744122663983486,null]],"C-d-iXNcQ_LYqEup9COaOA":[],"DjvAQ4p-Q78PR-6ohIs5kA":[[0.6540554818442975,0.6076554354243219,null],[0.06760736362981357,0.5327462069682204,null],[0.15940167071852562,0.5679262380860315,null],[0.34147786707859096,0.5659414222652196,null]],"EIUNEZz3tbipOJ72-2dQzg":[[0.9014409454944976,0.6013870319564544,null]],"G2Cvex9yWgd38gTaLEXxtA":[[0.08678425930834271,0.5853802685862366,null]],"G7_klPoeLoxXAdoW6affDA":[[0.8327069334388573,0.5878098042922595,null],[0.8778350896855671,0.5867608564203215,null],[0.2145842983854392,0.5620591117902213,null]],"GBSPMaXXCAqOdrozYcHR3w":[[0.612243684535004,0.55675700611771,null],[0.3253409342356941,0.5440484124846818,null]],"GurIhAJuG_H88kt0_10sIQ":[[0.05138709642016409,0.5301270363212971,null]],"HFYowXmV7u1c0TMp-p_pgQ":[[0.4433353264165396,0.538621086409594,null],[0.5324104814820669,0.5258335101444116,null]],"HHjmf0kWHmcwWAOfg7b4tw":[[0.41173045734166547,0.5517915874602142,null],[0.5518412504252054,0.5549662272581216,null],[0.6353613881611336,0.5678313829011689,null]],"IXCSjRuAqY4_iyOU_2rxeQ":[[0.5237214038194717,0.5308976525062328,null]],"L8lyOJwUI8j3Mk_cV-R80Q":[[0.9363984980722928,0.5419824306279576,null],[0.06997144963727558,0.5448391434948536,null],[0.1405910515844322,0.5798011302016378,null]],"NqOWmuzYX0M4vXF6ioTD7Q":[[0.039831588153492914,0.5302744779685684,null],[0.1813167009363223,0.5677157007146423,null]],"OhdI54JArKc1UopNZiFoZg":[[0.42919154411610655,0.5296197286255304,null],[0.8519840653221078,0.5757334208344971,null],[0.8997935274283501,0.5543224789914591,null],[0.13712277841006465,0.5475793076458837,null]],"PM92O1ji0i42GTuhT_ll5A":[[0.4127301878516226,0.5342864855561024,null],[0.7346676795765317,0.5569046773472296,null]],"PS-Atz2shq5v6QtlJs6OlA":[[0.7529477556813369,0.6073927803135007,null],[0.91628189227894,0.5369155998525851,null],[0.020160273843465748,0.5411141060317648,null],[0.18225488395406042,0.6354121556415806,null],[0.4024595235563485,0.5417590850674754,null]],"Q5D74UJFt5D2PVhGxRh1sw":[[0.6248524075052984,0.5803798415633203,null],[0.3867656192906903,0.5308580713149849,null],[0.32569584577827115,0.5853838064880571,null]],"QBTZFMjfQRfbiv9e7gAG2g":[],"R9WHDEVRAq5IdtLkLFhL_A":[[0.5467555784584559,0.5440325548087745,null],[0.7361373789988632,0.6367233542686543,null],[0.9313048867580012,0.5579050882007359,null],[0.4259947063177769,0.5324561665473945,null]],"RGAxlzGeyK3ncjFZ9VXpBA":[[0.9131307857804345,0.5671476651563573,null],[0.11583247412145087,0.5521863784881267,null]],"S06PQMrwEby-_62F0U4Oww":[[0.959661343645851,0.562069391545232,null]],"TFX8xO7By2dnr_cYK59_Gw":[[0.074789719102764,0.6795406945490835,null]],"TO2IeKCGXF_kk9hxE-_Nkg":[[0.551645396521241,0.5330644425234149,null],[0.4269994850799919,0.5303559208918068,null]],"Tt-ZblLT07az2ybbNYiaKg":[],"Tyc7Xoi2Qu6tNrHVre2kNA":[[0.37586256894531495,0.5901785813934333,null],[0.5483207654139595,0.5755603494229438,null]],"U6iQcIYtWc0Xbflaeshkdg":[[0.5680866607322257,0.5938853514099902,null],[0.5911978026051794,0.5445779995325372,null],[0.9508843925345084,0.5716662216506214,null],[0.38591742462899475,0.5320131817507122,null]],"UHcgUZHk3c4QDbrWSH--rw":[],"UOMaPWpm2CRHbfqMgY_hbQ":[[0.6189981027640699,0.5845534861408201,null],[0.9114702624465614,0.5752610008227439,null],[0.8826080627702542,0.6071678272076373,null],[0.11175485052684264,0.5465908338637846,null]],"Uh1qGHtgaFQVFpiBQyeN8Q":[[0.27511704028954664,0.5377052030160423,null]],"WDlpinKjYLWq0r40CkoGrw":[],"WJObi8mVH47hu4RDg_PEOQ":[[0.4113357440980094,0.5446458801884648,null],[0.5344469181631919,0.5542056131737312,null],[0.696842362670896,0.6202100958006473,null],[0.7081961710979181,0.6637941074340326,null],[0.8712986197968666,0.6424249231274742,null],[0.12206598767522658,0.6020378426977517,null],[0.32156336951596565,0.5899185375517942,null]],"WNiUHjkafGBEnMc0nc4wkQ":[[0.949258350577163,0.5497994026414954,null],[0.07818796521242022,0.5420308065265367,null]],"WX-82gCn8WVM76EBN-JjlQ":[],"WoAP6tTVMjKMFEwdCi7YaQ":[[0.11006765603123098,0.54119830057907,null]],"X3ppzLhvsS7aGu9Lv8GdBA":[[0.10492360777621318,0.5710228835215398,null]],"XRsv_X3jYvAM1BPI-oQJUg":[],"Xs3lDDc1o50wJmW9eGx_WA":[[0.740325331723171,0.6641207907450306,null],[0.7893588342117315,0.616917310302523,null],[0.9372304290676805,0.5485031409292325,null],[0.08448881109016404,0.5513193324418852,null],[0.2599156901248775,0.6092084014406248,null]],"Yg0X7I5adOM5rem7YVbwAg":[[0.9621897374845558,0.5281202164474709,null]],"Z929dNP7sDp8cU6sjIeKAw":[[0.62984873554851,0.6164906629832205,null],[0.46217391907895067,0.5366964901365355,null],[0.6577125494005818,0.6220210167829381,null],[0.25886630930912163,0.6080353532880959,null],[0.30234672027478826,0.6051906695748751,null]],"ZcGyifqcyvUwLtCYGTr3oA":[[0.8237120634198076,0.6024046455574035,null],[0.12548836852127537,0.6045920814562888,null],[0.0321987808208154,0.5393090336105663,null],[0.21689540906726185,0.6333397871196549,null]],"Zu314hB3XfshUEt2hsd0RQ":[[0.8883586661404281,0.630884359126006,null],[0.3387785583362166,0.5673931128514031,null]],"_8m5h_Iu713wwh652Kq9VQ":[[0.8544506544827766,0.7520142100497786,null]],"_FfdhRGlkbTABymQR3t1_A":[[0.38133531141239013,0.5597630450841868,null],[0.6152444275222285,0.5558344776803479,null]],"aZxwqPZnkC3sv5dTZOEomA":[[0.5550034079843664,0.5675654704212816,null],[0.43363621758156706,0.5218067061522154,null],[0.94828263849513,0.5946472883209397,null],[0.3784238268661279,0.5452011249573708,null]],"aon4zuny7-LGeKV7JympCw":[[0.44102678011817853,0.5624498779398005,null],[0.7107930198563679,0.6155847560408603,null],[0.2924844009255842,0.6460537231435222,null]],"as5xCAJAYMT35sgs0rx2_w":[[0.6243408693692092,0.5945298050855633,null],[0.42585351151642525,0.532114515078555,null],[0.5422806653024682,0.5254548529087559,null],[0.32266585063926756,0.5723686922865188,null]],"bfc5Mq-kfwj29vGpOPKS2A":[[0.8992935805256586,0.6098785964628803,null],[0.12363115458887669,0.5904469089295719,null],[0.16430601593208483,0.6313622803957704,null]],"bwpW8jgnlIbAQC0HRy99JA":[[0.7919307081759464,0.587956073298149,null],[0.941104309048606,0.5366892069913449,null],[0.07681379522599185,0.5367533047351586,null],[0.2287181665628325,0.5627672859074092,null]],"bxdL0Guz_zm-8O1E1Gn78A":[[0.9293609811379088,0.5377212472311914,null],[0.10465583580538135,0.5311856274918935,null]],"cuFkPrv4UsqzeJCdibl1FQ":[[0.566093399726537,0.5639310820726349,null]],"e2tMNl4AQvYcYtzyd9aTOg":[[0.9255402559819597,0.5641613458637362,null],[0.09323101785513871,0.580011788453985,null]],"eGWz6exPVDlv_S_DowBPfA":[[0.936538311200273,0.5558675936281664,null],[0.08203399417077711,0.5578939223572603,null]],"eIw8Y1zl7qgUW6bTPzk3gA":[[0.5236901102379574,0.5301341071801527,null],[0.8919026314589611,0.5651571772688673,null],[0.09159819824330023,0.5729660564186145,null]],"fGd6JE4qvoG1uOVvOjBBmg":[[0.5018393525299902,0.5281588716701398,null],[0.38918347505039796,0.5269991113717204,null],[0.6844388731566762,0.6297290759613828,null],[0.7258641391802393,0.6600772767142307,null],[0.20471629642447353,0.5777889952049509,null]],"fSQ5B3z1vT-Ym1-siL8iOA":[[0.5629616754637732,0.5639332761462263,null],[0.5354761117941337,0.5619735916225216,null],[0.8212623193179731,0.6127920433972123,null],[0.19564170161113675,0.5652411531903969,null],[0.3853422596380604,0.5476710981351068,null]],"fxOJWtper8id-wMq0TZoww":[],"gZ3Acgi38PrzOVBkW-Il4Q":[[0.8047648601205853,0.6539692226043587,null],[0.8858562442014595,0.5952888010477823,null],[0.9773417993083917,0.5394556573234552,null]],"hMx_5PxQtlixa9LRSNnKFQ":[[0.8301728117120086,0.6545234879590138,null],[0.8672539599775736,0.6327856575792299,null],[0.2392670296432718,0.5832257028450061,null]],"hV6LwTvkCzyX_vQF3ZG5qQ":[[0.603734717165318,0.5880463192088239,null],[0.27310588842329997,0.5527608007183908,null]],"hvegP5rTGyRrsRQb3h-rog":[[0.18973351933490185,0.5573571518385426,null],[0.3523442895992609,0.5408871831932643,null]],"iPgLrrQ5XDrWWlTXzI5MLQ":[[0.8675672447869209,0.6319984085262251,null],[0.8517211072313786,0.6759682610029998,null],[0.983827260547687,0.5264937472620177,null],[0.24660503677023105,0.6069324261461398,null]],"ijlwBkjXDfD2wCO_jSjCGA":[[0.9424441880506214,0.5399081268009154,null],[0.08235659097237741,0.534220697951389,null],[0.04212474567416602,0.5269381914576102,null]],"ivSZN64YSxp5R8Kqf1vr1A":[],"ixm9RoZTI443YB7QoJ6nLQ":[[0.42135738840965437,0.5482210327266622,null],[0.580962568876456,0.5447256291267186,null],[0.39431666012321787,0.5587480352286088,null]],"j2HS3aVFBaq_x66hiHVVRg":[],"jpunVR46zc0yifTPUihF2A":[[0.3795683476724827,0.5719636058170896,null],[0.582231213006788,0.5781143267512664,null]],"jtPd8YX0gkDeQe6dq9mk4A":[[0.5651406249253135,0.5390304988253087,null],[0.40850685349922905,0.5301795776126047,null],[0.9759370812093513,0.5603183826328931,null]],"k3ipUDBk_fu7EXSVSNGaig":[[0.4226682925245716,0.5336462122134525,null],[0.5420541497102344,0.535183296109776,null],[0.6560855638591487,0.5847168986260846,null],[0.3169534700524881,0.5815798535443317,null]],"kGSkdgo9loUlSNdyjW-qZw":[],"kJpIgjL0BDI74K-jbGnbow":[[0.9398881761962142,0.5475592493969483,null]],"kpvi-7jlODMmozIBUAUD_A":[[0.6728323101307991,0.6405213274378561,null],[0.9031966821112727,0.5811890719492973,null],[0.9376373055503001,0.5709208687051923,null],[0.1793680825948885,0.596936124225478,null],[0.23428318712447754,0.5865924909139284,null]],"lswzBhE0aaemfGKiH3fIYA":[[0.5409365879306378,0.526991689949051,null],[0.6733636118922204,0.5325050949016056,null]],"mB1B8XeH3F7RpuvOP0-Klg":[],"nqA7oH-86S2CmsGA9hyVtA":[[0.5767705925062387,0.5988912761852951,null],[0.6692548235057683,0.6070550972315513,null],[0.8074795602621909,0.5414268390182724,null],[0.0991878553956646,0.5350186526903123,null],[0.9796007413318897,0.5397932290201766,null],[0.20279133258753596,0.5679563716887606,null],[0.2497956888392741,0.6119813195973982,null]],"o1gmSHwnW1ULATpgowy3ug":[[0.9482363736857738,0.5485786909676191,null]],"o5_tCsE_7cR4lTvyrqy8mg":[[0.43422716788360965,0.5705625236109101,null],[0.5668560632881041,0.5661483642572177,null],[0.7607763193576704,0.6560732340732061,null],[0.7214752256590142,0.6285741616936823,null],[0.2377834618278884,0.6478106223630151,null],[0.27255024440450376,0.6287206411482655,null]],"oqI2cRuz9LMxtKdmQugqJw":[],"q10dxfl-oC7RkRFUVKU_RA":[[0.43831676837066724,0.5239568729556883,null]],"qL5hz8oXZM0g4pKxGR3Yyw":[[0.597013217516537,0.6044264783940118,null],[0.38226689661622215,0.5482434276435645,null],[0.6476353499591512,0.6274870369983906,null],[0.3278892623981498,0.5839770869869578,null]],"rMFTOoH3naao6FlHDLVNXQ":[],"sZDQ4gAO200kPfWhz1st9g":[[0.9017565511139672,0.5843670566369061,null],[0.06763233235487504,0.5592697586326665,null]],"tekhQ4HQ9pcOqs_kGGpGaw":[[0.4076745956710831,0.5597703436172778,null],[0.5687632449560449,0.5647591113156941,null],[0.7747499235588704,0.6211148294983869,null],[0.8405798241606457,0.6448106829974578,null],[0.19765760745224314,0.5993008859112652,null],[0.2297825589560995,0.5925917178160758,null]],"vRVQrenH_9eL4n4O4Gp_oQ":[[0.5342572858760861,0.5389442988367353,null],[0.4116929253793373,0.5487527217003604,null]],"wNcM5HCEQeGv8ndYe9HMmw":[[0.6148482230696751,0.5869514534421928,null],[0.4164491312805546,0.5519765305958033,null],[0.853308183807134,0.6068841694765745,null],[0.8775813304651149,0.6018342658169251,null],[0.27400224782332405,0.5911974616453352,null]],"wVWZk8xcqrpV0l1CKpK45w":[[0.18031652407180176,0.5700054538584988,null],[0.27795517121147817,0.576080549492361,null],[0.24729801533243378,0.5809344721595169,null]],"y2Ec-NEfLgOTuhI_riqUHQ":[[0.6515441249137326,0.6051269135225302,null]],"y8Q6XDfEoPt5K6ONqi406w":[[0.5573866087550124,0.523973475272136,null],[0.7550901883005638,0.5985303092449105,null],[0.8823312528255993,0.5841550619686323,null]],"yc2j9hm9FtM1LXTmsXTcWA":[[0.47075493991834505,0.5332649021605416,null],[0.9158728311659672,0.5572988565660523,null],[0.24873896059195771,0.533717434784845,null]],"ycJKMD6I5D8glf_jMPqkDQ":[[0.3858692523821622,0.5680514672523009,null],[0.3722975606145958,0.6279561719311939,null],[0.65305175374496,0.7176760636998609,null],[0.6852428359280223,0.7351944043863715,null]],"yd34YwObnIjHhBZvTmK3UQ":[],"yeR5wU5cy_CvQFTUyBhO1Q":[],"zB7_9mtLQuVbRyZrttlMXg":[[0.06988489725377417,0.5525584371972377,null]],"zzWJ9LWjpfDZLBT7W8o8zw":[[0.6357113209953249,0.6568081449539781,null],[0.6689607983903979,0.6573300377047882,null],[0.712070085485074,0.6446516296072435,null],[0.929525008130472,0.5566953779516615,null],[0.1085040019128714,0.5607853804248653,null],[0.29417950520521946,0.6028615894904007,null]]},"model":"claude-opus-5","n_panos":125,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__richmond.json b/benchmark/model_detections/claude-opus-5-effort-low__richmond.json new file mode 100644 index 00000000..629bfbf5 --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__richmond.json @@ -0,0 +1 @@ +{"city":"richmond","detections":{"1002881108606016":[[0.43130019753598337,0.5071305522666658,null],[0.5751672920851102,0.5091348033755287,null],[0.7241774534454684,0.5796463952648959,null],[0.4091969107444706,0.5069755904220838,null]],"1021092976148975":[[0.44031883144928075,0.5307698651986896,null]],"1023400645666420":[[0.4217388126725398,0.5372828963196059,null],[0.6117410847623326,0.5526828348651471,null],[0.743321271287022,0.5774536104018939,null],[0.7671592312358573,0.5902158617195815,null],[0.2327010150508933,0.5815012901268731,null],[0.2794304084802802,0.583051780650458,null]],"1041352537628655":[[0.7628703812486093,0.6303816931570912,null],[0.8919289228627165,0.5505418277813726,null],[0.8933151495103965,0.7400377398778617,null],[0.1086064764024588,0.7329346925006345,null],[0.22942178948081088,0.5772989314838647,null]],"1044606157061812":[[0.5165803732213531,0.5888137265433749,null],[0.6472490839767451,0.5900998450050583,null],[0.7158588776847463,0.5810328792323007,null],[0.11553098260067407,0.552488768178566,null],[0.9374358647820247,0.5613781394298638,null],[0.2650603302217588,0.5325349535994442,null],[0.30195428234085864,0.5555001247841691,null],[0.37365997490973374,0.5495008045163194,null]],"1050503583329955":[[0.38303323579607024,0.6049191218171689,null],[0.6592229204235371,0.607559606447595,null],[0.7107872950518561,0.6441739406001973,null],[0.36394269667832946,0.6536533553735625,null]],"1051211967136526":[[0.40834976576964255,0.5415986535133079,null],[0.5627891465607345,0.5660678379703538,null],[0.7403265512000563,0.6594128300218813,null],[0.7665911739543094,0.6562432954415738,null],[0.88030800880992,0.6468540198986635,null],[0.2593651334903647,0.5904328472731057,null]],"1069666650973873":[[0.5419295668871273,0.5467527868933039,null]],"1071700701728910":[[0.6505475573628086,0.6182936421206242,null],[0.8520297066894602,0.5869448344565334,null],[0.11264903001385818,0.5410659038847461,null]],"1073049581231056":[[0.9483476231994422,0.5565620774724997,null],[0.8494470889912646,0.5194807748277771,null],[0.9827998352097034,0.578092975619397,null],[0.1941401223293654,0.6211529472770443,null]],"1078560817421587":[],"1085890173202735":[[0.5148643926771741,0.6400770628102469,null],[0.5909767174149851,0.5183003542003872,null],[0.6890782930421951,0.4891905921490458,null],[0.87343186319088,0.5302615334970281,null]],"1089573432664131":[[0.9591717850962338,0.5447519304545789,null],[0.09853795805579679,0.5310048582643812,null]],"1092133469495462":[[0.4205450744098744,0.5283580764503732,null],[0.5552046361774936,0.5367581824081769,null],[0.7000214638901205,0.6167747241398409,null],[0.27384149960800774,0.566276720711525,null]],"1107064934776754":[[0.4305017839753983,0.5589585091679411,null],[0.5531551372633701,0.5622062580684125,null],[0.5943489769932405,0.5819540483526042,null],[0.8595578381018233,0.5971617183694771,null],[0.906876382311922,0.6009604392328145,null],[0.10768825420417005,0.5996402780025454,null],[0.15067114029320255,0.5787253914400508,null]],"1108131584865233":[[0.8202407601897108,0.5860733946921105,null],[0.8446571051440279,0.574909162989967,null],[0.05866576731375478,0.5525880398898403,null],[0.19807444309933725,0.5453597328613027,null]],"1110314244387816":[[0.37889092482222975,0.6263116171694996,null],[0.8015018979760187,0.5301176109275849,null],[0.43069737740795644,0.5816617491082177,null]],"1119166219764581":[[0.5258275323868045,0.5161447789911785,null],[0.9517612594002398,0.5490736022160918,null],[0.09538700288914409,0.5528449426914673,null]],"1119718896242677":[[0.42625755426860834,0.564075577954924,null],[0.5469804741357958,0.5678528347928455,null],[0.7939645788727163,0.618284927152057,null],[0.8319647485139577,0.6396830242892907,null],[0.17020521937934696,0.5773652458356574,null],[0.3684781172388325,0.5661163161430763,null]],"1120239902858678":[[0.722358881510693,0.6575857113536994,null]],"1126712375884972":[[0.523817957296999,0.54361653900793,null],[0.44618001137110674,0.5371204622642133,null],[0.7397509407099406,0.6016495434405063,null],[0.8386865178844976,0.6451792018393225,null],[0.8979765870655385,0.6111039639035168,null],[0.1854474049556064,0.5608597302421585,null],[0.2207748640585655,0.5271503894725249,null]],"1130501775077894":[[0.5339339883637169,0.5502337542580671,null],[0.43218250009755055,0.5377730537333124,null],[0.5920279063313656,0.5526822258971209,null],[0.7798302358737101,0.6095224544803715,null],[0.565549713516454,0.5493069832219467,null],[0.9058701542298577,0.5697486139981878,null],[0.9647886385091683,0.5697051684336547,null],[0.17045598886484725,0.5235840151753443,null],[0.19943869892547783,0.5133710266472098,null],[0.3120654027082882,0.5325023114715995,null],[0.4025577559726725,0.5373803185114902,null]],"1148968923952346":[[0.3784345804015813,0.6587151111470977,null],[0.025608889365266785,0.6268215358169384,null]],"1154113539910333":[],"1179500609797636":[[0.5836770646554532,0.5924160835858532,null],[0.6202139365102616,0.5868108042426969,null],[0.8684781172388325,0.5661163161430763,null],[0.9809918907867143,0.5496702549543012,null],[0.10564373944735228,0.5458383197451367,null],[0.2862504210637177,0.5590165332761321,null]],"1180673306331763":[[0.5966292951559468,0.5727123954777488,null],[0.6628621707928104,0.5951301660485782,null],[0.2947635874165602,0.6029729840204499,null],[0.30082872995198906,0.5398118486985846,null]],"1185319949364085":[[0.5427902439329744,0.5453840993930616,null],[0.3837519410833357,0.5402694818160375,null]],"1187483045673326":[[0.394289485978964,0.5183895921656346,null],[0.8634114977462497,0.6277712284150888,null],[0.31498419021531454,0.5380219728662371,null]],"1193579455857033":[[0.5134161305693262,0.5283160850010319,null],[0.7211861357236332,0.6093122794439926,null],[0.7636561020962881,0.6699278651826965,null]],"1237376427883773":[[0.5403021668506467,0.5615304788331397,null],[0.6188981922606779,0.6648896298446751,null],[0.4230280050640453,0.5489753229411036,null],[0.6659401939102245,0.6731921225560198,null],[0.6897406140528283,0.7248568062071044,null],[0.3077552500069972,0.6186572292251111,null]],"1260990361757674":[[0.5667461249578515,0.5347441690340963,null],[0.9147944952818889,0.5580090078080954,null]],"1263975528308076":[[0.5740626857278421,0.5361280081164851,null],[0.6142019803999702,0.5462003999041837,null],[0.7845348317513547,0.5977920213387842,null],[0.9360223764301447,0.5491071829057255,null],[0.06842426718188466,0.5398908860334343,null],[0.09829477468742426,0.5362661307837099,null],[0.34340187488009966,0.5301516930604284,null],[0.4010942783129356,0.5247271729786264,null]],"1273933840289887":[[0.6094001551665691,0.541552915544146,null],[0.47748163614338196,0.5251081738269471,null],[0.41258014018951816,0.5390579378607977,null],[0.7658794804472911,0.582016496801413,null],[0.7424756647378272,0.57418115369018,null],[0.8945670363812515,0.5522594644207299,null],[0.0996629333780964,0.5586454261171934,null],[0.22083463788453156,0.6216622112206243,null],[0.1802351974560113,0.5933591980166077,null],[0.2853126656923791,0.5908800041756633,null]],"1277351374404839":[],"1279326226723010":[[0.0491828514144782,0.5410211905754903,null]],"1279553583800599":[[0.3968942943267417,0.5376178846715796,null],[0.8746641303370151,0.5723851710002714,null],[0.07905935703324396,0.5614882270427939,null],[0.38548345556473884,0.6092721986159556,null]],"1288163659019705":[[0.4523222001938136,0.5388558084905278,null],[0.5525845985132581,0.5496757939604447,null],[0.676526185684409,0.6564682620467024,null],[0.7776848188489642,0.6554942140305431,null],[0.26971050236666094,0.6450417488513505,null],[0.3056097221130307,0.629904827167739,null]],"1294684158910702":[[0.8419547557963625,0.5009810357046353,null],[0.12819271018673717,0.6226000964947023,null]],"1294705618516185":[[0.5811141272131476,0.5316107591365706,null],[0.6873638658588479,0.5911796065342375,null]],"1297225628792280":[[0.2663907802721084,0.49519587348775146,null]],"1309093164014619":[[0.5396162941034617,0.5471905703242472,null],[0.6091306962879789,0.5549547916968934,null],[0.8701547015573368,0.5939198767717204,null],[0.9237045529394754,0.5991460042108288,null],[0.8996313185286093,0.6032336181440119,null],[0.10736049639109702,0.54336224980137,null],[0.3370345727792956,0.5317494839261987,null]],"1315788070079167":[[0.5610538372437439,0.52375964717883,null],[0.3772779625831603,0.4730711437112353,null]],"1315844333300588":[[0.19451600313363837,0.5822256639693653,null]],"1327023138707493":[[0.4137985098461076,0.5305843894883296,null],[0.5741484142558725,0.5349354056652444,null],[0.6612136711471571,0.568238083422981,null],[0.09824649549438874,0.5731914935318586,null],[0.38909967009595786,0.5758442778069272,null]],"1335429861397399":[[0.3811443381519632,0.6286014089606086,null],[0.7435427057695655,0.5732189526621608,null],[0.031275549388156076,0.5700598398071557,null],[0.9390687282113286,0.5582352019110998,null],[0.24451309938366156,0.6413472593560221,null]],"1338631107629110":[[0.5573723955680189,0.5263593156926732,null],[0.947037702685601,0.5656984796840671,null],[0.40254468911539504,0.5321216798348368,null]],"1403202260886086":[],"1408179757063785":[[0.3774344145419562,0.615252787369391,null]],"1427673368387929":[[0.45228745426809647,0.5232660520088317,null],[0.4837804347776386,0.5232179503468393,null],[0.6242965318905045,0.5321457059863087,null],[0.7206762264796851,0.5905845309908475,null],[0.8864650745299403,0.610219084931893,null],[0.33749708730647543,0.5787947402463451,null]],"1441274599876577":[[0.9403480163896398,0.5299987538936378,null],[0.06770403461959595,0.5346510516610324,null]],"1473003156922360":[[0.05578653393677968,0.5202710663643015,null],[0.9354806756928877,0.5311064984196823,null]],"1483585545954157":[[0.511520018498251,0.5392250218966479,null],[0.5420752305417149,0.5299140376197544,null],[0.7208208399476138,0.5733782476189448,null],[0.12569562470561113,0.6003927595763537,null],[0.2660304328345351,0.6696546204265587,null]],"1589004208714002":[[0.916456222281163,0.5564445308809208,null],[0.08787153129717973,0.564754926576575,null],[0.07141254587948503,0.5340802301577657,null]],"1599497457585207":[[0.6006465836443174,0.5551377880674087,null],[0.765188201839996,0.6138207157490382,null],[0.9504511770185367,0.5583971657989428,null],[0.16139091022860425,0.5825305864929167,null]],"1701172050581812":[[0.6559775960681901,0.7329524422304186,null],[0.6928356421147693,0.7526398368833012,null],[0.7253708667392548,0.7384630630145729,null],[0.9755427036839578,0.5627758536317697,null],[0.10189329047639756,0.5571317079427794,null],[0.23430356238631928,0.5615888385942873,null]],"1739689546844035":[],"1759178911495771":[[0.5624805806728971,0.5746529676451997,null],[0.3854545867986385,0.5600234396614252,null]],"1809151763043139":[[0.5551112175988964,0.5540600776187008,null],[0.07470843227148521,0.5781877998239051,null],[0.2819816209886061,0.6210623827635835,null]],"1828524234403425":[[0.8742521744973961,0.6578229943646208,null],[0.2146960368391747,0.6048828531881306,null]],"1847752429062443":[[0.5606951991462792,0.5444669904036116,null],[0.38911125111437905,0.5298180182812935,null],[0.6175221964288967,0.5782175640456826,null],[0.6492258636186606,0.5656405264192431,null],[0.8376849491553021,0.5834137657764362,null],[0.9069573660447161,0.5529703220148932,null],[0.11598250640155089,0.5392981687576135,null],[0.15346813595326736,0.5436876368213628,null],[0.17898616010856594,0.5363293211730693,null],[0.31894741639556,0.5391680303631411,null]],"1905003193597285":[[0.5402994781836727,0.5235745422148423,null],[0.6822940922756517,0.5821936888026522,null],[0.7943008730289298,0.5726460717927216,null],[0.8868921272586803,0.5621527010019662,null],[0.35682552586201965,0.5497538695153783,null]],"1976970169406126":[[0.880094633656419,0.5888269071883163,null],[0.3771884282701953,0.5219764083108989,null]],"1991198475040883":[[0.5763917620332896,0.5615178773621284,null],[0.38397875731367126,0.5532108345904465,null],[0.27925902532729296,0.5205859168219977,null]],"1991876994661275":[[0.6120325239733282,0.5306476456690402,null],[0.8902801486918601,0.5825025832313252,null],[0.9786300912855317,0.5509231582311057,null],[0.045453494510912074,0.5258009299987636,null],[0.22885924337578828,0.5666965161559744,null],[0.3485080648263209,0.5947487005371067,null]],"2026086521187684":[[0.8914787459782089,0.5776161833139669,null],[0.08163313155655044,0.5800972174218999,null]],"2067733823644753":[[0.783744970975934,0.6038658792352484,null],[0.9405515216737805,0.5516666198984745,null],[0.10056532275298286,0.5635819491204968,null],[0.2418995595801154,0.600456076680055,null]],"2111278956050786":[[0.3813217682670448,0.5674954645245233,null],[0.6844478436857311,0.5978373714177002,null],[0.7277945766148539,0.548705594577364,null],[0.31341202459437945,0.7021655743048849,null]],"2141681583023863":[[0.4933342172451437,0.5551529382059626,null],[0.7937197891130334,0.6123177077827645,null]],"2147631415757748":[[0.5670191333003288,0.6296967147576463,null],[0.9488746971475984,0.56893396946484,null],[0.11260501906431886,0.5383482851599684,null],[0.31379971094854675,0.5480398028353669,null]],"2184370012043145":[[0.9263860277153544,0.5750379816359412,null],[0.9736006497264174,0.5668109464774143,null],[0.28763998842361893,0.5767159152955784,null]],"2263954690670646":[[0.39822767903766376,0.538288209388477,null],[0.9379313644290083,0.6010746931863651,null],[0.9432928036911782,0.5569598814318633,null],[0.06399220688673174,0.5318695072891895,null],[0.17113933582324592,0.5395752770713238,null],[0.37232934400214024,0.5379077998114624,null]],"24127338963626294":[],"2456904447834971":[],"27724402107145083":[[0.4130845811312524,0.5423744624650766,null],[0.5679603006747864,0.5465855221193223,null],[0.5919681053020045,0.5716140879878788,null],[0.3802254166383924,0.5703220590943587,null]],"372149922554566":[[0.909339725899805,0.5623543567140529,null],[0.03285076849440277,0.5288197183469338,null],[0.06628737465518397,0.5438214481358424,null]],"3808624879384438":[[0.38744656647065956,0.5371189033199513,null],[0.685263369521586,0.5837657603234071,null],[0.8299138805770621,0.5834251574324759,null]],"382838731271782":[[0.38567974851632053,0.538209248950094,null]],"383880874806933":[[0.8383062590336723,0.6022344291203404,null],[0.9012422515834914,0.55235898909928,null],[0.10688918873112419,0.531887054574236,null],[0.18521466274718495,0.6072909441230966,null],[0.23305639118804633,0.6111800137852919,null],[0.23305306660368486,0.5381126014017587,null]],"386363531219281":[[0.43940453056074064,0.5627314548728883,null],[0.45957387621018864,0.521617952374239,null],[0.6574043469483051,0.5510209348816385,null],[0.7383183276440732,0.5312649568436421,null]],"3901547360084831":[[0.5311198497434807,0.5321720367003511,null],[0.21617633183834567,0.5417570783267847,null],[0.30416937811314015,0.5542941860759364,null]],"3956582084664486":[[0.43993744028784504,0.5457953802724196,null],[0.5549953617321253,0.5499310335465869,null]],"3980741912187626":[[0.5856229277602947,0.5875719939321457,null],[0.632508618849977,0.6196517231305121,null],[0.8825903492063034,0.590054812923746,null],[0.369591701251942,0.570158517552392,null]],"4002322973363581":[],"4042244719362475":[],"4102292636718018":[[0.6297297934229207,0.6890771784897424,null],[0.1535351215701508,0.6239502633516555,null],[0.3199879610556442,0.5566535238963114,null]],"4116269558691031":[[0.3986118383407106,0.6549095497968691,null],[0.49285768080199654,0.6093704919801253,null],[0.6512491148308632,0.5457641234829467,null],[0.8448105587224832,0.5092678510573951,null]],"423989957171190":[[0.5354704374855125,0.5570881743575533,null],[0.3834573789525719,0.5418558003482661,null]],"424364943656028":[[0.6225544747733568,0.5931396258679537,null],[0.894576213167513,0.5811304892455317,null],[0.13400974971347096,0.5527227300526176,null],[0.32914526430982893,0.5606993311166925,null]],"432356313302579":[[0.6196798375168322,0.5512780903554311,null],[0.2737178656577469,0.6393419998661488,null],[0.27710454893734426,0.5327328295431535,null]],"433987753066526":[[0.5584660826675404,0.5374677243852198,null],[0.5318090364148029,0.5372610929548909,null],[0.6558298414806448,0.6164760271861393,null],[0.7711405954484076,0.7311547730979739,null],[0.9120034466340325,0.6485162298732245,null],[0.8744646075103375,0.6849531024952411,null],[0.23534136640244951,0.5707950415388551,null],[0.2092280909753499,0.5853215800087057,null]],"461898606342872":[[0.5607547894654068,0.5709356818107434,null],[0.40924692375073457,0.5356166844454562,null],[0.6380965368042764,0.6371037799542846,null],[0.7303024092601,0.5690971679036338,null],[0.9116513399918534,0.567988450992792,null],[0.7048042024024437,0.6666263410057263,null],[0.965238933303635,0.5547193703244412,null],[0.13832868054570702,0.5374154040331441,null],[0.27726532301509776,0.5296597585856893,null],[0.3070681309974897,0.5411473869354363,null]],"481381621684815":[],"514363767744061":[[0.4147517405619503,0.5187805317115478,null],[0.5465870493311157,0.5221079423817752,null],[0.6706398942010553,0.5510903273578067,null],[0.901210032750965,0.5569244691401355,null],[0.11572710184172441,0.5416299452682796,null],[0.3004095946259474,0.5206305575364452,null]],"522346373509256":[[0.38365273932632876,0.5454500843321766,null],[0.6029428087869848,0.5591669558860807,null],[0.6502343072474981,0.572436046222355,null],[0.8916077162861806,0.5472621106411737,null],[0.9441570042219446,0.539442804630335,null],[0.10354800434151251,0.5334696523930172,null],[0.33156552525124205,0.5516464966988458,null]],"552315380916113":[[0.5437410213985071,0.5641567162096617,null],[0.40561984819390723,0.5456894273320849,null],[0.9781912994852804,0.5394849783277962,null]],"559372759847379":[[0.5892943200814523,0.5452351821421136,null],[0.20816945024092426,0.5790573420532412,null]],"562637219504223":[[0.4425816150059755,0.5671997080358389,null],[0.6130453969859212,0.5995387474151113,null],[0.568561983845722,0.5614145571819548,null],[0.9340002753213823,0.5726438176395205,null]],"563961506051153":[[0.53731248341699,0.5266488963196373,null]],"591584463734236":[[0.9330069117251851,0.5567737581860898,null],[0.09649769355664334,0.5505553627183436,null]],"593421160518191":[[0.4443741619450735,0.6177894159392838,null],[0.577424307636118,0.560054283308592,null],[0.412357941605618,0.5591586433967185,null],[0.6341812601358106,0.5369715982478734,null],[0.8466832115372724,0.5195121262597706,null],[0.9566259779679115,0.5440434637214048,null],[0.07500526162572418,0.5456154868298471,null],[0.2307154909997972,0.6161390722679924,null]],"612758008575849":[],"639645829096010":[[0.3801277055994919,0.5767411313941131,null],[0.4597647488132738,0.5170363070704401,null],[0.7217255187617787,0.606896707412549,null],[0.752800226953219,0.6036526489232084,null],[0.054974569060940504,0.5468550181771069,null],[0.10126359498706239,0.534044125641017,null],[0.23195806057681,0.6129186676169046,null]],"662385816800591":[[0.5542857242395655,0.5479513446337597,null],[0.27135485267814435,0.5646192672846213,null]],"667461712321088":[[0.588813391348284,0.5146614453109848,null],[0.8979371156205695,0.4991888012921547,null],[0.34520998908052924,0.5720473711002689,null]],"681333794968062":[[0.61022140258139,0.5405287624548083,null],[0.9781819582231877,0.4718649863557565,null]],"700668799338315":[[0.5617922909330527,0.5628081297348916,null],[0.39380138227177597,0.5447432255254311,null]],"723487737079243":[[0.8765091310365862,0.6433363725851821,null],[0.9900912631338237,0.5458888674620513,null],[0.21078694421393135,0.565472941139973,null],[0.2798705083563974,0.5629264039264666,null],[0.23263729403328537,0.5448953841592877,null]],"724492966635485":[[0.3808221406880214,0.5711211471606584,null],[0.351334823961295,0.5743457092216688,null]],"761421883101179":[],"775828278323576":[[0.49036943557480706,0.6738667527905176,null],[0.5977601172126917,0.6868830909899265,null],[0.8730090877514252,0.7469196266017939,null],[0.048480827375007585,0.680477348101397,null],[0.4390491185967804,0.6913269283615022,null]],"780391337668333":[[0.8978631602198295,0.5425421062934124,null],[0.08681164252301521,0.5804488572615917,null]],"785854814030175":[[0.45358156737000876,0.4984874645389982,null]],"822353906794602":[[0.589590787420546,0.6299726471210784,null],[0.7351496660379653,0.7239289755819116,null],[0.12280657488954133,0.6501586384498279,null],[0.19052713042573483,0.7822389961496231,null]],"839655921736413":[[0.7245290650212437,0.5690043995297178,null]],"843075271274817":[[0.5198632135486103,0.5164770503621483,null],[0.4470661838190036,0.5149984712880157,null],[0.9576216270537877,0.6009856879284851,null],[0.3298529042087863,0.5424945576826439,null]],"8726984263984099":[[0.5787826974875525,0.544818221458168,null],[0.7558593177437867,0.5527979898565882,null],[0.8731847735147042,0.5548448420820862,null],[0.11459072342988674,0.5083434341163351,null],[0.21209057817534838,0.5204686315741733,null],[0.37255800528707006,0.5175443901122055,null]],"872856994942214":[[0.6263534198550419,0.5948085595035509,null],[0.6551185250844651,0.5946252639632768,null],[0.6992450517022958,0.5778047718448277,null],[0.8493263341580668,0.5673816440620213,null],[0.25953373620066134,0.5776033029033748,null],[0.29363848593307595,0.569526571659289,null]],"888167256831303":[[0.9445891346793927,0.5501421180500737,null]],"893023542323690":[[0.6114325398842845,0.595862561679153,null],[0.5442225148128047,0.5481319353790041,null],[0.2201311818007315,0.5957822365666462,null],[0.2935464734632179,0.6119450706143379,null],[0.3204067395758747,0.6073399882636802,null]],"896345989301021":[],"911337534530260":[[0.39366794043148634,0.5426291787537403,null],[0.6179505463726522,0.5897906930117482,null],[0.9119774395305492,0.5743576294505984,null],[0.8045870029335329,0.6173858873838924,null],[0.197029467411314,0.5256500825550442,null],[0.27586042494933183,0.529670865388402,null],[0.3526452176257527,0.5328104595135216,null]],"925077602773417":[[0.37790409019435034,0.6042075653420645,null],[0.7160910951713768,0.5840863551094674,null],[0.7519023125804237,0.5778101394257993,null],[0.06893934340448182,0.5328597463938177,null],[0.2554861882566547,0.5697350804581114,null],[0.34455136929034996,0.6107242757330368,null]],"934739365184374":[[0.43095913456110324,0.5832656264362812,null],[0.5906601717996925,0.5961309833920128,null],[0.5610814970038562,0.5767856013108797,null],[0.6685777528155165,0.5978129718606928,null],[0.8753561910111334,0.599254006723372,null],[0.9143503266514243,0.5972917866734966,null],[0.09583379332551734,0.5849680865873261,null],[0.122365483061873,0.5887631786653817,null]],"996959015535106":[[0.6011781247473342,0.549066857702724,null],[0.5371362193773964,0.509744210982859,null],[0.8210102079285739,0.624910330663749,null],[0.3718336270319708,0.5622463680888793,null]]},"model":"claude-opus-5","n_panos":124,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/benchmark/model_detections/claude-opus-5-effort-low__sao_paulo.json b/benchmark/model_detections/claude-opus-5-effort-low__sao_paulo.json new file mode 100644 index 00000000..544c3c7c --- /dev/null +++ b/benchmark/model_detections/claude-opus-5-effort-low__sao_paulo.json @@ -0,0 +1 @@ +{"city":"sao_paulo","detections":{"--g3UwVswwny7hOlxl77-w":[[0.16270434071158146,0.6191183560805389,null]],"-pcHlrGDC4hPPRHVxU3W3Q":[[0.7436919193846632,0.6483491320894049,null],[0.7700921398780474,0.6250554365406475,null],[0.706164968257885,0.6554028958457281,null],[0.935835688978548,0.5508529068272637,null],[0.0661470960840817,0.5493247997148678,null],[0.15541843926634635,0.5400197854759394,null],[0.2789024640281855,0.6062221998429963,null]],"-uqS0QNM8aUFtP5gwTMu4w":[[0.6211005011549587,0.6166904178741186,null],[0.7192319937117855,0.6294404686128422,null],[0.04268131198368735,0.5495713617609455,null],[0.195617018354097,0.6446984829569384,null],[0.16965644049069484,0.5907236001774091,null],[0.30409383453011885,0.6306162232827814,null]],"0aJabNkZdyIHXbiruS_yLw":[[0.4706370106637743,0.5274340142874282,null],[0.08311719287447156,0.5676489107207477,null]],"0rG2QcmPAKBBec7NcS_SUw":[],"1bF6KihYF1Rw8PIg_ZTh6g":[[0.37468498864233196,0.6018146612920819,null],[0.6320194416200551,0.6089498124467334,null],[0.46459786991702284,0.5373394067050378,null],[0.6976558917220836,0.6668208724194158,null],[0.7646016396931823,0.6759462332629497,null]],"23nSVF67ExPhVWfty9-4Qw":[[0.4305835748960168,0.5538897355298896,null],[0.5519791940955948,0.5581099118561736,null]],"2L64Zqq-r58lo_7JVUoJSA":[[0.3834068207148217,0.576292383466507,null],[0.6136779876603621,0.5718950208784873,null],[0.7827731504144009,0.5830511938754951,null],[0.8745603116743937,0.5921582950211587,null],[0.11067655643626123,0.596968761899599,null],[0.16123746406637485,0.5995467151749531,null],[0.23602433203675088,0.5510709679715019,null]],"2XdWe9r6z5m-9SmNQq7TGQ":[[0.4600732145917579,0.5476917247343506,null],[0.5975631038227823,0.5620347892197125,null],[0.7845307283739145,0.6559194410850386,null],[0.16860252716758278,0.655781397736441,null]],"2vVoZEbjn1wxGCCvCmgeUQ":[[0.37126022023785277,0.5957564410506399,null],[0.6265590199506468,0.6313355089548328,null],[0.6070053859139118,0.5916859974061721,null],[0.33128382354219554,0.6387518147892038,null]],"31AcIQn0F7vJCIRP7U4gyA":[[0.8250865838120149,0.630604745310269,null],[0.862508137685007,0.6098577450666377,null],[0.8951357583464754,0.6019766856555998,null],[0.12638246471440046,0.604280600040007,null],[0.1497851924575515,0.6107035449276251,null]],"3SnxAqXSf1WzcWlp76h_2Q":[[0.5889031962649204,0.5309368949835874,null],[0.7685264134900581,0.5651007757061103,null],[0.2540493079994014,0.5870176130993301,null],[0.2891180821726709,0.6044146716245986,null],[0.2320765856724663,0.6114703067497608,null]],"3lHC4L5HK6yvXRZfPYnUqg":[[0.5452635884044271,0.5564693442760896,null],[0.4018578719697381,0.55435585669625,null],[0.6001499968569716,0.5654241305120956,null],[0.9490294721804269,0.5558541388479256,null],[0.07884101426729434,0.5479465997061371,null],[0.3096948339904266,0.5988681618598917,null]],"4RXNxzYjFwN0EBvKh8aHSQ":[[0.6220039174106604,0.611385570824842,null],[0.6748667034074394,0.6392167132487341,null],[0.7574318413248671,0.5911113260039628,null],[0.9031956096808282,0.5459036304938847,null],[0.17061507286154748,0.5696685801975117,null],[0.32789192035535764,0.5836869873394956,null]],"4Z4bj21zN62_fBZJzoF6Xg":[[0.726429943424181,0.6173149174277945,null]],"4n8b3OMdbJZgeJXz_TxNBA":[[0.735253625962691,0.6688861984591264,null],[0.22740437072511555,0.6010106315087207,null]],"52VthEdI90oCcFRLa9Wy5g":[[0.11016646741679842,0.6294119731229189,null]],"53l0gQcJPK4NrZhJZhtKlg":[[0.946621440386248,0.5678029754608136,null],[0.03668159063019949,0.5337021877905487,null]],"5IK4jyU16o627F2yDHu6Gw":[[0.5867107446380934,0.5341323995479901,null],[0.05540040464419088,0.5430014689927273,null],[0.39099270402849284,0.5422760876590335,null]],"5b-MNFZDk9yhziWPr2u7Hg":[],"6a_UTgidEGA0mlGKPlurNw":[[0.37901638811864014,0.6565054598293424,null],[0.7911513351546737,0.632072388572944,null],[0.08230959159559792,0.5608116009950247,null],[0.9096193819440633,0.5402064856047682,null],[0.26509260276939706,0.6117359304764955,null]],"6fpdvTPOA_R4Jb9bCBCwug":[],"74L65cpBc6Sv_D15h1n4Rw":[[0.6835567216765434,0.6110032415383314,null],[0.9491117769635168,0.5607302488966865,null]],"7O-o_5mVjT3KdC0SV0shUw":[[0.7130461858803598,0.6841840870776614,null]],"7RmzjvAikTNecf6xoiPw4Q":[],"7UTz_5Xfnx4Hz_0gfN2WZA":[[0.5808417855285781,0.5863672784216929,null],[0.651102484711562,0.585668380378761,null],[0.9421065200974371,0.5837884906966408,null],[0.16880464621827773,0.7086823486872061,null],[0.20551588614546928,0.7209730925048641,null]],"7um0A6G_bSJ3Hn6nRbixvA":[[0.7787300812425678,0.7077535553587978,null],[0.742760277984427,0.7232993851133109,null],[0.8567703382387546,0.7172349626449837,null],[0.9457244847109015,0.5747231252227265,null],[0.22566817939392986,0.5807421209241802,null],[0.2704812773409939,0.5801062051209819,null]],"8PunOGXWhLW2Jcp-z1kSNg":[[0.8832856038088863,0.5863296751071985,null]],"8qkdHYilRAI7Vrdpg2XuDw":[],"9MaI1wktdWe6O8uAN_-zYg":[[0.8957544748139805,0.5806229012785218,null]],"9e52qBoSx6RlPRP2VHBTqA":[[0.41817154788629685,0.5751153405731881,null],[0.7041788491977579,0.5739261851579819,null],[0.33260478214276046,0.6747455329310179,null]],"9ogLSXwqkmRsXUlTLya0uQ":[[0.3938804424934435,0.5402923931728685,null],[0.5610511236336594,0.5479546872460733,null],[0.5173850438580431,0.5390949240201623,null],[0.4375262255416881,0.5361187614479999,null],[0.7171570689492689,0.6198773537521121,null],[0.7834874055922849,0.6393949569224059,null],[0.08953083356480374,0.606955757033452,null],[0.21499402299523912,0.6141659401422376,null],[0.25733962268170624,0.5872448455282778,null]],"AJMyOxILJ9hobRi9AdqnCw":[],"AJVnpStT1x3aP5rNLkUUyA":[[0.2267810077607495,0.6203763558926244,null]],"AyVE6TqZxahbZyzOG8KhPg":[[0.9390548314763969,0.5660395077356918,null],[0.12020482576551128,0.5934298898311449,null]],"BK29yXSNmXidi-3ggxm_Zg":[[0.4267729780903334,0.5906277342991969,null],[0.625561727242287,0.6295372239459428,null],[0.9177812782895718,0.596170419260256,null],[0.08073890140100676,0.5623939671878899,null],[0.23845025732300806,0.5975966800216891,null],[0.3276505099533269,0.626398856353691,null]],"Cuj3qJDAZnTKnMhQLXy1ig":[[0.6283094134269908,0.6207815027959875,null],[0.6588949451433629,0.645409543823691,null],[0.9031056706365306,0.6770109187447337,null],[0.28913221609924544,0.5936613399879291,null],[0.3534074120783615,0.6482553243685552,null]],"DSdxd4wO-H8l4g-O_IGfkg":[[0.5707842312051616,0.5656792436635704,null]],"E2-oeApmq0sW_GDdLFbnCQ":[[0.5637334197481492,0.5725413873824611,null],[0.7159545219451032,0.6183746993852868,null],[0.762316974907775,0.6531356722261656,null],[0.8713409592788635,0.6373417060318466,null],[0.2772709144386916,0.6160596216206007,null],[0.3038717093512,0.5904272800388661,null]],"FRKJomHWa7nYfbmGJRNuvA":[[0.5319496744665791,0.53984006088116,null]],"GnxqQAq5GVonmVKfcMcr6w":[[0.9407023907917965,0.5698429723208592,null],[0.07112947908229234,0.5526177803500723,null]],"H2cEJygbNd56-362PsICjA":[[0.5983761003368171,0.5909080256990777,null],[0.3685335328346123,0.576975525762224,null]],"I0LGgFwBsGIYNbIsIaLgpQ":[[0.6106358521374993,0.5423531339430864,null],[0.7073962232125026,0.5870784944688326,null],[0.7275383698372614,0.6160441033354322,null],[0.3487132162927271,0.6530071969021262,null]],"IAF46f7Lfz65Y2vjckHeMA":[[0.7433911701089346,0.6263049418608003,null],[0.7114352387706835,0.7541948705307235,null],[0.09069960446493447,0.6097257940541089,null],[0.14420487314854225,0.6135576422396134,null],[0.2336279130323316,0.5555362657831044,null],[0.2275383698372614,0.6160441033354322,null]],"JAwR7rGXLysrpeYgTzkLCQ":[[0.8914342562540247,0.6463645219986638,null],[0.13203802916473817,0.6787478822975934,null]],"JF3A8_EnhGARQxmfqdpANA":[[0.7079862648003581,0.578519433276036,null],[0.7392248274021533,0.5797300509857268,null],[0.2695063371226988,0.6117602852210359,null],[0.29209563307466446,0.6143349038538979,null]],"KD-uKFrh3oLPfzbwOStEWA":[[0.7779407328722472,0.7307438278229057,null],[0.8847682347207684,0.6663916959841152,null],[0.1202797600612307,0.6030603848965199,null],[0.15117009694096067,0.5950255852225647,null]],"LUW-LxIfrK_j7kCq16WoyQ":[[0.7512149741277115,0.6270381845430093,null],[0.8903260666813069,0.6152768418470471,null],[0.9517348432629095,0.5612136574413695,null],[0.23488313670051297,0.5783339786802916,null]],"LfPVRvshATNPaK78qOt-Ug":[[0.7014105741111238,0.5691943234307991,null]],"M5mjc-BAtPFJppa9npq4xA":[[0.6146178563353966,0.6279808284072438,null],[0.6652361847011422,0.5966307644394975,null],[0.7111607761460658,0.6423628539215525,null],[0.24549851142921414,0.5914897513585136,null],[0.26838491517121454,0.5771845650360888,null]],"MBuniYPiSz9apm9agSpqPw":[[0.6223398395621547,0.5720636861613748,null]],"MO1IiuyyjHfY4uBuEsgoeg":[[0.7413899668524395,0.6443540938351716,null],[0.7100537165719949,0.6650732278058323,null],[0.9202877958123465,0.5639859265681015,null],[0.9578981402884933,0.5629760134226552,null],[0.3025753420867363,0.6007958862361064,null]],"MTKGGsTjIZqzb5CJ4L_zKw":[[0.4500884043971539,0.5677440432781925,null],[0.5461722777189004,0.5299441836452198,null]],"MjzhJxGXvpsFuKgE1Mh-Xg":[[0.4488917971336113,0.5373355085384415,null],[0.5382002707247565,0.5361655164320663,null]],"N2dp00E-bFRP97GdlN6YgA":[[0.5967762436094826,0.5736515256456168,null],[0.27455759014552017,0.5995448215199758,null]],"NiIR-hk3d6gP7qp4s1BM4w":[[0.5695019542752627,0.5397674143671197,null],[0.825090552207947,0.6900493698289896,null],[0.9555628709839998,0.5528387710917702,null],[0.3162185006430902,0.6528615069751875,null]],"Nn8w6mmD-cbG_XScCiIjwA":[],"NoadtY-SJzlUAlm7E74VLw":[[0.5702902283565774,0.5465214960547907,null],[0.1574160478842746,0.5692930121425,null]],"OwHITjkTQwapTUEqLaf_HA":[[0.6660159361046707,0.6103746066684154,null],[0.7296429151728199,0.5858236894645001,null],[0.897415406457857,0.5817889108439629,null],[0.06294600806583689,0.5663053962397613,null],[0.2461194496741489,0.5854347713851609,null],[0.2838147603986362,0.5922560717453759,null]],"PLxw5KV6VDCzoMS7hmL8yw":[[0.5373820114412765,0.555856540915866,null]],"Pj3oiYfbY1b71tzLockamw":[],"Pvs7VOQJOX9mw2Ru9PrQZw":[[0.957985529509171,0.5498900448056472,null],[0.03667323179136883,0.5357374904383007,null]],"PyBg7R0KLocC6zqVFijKSg":[[0.063782588373547,0.5333313355510156,null]],"RFpFHgDV38a8Q-_kF4VJMg":[[0.6382488567036668,0.5627964725092057,null]],"S2IuBk3pE9q76j1Z4hQC8A":[[0.41698253463575047,0.5669153318962297,null],[0.52322381591208,0.5394355173300548,null]],"S4WdqpLlooYt4ODkL03k5g":[[0.5617980631164506,0.577212436479547,null],[0.6115783498077335,0.5771487492808782,null],[0.9468786439061708,0.5858590319476878,null],[0.2862012624230179,0.7091113008748362,null],[0.3656374516900675,0.5727645448094494,null]],"SL1kkNP2VXfJkTD7YQf4tQ":[[0.3933986559516969,0.5790669368198805,null],[0.5919952403061157,0.5985363916751772,null],[0.4114865771223683,0.5517198247289364,null]],"Sg0GiqhXxeFwuhRVQZGWfw":[[0.4469710962415272,0.5187419788984787,null],[0.2857778915127185,0.6166832550252178,null],[0.29709862862409187,0.5419391867969322,null]],"SkpVKN34vuC9ItNhaZfFwQ":[],"TRSZB4htf-pfJEgn8NdnFA":[],"VEIsNEdoZ793WdrHU57t2g":[[0.558335406788967,0.564131506396625,null],[0.4447953638225064,0.5367581824081769,null]],"WC5s83YNe-hRPVOd0KZQrA":[[0.6191482666917468,0.6174184458742796,null],[0.8573067418257169,0.5802474790826642,null],[0.8958881402503285,0.5380685656250611,null],[0.9589968675632543,0.5317176271770261,null],[0.14933911674504524,0.577228747526677,null]],"WShTUyMV7KcYdpINkKU6zw":[[0.3995126511980671,0.6041546643046863,null]],"XnnGXt94GGVxBt_3oSk2Uw":[[0.03335612714502778,0.5377013760544012,null]],"ZdSe6D-Wu285r762n01uOg":[[0.6460398293884373,0.5759055942515153,null],[0.8703810643749315,0.6049732346848871,null],[0.025704304732515282,0.5380399832981807,null]],"Zx5RLbncodJhmxWdJwAaEA":[[0.9176737676083508,0.5845550848499046,null],[0.2070954164752966,0.5489663424957552,null]],"_E3aWogWLo5SoGkPdihyAQ":[[0.5851790716551022,0.5696700561157287,null],[0.36221453323612796,0.5942657170697057,null]],"_fPo4iIbhE1HLT_ixD57_Q":[[0.38182752492546385,0.6167573780538457,null],[0.7305523223718704,0.6078845734112369,null],[0.8074564211513866,0.63287563136397,null],[0.8900247663454585,0.6138192854642195,null]],"_zoPpzbkTA4aS166T26J_Q":[[0.5517061558593847,0.5534102181430155,null],[0.6378653928269089,0.5616698927458277,null],[0.06250001566706881,0.5435158912290309,null]],"a70PDmeTUAplnAAszg96rQ":[[0.600071811910056,0.5441266225852017,null],[0.23096719663569626,0.5791561730526981,null],[0.2661075351498151,0.5801619262758326,null]],"bNHPwV-rN_X3TENKUQgmNg":[[0.38269017318946325,0.5630921321207916,null],[0.14331982881154043,0.5645027710524638,null]],"bhyqT91fwl6p064hY4GxSA":[],"dZ3b0KCyXPcdVPHv5lCR2Q":[[0.5378926253416951,0.5462661678749301,null],[0.7174892438208821,0.6113024547109447,null],[0.15870626946758914,0.5862485817952248,null]],"dZHjPLWAOhyUatavJJEPzA":[[0.9780635645672421,0.5428974395514784,null],[0.059551775803464546,0.5615742133040115,null]],"e6i7jzmjStkNCJ7jp_KV5Q":[[0.6234074384042353,0.6184466101750731,null],[0.9506179538414872,0.5680860284936721,null],[0.04418272933537948,0.5531267445634092,null]],"eHzL2wfpN6POl85bAin9VQ":[],"el13lYhzqKSZJwWb5w5vUg":[[0.3951068945090054,0.5846097375121027,null],[0.617426063523378,0.6051601374677387,null],[0.6694228876185634,0.6436849834991296,null],[0.8221438349214589,0.6526751406644777,null],[0.10806831564438019,0.5821310866997296,null],[0.9134213690073261,0.6005290542555706,null],[0.3224494181451902,0.601462727606914,null]],"fto2w3ZBO7XYUyIPzctfxw":[[0.6121017558781188,0.5734211216838278,null],[0.212174350831067,0.6295836796740354,null],[0.28494696910075884,0.6096494898867633,null]],"fwvaPMif_uwbx3U5SrHM2Q":[],"gGUeCVHGs16weP-RfRckug":[],"gcGy5uR3kxFRCiUdSal-Nw":[],"iGerSDCCTaZmeNVguzT7mg":[[0.40416536143588944,0.645829969623336,null],[0.6329415365122731,0.7112856515841053,null],[0.7572759130735436,0.6124163231811848,null],[0.4273379195180877,0.6328622854371493,null]],"iXAvtzzMg2Vh5aZPuGIk_Q":[[0.8517846076771514,0.6459454998940752,null],[0.877354518155171,0.6171976752109533,null]],"ieZ8VA3zKLMrd7to7HHB5A":[[0.3860628658606913,0.5776178121293396,null]],"ilpqlnL95rHOAj1Q46fLuw":[[0.8496186585037313,0.5781385056760684,null],[0.11723084626528552,0.5849049750905047,null]],"jtmPZ_CIVoN69mYjj_JlFw":[],"kJesVaz2BfulGS7gWiNsNg":[[0.92377510278167,0.5697989090672384,null],[0.9678047497021356,0.5367280774814635,null],[0.06967962268904798,0.5361295731863283,null]],"lASBXjG5VnksPRJajz2FKw":[[0.3765430865582109,0.6171653543910636,null],[0.12058170376157207,0.5908543984176303,null],[0.17431163550851997,0.5859654456465201,null],[0.23629852894905606,0.5656378530223979,null]],"lOAN4VLX5ttywD8N5Be0wg":[[0.5460515776776802,0.5577193646329296,null]],"lcrZHovEffd0lTN2XIsnZA":[[0.5775297173513573,0.5754083049767388,null],[0.3932654422320423,0.5678230683895548,null],[0.9150957518956582,0.5951660909578422,null],[0.15493179149715103,0.596092333061511,null],[0.3428388057493268,0.5570247585193228,null],[0.43004592743748715,0.5579838384605558,null]],"loCxSX72-DYrDrDwY0cVGw":[[0.39052425138490154,0.5938001648059223,null],[0.7199224019213787,0.7149481277899277,null],[0.7386410231670714,0.689134101259873,null]],"n-DwmldMs-ZZVYS1gF_qYQ":[],"nBEV4rFOanW4bZa31JM0Sw":[[0.3967974675116893,0.5694409018987806,null],[0.5200062848323552,0.6952328327350167,null],[0.6499044180277097,0.6065183547382125,null],[0.9432359723528037,0.5704884041784821,null],[0.07620206844438732,0.5658842351853147,null],[0.2867610086629043,0.572102583336839,null],[0.14069361320528873,0.5604809529072763,null],[0.3232994831869592,0.589104708477156,null]],"nSGBAhZBqn92K0X6YMt3eg":[[0.6282105038578628,0.6250759544930062,null],[0.0635723363431534,0.5622630749374213,null]],"oPWPyI-Hrg7Ujq6yrFNyoQ":[[0.5657337484497229,0.5775135838373013,null],[0.5884945377163149,0.5611646582394548,null],[0.8667290985375325,0.6288008007861704,null],[0.9416476794464785,0.6095393120183088,null],[0.8916444162374528,0.6304934634288186,null],[0.11927712009335811,0.6110575868730849,null],[0.32409638057652684,0.6189694923306104,null]],"oV7Q7EUJPH-5ZMAT16aI7A":[[0.5434366332462844,0.5286047074573603,null]],"oubzmEZebW0xt9hWWIgN3Q":[[0.3840015375564759,0.5730722911592225,null],[0.5851018937778951,0.5812058595653327,null]],"p3WPC3r3OZij0pZr8jyR-A":[[0.04900203928797742,0.5651572783469989,null]],"pEfwvmb_FBOmLOnckeIGFA":[[0.4193532878966159,0.5232969384242431,null],[0.5969374561073313,0.5222770991562644,null],[0.6657553189899017,0.5696882399930215,null],[0.20841435452142043,0.6083491237185128,null]],"pTT4eaIPPMz9c1LU7IbE-w":[],"pUohf38G_xUQJ2nFzTGazg":[],"qZhtfaweaHDmUgGJtgVKTQ":[],"r0oKhyEUSiK2FMKwXE7Z4g":[[0.924162237453205,0.5794969957519847,null],[0.1588009394291079,0.5676328788229973,null]],"rZtL0FIlqaYtANWu_nDSjQ":[[0.5696591290917876,0.5463598501931762,null],[0.40911677625633835,0.5976324000544818,null]],"rtBqqxWOVVb3_RFgloXeAA":[],"thO-ovLo5h3IcEIBiZSBmA":[[0.9116174159970611,0.5783601744364313,null],[0.07257901599082484,0.5648165148049369,null],[0.22134302893498448,0.6437810732375184,null]],"uVUc_s2SzXj8SYWwiFlbzA":[[0.5670554346680529,0.5679477680890813,null],[0.2578404693518218,0.6264683906786244,null]],"vR2DJ0kQMRDnmCk0ICH3mQ":[],"vsYI9YRnGs7H2rchsFS44A":[[0.3763220885101751,0.6182668270766637,null],[0.7005953485386752,0.5859188567431503,null],[0.14585275549882076,0.7136839295408395,null],[0.8939949959512631,0.6397069911596522,null],[0.28349668964700114,0.7342486685642307,null],[0.3401193460058456,0.6932015178013831,null]],"wX8uKBQfioiFHogENC9FbQ":[[0.7217081778271659,0.6442762115472797,null],[0.06409002511162565,0.5549403900760805,null],[0.3499319852140296,0.6295160211091622,null]],"wgL3E2GPYntQeDbJjkaQsw":[],"xfnPV8p9liS6w_thn5Juqw":[[0.42202474343495283,0.5739993125812257,null],[0.5751135896235732,0.566109969835568,null],[0.8427927746669661,0.5923477663883209,null],[0.9326142433539037,0.5436883087500661,null],[0.06944565564339222,0.5466359078570889,null],[0.22786296555336294,0.5874658122397647,null]],"xyyjv_yjywf2p-XYB-9qDg":[[0.4381187797521368,0.5460903503857091,null],[0.7888518644919167,0.6555492547996944,null],[0.8749378633292058,0.6250932626494472,null],[0.24227198785122556,0.577181825092935,null]],"zo9eQGzGOdA57uZQE89rOA":[[0.6240530397982029,0.6105948113035092,null],[0.7597679493840187,0.6260263143173657,null],[0.9404392590841303,0.5543838537961203,null],[0.07789095401171536,0.5478549820852755,null],[0.2587850348473332,0.6091430924516503,null]],"zrChULSCpDxHp7XG9HdGzw":[[0.5709492881553944,0.5357557975457423,null],[0.6975991332309692,0.6720656159340218,null],[0.72372255050449,0.6518430520525894,null],[0.25088837684472126,0.5420948450090541,null],[0.2782178489835737,0.6011431333389678,null]]},"model":"claude-opus-5","n_panos":125,"n_uncached":0,"published_as":"claude-opus-5-effort-low","signature":{"box_tool":"{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}","effort":"low","max_edge":1568,"model_id":"claude-opus-5","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.","provider":"claude","source_max_edge":4096,"tile":true,"tool_choice":"auto","views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}} \ No newline at end of file diff --git a/docs/data/vertex_minute_series/claude-opus-5_2026-08-15.json b/docs/data/vertex_minute_series/claude-opus-5_2026-08-15.json new file mode 100644 index 00000000..fd0749e2 --- /dev/null +++ b/docs/data/vertex_minute_series/claude-opus-5_2026-08-15.json @@ -0,0 +1,91 @@ +{ + "model": "claude-opus-5", + "metric": "aiplatform.googleapis.com/publisher/online_serving/token_count", + "alignment_period": "60s", + "interval_start": "2026-08-15T17:00:00Z", + "interval_end": "2026-08-15T21:30:00Z", + "fetched_utc": "2026-09-03T13:22:18Z", + "columns": [ + "end_time", + "input_tokens", + "output_tokens" + ], + "rows": [ + ["2026-08-15T17:51:00Z", 5763, 394], + ["2026-08-15T17:52:00Z", 36633, 2133], + ["2026-08-15T17:53:00Z", 56868, 3866], + ["2026-08-15T17:54:00Z", 61471, 4229], + ["2026-08-15T17:55:00Z", 61540, 4057], + ["2026-08-15T17:56:00Z", 58899, 4118], + ["2026-08-15T17:57:00Z", 55446, 4107], + ["2026-08-15T17:58:00Z", 55108, 4069], + ["2026-08-15T17:59:00Z", 53415, 4316], + ["2026-08-15T18:00:00Z", 56530, 4289], + ["2026-08-15T18:01:00Z", 56326, 4138], + ["2026-08-15T18:02:00Z", 57410, 4194], + ["2026-08-15T18:03:00Z", 59779, 4094], + ["2026-08-15T18:04:00Z", 56597, 3742], + ["2026-08-15T18:05:00Z", 53686, 3845], + ["2026-08-15T18:06:00Z", 56259, 4045], + ["2026-08-15T18:07:00Z", 63502, 3986], + ["2026-08-15T18:08:00Z", 63571, 3938], + ["2026-08-15T18:09:00Z", 64382, 3897], + ["2026-08-15T18:10:00Z", 63571, 4263], + ["2026-08-15T18:11:00Z", 59779, 4356], + ["2026-08-15T18:12:00Z", 60050, 4101], + ["2026-08-15T18:13:00Z", 58628, 4163], + ["2026-08-15T18:14:00Z", 58019, 3800], + ["2026-08-15T18:15:00Z", 58899, 3627], + ["2026-08-15T18:16:00Z", 61200, 4064], + ["2026-08-15T18:17:00Z", 53754, 3989], + ["2026-08-15T18:18:00Z", 51317, 3414], + ["2026-08-15T18:19:00Z", 58290, 3466], + ["2026-08-15T18:20:00Z", 63231, 3967], + ["2026-08-15T18:21:00Z", 60389, 4008], + ["2026-08-15T18:22:00Z", 62622, 4003], + ["2026-08-15T18:23:00Z", 68174, 3867], + ["2026-08-15T18:24:00Z", 62149, 3918], + ["2026-08-15T18:25:00Z", 54566, 4158], + ["2026-08-15T18:26:00Z", 58560, 4027], + ["2026-08-15T18:27:00Z", 66414, 3962], + ["2026-08-15T18:28:00Z", 60998, 4019], + ["2026-08-15T18:29:00Z", 54837, 4130], + ["2026-08-15T18:30:00Z", 53686, 4057], + ["2026-08-15T18:31:00Z", 41297, 3387], + ["2026-08-15T18:32:00Z", 26742, 2838], + ["2026-08-15T18:33:00Z", 20039, 2857], + ["2026-08-15T18:34:00Z", 19430, 2842], + ["2026-08-15T18:35:00Z", 20310, 2465], + ["2026-08-15T18:36:00Z", 22612, 2286], + ["2026-08-15T18:37:00Z", 22070, 2654], + ["2026-08-15T18:38:00Z", 21461, 2608], + ["2026-08-15T18:39:00Z", 23492, 2619], + ["2026-08-15T18:40:00Z", 22070, 2526], + ["2026-08-15T18:41:00Z", 19159, 2314], + ["2026-08-15T18:42:00Z", 21732, 2452], + ["2026-08-15T18:43:00Z", 22070, 2671], + ["2026-08-15T18:44:00Z", 20310, 2757], + ["2026-08-15T18:45:00Z", 22612, 2363], + ["2026-08-15T18:46:00Z", 23221, 2371], + ["2026-08-15T18:47:00Z", 21190, 2525], + ["2026-08-15T18:48:00Z", 21461, 2481], + ["2026-08-15T18:49:00Z", 20039, 2645], + ["2026-08-15T18:50:00Z", 19430, 2561], + ["2026-08-15T18:51:00Z", 21461, 2615], + ["2026-08-15T18:52:00Z", 18888, 2692], + ["2026-08-15T18:53:00Z", 18550, 2484], + ["2026-08-15T18:54:00Z", 21461, 2654], + ["2026-08-15T18:55:00Z", 21190, 2655], + ["2026-08-15T18:56:00Z", 22612, 2515], + ["2026-08-15T18:57:00Z", 20919, 2850], + ["2026-08-15T18:58:00Z", 19430, 2884], + ["2026-08-15T18:59:00Z", 22612, 2560], + ["2026-08-15T19:00:00Z", 22070, 2474], + ["2026-08-15T19:01:00Z", 20310, 2689], + ["2026-08-15T19:02:00Z", 20310, 2749], + ["2026-08-15T19:03:00Z", 20310, 2491], + ["2026-08-15T19:04:00Z", 23763, 2478], + ["2026-08-15T19:05:00Z", 22950, 2344], + ["2026-08-15T19:06:00Z", 8801, 1050] + ] +} diff --git a/docs/data/vertex_minute_series/claude-opus-5_2026-08-18.json b/docs/data/vertex_minute_series/claude-opus-5_2026-08-18.json new file mode 100644 index 00000000..8a7713da --- /dev/null +++ b/docs/data/vertex_minute_series/claude-opus-5_2026-08-18.json @@ -0,0 +1,98 @@ +{ + "model": "claude-opus-5", + "metric": "aiplatform.googleapis.com/publisher/online_serving/token_count", + "alignment_period": "60s", + "interval_start": "2026-08-18T00:00:00Z", + "interval_end": "2026-08-19T12:00:00Z", + "fetched_utc": "2026-09-03T13:21:42Z", + "columns": [ + "end_time", + "input_tokens", + "output_tokens" + ], + "rows": [ + ["2026-08-18T23:29:00Z", 5754, 390], + ["2026-08-18T23:30:00Z", 9005, 676], + ["2026-08-18T23:31:00Z", 20783, 986], + ["2026-08-18T23:32:00Z", 13202, 700], + ["2026-08-18T23:54:00Z", 51790, 2439], + ["2026-08-18T23:55:00Z", 161599, 6864], + ["2026-08-18T23:56:00Z", 233700, 9257], + ["2026-08-18T23:57:00Z", 238575, 9562], + ["2026-08-18T23:58:00Z", 232685, 9357], + ["2026-08-18T23:59:00Z", 250828, 9564], + ["2026-08-19T00:00:00Z", 257125, 9337], + ["2026-08-19T00:01:00Z", 247240, 8735], + ["2026-08-19T00:02:00Z", 242569, 8522], + ["2026-08-19T00:03:00Z", 235935, 8307], + ["2026-08-19T00:04:00Z", 241892, 9064], + ["2026-08-19T00:05:00Z", 253266, 9296], + ["2026-08-19T00:06:00Z", 257056, 8771], + ["2026-08-19T00:07:00Z", 256787, 8957], + ["2026-08-19T00:08:00Z", 260509, 9207], + ["2026-08-19T00:09:00Z", 265181, 9129], + ["2026-08-19T00:10:00Z", 251099, 8767], + ["2026-08-19T00:11:00Z", 232753, 8058], + ["2026-08-19T00:12:00Z", 244735, 8436], + ["2026-08-19T00:13:00Z", 261390, 8845], + ["2026-08-19T00:14:00Z", 259427, 8794], + ["2026-08-19T00:15:00Z", 254755, 9123], + ["2026-08-19T00:16:00Z", 243517, 8525], + ["2026-08-19T00:17:00Z", 247105, 8441], + ["2026-08-19T00:18:00Z", 259358, 8819], + ["2026-08-19T00:19:00Z", 264301, 8949], + ["2026-08-19T00:20:00Z", 256854, 8812], + ["2026-08-19T00:21:00Z", 263624, 8890], + ["2026-08-19T00:22:00Z", 263827, 9004], + ["2026-08-19T00:23:00Z", 260509, 8871], + ["2026-08-19T00:24:00Z", 264030, 8635], + ["2026-08-19T00:25:00Z", 261728, 8257], + ["2026-08-19T00:26:00Z", 262270, 8169], + ["2026-08-19T00:27:00Z", 260577, 8394], + ["2026-08-19T00:28:00Z", 216506, 7222], + ["2026-08-19T00:29:00Z", 142238, 4898], + ["2026-08-19T00:30:00Z", 98165, 3629], + ["2026-08-19T00:31:00Z", 80157, 2796], + ["2026-08-19T00:32:00Z", 46646, 1596], + ["2026-08-19T00:33:00Z", 11441, 429], + ["2026-08-19T00:37:00Z", 34527, 1249], + ["2026-08-19T00:38:00Z", 74740, 2539], + ["2026-08-19T00:39:00Z", 88755, 2730], + ["2026-08-19T00:40:00Z", 83339, 2820], + ["2026-08-19T00:41:00Z", 79479, 2767], + ["2026-08-19T00:42:00Z", 90447, 2896], + ["2026-08-19T00:43:00Z", 91734, 2752], + ["2026-08-19T00:44:00Z", 85031, 2650], + ["2026-08-19T00:45:00Z", 84422, 2960], + ["2026-08-19T00:46:00Z", 86453, 3034], + ["2026-08-19T00:47:00Z", 85031, 3137], + ["2026-08-19T00:48:00Z", 78668, 2968], + ["2026-08-19T00:49:00Z", 86656, 2943], + ["2026-08-19T00:50:00Z", 92004, 3130], + ["2026-08-19T00:51:00Z", 90515, 3141], + ["2026-08-19T00:52:00Z", 97149, 3100], + ["2026-08-19T00:53:00Z", 92343, 3037], + ["2026-08-19T00:54:00Z", 85302, 3094], + ["2026-08-19T00:55:00Z", 86453, 2857], + ["2026-08-19T00:56:00Z", 78126, 2552], + ["2026-08-19T00:57:00Z", 68783, 2371], + ["2026-08-19T00:58:00Z", 80834, 2603], + ["2026-08-19T00:59:00Z", 91395, 2891], + ["2026-08-19T01:00:00Z", 94847, 3021], + ["2026-08-19T01:01:00Z", 87131, 2871], + ["2026-08-19T01:02:00Z", 76907, 2569], + ["2026-08-19T01:03:00Z", 80901, 2731], + ["2026-08-19T01:04:00Z", 88755, 2933], + ["2026-08-19T01:05:00Z", 82188, 2989], + ["2026-08-19T01:06:00Z", 75147, 2633], + ["2026-08-19T01:07:00Z", 80901, 2443], + ["2026-08-19T01:08:00Z", 70341, 2386], + ["2026-08-19T01:09:00Z", 76162, 2619], + ["2026-08-19T01:10:00Z", 84760, 3031], + ["2026-08-19T01:11:00Z", 81240, 3011], + ["2026-08-19T01:12:00Z", 61675, 2313], + ["2026-08-19T01:13:00Z", 44412, 1520], + ["2026-08-19T01:14:00Z", 35745, 1242], + ["2026-08-19T01:15:00Z", 13202, 491] + ] +} diff --git a/docs/data/vertex_minute_series/claude-sonnet-5_2026-08-15.json b/docs/data/vertex_minute_series/claude-sonnet-5_2026-08-15.json new file mode 100644 index 00000000..17b795d6 --- /dev/null +++ b/docs/data/vertex_minute_series/claude-sonnet-5_2026-08-15.json @@ -0,0 +1,70 @@ +{ + "model": "claude-sonnet-5", + "metric": "aiplatform.googleapis.com/publisher/online_serving/token_count", + "alignment_period": "60s", + "interval_start": "2026-08-15T12:00:00Z", + "interval_end": "2026-08-15T21:30:00Z", + "fetched_utc": "2026-09-03T13:21:39Z", + "columns": [ + "end_time", + "input_tokens", + "output_tokens" + ], + "rows": [ + ["2026-08-15T16:13:00Z", 5, 2], + ["2026-08-15T16:14:00Z", 515, 2], + ["2026-08-15T16:15:00Z", 1008, 0], + ["2026-08-15T16:23:00Z", 4, 0], + ["2026-08-15T16:24:00Z", 26, 9], + ["2026-08-15T16:25:00Z", 87, 33], + ["2026-08-15T16:26:00Z", 536, 66], + ["2026-08-15T16:27:00Z", 864, 50], + ["2026-08-15T16:28:00Z", 9659, 216], + ["2026-08-15T16:29:00Z", 19318, 264], + ["2026-08-15T16:30:00Z", 13374, 318], + ["2026-08-15T16:31:00Z", 26748, 388], + ["2026-08-15T16:58:00Z", 16345, 358], + ["2026-08-15T16:59:00Z", 38637, 610], + ["2026-08-15T17:00:00Z", 15473, 503], + ["2026-08-15T17:01:00Z", 7170, 355], + ["2026-08-15T17:02:00Z", 4198, 189], + ["2026-08-15T17:03:00Z", 28686, 1124], + ["2026-08-15T17:04:00Z", 55973, 2113], + ["2026-08-15T17:05:00Z", 44079, 2191], + ["2026-08-15T17:06:00Z", 40581, 2153], + ["2026-08-15T17:07:00Z", 41280, 2074], + ["2026-08-15T17:08:00Z", 43379, 2175], + ["2026-08-15T17:09:00Z", 48278, 2391], + ["2026-08-15T17:10:00Z", 47577, 2166], + ["2026-08-15T17:11:00Z", 70666, 2926], + ["2026-08-15T17:12:00Z", 102151, 3773], + ["2026-08-15T17:13:00Z", 102851, 3845], + ["2026-08-15T17:14:00Z", 106349, 4049], + ["2026-08-15T17:15:00Z", 102851, 3811], + ["2026-08-15T17:16:00Z", 100053, 3611], + ["2026-08-15T17:17:00Z", 103550, 3491], + ["2026-08-15T17:18:00Z", 104950, 3788], + ["2026-08-15T17:19:00Z", 104950, 3978], + ["2026-08-15T17:20:00Z", 103551, 3735], + ["2026-08-15T17:21:00Z", 100052, 3301], + ["2026-08-15T17:22:00Z", 102851, 3482], + ["2026-08-15T17:23:00Z", 109848, 3717], + ["2026-08-15T17:24:00Z", 104950, 3393], + ["2026-08-15T17:25:00Z", 104250, 3706], + ["2026-08-15T17:26:00Z", 109148, 3846], + ["2026-08-15T17:27:00Z", 104950, 3456], + ["2026-08-15T17:28:00Z", 107049, 3660], + ["2026-08-15T17:29:00Z", 110547, 4189], + ["2026-08-15T17:30:00Z", 104951, 4092], + ["2026-08-15T17:31:00Z", 98653, 3565], + ["2026-08-15T17:32:00Z", 102851, 3316], + ["2026-08-15T17:33:00Z", 101451, 3536], + ["2026-08-15T17:34:00Z", 93756, 3597], + ["2026-08-15T17:35:00Z", 74864, 2404], + ["2026-08-15T17:36:00Z", 58772, 2099], + ["2026-08-15T17:37:00Z", 60871, 2288], + ["2026-08-15T17:38:00Z", 58073, 1696], + ["2026-08-15T17:39:00Z", 55973, 1600], + ["2026-08-15T17:40:00Z", 30786, 770] + ] +} diff --git a/docs/data/vertex_minute_series/vertex_usage_daily_2026-09-03.json b/docs/data/vertex_minute_series/vertex_usage_daily_2026-09-03.json new file mode 100644 index 00000000..9c243600 --- /dev/null +++ b/docs/data/vertex_minute_series/vertex_usage_daily_2026-09-03.json @@ -0,0 +1,22 @@ +{ + "metric": "aiplatform.googleapis.com/publisher/online_serving/token_count", + "alignment_period": "86400s", + "lookback_days": 25.0, + "fetched_utc": "2026-09-03T13:21:45Z", + "rows": [ + {"window_end": "2026-08-15", "model": "gemini-3.6-flash", "tokens": {"input": 6, "output": 97}}, + {"window_end": "2026-08-15", "model": "gemini-3.7-flash", "tokens": {"input": 14597132, "output": 3584875}}, + {"window_end": "2026-08-16", "model": "claude-opus-5", "tokens": {"cache_read_input": 0, "cache_write_1h_input": 0, "cache_write_input": 0, "input": 3058702, "output": 247222}}, + {"window_end": "2026-08-16", "model": "claude-sonnet-5", "tokens": {"cache_read_input": 0, "cache_write_1h_input": 0, "cache_write_input": 0, "input": 3300368, "output": 118471}}, + {"window_end": "2026-08-16", "model": "gemini-3.6-flash", "tokens": {"input": 4, "output": 4}}, + {"window_end": "2026-08-16", "model": "gemini-3.7-flash", "tokens": {"input": 1549378, "output": 425879}}, + {"window_end": "2026-08-19", "model": "claude-opus-5", "tokens": {"cache_read_input": 0, "cache_write_1h_input": 0, "cache_write_input": 0, "input": 11988993, "output": 418503}}, + {"window_end": "2026-08-19", "model": "claude-sonnet-5", "tokens": {"cache_read_input": 0, "cache_write_1h_input": 0, "cache_write_input": 0, "input": 12594, "output": 480}}, + {"window_end": "2026-09-01", "model": "gemini-3.1-pro-preview", "tokens": {"input": 719664, "output": 8201}}, + {"window_end": "2026-09-01", "model": "gemini-3.6-flash", "tokens": {"input": 719664, "output": 222444}}, + {"window_end": "2026-09-01", "model": "gemini-3.7-flash", "tokens": {"input": 526967, "output": 108039}}, + {"window_end": "2026-09-02", "model": "gemini-3.1-pro-preview", "tokens": {"input": 658416, "output": 5329}}, + {"window_end": "2026-09-02", "model": "gemini-3.6-flash", "tokens": {"input": 658416, "output": 196646}}, + {"window_end": "2026-09-02", "model": "gemini-3.7-flash", "tokens": {"input": 851113, "output": 163413}} + ] +} diff --git a/docs/figures/scoreboard_by_split.png b/docs/figures/scoreboard_by_split.png index 7c7e429e..45e6a623 100644 Binary files a/docs/figures/scoreboard_by_split.png and b/docs/figures/scoreboard_by_split.png differ diff --git a/docs/figures/scoreboard_f1.png b/docs/figures/scoreboard_f1.png index aa561d9b..72fb3ccc 100644 Binary files a/docs/figures/scoreboard_f1.png and b/docs/figures/scoreboard_f1.png differ diff --git a/docs/figures/scoreboard_pr.png b/docs/figures/scoreboard_pr.png index fd4dd3b9..5d64bf6a 100644 Binary files a/docs/figures/scoreboard_pr.png and b/docs/figures/scoreboard_pr.png differ diff --git a/docs/figures/scoreboard_pr_curves.png b/docs/figures/scoreboard_pr_curves.png index f58cd09b..5702e39f 100644 Binary files a/docs/figures/scoreboard_pr_curves.png and b/docs/figures/scoreboard_pr_curves.png differ diff --git a/docs/model_comparison.md b/docs/model_comparison.md index aa76a302..373207ca 100644 --- a/docs/model_comparison.md +++ b/docs/model_comparison.md @@ -77,14 +77,33 @@ the matrix above exists to make, stated here because the artifact is already in would otherwise read as a withheld result. Promoting it is one field in the registry plus a re-run; since #122 froze the #46 witness pool, doing so no longer disturbs that human pass. -**Two more off-roster models, `claude-sonnet-5` and `claude-opus-5`, have been run on -annapolis only** — four legs, both models × effort `low`/`high` (#122). They are likewise -absent from the roster tables, so no number below moves. Detections are published and -verified (`benchmark/model_detections/claude-*-effort-*__annapolis.json`, 4/4 pairs -identical to the cache) and the write-up is the "Claude on Vertex" section further down, -where the whole result table is re-derived from those files by -`tests/test_claude_annapolis_leg.py`. **The other nine splits have not been run** — that is -a gap in coverage, not a withheld result, and closing it costs about $57 at `low` effort. +**Two more off-roster models, `claude-sonnet-5` and `claude-opus-5`, have been run** — four +legs, both models × effort `low`/`high` (#122). They are absent from the roster tables +below, so no number in this document moves; `claude-opus-5` at `low` **is** scored in +[`model_scoreboard.md`](model_scoreboard.md), which keys off coverage rather than +`standing`. Detections are published +(`benchmark/model_detections/claude-*-effort-*__*.json`, 14 files) and 12 of the 14 are +verified identical to the desktop cache with `export_model_cache.py --verify` (the four +annapolis legs and the eight #139 splits); the two Laurens files were exported with +`n_uncached` 0 on the machine that ran #151, which never had a recorded `--verify`, and +are re-scored from the committed detections by `tests/test_claude_published_legs.py` — a +different guarantee, stated as such. The write-up is the "Claude on Vertex" section +further down, where the +annapolis result table is re-derived from those files by +`tests/test_claude_published_legs.py`. + +Coverage is no longer uniform across the four, so it is stated per leg: + +- **`claude-opus-5` at `low` has run eleven splits** — everything but `manual_gold`: the + nine of #139 plus both Laurens arms (#151). The `manual_gold` absence is a decision, not a + pending run: `gemini-3.1-pro-preview` has no `manual_gold` row either, so a Claude-only + run there would have no peer to compare against (#144). +- **The other three legs have run annapolis only.** A gap in coverage, not a withheld + result. It was estimated at "about $57 at `low` effort" before anyone ran it; the nine-split + Opus leg then **measured $70.41 for eight splits** (§"Cost accounting" below), so budget + roughly **$8.80 per split** for Opus and re-derive Sonnet from its own $3.60 annapolis leg + rather than from that average. Re-running `high` comprehensively would roughly double the + bill to re-measure a result we already have, which is why it stays at one split. **The three `y*_pano` rows are the supervised YOLO baseline** (#51), the one part of the registry that is not zero-shot. They have run on all ten splits and are scored, but under the @@ -2073,6 +2092,11 @@ reading anything into the numbers — predictions sit tight on the ramps, no off | **claude-opus-5** | **low** | 0.572 | 0.605 | **0.588** | 178/133/116 | 523 | $8.94 | | claude-opus-5 | high | 0.430 | 0.656 | 0.520 | 193/256/101 | 127,227 | $12.46 | +The two Opus costs are the only ones here with independent corroboration: they were recorded +from console output at run time, and Cloud Monitoring's minute series was later solved for +the same split and returned $8.95 / $12.47 (§"Splitting a two-leg day by effort"). The Sonnet +pair has no such check — that day's telemetry does not separate. + **Every number in this table is re-derivable from committed files**, with no `.model_cache`, no API key and no GPU: the per-panorama detections are published under `benchmark/model_detections/claude-*-effort-*__annapolis.json`, and @@ -2095,6 +2119,24 @@ tokens to lose 0.068 F1. precision** (0.589 → 0.572), which is a capability difference rather than a threshold shift. RampNet still leads it by **0.251** (0.839 vs 0.588). +> **Superseded on the pooled board, 2026-08-19 (#139); re-pooled over eight splits +> 2026-09-17 after #151 added `laurens_mapillary`.** That displacement is an annapolis +> result and it does not generalise. Run on the other splits, `claude-opus-5` at `low` pools +> to **F1 0.568 against `gemini-3.1-pro`'s 0.575** over the eight US city splits — an +> annapolis lead of +0.021 becoming a pooled gap of −0.007, inside any reading of a tie. +> **`gemini-3.1-pro` still tops the table, by less than 0.01.** Read per split rather than +> pooled, the two trade wins — four each on the eight pooled splits, six of eleven overall +> for Opus, with laurens_mapillary (+0.086) the largest pooled gap in Opus's favour (the +> held-out laurens_gsv is wider still, +0.158) and gainesville (−0.069) and richmond +> (−0.066) the largest against — so the honest reading is +> not "Opus is worse" but **"per-split gaps whose range is 0.156 swamped a +0.021 lead"**. +> What survives is the shape rather than the ranking: Opus trades **−0.077 precision for +> +0.052 recall**, the highest recall of any chat VLM with full pooled coverage (0.586). The +> pooled table is in [`model_scoreboard.md`](model_scoreboard.md); the paragraph above is +> left as written because the annapolis numbers in it are still correct and are what the rest +> of this section analyses. (On the seven-split board this note was first written against, +> the same detections pooled to 0.588 against 0.608, 3 wins of 7; the eighth split moved it.) + **Correction, 2026-08-18 — the sonnet/low row originally used a different denominator.** As first published it read 0.587 / 0.372 / 0.456 on `108/76/182`, which is **290** GT ramps, not 294: one panorama (`annapolis:1528518111324684`) was lost when a malformed tool @@ -2162,21 +2204,163 @@ for m in claude-sonnet-5 claude-opus-5; do for e in low high; do python scripts/analysis/export_model_cache.py --verify --splits annapolis \ --models claude:$m --claude-effort $e --publish-as $m-effort-$e done; done + +# and the eleven-split opus/low leg (#139, #151), which takes no --splits: +python scripts/analysis/export_model_cache.py --verify \ + --models claude:claude-opus-5 --claude-effort low \ + --publish-as claude-opus-5-effort-low +# -> on a cache holding every run, "11 pair(s): published detections score IDENTICALLY +# to the cache". Nobody has run it on such a cache: observed 2026-09-17 on the desktop +# cache, which never held the two Laurens runs, it printed "compared 9" and flagged +# both Laurens files as "NOTHING was compared" -- unverified, not verified. ``` **The gap, stated plainly: the four original legs' token counts were never written to -`analysis_out/usage_log.jsonl`, and they cannot be recovered.** The $28.82 total and the -per-leg costs in the table above come from the runs' console output, not from a committed -record. A re-run cannot back-fill them either — the detections are cached, so a repeat run -makes zero API calls and has no usage to report. Only the 2026-08-18 single-panorama -re-run ($0.03) is in the log. +`analysis_out/usage_log.jsonl`.** The $28.82 total and the per-leg costs in the table above +come from the runs' console output, not from a committed record. A re-run cannot back-fill +them — the detections are cached, so a repeat run makes zero API calls and has no usage to +report. Only the 2026-08-18 single-panorama re-run ($0.03) is in the log. + +**Recovered from Cloud Monitoring, 2026-08-19 — this paragraph previously said the counts +"cannot be recovered", and that was wrong.** A re-run cannot back-fill them, but the +server-side metrics can: `vertex_usage.py --days 7` returns billed tokens per model per day, +and the #122 legs are four days inside the ~6-week retention window. They ran 2026-08-15 and +land in the row labelled 2026-08-16, because each row is a 24 h window ending at the query's +time of day rather than a calendar day. + +| model (both efforts, one row each) | input | output | billed | console figures | +|---|---:|---:|---:|---:| +| `claude-opus-5` (low + high) | 3,058,702 | 247,222 | **$21.47** | $8.94 + $12.46 = $21.40 | +| `claude-sonnet-5` (low + high) | 3,300,368 | 118,471 | **$7.79** | $3.60 + $3.82 = $7.42 | +| **total** | | | **$29.26** | **$28.82** | + +So the console numbers were right to within **1.5%**, and the table above stands as +published. Two things this changes, and one it does not: + +- **The Opus per-leg split is recoverable too, at minute resolution.** The daily row is + per model, so `low` and `high` land in one number — but the metric can be aligned to 60 s + instead of 86,400 s, and the two legs leave different traces. `vertex_effort_split.py` + does this and confirms the console figures **to 0.1%**; the working is below. +- **The Sonnet split is not recoverable, and the tool says so rather than guessing.** + Whether a per-effort split survives depends on whether effort actually changed the + model's behaviour, which makes this a property of the *result*, not of the telemetry. +- **The method validated itself against the one leg that did log.** The 2026-08-18 Sonnet + re-run appears in monitoring as 12,594 input / 480 output — token-for-token identical to + its committed `usage_log.jsonl` record. Layer 3 reproducing layer 1 exactly, on the one + case where both exist, is what makes the recovered figures above trustworthy. +- **It does not make the loss cheap.** Recovery worked because someone looked within six + weeks. Past that window this paragraph's original claim becomes true retroactively. + +#### Splitting a two-leg day by effort + +Cloud Monitoring has **no `effort` label** — effort is a request parameter and never +reaches the metric, whose labels are `type`, `request_type`, `shared_request_type`, +`source`, `explicit_caching` plus the resource's `model_user_id` / `model_version_id` / +`publisher` / `location`. The daily alignment is a *query* choice, though, not a property +of the data, so the lever is time plus two facts this repo already holds: + +1. **Input is deterministic** — 12,186 tokens per Opus panorama (6 views × 2,031). Total + input therefore pins the pano count exactly: the 08-15 Opus day is **251.00 panos** — + 250 leg panos plus one panorama's worth of input (12,186 tokens, about $0.06) whose + origin is not in the record. The only single-panorama re-run in + `analysis_out/usage_log.jsonl` is Sonnet's, on 08-18, so it is not that; the likeliest + source is a smoke call or a 404 retry from the #122 enablement window (the "12/12 + identical calls … 3 of 5 panos 404'd" measurement above, which did not name a model). + The geometric split drops it from both legs, which is why the two anchors sum to + $21.41 against the day's $21.47. The input half of the split needs no inference at all. +2. **Effort bills as output, not input.** A high-effort leg has a higher output/input ratio + *and* a lower throughput, so when the fast leg finishes, both change at once. + +That leaves one unknown — how the output divides — and the minute series shows exactly the +predicted shape. The legs ran **concurrently**, not back to back: throughput holds at ~5 +panos/min until **18:32 UTC**, then drops **2.54×** to ~1.7 while the output ratio doubles +(0.0675 → 0.1203). That is the `low` leg finishing and leaving `high` running alone. + +| anchor | low effort | high effort | sum | +|---|---:|---:|---:| +| tail ratio = pure high (0.1203) | $9.21 | $12.20 | $21.41 | +| low ratio = 0.0349, measured on the 984-pano #139 leg | **$8.95** | **$12.47** | $21.41 | +| **console output, recorded at run time** | **$8.94** | **$12.46** | $21.40 | + +**The rate-anchored solve reproduces the console figures to 0.1%, from a completely +independent source.** The anchor was taken from the #139 leg's measured output rate before +either number was compared, so the agreement is a check, not a fit. Quote **$8.94 / $12.46** +— the run-time record — and treat this as the corroboration that they are right. + +```bash +python scripts/analysis/vertex_effort_split.py --model claude-opus-5 \ + --start 2026-08-15T17:00:00Z --end 2026-08-15T21:30:00Z \ + --per-pano-input 12186 --anchor-low-ratio 0.034908 \ + --save-series docs/data/vertex_minute_series/claude-opus-5_2026-08-15.json +``` + +**The same command on `claude-sonnet-5` refuses to answer, and that is the more +transferable result.** Sonnet's ratio is flat across its whole run — throughput drops only +1.78× and the ratio moves the *wrong way* (0.0363 → 0.0273) — so there is no second +component to find and the script prints `NOT SEPARABLE`. (First published as 1.63× and +0.0365 → 0.0281: the changepoint search stopped one position short of the last full window, +which is exactly where this series' largest drop sits, so the cut landed a minute early at +17:35 instead of 17:36. Fixed 2026-09-17; the verdict does not move.) The reason is in the result table +above: Sonnet's high leg spent **17,820** thinking tokens against Opus's **127,227**, so the +dial that this method reads barely moved. A mixture solver run on that series returns "high +effort cost less than low", which is false; the guard exists because the wrong answer is the +plausible-looking one. **A per-effort split is recoverable exactly when effort changed the +model enough to be worth splitting** — the telemetry is not the limiting factor. + +**The minute series is committed, so this section no longer has an expiry date.** +Everything above was read out of telemetry with ~6 weeks of retention: the +2026-08-15 series would have aged out around **2026-09-26** and the 2026-08-18 day +around **2026-09-29**, after which nobody, with or without access to the project, +could re-derive a number in it. `--save-series` writes the fetched rows to JSON and +`--from-series` replays one, which needs no credentials, no project and no network: + +```bash +python scripts/analysis/vertex_effort_split.py --model claude-opus-5 \ + --from-series docs/data/vertex_minute_series/claude-opus-5_2026-08-15.json \ + --per-pano-input 12186 --anchor-low-ratio 0.034908 +python scripts/analysis/vertex_effort_split.py --model claude-sonnet-5 \ + --from-series docs/data/vertex_minute_series/claude-sonnet-5_2026-08-15.json +``` + +Four snapshots are committed under `docs/data/vertex_minute_series/`, all fetched +2026-09-03, and replaying them reproduces every figure published above exactly: + +| file | window (UTC) | active minutes | input | output | +|---|---|---:|---:|---:| +| `claude-opus-5_2026-08-15.json` | 08-15 17:00-21:30, both effort legs | 76 | 3,058,702 | 247,222 | +| `claude-sonnet-5_2026-08-15.json` | 08-15 12:00-21:30, both effort legs | 55 | 3,300,368 | 118,470 | +| `claude-opus-5_2026-08-18.json` | 08-18 00:00 - 08-19 12:00, the #139 leg | 83 | 11,988,993 | 418,503 | +| `vertex_usage_daily_2026-09-03.json` | the daily rows, 25-day lookback | - | - | - | + +Four details worth knowing before re-running any of it. The Sonnet window is wider +than the Opus one because that leg started before 17:00 — the narrower window clips +it to 3,157,769 input and moves the head ratio to 0.0374, which is why the figures +quoted above need the wide one. Sonnet's output is one token under the daily row's +118,471, and the cause is not identified: the 60 s deltas over a window that holds the +whole run should sum to the daily delta (the Opus series matches its row to the token), +and the fetch that wrote these files kept only minutes with input tokens, so a minute +holding a single output token and no input — a response finishing after its request +was counted, the shape of the 2-token smoke minutes at 16:13–16:14 — would have been +dropped before the file was saved. That filter is gone (`minute_rows` keeps every +minute with any tokens), but the committed series were fetched under it, so a re-fetch +inside the retention window could carry one more row than these do. The daily-row +snapshot, written by `vertex_usage.py --save-rows`, carries every token type including +the three cache buckets that are zero here: a snapshot that quietly dropped a billed +bucket would be worse than no snapshot. Only `fetched_utc` moves between regenerations; +the rows are byte-stable, and `tests/test_vertex_effort_split.py` asserts that for all +four files — `test_the_daily_snapshot_backs_the_published_cost_table` opens the daily +file, pins the four Claude rows, prices them to the $21.47 / $7.79 / $70.41 / $0.03 in +the table above and round-trips it through `write_json`. Two guards now stand where that went wrong, and the order matters. `compare.py` **refuses to start** a paid leg under `--usage-log none` (override: `--allow-unrecorded-spend`), which is the check that fires while the money is still unspent; `report_usage` still warns loudly at the end of a leg that logged nothing, for the case where the log path existed but could not -be written. A warning after the fact could not have saved these four legs — by the time it -prints, the tokens are bought and the counts are already unrecoverable. +be written. A warning after the fact would still have been worth having: everything above was +reconstructed five days late, and the only reason it worked is that nobody waited six weeks. +What the reconstruction cannot give you is a *guarantee* — Opus separated because its effort +dial moved 127k thinking tokens, Sonnet's did not separate at all, and which case you are in +is not knowable until after the money is spent. Layer 1 is the only layer that always works. ## Cost accounting: what a run cost in time and money @@ -2340,6 +2524,74 @@ Models differ by an order of magnitude in how much they do that (gemini-3.6-flas flash legs are not as cheap relative to pro as the rate card suggests — the 3.6-flash / 3.1-pro gap is $1.98 vs $2.31 per leg, not the 2.7× the input rates alone imply. +### The claude-opus-5 nine-split leg: layer 1 failed, layer 3 recovered it (#139) + +**This is the case the three-layer scheme was designed for, and it is worth reading as a +worked example rather than a footnote.** The eight-split `claude-opus-5` run of 2026-08-18 +(984 panoramas: bend, budapest_district5, clovis, gainesville, morgantown, paterson, +richmond, sao_paulo) **wrote no record to `analysis_out/usage_log.jsonl`.** As of +2026-08-19 the ledger held three entries totalling $0.34 — one `claude-sonnet-5` annapolis +leg and two richmond smoke tests — and nothing for the run itself. Layer 1 simply did not +fire. (The three Laurens rows of 2026-09-04, $12.78, came later and are #151's.) + +Layer 3 recovered the ground truth the next day: + +| model | input tokens | output tokens | est. cost | +|---|---:|---:|---:| +| claude-opus-5 (the eight splits, 984 panos, 08-18) | 11,988,993 | 418,503 | **$70.41** | +| claude-sonnet-5 (re-run remnant) | 12,594 | 480 | $0.03 | + +`python scripts/analysis/vertex_usage.py --days 3`, run 2026-08-19. + +**Wall-clock, recovered 2026-09-03 from the same metric at 60 s alignment: +2026-08-18 23:29 to 2026-08-19 01:15 UTC — a 106-minute span, 83 of those +minutes active, 9.3 panos/min.** Layer 1 would have recorded that at run time; +layer 3 gives it back only because someone looked inside the retention window, so +the series is committed as +`docs/data/vertex_minute_series/claude-opus-5_2026-08-18.json` and the span is +re-derivable from the repo alone. The 425 output tokens/pano below come from the +same file. + +**The four richmond +smoke panos are inside that 984, not additional to it** — both smoke records carry +`bundle: richmond`, the export covers all 124 richmond panos, and the main run found those +four already cached and re-billed nothing. For the same reason the ledger's $0.31 of smoke +spend is a *subset* of the $70.41 above, not a line to add to it. Four things follow, and +the last one is the one that bites: + +- **Input tokens are deterministic, so the input half is checkable without any cloud + access.** An Opus pano is exactly **12,186 tokens** (6 views × 2,031), identical in both + smoke records. 984 panos predicts 11,991,024 against 11,988,993 billed — 0.02% out. + Precisely: the billed figure is 2,031 × **5,903**, one view short of the 5,904 the + geometry demands, and that single missing call is unexplained. Anyone can re-derive this + from the committed detections; it is $59.94 of the $70.41. A second, fully independent + check agrees — the annapolis leg's measured **$8.94/125 panos** scales to $70.4 for 984. +- **The output half cannot be reconstructed this way.** Extrapolating the smoke runs' + 688 output tokens/pano (2,752 tokens over four panos, across two records) gives $16.92 + where the true figure is $10.46 — **62% high**, because output is thinking plus box count + and those four panos were unusually verbose. Actual: 425 output tokens/pano. Estimate + input from geometry; never estimate output. +- **Recovery is per-model per-day, not per-split.** Cloud Monitoring cannot say what + richmond cost as distinct from clovis. That attribution is permanently gone, which is + tolerable here only because the eight splits were one contiguous run of one model. +- **The recovery window is ~6 weeks and then it is not.** Had this gone unnoticed until + October the number would have been unrecoverable at any price. **A missing layer-1 record + is an emergency with a deadline, not a paperwork error** — run `vertex_usage.py` the + moment a paid leg finishes without logging, not when someone next reads the doc. + +The `--usage-log none` guard added in #119 refuses to start a paid leg with logging +disabled, so the likely mechanism is a run whose `REPO_ROOT` resolved to a scratch worktree +that was later deleted, taking the ledger with it. That is a real hole: the guard proves a +log path was *accepted*, not that the file it wrote still exists. See #143. + +**Corroborating evidence for that mechanism, found while reviewing this section.** +`REPO_ROOT` is derived from `__file__`, so *every* repo-root path breaks the same way in a +worktree — including the read side. Run `vertex_usage.py` from a worktree and it exits with +"no project: pass `--project`, or set `GOOGLE_CLOUD_PROJECT`… in a repo-root `.env`", +because the `.env` sits in the main checkout. Same root cause, both directions: a leg run +from a worktree writes its ledger somewhere disposable, and the tool that would recover the +spend cannot even find the project id. Pass `--project` explicitly when running from one. + ## Running it ```bash @@ -2672,6 +2924,10 @@ What this split adds to the story: - `scripts/analysis/vertex_usage.py` — server-side reconciliation: actual billed tokens per model from Cloud Monitoring. Needs ADC on the billing project, so only its output is replicable from this repo. +- `scripts/analysis/vertex_effort_split.py` — divides one such daily total between two legs + of the same model (the #122 low/high pairs) using minute alignment plus the deterministic + input geometry. Refuses with `NOT SEPARABLE` when the legs leave no distinguishable + trace, which is the Sonnet case. Same ADC requirement. - `analysis_out/usage_log.jsonl` — committed, append-only record of what each paid run spent. - `requirements-vlm.txt` — optional VLM deps. - `tests/test_detection_eval.py`, `tests/test_model_comparison.py`, diff --git a/docs/model_scoreboard.md b/docs/model_scoreboard.md index 9aa59028..9890e16d 100644 --- a/docs/model_scoreboard.md +++ b/docs/model_scoreboard.md @@ -46,6 +46,7 @@ model, and the reasons are in "How to read this" below. | YOLO11x (pano) | supervised baseline | 0.25 | **0.967** | 0.409 | 0.569 | -0.222 | 0.723 | 0.0 | 0.40–0.71 | | YOLO26 (pano) | supervised baseline | 0.25 | 0.744 | 0.447 | 0.553 | -0.238 | 0.606 | 0.4 | 0.45–0.68 | | Gemini 3.1 Pro | chat VLM | no score | 0.638 | 0.533 | 0.575 | -0.217 | – | 0.7 | 0.34–0.68 | +| Claude Opus 5 (low) | chat VLM | no score | 0.562 | 0.586 | 0.568 | -0.224 | – | 1.1 | 0.43–0.65 | | Gemini 3.7 Flash | chat VLM | no score | 0.679 | 0.458 | 0.539 | -0.252 | – | 0.5 | 0.28–0.66 | | Gemini 3.6 Flash | chat VLM | no score | 0.571 | 0.505 | 0.528 | -0.264 | – | 0.9 | 0.28–0.63 | | Qwen3-VL-8B | chat VLM | no score | 0.312 | 0.340 | 0.322 | -0.469 | – | 1.8 | 0.21–0.41 | @@ -80,6 +81,35 @@ operating point anyone has committed to, and on ground truth that never saw a Ra model here can be made precise. Finding the ramps is the hard part, which is why the project's operating-point work optimizes recall-first ([`operating_point.md`](operating_point.md)). +4. **A one-split lead of this size is indistinguishable from split noise.** `claude-opus-5` + at low effort beat `gemini-3.1-pro` on annapolis by **+0.021 F1** — the only time any model + has displaced the top challenger — so #139 ran it on the other splits. Pooled over the + eight US city splits it is **within 0.01: 0.568 against 0.575**, a −0.007 gap that is far + inside the spread below. The headline row is unchanged — `gemini-3.1-pro` still tops the + table — but the claim "nothing has displaced it" is now a tested tie rather than an + unchallenged lead. + + **The pooled number is the weaker half of that result.** Per split, the two trade wins: + Opus takes laurens_mapillary (+0.086, the largest gap of the eight pooled splits), + clovis (+0.037), annapolis (+0.022) and morgantown (+0.006); Gemini takes gainesville + (−0.069), richmond (−0.066), paterson (−0.039) and bend (−0.033) — **four wins each on + the eight pooled splits, six of eleven overall for Opus** (it also takes laurens_gsv, by + +0.158, the largest gap on the whole board, and sao_paulo by +0.015, and loses budapest + by 0.004). The range of the eight pooled gaps is 0.156, roughly **7× the annapolis lead + that motivated the run**, and the largest single pooled gap (0.086) is 4× it; the + annapolis lead was smaller in magnitude than six of the other seven. It was never + evidence of a real difference. That generalizes past these two models: a single-split + margin under ~0.09 F1 — the largest gap two models on this board have shown on a pooled + split, and under 0.16 counting the held-out laurens_gsv — should be treated as + unresolved until it is pooled, whichever direction it points. + + The F1 tie also hides two models that are not alike, and by this board's own third + finding the difference matters: **Opus trades −0.077 precision for +0.052 recall**, the + **highest recall of any chat VLM with full pooled coverage** (0.586, above every Gemini + leg; `claude-opus-5` at *high* effort reaches 0.656 but on annapolis alone — see the + partial table below). It ties on the aggregate and wins on the axis the operating-point + work says to optimize. Quote the ranking without that and you have the direction right and + the reason wrong. ![Precision vs recall](figures/scoreboard_pr.png) @@ -93,8 +123,8 @@ and F1 cannot tell you. ## Legs that have not run every pooled split -Six legs have run one split each, so they have no pooled mean to put in the table above — -a one-city average printed beside a seven-city one is exactly the comparison the coverage +Five legs have run one split each, so they have no pooled mean to put in the table above — +a one-city average printed beside an eight-city one is exactly the comparison the coverage column exists to prevent. They are reported per split instead, at the split they ran on: @@ -104,9 +134,6 @@ column exists to prevent. They are reported per split instead, at the split they | Mask2Former Vistas (curb cut) | supervised transfer | `richmond` | 0.411 | 0.697 | 0.517 | 0.513 | 2.5 | 216/309/94 | | Mask2Former Vistas (+curb) | supervised transfer | `richmond` | 0.126 | 0.648 | 0.210 | 0.089 | 11.3 | 201/1399/109 | | Claude Opus 5 (high) | chat VLM | `annapolis` | 0.430 | 0.656 | 0.520 | – | 2.0 | 193/256/101 | -| Claude Opus 5 (low) | chat VLM | `annapolis` | 0.572 | 0.605 | 0.588 | – | 1.1 | 178/133/116 | -| Claude Opus 5 (low) | chat VLM | `laurens_mapillary` | 0.485 | 0.386 | 0.430 | – | 1.1 | 96/102/153 | -| Claude Opus 5 (low) | chat VLM | `laurens_gsv` | 0.489 | 0.395 | 0.437 | – | 1.1 | 87/91/133 | | Claude Sonnet 5 (low) | chat VLM | `annapolis` | 0.589 | 0.381 | 0.463 | – | 0.6 | 112/78/182 | | Claude Sonnet 5 (high) | chat VLM | `annapolis` | 0.506 | 0.415 | 0.456 | – | 1.0 | 122/119/172 | @@ -115,10 +142,14 @@ column exists to prevent. They are reported per split instead, at the split they Two things worth carrying out of that table, both from splits where the roster's own numbers are directly above them in `model_comparison.md`: -- **Claude Opus 5 at low effort is the strongest challenger measured on annapolis** (F1 - 0.588, against gemini-3.1-pro's 0.567) — and **more thinking makes it worse** (0.520 at - high effort). The same direction holds for Sonnet 5 (0.463 → 0.456). Effort moves the - operating point; it does not raise the ceiling (#122). +- **More thinking makes it worse.** Claude Opus 5 drops from 0.588 at low effort to 0.520 at + high on annapolis, and the same direction holds for Sonnet 5 (0.463 → 0.456). Effort moves + the operating point; it does not raise the ceiling (#122). The low-effort leg has since run + eleven splits (#139, #151) and is in the headline table above — **and its annapolis lead + over `gemini-3.1-pro` did not survive the other seven pooled splits**; the two tie within + 0.01, see the note under that table. The high-effort leg stays here, at one split, because + re-running it comprehensively would roughly double the bill to re-measure a result we + already have. - **Supervised transfer fixes most of the precision problem and still loses.** Mask2Former reading Vistas' `Curb Cut` class scores 0.517 on richmond with **12.4× OWLv2's precision** and no training at all — but RampNet's 0.855 on that split is 0.337 clear of @@ -139,6 +170,7 @@ numbers are directly above them in `model_comparison.md`: | YOLO11x (pano) | 0.547 | 0.710 | 0.551 | 0.686 | 0.397 | 0.635 | 0.499 | 0.529 | 0.569 | 0.568 | 0.221 | 0.659 | 0.851 | | YOLO26 (pano) | 0.491 | 0.637 | 0.552 | 0.681 | 0.450 | 0.591 | 0.451 | **0.574** | 0.553 | 0.538 | 0.277 | 0.605 | 0.739 | | Gemini 3.1 Pro | 0.667 | 0.638 | 0.514 | 0.643 | 0.567 | 0.681 | 0.548 | 0.343 | 0.575 | 0.279 | 0.381 | 0.454 | – | +| Claude Opus 5 (low) | 0.601 | 0.604 | 0.550 | 0.649 | 0.588 | 0.642 | 0.479 | 0.430 | 0.568 | 0.437 | 0.378 | 0.468 | – | | Gemini 3.7 Flash | 0.664 | 0.639 | 0.504 | 0.595 | 0.565 | 0.609 | 0.456 | 0.281 | 0.539 | 0.261 | 0.338 | 0.358 | 0.527 | | Gemini 3.6 Flash | 0.634 | 0.597 | 0.483 | 0.633 | 0.554 | 0.608 | 0.438 | 0.277 | 0.528 | 0.274 | 0.336 | 0.346 | – | | Qwen3-VL-8B | 0.377 | 0.359 | 0.257 | 0.340 | 0.327 | 0.405 | 0.302 | 0.210 | 0.322 | 0.161 | 0.169 | 0.219 | 0.386 | @@ -149,7 +181,6 @@ numbers are directly above them in `model_comparison.md`: | Mask2Former Vistas (curb cut) | 0.517 | – | – | – | – | – | – | – | – | – | – | – | – | | Mask2Former Vistas (+curb) | 0.210 | – | – | – | – | – | – | – | – | – | – | – | – | | Claude Opus 5 (high) | – | – | – | – | 0.520 | – | – | – | – | – | – | – | – | -| Claude Opus 5 (low) | – | – | – | – | 0.588 | – | – | 0.430 | – | 0.437 | – | – | – | | Claude Sonnet 5 (low) | – | – | – | – | 0.463 | – | – | – | – | – | – | – | – | | Claude Sonnet 5 (high) | – | – | – | – | 0.456 | – | – | – | – | – | – | – | – | @@ -382,9 +413,19 @@ Omissions are content, so they are named rather than left as blanks: because it is untested. - **`manual_gold` has no null-recall pass** (O(n²) in panos), so the open detectors' recall discount is unmeasured on that split. -- **Six legs have one split each**, so they are in the partial table rather than the - headline: the two Vistas arms (richmond) and the four Claude legs (annapolis). Extending - either to the full pool is a run, not a code change. +- **Five legs have one split each**, so they are in the partial table rather than the + headline: the two Vistas arms (richmond), and Claude Opus 5 (high) plus both Sonnet 5 legs + (annapolis). Extending either to the full pool is a run, not a code change. +- **`claude-opus-5-effort-low` is scored here but is not a standing roster entry.** It has + full 8/8 coverage, so it appears in every table above; `standing` stays `False` because + `roster.py` forbids a *pinned* leg from being standing — a scored entry has to be what a + bare `--models` spec reproduces, and both efforts of `claude:claude-opus-5` share one spec. + The consequence is narrow and deliberate: it is absent from the `fp_taxonomy` / + `null_recall` defaults and from the frozen `WITNESS_POOL_46`, not from the scoreboard. The + three YOLO pano arms and `gemini-3.7-flash` sit in exactly the same position. +- **`claude-opus-5` has no `manual_gold` row, and that is a decision rather than a pending + run** — `gemini-3.1-pro-preview` has none either, so a Claude-only run there would have no + peer to compare against. Stated in full in #144. - **Nothing else in the registry is missing.** The board is driven by `rampnet/roster.py`, and `unregistered_exports` is empty — every published detections file is claimed by a roster entry and scored here. diff --git a/docs/replication.md b/docs/replication.md index 40be88c4..3e99b7f4 100644 --- a/docs/replication.md +++ b/docs/replication.md @@ -21,7 +21,7 @@ lives on one machine. | `benchmark/miss_taxonomy_46/*.json` (human verdicts) | small | **committed** | ✅ | | RampNet model weights | — | HF `projectsidewalk/rampnet-model` | ✅ | | Stage 1 dataset | **463 GB** (test split ~44 GB) | HF `projectsidewalk/rampnet-dataset` | ✅ | -| `benchmark/model_detections/` (challenger detections) | 25.1 MB (138 files) | **committed** ✅ | ✅ | +| `benchmark/model_detections/` (challenger detections) | 25.2 MB (146 files) | **committed** ✅ | ✅ | | **`location_data/` (the paper's government inventories)** | 71.8 MB | **committed** ✅ | ✅ | | **`street_data/` derivative (what the pipeline actually reads)** | 18.7 MB | **committed** ✅ | ✅ | | `street_data/` raw downloads (NY file alone is 669 MB) | 801 MB | git-ignored; HF #21 pending | ⚠️ superseded by the derivative | @@ -42,7 +42,7 @@ in this sentence — the list here was one of the things that drifted. single-panorama shards keyed by an opaque SHA-1 of (label, signature, city, pano), unreadable without reconstructing detector signatures. `scripts/analysis/export_model_cache.py` consolidates it into human-readable files, one per (model, split), keyed by panorama id with the detector -signature recorded inside. As of 2026-09-04 that is **138 files, 25.1 MB**, and every one of +signature recorded inside. As of 2026-09-17 that is **146 files, 25.2 MB**, and every one of them belongs to a registered leg: | what | files | where it is written up | @@ -50,7 +50,8 @@ them belongs to a registered leg: | the standing zero-shot roster, twelve splits each (two Gemini legs are absent on `manual_gold`) | 82 | the roster tables in [`model_comparison.md`](model_comparison.md) | | `gemini-3.7-flash`, twelve splits, published ahead of its write-up (#120) | 12 | §below | | the supervised YOLO pano trio, twelve splits each (#51) | 36 | [`model_comparison.md` §supervised baseline](model_comparison.md), and the [training record](../scripts/model_comparison/yolo_baseline/README.md) | -| the four annapolis Claude legs (#122) | 4 | [`model_comparison.md` §Claude](model_comparison.md) | +| `claude-opus-5` at `low` effort, eleven splits (#122; the pool by #139, both Laurens arms by #151) | 11 | [`model_comparison.md` §Claude](model_comparison.md) | +| the other three Claude legs, annapolis only (#122) | 3 | [`model_comparison.md` §Claude](model_comparison.md) | | the two Mapillary Vistas class-set arms, richmond only (#126) | 2 | [`model_comparison.md` §Vistas](model_comparison.md) | `rampnet` is a row in every results table and has no file here: it is read from each bundle's @@ -184,17 +185,18 @@ longer touches is the #46 human pass.** `silent_witness.py` defaults to `roster.WITNESS_POOL_46`, the pool frozen at the state the pass was rated under, and both committed artifacts record the pool they ran over. -#### The four Claude legs are published, annapolis only, one file per effort level +#### The four Claude legs are published, one file per effort level per split -`benchmark/model_detections/claude-{sonnet,opus}-5-effort-{low,high}__annapolis.json` — four -files, 125 panoramas each, 0 uncached (#122). - -**`claude-opus-5` at effort low now covers three splits**, not one: -`claude-opus-5-effort-low__{annapolis,laurens_mapillary,laurens_gsv}.json` (125 / 94 / 86 -panoramas, 0 uncached each). The two Laurens arms were run 2026-09-04 to answer #151 — whether -Laurens is hard because it is rural or because of the imagery rig — and needed the strongest -zero-shot model in the benchmark to make that test sharp. The other seven splits, and all three -remaining Claude legs on every split but annapolis, are still a stated gap. +`benchmark/model_detections/claude-{sonnet,opus}-5-effort-{low,high}__*.json` — **14 files**, +0 uncached. Three of the four legs ran annapolis only (125 panoramas, #122) and the other +splits are a stated gap for them. **`claude-opus-5` at `low` ran eleven splits** — 1,289 +panoramas: the nine of #139 (annapolis, bend, budapest_district5, clovis, gainesville, +morgantown, paterson, richmond, sao_paulo; 1,109 panoramas) plus both Laurens arms +(`laurens_mapillary` 94, `laurens_gsv` 86), run 2026-09-04 to answer #151 — whether Laurens +is hard because it is rural or because of the imagery rig — with the strongest zero-shot +model in the benchmark. `manual_gold` is deliberately absent from all four: no +`gemini-3.1-pro-preview` row exists there either, so a Claude-only run would have no peer +(#144). ⚠️ **The mapillary arm needed a second pass.** Two panoramas (`2102336717175440`, `2281219182305735`) died on Vertex's transient 404 even after the detector's @@ -212,6 +214,7 @@ because it is baked into keys that have already been paid for), and `export_mode now refuses outright to overwrite a file whose recorded signature differs from the run's: ```bash +# the three annapolis-only legs for m in claude-sonnet-5 claude-opus-5; do for e in low high; do python scripts/analysis/export_model_cache.py --splits annapolis \ --models claude:$m --claude-effort $e --publish-as $m-effort-$e @@ -219,25 +222,67 @@ for m in claude-sonnet-5 claude-opus-5; do for e in low high; do --models claude:$m --claude-effort $e --publish-as $m-effort-$e done; done # -> 4 x "compared 1 (model, split) pair(s); published detections score IDENTICALLY" + +# the eleven-split opus/low leg (#139, #151) -- no --splits, so it covers every bundle +python scripts/analysis/export_model_cache.py --verify \ + --models claude:claude-opus-5 --claude-effort low \ + --publish-as claude-opus-5-effort-low +# What a cache holding every run prints: +# -> "compared 11 (model, split) pair(s) ... 11 pair(s): published detections score +# IDENTICALLY to the cache", plus "claude-opus-5 / manual_gold: no published +# export to check" -- the #144 decision, showing up as a named absence. +# What was actually observed, 2026-09-17, on the desktop cache (which never held the +# two Laurens runs, made on the #151 machine): "compared 9 ... 9 pair(s): published +# detections score IDENTICALLY", with both Laurens files flagged "NOTHING was +# compared". So 12 of the 14 Claude files have a recorded cache-identity check; the +# two Laurens files have n_uncached 0 at export and the CI re-score in +# tests/test_claude_published_legs.py, which is a different guarantee. ``` +Add `--cache-dir ` when running from a git worktree: the default resolves against +`REPO_ROOT`, which is the worktree, not the checkout holding `.model_cache`. + Neither Claude spec is in `CHALLENGERS`, so the same caveat as `gemini-3.7-flash` applies: the default `--verify` never opens these files, and `fp_taxonomy.py` / `silent_witness.py` cannot reach them without the explicit `--models`. Unlike every other published leg, these four can also be checked with **no cache and no -credentials at all** — `tests/test_claude_annapolis_leg.py` recomputes the entire published -result table from the committed detections plus the committed annapolis bundle, and runs in -CI. That is the strongest form this ledger's promise can take, and it is the pattern worth -copying to the other legs. - -**Known gap, unrecoverable: the four legs' token counts were never written to +credentials at all** — `tests/test_claude_published_legs.py` recomputes the entire published +result table from the committed detections plus the committed bundles, and runs in CI. That +is the strongest form this ledger's promise can take, and it is the pattern worth copying to +the other legs. The opus/low leg's other splits get the same treatment one level up: +`tests/test_scoreboard.py` builds the whole board by re-scoring every committed detections +file against every committed bundle, so those numbers are re-derived in CI too rather than +read back from `analysis_out/scoreboard.json`. + +**Known gap, partly recovered: the four legs' token counts were never written to `analysis_out/usage_log.jsonl`.** The $28.82 figure and the per-leg costs quoted in -`docs/model_comparison.md` come from the runs' console output. They cannot be back-filled, -because a re-run reads the detection cache, makes zero API calls and therefore has no usage -to record — the cost record is the one artifact here that is write-once. Only the -2026-08-18 single-panorama re-run ($0.03) is in the log. `compare.report_usage` now prints -a loud warning when a leg that spent money finishes with no log destination. +`docs/model_comparison.md` come from the runs' console output. A *re-run* cannot back-fill +them, because it reads the detection cache, makes zero API calls and therefore has no usage +to record. Only the 2026-08-18 single-panorama re-run ($0.03) is in the log. + +**Cloud Monitoring can, and did (2026-08-19).** `scripts/analysis/vertex_usage.py --days 7` +recovered the billed daily totals — `claude-opus-5` $21.47 and `claude-sonnet-5` $7.79, +$29.26 against the console's $28.82, so the published figures are right to 1.5%. + +The **per-leg** split goes one level further and only half works. +`scripts/analysis/vertex_effort_split.py` re-queries the same metric at 60 s instead of +daily alignment: the two Opus legs ran concurrently and separate at their changepoint +(18:32 UTC, throughput /2.54), giving **$8.95 low / $12.47 high** against the console's +**$8.94 / $12.46** — 0.1%, from an independent source. **Sonnet does not separate**, and +the script prints `NOT SEPARABLE` rather than a number: its high leg spent 17,820 thinking +tokens to Opus's 127,227, so the signal this method reads is not there. Numbers, method and +the exact commands are in `docs/model_comparison.md` §"Splitting a two-leg day by effort". + +**Retention is ~6 weeks**, so all of this was recoverable only because someone looked within +it; treat a missing usage record as having a deadline, not as paperwork. The 2026-08-15 +series would have aged out around **2026-09-26** and the 2026-08-18 day around +**2026-09-29**, so the minute rows behind both are now committed under +`docs/data/vertex_minute_series/` — `vertex_effort_split.py --from-series` replays them +with no credentials and no project, and `tests/test_vertex_effort_split.py` checks that the +published figures still fall out of them. +`compare.report_usage` now prints a loud warning when a leg that spent money finishes with +no log destination. Downstream scripts prefer the published files over `.model_cache`, and the label a `--models` spec resolves to is derived *without* building a detector, so **a clean clone reproduces these numbers diff --git a/rampnet/roster.py b/rampnet/roster.py index 58a071d9..10372bfc 100644 --- a/rampnet/roster.py +++ b/rampnet/roster.py @@ -181,23 +181,35 @@ def slug(label): "at 0.05, roughly double the YOLO11 arms -- which is why it leads only " "on budapest, where firing at all is the binding constraint."), - # The Claude legs (#122). Two model ids x two efforts, all four on annapolis; - # the opus/low leg also covers both Laurens arms (#151, 2026-09-04). Still far - # short of the ten splits the standing rows report, so they stay off the roster - # tables and the write-up scopes each number to the splits it was measured on. - # The first provider whose knob splits one id into several legs, hence `pins` - # and `published_as`. + # The Claude legs (#122). Two model ids x two efforts, all four on annapolis. + # The opus/low leg now covers eleven splits -- the nine of #139 plus both + # Laurens arms (#151, 2026-09-04) -- so it is a complete leg on the scoreboard; + # the other three are annapolis only and stay off every pooled table, with the + # write-up scoping each number to the split it was measured on. None is + # `standing`: a pinned leg cannot be, because both efforts of one model id share + # a spec (see the check below). The first provider whose knob splits one id + # into several legs, hence `pins` and `published_as`. Challenger( spec="claude:claude-opus-5", label="claude-opus-5", provider="claude", density="sparse", standing=False, added="2026-08-15", pins=(("claude_effort", "low"),), published_as="claude-opus-5-effort-low", - note="Top challenger on annapolis (F1 0.588), the first model to displace " - "gemini-3.1-pro. 2.56 boxes/pano. Effort low is the provider default, " - "so this is what a bare `claude:claude-opus-5` reproduces. Also the " - "best zero-shot model on BOTH Laurens arms (0.430 mapillary, 0.437 " - "gsv) -- and flat across them (+0.007) where RampNet gains +0.115, " - "which is what makes #151's rig-not-town reading sharp."), + note="Eleven splits: nine in #139, both Laurens arms in #151. Pooled F1 0.568 " + "over the eight city splits, within 0.01 of gemini-3.1-pro's 0.575 -- its " + "+0.021 annapolis lead, the only time anything displaced the top " + "challenger, did not survive pooling, but neither did a deficit: the two " + "split 4 wins each on the pooled eight (Opus 6 of 11 overall), and the " + "pooled per-split gaps range from -0.069 (gainesville) to +0.086 " + "(laurens_mapillary; the held-out laurens_gsv is wider, +0.158), so the " + "annapolis lead was split noise, not a difference. Highest recall of any " + "fully-pooled chat VLM " + "(0.586), trading -0.077 precision for +0.052 recall. Best zero-shot model " + "on BOTH Laurens arms (0.430 mapillary, 0.437 gsv) and flat across them " + "(+0.007) where RampNet gains +0.115, which is what makes #151's " + "rig-not-town reading sharp. 2.54 boxes/pano over eleven splits (2.56 on " + "annapolis alone). No manual_gold row, deliberately: #144. Effort low is " + "the provider default, so this is what a bare `claude:claude-opus-5` " + "reproduces."), Challenger( spec="claude:claude-opus-5", label="claude-opus-5", provider="claude", density="sparse", standing=False, added="2026-08-15", diff --git a/scripts/analysis/vertex_effort_split.py b/scripts/analysis/vertex_effort_split.py new file mode 100644 index 00000000..82260e61 --- /dev/null +++ b/scripts/analysis/vertex_effort_split.py @@ -0,0 +1,297 @@ +"""Split one day's billed Vertex tokens between two legs of the same model (#139, #143). + +``vertex_usage.py`` recovers spend **per model per day**, which is where the recovery +stops when two legs of one model ran the same day. That is exactly the #122 case: both +effort levels of ``claude-opus-5`` and of ``claude-sonnet-5`` ran on 2026-08-15, so the +daily row is a sum of two runs we would like to price separately. + +Cloud Monitoring carries no ``effort`` label -- the labels on +``publisher/online_serving/token_count`` are ``type``, ``request_type``, +``shared_request_type``, ``source``, ``explicit_caching`` and the resource's +``model_user_id`` / ``model_version_id`` / ``publisher`` / ``location``. Effort is a +request parameter and never reaches the metric. So the only lever is **time**, at +minute resolution, plus two facts this repo already holds: + +1. **Input is deterministic.** A pano is a fixed number of views at a fixed size, so + input tokens per pano are constant (Opus: 12,186 = 6 x 2,031). Total input therefore + pins the pano count exactly, and a two-leg day splits its input 50/50 by geometry + with no inference at all. +2. **Effort shows up in output, not input.** Thinking bills as output, so a + high-effort leg has a higher output/input ratio and a lower throughput than a + low-effort leg of the same model. + +That makes the day a two-component mixture with a known input split, and the only +unknown is how the output divides. This script solves it two ways and reports both, +because agreement between them is the whole basis for trusting the answer: + +* **tail anchor** -- find the changepoint where throughput drops (the faster leg + finishing, leaving the slower one running alone) and take the tail's ratio as the + slow leg's pure ratio. +* **rate anchor** (``--anchor-low-ratio``) -- use an output/input ratio measured for + the *same model at the same effort* on some other, cleanly-attributed run. + +**It does not always work, and it says so.** Separability needs the effort dial to have +actually changed the model's behaviour. It did for Opus (127,227 thinking tokens across +the high leg, a 2.5x throughput drop, ratios 0.035 vs 0.127) and it did not for Sonnet +(17,820 thinking tokens, a flat ratio across the whole run). When the tail ratio is not +meaningfully above the head ratio there is no separation to find, and this prints +NOT SEPARABLE rather than a confident wrong number -- a mixture solver handed a flat +series will happily return "high effort cost less than low", which is the failure this +guard exists to catch. + +Read-only. Needs the same ADC and project as ``vertex_usage.py``. + + python scripts/analysis/vertex_effort_split.py --model claude-opus-5 \ + --start 2026-08-15T17:00:00Z --end 2026-08-15T21:30:00Z \ + --per-pano-input 12186 --anchor-low-ratio 0.034908 + +**Replication note:** like ``vertex_usage.py`` this reads one cloud project's billing +telemetry, so only someone with access to that project can re-derive it, and Google's +metric retention is ~6 weeks. ``--save-series`` writes the fetched minute rows to a JSON +file and ``--from-series`` replays one, which is what takes the result off that clock: +the series committed under ``docs/data/vertex_minute_series/`` re-runs the whole analysis +with no cloud access at all, long after the metric has aged out. + + python scripts/analysis/vertex_effort_split.py --model claude-opus-5 \\ + --from-series docs/data/vertex_minute_series/claude-opus-5_2026-08-15.json \\ + --per-pano-input 12186 --anchor-low-ratio 0.034908 + +The numbers it produced are transcribed into ``docs/model_comparison.md`` section +"Reproducing these four legs". +""" +import argparse +import json +import os +import sys +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "model_comparison")) +from pricing import estimate_cost, price_for # noqa: E402 +from vertex_usage import ( # noqa: E402 (same query, same snapshot format) + TOKEN_METRIC, _load_dotenv, fetch_series, write_json) + +#: A tail ratio must exceed the head ratio by this factor before the two legs are +#: called separable. Below it the series is flat and any split is fitting noise. +MIN_RATIO_LIFT = 1.25 +#: Minutes dropped either side of the changepoint, which is a blend of both legs. +GUARD_MINUTES = 3 + + +def minute_rows(series): + """Cloud Monitoring time series -> sorted (end_time, input, output) rows. + + **Every minute with any tokens is kept.** A minute with output and no input is a + long response completing after its request was counted (a thinking-heavy call + that spans the 60 s boundary), and it is still billed. The first version of this + filtered on input alone, which threw such minutes away before ``save_series`` ran + -- so the committed series could never be checked against the daily row for that + class of loss. Zero-input minutes are handled downstream by ``find_changepoint`` + (they cannot anchor a window) rather than by dropping them here. + """ + buckets = defaultdict(lambda: defaultdict(int)) + for s in series: + ttype = s.get("metric", {}).get("labels", {}).get("type", "?") + for pt in s.get("points", []): + n = int(pt["value"].get("int64Value", 0) or 0) + if n: + buckets[pt["interval"]["endTime"]][ttype] += n + return sorted((ts, d.get("input", 0), d.get("output", 0)) + for ts, d in buckets.items() + if d.get("input", 0) or d.get("output", 0)) + + +def fetch_minute_series(project, model, start, end): + """Minute-aligned (input, output) deltas for one model, oldest first.""" + series = fetch_series( + project, + f'metric.type = "{TOKEN_METRIC}" AND resource.labels.model_user_id = "{model}"', + start, end, "60s", ["metric.labels.type"], timeout=90, page_size=2000) + return minute_rows(series) + + +def save_series(path, model, start, end, rows): + """Write the fetched minute rows so the analysis outlives metric retention.""" + doc = { + "model": model, + "metric": TOKEN_METRIC, + "alignment_period": "60s", + "interval_start": start, + "interval_end": end, + "fetched_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "columns": ["end_time", "input_tokens", "output_tokens"], + "rows": [[ts, int(i), int(o)] for ts, i, o in rows], + } + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + write_json(path, doc, "rows") + return doc + + +def load_series(path): + """Replay a saved series. Returns (rows, model, start, end).""" + doc = json.loads(Path(path).read_text(encoding="utf-8")) + rows = [(ts, int(i), int(o)) for ts, i, o in doc["rows"]] + return (rows, doc.get("model"), doc.get("interval_start"), + doc.get("interval_end")) + + +def find_changepoint(rows, window=5): + """Index of the largest sustained drop in throughput, and the drop factor. + + Returns ``(None, 0.0)`` when no position has input on both sides of it, which + is what a series of output-only minutes looks like; callers must not slice on + a ``None`` cut. The last valid position is ``len(rows) - window`` -- the one + whose "after" window is exactly the final ``window`` minutes -- so the range is + inclusive of it. An earlier version stopped one short, which put a cliff that + sat ``window`` minutes from the end one minute early, with a diluted "after". + """ + best, best_drop = None, 0.0 + for i in range(window, len(rows) - window + 1): + before = sum(r[1] for r in rows[i - window:i]) / window + after = sum(r[1] for r in rows[i:i + window]) / window + if before and after and before / after > best_drop: + best, best_drop = i, before / after + return best, best_drop + + +def main(): + _load_dotenv() + ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + ap.add_argument("--project", default=os.environ.get("GOOGLE_CLOUD_PROJECT"), + help="GCP project (default: $GOOGLE_CLOUD_PROJECT / repo-root .env). " + "Pass explicitly when running from a git worktree.") + ap.add_argument("--model", required=True, help="model_user_id, e.g. claude-opus-5") + ap.add_argument("--start", help="RFC3339, e.g. 2026-08-15T17:00:00Z. Required " + "unless --from-series is given.") + ap.add_argument("--end", help="RFC3339. Required unless --from-series is given.") + ap.add_argument("--save-series", metavar="PATH", + help="Write the fetched minute rows to PATH as JSON, so the " + "analysis survives the ~6-week metric retention.") + ap.add_argument("--from-series", metavar="PATH", + help="Replay a saved series instead of querying Cloud " + "Monitoring. Needs no credentials and no project.") + ap.add_argument("--per-pano-input", type=int, default=0, + help="Deterministic input tokens per panorama; pins the pano count.") + ap.add_argument("--legs", type=int, default=2, + help="Legs of this model that ran in the window (default 2).") + ap.add_argument("--anchor-low-ratio", type=float, default=0.0, + help="output/input ratio for the FAST leg, measured on a cleanly " + "attributed run of the same model at the same effort.") + args = ap.parse_args() + + if args.from_series: + rows, saved_model, start, end = load_series(args.from_series) + # A series is a per-model file; replaying one under a different --model would + # price the wrong rate card against it and say nothing. + if saved_model and saved_model != args.model: + raise SystemExit(f"{args.from_series} holds {saved_model}, not " + f"{args.model} — pass --model {saved_model}.") + print(f"(replaying {args.from_series}: {len(rows)} minutes, " + f"{start} -> {end}, no cloud query)") + else: + if not args.project: + raise SystemExit("no project: pass --project, or set GOOGLE_CLOUD_PROJECT " + "in the environment or a repo-root .env.") + if not (args.start and args.end): + raise SystemExit("a cloud query needs --start and --end (or replay a " + "saved window with --from-series).") + rows = fetch_minute_series(args.project, args.model, args.start, args.end) + if args.save_series: + save_series(args.save_series, args.model, args.start, args.end, rows) + print(f"(wrote {len(rows)} minute rows to {args.save_series})") + + if len(rows) < 4 * GUARD_MINUTES: + raise SystemExit(f"only {len(rows)} active minute(s) in the window — widen it") + tin = sum(r[1] for r in rows) + tout = sum(r[2] for r in rows) + if not tin: + # Reachable since output-only minutes are kept: a window that caught only + # the tail of a response has tokens but nothing to split on. + raise SystemExit(f"no input tokens in the window ({len(rows)} output-only " + f"minute(s), {tout:,} output tokens) -- widen it.") + + print(f"== {args.model} {rows[0][0]} -> {rows[-1][0]}") + print(f" {len(rows)} active minutes, input {tin:,}, output {tout:,}, " + f"blended ratio {tout / tin:.4f}") + if args.per_pano_input: + n = tin / args.per_pano_input + # A non-integer pano count means the window is clipping a run or the + # per-pano rate is wrong, and either way the 50/50 input split below is + # unsound. Say so rather than dividing anyway. + verdict = (f"{round(n)} exactly" if abs(n - round(n)) < 0.02 else + "NOT an integer -- the window or the rate is off") + print(f" panos {n:.2f} at {args.per_pano_input:,} input/pano ({verdict})") + + cut, drop = find_changepoint(rows) + if cut is None: + raise SystemExit("no changepoint: no position in the window has input on both " + "sides of it. Widen the window, or the series is not a run.") + head, tail = rows[:cut - GUARD_MINUTES], rows[cut + GUARD_MINUTES:] + hi, ho = sum(r[1] for r in head), sum(r[2] for r in head) + ti, to = sum(r[1] for r in tail), sum(r[2] for r in tail) + if not (hi and ti): + # A cut this close to an edge leaves a guard-trimmed side with no input, + # and a ratio over zero input is not a ratio. + raise SystemExit(f"changepoint {rows[cut][0]} leaves a side with no input " + f"(head {hi:,}, tail {ti:,}) -- widen the window.") + r_head, r_tail = ho / hi, to / ti + print(f" changepoint {rows[cut][0]}: throughput /{drop:.2f}, " + f"output ratio {r_head:.4f} -> {r_tail:.4f}") + + if r_tail < r_head * MIN_RATIO_LIFT: + print(f"\n NOT SEPARABLE: the tail ratio is not {MIN_RATIO_LIFT}x the head's, " + f"so there is no\n second component to find. Either the legs did not " + f"overlap the way this\n assumes, or the effort dial did not move this " + f"model's output enough to\n leave a trace. Report the daily total and " + f"say the split is unrecovered.") + return 0 + + # Input divides by geometry, not inference: every leg made one pass over the split. + if args.per_pano_input: + per_leg_in = (round(tin / args.per_pano_input) // args.legs) * args.per_pano_input + else: + per_leg_in = tin / args.legs + if per_leg_in <= 0: + # Fewer than `legs` panos at this rate: the rate is wrong for the series, and + # every ratio in report() would divide by it. + raise SystemExit(f"per-leg input is zero: fewer than {args.legs} panos at " + f"{args.per_pano_input:,} input/pano -- the rate is wrong for " + f"this series.") + print(f"\n per-leg input (geometry, {args.legs} legs): {per_leg_in:,.0f}") + + # estimate_cost returns None for an id that is not in the verified rate card, and + # None cannot be formatted as a dollar figure. Decide once, up front, so an + # unpriced model reports its token split instead of raising inside report(). + priced = price_for(args.model) is not None + if not priced: + print(f" (no verified price for {args.model}; token splits only — add a " + f"verified entry to scripts/model_comparison/pricing.py)") + + def report(tag, out_slow): + out_fast = tout - out_slow + print(f" [{tag}]") + for label, out in (("low ", out_fast), ("high", out_slow)): + cost = (f" ${estimate_cost(args.model, per_leg_in, out):7.2f}" + if priced else "") + print(f" {label} effort: output {out:>10,.0f} " + f"ratio {out / per_leg_in:.4f}{cost}") + if priced: + total = (estimate_cost(args.model, per_leg_in, out_fast) + + estimate_cost(args.model, per_leg_in, out_slow)) + print(f" sum ${total:7.2f} vs billed day " + f"${estimate_cost(args.model, tin, tout):.2f}") + + report("tail anchor", r_tail * per_leg_in) + if args.anchor_low_ratio: + report(f"rate anchor (low ratio {args.anchor_low_ratio:.4f})", + tout - args.anchor_low_ratio * per_leg_in) + print("\n Two anchors that disagree are two estimates, not one answer -- quote " + "the spread.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/analysis/vertex_usage.py b/scripts/analysis/vertex_usage.py index 182d1b21..e3bf2b18 100644 --- a/scripts/analysis/vertex_usage.py +++ b/scripts/analysis/vertex_usage.py @@ -16,7 +16,10 @@ python scripts/analysis/vertex_usage.py --days 42 --project my-project **Replication note:** this reads one specific cloud project's billing telemetry, so -only someone with access to that project can re-derive its output. The numbers it +only someone with access to that project can re-derive its output, and only inside the +retention window. `--save-rows` writes the daily rows to a JSON file, which is how a +recovered figure stops depending on that window; the snapshots behind the #122 and #139 +cost tables are committed under docs/data/vertex_minute_series/. The numbers this produced are transcribed into docs/model_comparison.md; the per-run token counts in analysis_out/usage_log.jsonl are the committed, checkable half. @@ -25,6 +28,7 @@ labeled the 15th. Attribute rows to legs by the run record, not by eye. """ import argparse +import json import os import sys from collections import defaultdict @@ -43,24 +47,79 @@ KNOWN_TOKEN_TYPES = ("input", "output") +def write_json(path, doc, compact_key=None): + """Write a committed telemetry snapshot: LF-only, one row per line. + + Plain ``indent=2`` puts every integer of a 76-row series on its own line, which + buries a real change in several hundred lines of noise, so the list named by + ``compact_key`` is emitted one entry per line instead. Everything written here is + an integer token count or a string, so there is nothing to round and a regenerated + copy is provably byte-identical to the committed one (only ``fetched_utc`` moves). + """ + if compact_key is None: + text = json.dumps(doc, indent=2) + else: + marker = "__ROWS_PLACEHOLDER__" + entries = [json.dumps(r, separators=(", ", ": ")) for r in doc[compact_key]] + rendered = ("[\n " + ",\n ".join(entries) + "\n ]") if entries else "[]" + text = json.dumps(dict(doc, **{compact_key: marker}), indent=2).replace( + json.dumps(marker), rendered) + with open(path, "w", encoding="utf-8", newline="") as f: + f.write(text + "\n") + + +def _parse_dotenv(path): + """KEY=VALUE lines from ``path`` into os.environ, never overriding a set var. + + The same minimal parser as ``compare.load_dotenv``, kept here so this module + and ``vertex_effort_split.py`` still read the repo-root .env when the detector + stack is not importable (a bare ``pip install -r requirements-vlm.txt`` clone). + """ + if not os.path.exists(path): + return + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, val = line.split("=", 1) + os.environ.setdefault(key.strip(), val.strip().strip('"').strip("'")) + + def _load_dotenv(): - """Reuse compare.py's .env loader so both halves of the harness read the same - credentials file, from this checkout and from the main checkout it belongs to. - Imported inside the function (the export_model_cache idiom) so the module still - imports without the detector stack on the path. + """Same repo-root .env the rest of the harness reads. + + Prefers compare.py's loader so both halves of the harness read one credentials + file the same way, from this checkout and from the main checkout it belongs to; + imported inside the function (the export_model_cache idiom) so the module still + imports without the detector stack on the path, in which case the local parser + above does the same job rather than silently loading nothing. The main checkout matters here for the same reason the ledger does (#143): run from a scratch worktree, which carries no git-ignored `.env`, this script would - otherwise exit with "no project" — the tool whose job is to recover a lost spend - failing in exactly the situation that loses one.""" + otherwise exit with "no project" -- the tool whose job is to recover a lost spend + failing in exactly the situation that loses one. REPO is derived from __file__, + so a worktree looks in the worktree: pass --project explicitly there. + """ try: from compare import load_dotenv_for_run except ImportError: + _parse_dotenv(os.path.join(str(REPO), ".env")) return load_dotenv_for_run(REPO) -def fetch_token_series(project, days): +def fetch_series(project, filter_, start, end, alignment_period, group_by, + timeout=60, page_size=1000): + """Every ALIGN_DELTA / REDUCE_SUM time series for one Cloud Monitoring query. + + One paging loop for both the daily (``vertex_usage.py``) and the minute + (``vertex_effort_split.py``) queries, so the two cannot drift apart. ``start`` + and ``end`` are RFC3339 strings; ``group_by`` is the list of label paths the + reducer keeps. Follows ``nextPageToken`` to the end: a dropped page is a SILENT + UNDERCOUNT in the one tool whose job is server-side ground truth, and "the total + came back low" has no symptom a reader could notice. + """ try: import google.auth import google.auth.transport.requests @@ -73,26 +132,21 @@ def fetch_token_series(project, days): creds.refresh(google.auth.transport.requests.Request()) headers = {"Authorization": f"Bearer {creds.token}", "x-goog-user-project": project} - end = datetime.now(timezone.utc) params = { - "filter": f'metric.type = "{TOKEN_METRIC}"', - "interval.startTime": (end - timedelta(days=days)).isoformat(), - "interval.endTime": end.isoformat(), - "aggregation.alignmentPeriod": "86400s", + "filter": filter_, + "interval.startTime": start, + "interval.endTime": end, + "aggregation.alignmentPeriod": alignment_period, "aggregation.perSeriesAligner": "ALIGN_DELTA", "aggregation.crossSeriesReducer": "REDUCE_SUM", - "aggregation.groupByFields": ["resource.labels.model_user_id", - "metric.labels.type"], - "pageSize": 1000, + "aggregation.groupByFields": list(group_by), + "pageSize": page_size, } - # Follow nextPageToken. A dropped page is a SILENT UNDERCOUNT in the one tool - # whose job is server-side ground truth, and "the total came back low" has no - # symptom a reader could notice. url = f"https://monitoring.googleapis.com/v3/projects/{project}/timeSeries" series, token, pages = [], None, 0 while True: page_params = dict(params, **({"pageToken": token} if token else {})) - r = requests.get(url, params=page_params, headers=headers, timeout=60) + r = requests.get(url, params=page_params, headers=headers, timeout=timeout) if r.status_code != 200: raise SystemExit(f"Cloud Monitoring query failed ({r.status_code}): " f"{r.text[:500]}") @@ -104,12 +158,20 @@ def fetch_token_series(project, days): break if pages >= 50: # runaway guard; say so rather than truncating quietly raise SystemExit(f"stopped after {pages} pages with more remaining — " - f"narrow --days and re-run, or the totals would be partial") + f"narrow the window and re-run, or the totals would be " + f"partial") if pages > 1: print(f"(fetched {len(series)} time series across {pages} pages)") return series +def fetch_token_series(project, days): + """Daily-aligned token counts per (model_user_id, type) over the last ``days``.""" + end = datetime.now(timezone.utc) + return fetch_series( + project, f'metric.type = "{TOKEN_METRIC}"', + (end - timedelta(days=days)).isoformat(), end.isoformat(), "86400s", + ["resource.labels.model_user_id", "metric.labels.type"]) def ledger_totals_by_model(rows, since=None): """Per-model token totals from usage_log.jsonl rows, for reconciliation. @@ -244,6 +306,10 @@ def main(): "which compare.py also reads from a repo-root .env).") ap.add_argument("--days", type=float, default=30, help="Lookback window (metric retention is ~6 weeks).") + ap.add_argument("--save-rows", metavar="PATH", + help="Write the daily rows to PATH as JSON, so a recovered " + "figure survives the ~6-week metric retention. Every row " + "is written, including ones --min-tokens hides.") ap.add_argument("--reconcile", action="store_true", help="Compare these billed totals against what " "analysis_out/usage_log.jsonl recorded, per model. The only " @@ -287,6 +353,24 @@ def main(): f"{extra:,.0f} tokens are excluded from every figure below. " f"Price them in pricing.py or the total understates real spend.\n") + if args.save_rows: + # Every row and every token type, not just the two that get priced below: + # a snapshot that silently drops a bucket is worse than no snapshot. + doc = { + "metric": TOKEN_METRIC, + "alignment_period": "86400s", + "lookback_days": args.days, + "fetched_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "rows": [{"window_end": day, "model": model, + "tokens": {t: int(round(v)) + for t, v in sorted(daily[(day, model)].items())}} + for (day, model) in sorted(daily)], + } + out = Path(args.save_rows) + out.parent.mkdir(parents=True, exist_ok=True) + write_json(out, doc, "rows") + print(f"(wrote {len(doc['rows'])} daily rows to {args.save_rows})\n") + print(f"{'window end':12s} {'model':26s} {'input':>14s} {'output':>12s}") for (day, model) in sorted(daily): t = daily[(day, model)] diff --git a/tests/test_export_model_cache.py b/tests/test_export_model_cache.py index cb736ee0..34caf061 100644 --- a/tests/test_export_model_cache.py +++ b/tests/test_export_model_cache.py @@ -336,6 +336,19 @@ def test_the_ledger_count_matches_the_directory(): assert row and int(row.group(1)) == len(published), ( "the 'Status by input' table's file count disagrees with the directory") + # The per-leg breakdown is the half a reader uses to find a given file's write-up, + # and it is the half that drifts: #139 published eight files and updated only the + # total, leaving rows that summed to 114 under a heading that said 122. A total + # nobody can decompose is not a ledger, so check the decomposition too. + block = re.search(r"^\| what \| files \|.*?\n\n", text, re.S | re.M) + assert block, "docs/replication.md no longer has a '| what | files |' breakdown table" + rows = re.findall(r"^\|[^|\n]+\|\s*(\d+)\s*\|", block.group(0), re.M) + assert rows, "the breakdown table has no countable rows" + assert sum(int(n) for n in rows) == len(published), ( + f"the per-leg breakdown in docs/replication.md sums to " + f"{sum(int(n) for n in rows)} ({'+'.join(rows)}), but {em.PUBLISHED_DIR} holds " + f"{len(published)}. A row is missing or stale — the total alone is not the ledger.") + def test_every_published_file_is_in_canonical_form(): """A regenerated copy must be provably identical, not merely equivalent. diff --git a/tests/test_scoreboard.py b/tests/test_scoreboard.py index ca9d3392..4df5332f 100644 --- a/tests/test_scoreboard.py +++ b/tests/test_scoreboard.py @@ -423,6 +423,78 @@ def test_a_pinned_leg_loads_its_own_detections(board): assert low["f1"] == pytest.approx(0.588, abs=0.0006) +def test_the_annapolis_displacement_does_not_survive_pooling(board): + """#139's whole result, pinned: the one split that flattered Opus was the one split. + + The doc-currency tests below catch a *forgotten* regeneration. They pass happily on a + regenerated wrong number, because the doc and the JSON would both move together. This + is the assertion that has to be edited deliberately, so it names the claim instead of + the artifact: on annapolis Opus leads by +0.021; pooled over the eight city splits the + two are within 0.01 of each other (0.568 vs 0.575, Gemini ahead); and the split-to-split + spread is what swamps the lead. + + History, because the numbers moved once already: on the seven-split board this was + 0.588 vs 0.608, a -0.021 deficit with 3 wins of 7. Registering laurens_mapillary as the + eighth pooled split (#151) -- the split where Opus leads Gemini by the most, +0.086 -- + pulled the gap to -0.007 and the wins to 4 each. The claim that survives both boards is + "the annapolis lead was noise", not "Opus trails"; the win count is deliberately not + pinned, because it is the number most likely to churn with the next split. + """ + opus = _summary(board, "claude-opus-5-effort-low") + gem = _summary(board, "gemini-3.1-pro-preview") + + assert opus["coverage"] == "8/8" and opus["complete"] is True + assert opus["n_splits_run"] == 11 # 8 pooled + laurens_gsv + budapest + sao_paulo + assert opus["f1"] == pytest.approx(0.568, abs=0.0006) + assert opus["precision"] == pytest.approx(0.562, abs=0.0006) + assert opus["recall"] == pytest.approx(0.586, abs=0.0006) + assert gem["f1"] == pytest.approx(0.575, abs=0.0006) + + # The ranking claim itself, not just the two numbers behind it: Gemini still tops the + # table, and by less than the 0.01 the doc calls a tie. + assert opus["f1"] < gem["f1"], "gemini-3.1-pro is no longer the best challenger" + assert abs(opus["f1"] - gem["f1"]) < 0.01 + assert _cell(board, "claude-opus-5-effort-low", "annapolis")["f1"] > \ + _cell(board, "gemini-3.1-pro-preview", "annapolis")["f1"] + + # ...and the reason the lead did not generalise: the per-split gaps span a range far + # wider than the annapolis margin, and both signs occur. The doc quotes the range and + # the largest single gap as multiples of the lead, so both statistics are named here. + deltas = {s: _cell(board, "claude-opus-5-effort-low", s)["f1"] - _cell(board, gem["model"], s)["f1"] + for s in opus["pooled_splits"]} + assert any(d > 0 for d in deltas.values()) and any(d < 0 for d in deltas.values()), deltas + spread = max(deltas.values()) - min(deltas.values()) + assert spread == pytest.approx(0.156, abs=0.001), deltas + assert spread > 7 * abs(deltas["annapolis"]) + assert max(abs(d) for d in deltas.values()) == pytest.approx(0.086, abs=0.001) + assert max(deltas, key=deltas.get) == "laurens_mapillary" + # "smaller than six of the other seven": every pooled gap but morgantown's exceeds it. + assert sum(abs(d) > abs(deltas["annapolis"]) + for s, d in deltas.items() if s != "annapolis") == 6, deltas + + # The pooled scope matters: the held-out laurens_gsv is a wider gap still, and the + # doc must say "of the eight pooled splits" for 0.086 and name laurens_gsv (+0.158) + # as the largest on the whole board -- not call 0.086 the board's largest. + city_deltas = {s: _cell(board, "claude-opus-5-effort-low", s)["f1"] + - _cell(board, gem["model"], s)["f1"] + for s in board["city_splits"]} + assert max(city_deltas, key=lambda s: abs(city_deltas[s])) == "laurens_gsv" + assert city_deltas["laurens_gsv"] == pytest.approx(0.158, abs=0.001) + assert sum(d > 0 for d in city_deltas.values()) == 6 # six of eleven overall + with open(sb.DEFAULT_DOC, encoding="utf-8") as f: + prose = re.sub(r"\s+", " ", f.read()) # the doc wraps at 90 columns + assert "+0.086, the largest gap of the eight pooled splits" in prose + assert "laurens_gsv, by +0.158, the largest gap on the whole board" in prose + assert "the largest gap on the board)" not in prose + assert "smaller in magnitude than six of the other seven" in prose + + # Opus is the highest-recall chat VLM that has run the full pool -- the axis + # operating_point.md says to optimize, and the reason the near-tie is not a wash. + pooled_vlms = [m for m in board["models"] + if m["class"] == "chat-vlm" and m["complete"]] + assert max(pooled_vlms, key=lambda m: m["recall"])["model"] == "claude-opus-5-effort-low" + + def test_partial_coverage_is_reported_not_averaged_away(board): """The two Gemini legs have city detections but no published manual_gold. @@ -438,22 +510,25 @@ def test_partial_coverage_is_reported_not_averaged_away(board): def test_single_split_legs_stay_out_of_the_pooled_tables(board): - """Vistas ran richmond only; the Claude legs cover one or two cities, not eight. + """Vistas ran richmond only; three of the four Claude legs ran annapolis only. + + claude-opus-5-effort-low is deliberately NOT in this set any more: #139 took it to + nine splits and #151 to both Laurens arms, so it is a complete leg and belongs in + the pooled tables. If it ever reappears here, a leg lost coverage rather than a + test needing a nudge. - A one- or two-city macro-mean in the pooled column would be read as an eight-city - one. It is computed (the number is real, for those cities) but must not reach the - headline table or the pooled column of the matrix. + A one-city macro-mean in the pooled column would be read as an eight-city one. It + is computed (the number is real, for that city) but must not reach the headline + table or the pooled column of the matrix. - Coverage is pinned per leg rather than as one number, because it is no longer - uniform: `claude-opus-5` at effort low was run on both Laurens arms for #151, so it - now covers 2 of the 8 pooled splits (annapolis + laurens_mapillary -- laurens_gsv is - held out of the pool as the second arm of a city already in it). Partial coverage - still means excluded; the point of this test is the exclusion, not the number. + Coverage is pinned per leg rather than as one number, so a leg that gains a split + without reaching the full pool is noticed here rather than averaged in. Partial + coverage still means excluded; the point of this test is the exclusion, not the + number. """ want_coverage = { "mask2former-vistas-curb-cut": "1/8", "mask2former-vistas-curb-cut+curb": "1/8", - "claude-opus-5-effort-low": "2/8", "claude-opus-5-effort-high": "1/8", "claude-sonnet-5-effort-low": "1/8", "claude-sonnet-5-effort-high": "1/8", @@ -480,8 +555,10 @@ def test_single_split_legs_stay_out_of_the_pooled_tables(board): def test_partial_table_names_the_split_every_number_came_from(board): table = sr.partial_table(board) assert "`richmond`" in table and "`annapolis`" in table - assert "Claude Opus 5 (low)" in table + assert "Claude Opus 5 (high)" in table assert "Mask2Former Vistas (curb cut)" in table + # The low-effort leg went to nine splits in #139, so it is pooled now, not partial. + assert "Claude Opus 5 (low)" not in table def test_a_leg_from_an_unmapped_provider_is_classified_not_dropped(): diff --git a/tests/test_vertex_effort_split.py b/tests/test_vertex_effort_split.py new file mode 100644 index 00000000..82da87fc --- /dev/null +++ b/tests/test_vertex_effort_split.py @@ -0,0 +1,423 @@ +"""Guards on the two-leg effort split (#139, #143). + +The cloud query needs credentials and is not testable here. The *decision* logic is, +and it is the part that can hand back a confident wrong number: a mixture solver +pointed at a flat series will cheerfully report that high effort cost less than low. +These pin the changepoint detector and the separability rule against series whose +right answer is known by construction. +""" +import json +import os +import sys + +import pytest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "scripts", "analysis")) +sys.path.insert(0, os.path.join(REPO, "scripts", "model_comparison")) + +ves = pytest.importorskip("vertex_effort_split") +vu = pytest.importorskip("vertex_usage") +pricing = pytest.importorskip("pricing") + +SERIES_DIR = os.path.join(REPO, "docs", "data", "vertex_minute_series") +DAILY_SNAPSHOT = os.path.join(SERIES_DIR, "vertex_usage_daily_2026-09-03.json") + + +@pytest.fixture +def no_dotenv(monkeypatch): + """main() reads the repo-root .env before argparse. On a developer checkout that + file holds credentials, and a test that calls main() would load them into the + pytest process for the rest of the run (S7). Every main() test takes this.""" + monkeypatch.setattr(ves, "_load_dotenv", lambda: None) + + +def _point(end_time, ttype, n): + """One Cloud Monitoring time series carrying one ALIGN_DELTA point.""" + return {"metric": {"type": ves.TOKEN_METRIC, "labels": {"type": ttype}}, + "points": [{"interval": {"endTime": end_time}, + "value": {"int64Value": str(n)}}]} + + +def _series(spec): + """[(n_minutes, input_per_min, ratio)] -> the (ts, input, output) rows.""" + rows, minute = [], 0 + for n, inp, ratio in spec: + for _ in range(n): + rows.append((f"2026-08-15T{minute // 60:02d}:{minute % 60:02d}:00Z", + inp, round(inp * ratio))) + minute += 1 + return rows + + +def test_changepoint_finds_the_throughput_cliff(): + """Fast leg finishes, slow leg runs on alone: throughput drops, and the index + returned must be the first minute of the slow phase, not somewhere in the middle.""" + rows = _series([(30, 60_000, 0.035), (30, 20_000, 0.127)]) + cut, drop = ves.find_changepoint(rows) + assert cut == 30 + assert drop == pytest.approx(3.0, abs=0.01) + + +def test_changepoint_can_land_on_the_last_full_window(): + """S6: a cliff exactly `window` minutes from the end. rows[i:i+window] is a full + window up to i == len(rows) - window, and the search has to include it -- the + first version stopped one short, put the cut a minute early and read a diluted + "after" window (3.57x here instead of the real 10x). This is not hypothetical: + the committed Sonnet series has its largest drop at exactly that position.""" + rows = _series([(15, 100, 0.05), (5, 10, 0.05)]) + cut, drop = ves.find_changepoint(rows) + assert cut == 15 + assert drop == pytest.approx(10.0, abs=0.01) + # ...and one more tail minute, which the old range did search, agrees. + cut, drop = ves.find_changepoint(rows + [("2026-08-15T00:20:00Z", 10, 1)]) + assert (cut, drop) == (15, pytest.approx(10.0, abs=0.01)) + + +def test_changepoint_refuses_a_series_with_no_input_to_anchor_on(): + """S2 made zero-input minutes reachable, so the detector must say "none" rather + than slice on it: (None, 0.0), and main() turns that into a message, not a + TypeError.""" + rows = [(f"2026-08-15T00:{m:02d}:00Z", 0, 50) for m in range(20)] + assert ves.find_changepoint(rows) == (None, 0.0) + + +def test_output_only_minutes_survive_the_bucket_step(): + """S2: a minute with output tokens and no input tokens is a long response + completing after its request was counted. It is billed, and the first version + dropped it before save_series ran, so the committed file could never be checked + against the daily row for that loss. Every minute with any tokens is kept.""" + series = [_point("2026-08-15T17:51:00Z", "input", 60_000), + _point("2026-08-15T17:51:00Z", "output", 2_000), + _point("2026-08-15T17:52:00Z", "output", 5), # output only + _point("2026-08-15T17:53:00Z", "input", 60_000), + _point("2026-08-15T17:54:00Z", "input", 0)] # a zero point + rows = ves.minute_rows(series) + assert rows == [("2026-08-15T17:51:00Z", 60_000, 2_000), + ("2026-08-15T17:52:00Z", 0, 5), + ("2026-08-15T17:53:00Z", 60_000, 0)] + assert sum(r[2] for r in rows) == 2_005 # nothing lost + + +def test_an_output_only_series_replays_to_a_message_not_a_traceback( + tmp_path, monkeypatch, no_dotenv): + """The other half of S2: now that such minutes are kept, a replayed series can + have no input anywhere, and main() must exit with a sentence.""" + def replay(rows): + path = tmp_path / "series.json" + ves.save_series(path, "claude-opus-5", "s", "e", rows) + monkeypatch.setattr(sys, "argv", ["vertex_effort_split.py", "--model", + "claude-opus-5", "--from-series", str(path)]) + with pytest.raises(SystemExit) as e: + ves.main() + return str(e.value) + + # Nothing but output: refused before the blended ratio would divide by zero. + assert "no input tokens" in replay( + [(f"2026-08-15T00:{m:02d}:00Z", 0, 50) for m in range(20)]) + # A per-pano rate too large for the series: per_leg_in rounds to 0 and every + # ratio in report() would divide by it. The "NOT an integer" verdict prints and + # continues, so this guard is the one that has to fire. + monkeypatch.setattr(sys, "argv", [ + "vertex_effort_split.py", "--model", "claude-opus-5", "--per-pano-input", + "3000000", "--from-series", + os.path.join(SERIES_DIR, "claude-opus-5_2026-08-15.json")]) + with pytest.raises(SystemExit) as e: + ves.main() + assert "per-leg input is zero" in str(e.value) + # Some input, but never on both sides of a window: the detector returns None + # and main() must say so rather than slice rows[:None - 3]. + assert "no changepoint" in replay( + [("2026-08-15T00:00:00Z", 12_186, 400), ("2026-08-15T00:01:00Z", 12_186, 400)] + + [(f"2026-08-15T00:{m:02d}:00Z", 0, 50) for m in range(2, 20)]) + + +def test_a_flat_series_is_reported_as_not_separable(): + """Sonnet's real shape: both legs at effectively one ratio. + + The detector still returns *a* changepoint -- there is always a largest drop -- + so the separability rule, not the detector, is what has to refuse. If this ever + starts passing the lift threshold, the split it produces is noise. + """ + rows = _series([(30, 100_000, 0.036), (25, 60_000, 0.035)]) + cut, _ = ves.find_changepoint(rows) + g = ves.GUARD_MINUTES + head, tail = rows[:cut - g], rows[cut + g:] + r_head = sum(r[2] for r in head) / sum(r[1] for r in head) + r_tail = sum(r[2] for r in tail) / sum(r[1] for r in tail) + assert r_tail < r_head * ves.MIN_RATIO_LIFT + + +def test_a_real_two_component_day_clears_the_threshold(): + """The Opus shape, so the guard is not simply refusing everything.""" + rows = _series([(40, 58_000, 0.0675), (35, 21_000, 0.1203)]) + cut, _ = ves.find_changepoint(rows) + g = ves.GUARD_MINUTES + head, tail = rows[:cut - g], rows[cut + g:] + r_head = sum(r[2] for r in head) / sum(r[1] for r in head) + r_tail = sum(r[2] for r in tail) / sum(r[1] for r in tail) + assert r_tail >= r_head * ves.MIN_RATIO_LIFT + + +def test_the_input_split_is_geometry_not_inference(): + """Input per leg comes from the pano count, so it must not depend on the + output at all -- that is what makes the mixture solvable with one unknown.""" + per_pano, legs = 12_186, 2 + total_in = 251 * per_pano + per_leg = (round(total_in / per_pano) // legs) * per_pano + assert per_leg == 125 * per_pano # the odd pano is not silently halved + + +def test_rate_anchor_and_tail_anchor_bracket_the_published_opus_split(): + """The 2026-08-15 Opus day, as recovered. Both anchors must stay on the same + side of the story: high effort costs more, and the two agree within ~5%.""" + total_in, total_out = 3_058_702, 247_222 + per_leg_in = 125 * 12_186 + for out_high in (0.1203 * per_leg_in, # tail anchor + total_out - 0.034908 * per_leg_in): # rate anchor + out_low = total_out - out_high + assert out_high > out_low + # Priced through pricing.py, not a hand-copied rate: if the claude-opus-5 + # row ever moves, the script's output moves with it and so must this test, + # rather than passing on numbers the script no longer prints. + cost_low = pricing.estimate_cost("claude-opus-5", per_leg_in, out_low) + cost_high = pricing.estimate_cost("claude-opus-5", per_leg_in, out_high) + assert cost_low == pytest.approx(9.1, abs=0.3) + assert cost_high == pytest.approx(12.3, abs=0.3) + assert cost_low + cost_high == pytest.approx(21.41, abs=0.05) + + +# --- the committed minute series (F1) --------------------------------------- +# +# Cloud Monitoring keeps this metric about six weeks, so every figure in +# docs/model_comparison.md section "Splitting a two-leg day by effort" was, until these +# files were committed, derivable only from one cloud project inside one month. These +# replay the committed series and pin the published answers to them, which is what makes +# that section reproducible from a clean clone with no credentials. + +def _replay(name): + return ves.load_series(os.path.join(SERIES_DIR, name)) + + +def test_the_committed_opus_series_replays_the_published_effort_split(): + """The 2026-08-15 Opus day, from the committed rows: 251 panos, the 18:32 + changepoint, and the two anchors that bracket $8.94 / $12.46.""" + rows, model, _, _ = _replay("claude-opus-5_2026-08-15.json") + assert model == "claude-opus-5" + assert len(rows) == 76 + tin = sum(r[1] for r in rows) + tout = sum(r[2] for r in rows) + assert (tin, tout) == (3_058_702, 247_222) # == the billed daily row + + per_pano = 12_186 + assert tin / per_pano == pytest.approx(251.0, abs=0.02) + + cut, drop = ves.find_changepoint(rows) + assert rows[cut][0] == "2026-08-15T18:32:00Z" + assert drop == pytest.approx(2.54, abs=0.01) + + g = ves.GUARD_MINUTES + head, tail = rows[:cut - g], rows[cut + g:] + r_head = sum(r[2] for r in head) / sum(r[1] for r in head) + r_tail = sum(r[2] for r in tail) / sum(r[1] for r in tail) + assert r_head == pytest.approx(0.0675, abs=0.0001) + assert r_tail == pytest.approx(0.1203, abs=0.0001) + assert r_tail >= r_head * ves.MIN_RATIO_LIFT # separable + + # Both anchors, priced off the rate card, straddle the run-time console figures. + per_leg_in = (round(tin / per_pano) // 2) * per_pano + for out_high in (r_tail * per_leg_in, tout - 0.034908 * per_leg_in): + out_low = tout - out_high + low = pricing.estimate_cost("claude-opus-5", per_leg_in, out_low) + high = pricing.estimate_cost("claude-opus-5", per_leg_in, out_high) + assert low == pytest.approx(9.1, abs=0.3) # console: $8.94 + assert high == pytest.approx(12.3, abs=0.3) # console: $12.46 + + +def test_the_committed_sonnet_series_still_refuses_to_separate(): + """Sonnet is the negative result, and it has to stay negative: a future change + that made this series look separable would publish a wrong split.""" + rows, model, _, _ = _replay("claude-sonnet-5_2026-08-15.json") + assert model == "claude-sonnet-5" + assert sum(r[1] for r in rows) == 3_300_368 # == the billed daily row + cut, drop = ves.find_changepoint(rows) + # The largest drop sits exactly `window` minutes from the end -- the position the + # S6 off-by-one used to exclude. It read 17:35 and 1.63x before that fix. + assert rows[cut][0] == "2026-08-15T17:36:00Z" + assert drop == pytest.approx(1.78, abs=0.01) + g = ves.GUARD_MINUTES + head, tail = rows[:cut - g], rows[cut + g:] + r_head = sum(r[2] for r in head) / sum(r[1] for r in head) + r_tail = sum(r[2] for r in tail) / sum(r[1] for r in tail) + assert r_head == pytest.approx(0.0363, abs=0.0001) + assert r_tail == pytest.approx(0.0273, abs=0.0001) + assert r_tail < r_head # the ratio moves the wrong way + assert r_tail < r_head * ves.MIN_RATIO_LIFT # NOT SEPARABLE + + +def test_the_committed_139_series_carries_the_leg_wall_clock(): + """The $70.41 leg. The money was recovered at the time; the wall-clock is only in + this series, which is one reason it is committed.""" + from datetime import datetime + + rows, model, _, _ = _replay("claude-opus-5_2026-08-18.json") + assert model == "claude-opus-5" + tin = sum(r[1] for r in rows) + tout = sum(r[2] for r in rows) + assert (tin, tout) == (11_988_993, 418_503) # == the billed daily row + assert pricing.estimate_cost("claude-opus-5", tin, tout) == pytest.approx( + 70.41, abs=0.01) + + fmt = "%Y-%m-%dT%H:%M:%SZ" + first = datetime.strptime(rows[0][0], fmt) + last = datetime.strptime(rows[-1][0], fmt) + assert rows[0][0] == "2026-08-18T23:29:00Z" + assert rows[-1][0] == "2026-08-19T01:15:00Z" + assert len(rows) == 83 # active minutes + assert (last - first).total_seconds() / 60 == 106.0 + assert tout / 984 == pytest.approx(425.3, abs=0.1) # output tokens per pano + + +def test_every_committed_series_round_trips_through_save_and_load(tmp_path): + """save_series/load_series are the committed artifacts' only writer and reader, so + a change to either must not silently reshape the files already in git.""" + for name in ("claude-opus-5_2026-08-15.json", + "claude-sonnet-5_2026-08-15.json", + "claude-opus-5_2026-08-18.json"): + src = os.path.join(SERIES_DIR, name) + rows, model, start, end = ves.load_series(src) + out = tmp_path / name + ves.save_series(out, model, start, end, rows) + with open(src, encoding="utf-8", newline="") as f: + original = f.read() + with open(out, encoding="utf-8", newline="") as f: + written = f.read() + # fetched_utc is provenance and moves; every other byte must not. + keep = lambda t: [x for x in t.splitlines() if "fetched_utc" not in x] + assert keep(written) == keep(original) + # LF-only, and one row per line rather than one integer per line -- which is + # what keeps a real change to a series visible in a diff. + assert "\r" not in written + assert sum(1 for x in written.splitlines() + if x.startswith(' ["')) == len(rows) + + +def test_the_daily_snapshot_backs_the_published_cost_table(tmp_path): + """S4: the four Claude rows in docs/model_comparison.md's recovery table -- $21.47, + $7.79, $70.41 and the $0.03 re-run -- come from the committed daily snapshot, and + until this test nothing opened that file. The rows are pinned, priced through + pricing.py to the published figures, and round-tripped through write_json (the + --save-rows path, dict rows, not save_series) byte-for-byte except fetched_utc.""" + with open(DAILY_SNAPSHOT, encoding="utf-8", newline="") as f: + original = f.read() + doc = json.loads(original) + assert doc["alignment_period"] == "86400s" + claude = {(r["window_end"], r["model"]): r["tokens"] for r in doc["rows"] + if r["model"].startswith("claude-")} + want = { # window_end, model -> input, output, $ + ("2026-08-16", "claude-opus-5"): (3_058_702, 247_222, 21.47), + ("2026-08-16", "claude-sonnet-5"): (3_300_368, 118_471, 7.79), + ("2026-08-19", "claude-opus-5"): (11_988_993, 418_503, 70.41), + ("2026-08-19", "claude-sonnet-5"): (12_594, 480, 0.03), + } + assert set(claude) == set(want) + for key, (tin, tout, dollars) in want.items(): + tokens = claude[key] + assert (tokens["input"], tokens["output"]) == (tin, tout), key + # Every token type is carried, including the cache buckets that are zero here: + # a snapshot that dropped a billed bucket would be worse than no snapshot. + assert {"cache_read_input", "cache_write_1h_input", "cache_write_input"} <= set(tokens) + assert pricing.estimate_cost(key[1], tin, tout) == pytest.approx(dollars, abs=0.005) + # The 08-15 Opus and Sonnet minute series are these two daily rows, re-read at 60 s. + opus_rows, _, _, _ = _replay("claude-opus-5_2026-08-15.json") + assert (sum(r[1] for r in opus_rows), sum(r[2] for r in opus_rows)) == (3_058_702, 247_222) + sonnet_rows, _, _, _ = _replay("claude-sonnet-5_2026-08-15.json") + assert sum(r[1] for r in sonnet_rows) == 3_300_368 + assert sum(r[2] for r in sonnet_rows) == 118_471 - 1 # the one-token gap, see the doc + + out = tmp_path / "daily.json" + vu.write_json(out, doc, "rows") + with open(out, encoding="utf-8", newline="") as f: + written = f.read() + keep = lambda t: [x for x in t.splitlines() if "fetched_utc" not in x] + assert keep(written) == keep(original) + assert "\r" not in written + assert sum(1 for x in written.splitlines() if x.startswith(' {"window_end"')) == \ + len(doc["rows"]) + + +def test_an_unpriced_model_reports_tokens_instead_of_raising(): + """F3: estimate_cost returns None for a model that is not in the rate card, and + None cannot be formatted with :7.2f. The script has to decide that before it + formats anything, or its "no verified price" branch is unreachable code.""" + assert pricing.price_for("claude-opus-5") is not None + unpriced = "no-such-model-9" + assert pricing.price_for(unpriced) is None + assert pricing.estimate_cost(unpriced, 1_000_000, 1_000) is None + with pytest.raises(TypeError): + # the shape of the bug: this is what report() used to do unconditionally + "{:7.2f}".format(pricing.estimate_cost(unpriced, 1_000_000, 1_000)) + + +def test_replaying_a_series_under_the_wrong_model_is_refused(monkeypatch, no_dotenv): + """A series file is per-model. Replaying Sonnet's series as Opus would price the + wrong rate card against it and print a confident wrong number.""" + monkeypatch.setattr(sys, "argv", [ + "vertex_effort_split.py", "--model", "claude-opus-5", "--from-series", + os.path.join(SERIES_DIR, "claude-sonnet-5_2026-08-15.json")]) + with pytest.raises(SystemExit) as e: + ves.main() + assert "claude-sonnet-5" in str(e.value) + + +def test_a_cloud_query_still_needs_a_window(monkeypatch, no_dotenv): + """--start/--end stopped being argparse-required so --from-series could omit them. + The check has to survive by hand, or a windowless query reaches the API.""" + monkeypatch.setattr(sys, "argv", [ + "vertex_effort_split.py", "--model", "claude-opus-5", "--project", "p"]) + with pytest.raises(SystemExit) as e: + ves.main() + assert "--start and --end" in str(e.value) + + +def test_the_minute_query_and_the_daily_query_share_one_fetch(monkeypatch): + """S8: vertex_effort_split.py had its own copy of vertex_usage.py's paging loop + and .env parser, differing only in timeout and page size. One fetch_series now + serves both, so a paging fix lands in both queries. Checked without HTTP: the + minute fetch must call the shared function with a 60 s alignment and the + per-model filter, and hand its result to minute_rows.""" + calls = [] + + def fake_fetch(project, filter_, start, end, alignment_period, group_by, **kw): + calls.append((project, filter_, start, end, alignment_period, list(group_by), kw)) + return [_point(end, "input", 12_186), _point(end, "output", 400)] + + monkeypatch.setattr(ves, "fetch_series", fake_fetch) + rows = ves.fetch_minute_series("proj", "claude-opus-5", "S", "E") + assert rows == [("E", 12_186, 400)] + (project, filter_, start, end, alignment, group_by, kw), = calls + assert (project, start, end, alignment) == ("proj", "S", "E", "60s") + assert ves.TOKEN_METRIC in filter_ and 'model_user_id = "claude-opus-5"' in filter_ + assert group_by == ["metric.labels.type"] + assert kw == {"timeout": 90, "page_size": 2000} + assert ves.fetch_series is not vu.fetch_series # the patch took + assert ves._load_dotenv is vu._load_dotenv # one .env reader, not two + + +def test_the_dotenv_fallback_reads_the_same_file_the_same_way(tmp_path, monkeypatch): + """vertex_usage._load_dotenv prefers compare.load_dotenv; when the detector stack + is not importable it must still read .env rather than silently load nothing, + and the two parsers must agree on what a line means.""" + env = tmp_path / ".env" + env.write_text('# comment\nGOOGLE_CLOUD_PROJECT="proj-a"\nOTHER=x=y\n\nBAD LINE\n', + encoding="utf-8") + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + monkeypatch.delenv("OTHER", raising=False) + vu._parse_dotenv(str(env)) + assert os.environ["GOOGLE_CLOUD_PROJECT"] == "proj-a" + assert os.environ["OTHER"] == "x=y" # split on the first '=' only + compare = pytest.importorskip("compare") + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + monkeypatch.delenv("OTHER", raising=False) + compare.load_dotenv(str(tmp_path)) + assert (os.environ["GOOGLE_CLOUD_PROJECT"], os.environ["OTHER"]) == ("proj-a", "x=y")