One challenger registry, and stop a new model moving the #46 human pass (#122)

analysis_out/fp_taxonomy.json

@@ -10,6 +10,41 @@
     "paterson",
     "gainesville"
   ],
+  "models": {
+    "pool": null,
+    "specs": [
+      "rampnet",
+      "gemini:gemini-3.6-flash",
+      "gemini:gemini-3.1-pro-preview",
+      "qwen:Qwen/Qwen3-VL-8B-Instruct",
+      "qwen:Qwen/Qwen3-VL-32B-Instruct",
+      "molmo:allenai/Molmo2-8B",
+      "owlv2",
+      "gdino"
+    ],
+    "labels": [
+      "rampnet",
+      "gemini-3.6-flash",
+      "gemini-3.1-pro-preview",
+      "Qwen/Qwen3-VL-8B-Instruct",
+      "Qwen/Qwen3-VL-32B-Instruct",
+      "allenai/Molmo2-8B",
+      "google/owlv2-large-patch14-ensemble",
+      "IDEA-Research/grounding-dino-base"
+    ],
+    "sparse": [
+      "rampnet",
+      "gemini:gemini-3.6-flash",
+      "gemini:gemini-3.1-pro-preview",
+      "qwen:Qwen/Qwen3-VL-8B-Instruct",
+      "qwen:Qwen/Qwen3-VL-32B-Instruct",
+      "molmo:allenai/Molmo2-8B"
+    ],
+    "dense": [
+      "owlv2",
+      "gdino"
+    ]
+  },
   "per_model": {
     "rampnet @0.30": {
       "n_fp": 181,

analysis_out/silent_witness.json

@@ -2,6 +2,38 @@
   "bucket": "silent",
   "threshold": 0.3,
   "n": 128,
+  "models": {
+    "pool": "WITNESS_POOL_46",
+    "specs": [
+      "gemini:gemini-3.6-flash",
+      "gemini:gemini-3.1-pro-preview",
+      "qwen:Qwen/Qwen3-VL-8B-Instruct",
+      "qwen:Qwen/Qwen3-VL-32B-Instruct",
+      "molmo:allenai/Molmo2-8B",
+      "owlv2",
+      "gdino"
+    ],
+    "labels": [
+      "gemini-3.6-flash",
+      "gemini-3.1-pro-preview",
+      "Qwen/Qwen3-VL-8B-Instruct",
+      "Qwen/Qwen3-VL-32B-Instruct",
+      "allenai/Molmo2-8B",
+      "google/owlv2-large-patch14-ensemble",
+      "IDEA-Research/grounding-dino-base"
+    ],
+    "sparse": [
+      "gemini:gemini-3.6-flash",
+      "gemini:gemini-3.1-pro-preview",
+      "qwen:Qwen/Qwen3-VL-8B-Instruct",
+      "qwen:Qwen/Qwen3-VL-32B-Instruct",
+      "molmo:allenai/Molmo2-8B"
+    ],
+    "dense": [
+      "owlv2",
+      "gdino"
+    ]
+  },
   "per_model": {
     "gemini-3.6-flash": {
       "witnessed": 33,

benchmark/model_detections/Qwen__Qwen3-VL-32B-Instruct__budapest_district5.json

@@ -1 +1 @@
-{"city":"budapest_district5","detections":{"1003827842477344":[],"1007938414942666":[],"1019095247256685":[],"1027077646394579":[],"1037039761997906":[[0.753661650183105,0.5716294489663263,null],[0.8690434934396101,0.5921698323046172,null],[0.09097323783957839,0.6033745303075791,null]],"1037329802140159":[],"1037999235592907":[],"1038280075426836":[],"1042570498111893":[],"1043138107118713":[],"1046671274052950":[],"1137646927290075":[],"1170370375008478":[],"1188947286474652":[],"1216426636963136":[],"1234836825326131":[],"1263700272280191":[],"1279768990703921":[],"1290858892857927":[],"1320252873424067":[],"1321969539605547":[],"1328862755794452":[],"1330282068787409":[],"1338136654317995":[],"1358450323019655":[],"1382388957125864":[],"1402190408446458":[],"1402259001719568":[],"1415048523727047":[[0.6320770814797556,0.7170321397183919,null],[0.88077008125954,0.6010319929714012,null],[0.9297388777498468,0.6420170415471849,null],[0.06714662158324852,0.6606906380480676,null]],"1479510223377234":[[0.9457378830965617,0.6280053252900943,null],[0.29968810543055785,0.6572907027671258,null]],"1479649270603872":[],"1485983376024324":[[0.7698335990760192,0.6409366512101438,null]],"1493807495453961":[],"1503933797776035":[],"1507586557510184":[],"1526153925847560":[],"1526644485012786":[],"1526903691607662":[],"1531415762060842":[],"1531940155140947":[],"1533099125112846":[],"1547490957023264":[],"1548551026941868":[],"1550619426690125":[],"1554553479649915":[],"1560573605574046":[],"1579465080398346":[],"1621880408877798":[],"1628684669262976":[],"1642079667427179":[],"1644929466791193":[],"1665066454545157":[],"1665856994688538":[],"1679632769964162":[],"1688417845615915":[],"1708279679842703":[[0.9462482876138417,0.6519850461086084,null]],"1711104372904021":[],"1726281301363646":[],"1736795824019170":[],"1743265983784166":[[0.39343845166225333,0.6300363753399092,null]],"1743535723318388":[],"1743866716581524":[],"1762265868113165":[],"1769629404195835":[],"1779415216757575":[[0.8963378474565633,0.6469744991149612,null]],"1872865880061015":[],"1875441803044127":[],"1892190831489561":[],"1920855128875530":[],"1933608350737216":[],"1980881726130460":[[0.5582762343643185,0.5965172794337936,null],[0.18235903528396297,0.7148587194472441,null]],"2002394670457045":[[0.9616835102845476,0.7058177644912155,null]],"2039237576963899":[],"2055080831885811":[[0.7653392159053829,0.7122298586128143,null],[0.832893695338722,0.7547468732372489,null]],"2064297630816443":[[0.059470857403150224,0.5851085348855025,null]],"2080767922502123":[],"2088155352056225":[],"2185625018892068":[],"2216048919129703":[],"2221274008612179":[],"2243369796465669":[],"2392893427787498":[],"2511246802645139":[],"2641788216158646":[],"27177228765230843":[],"27260772950175386":[],"28215663224697246":[],"2962988900572350":[],"3015816262094291":[],"3140904756109373":[[0.847669357233338,0.5785735518060254,null]],"3296688647144785":[[0.2796097390360186,0.6693030721503049,null],[0.3173010350203889,0.6567633261001978,null]],"335801701340862":[],"3688080354666548":[[0.6726836988211788,0.5502044715604599,null],[0.5884391232626168,0.5791743706402792,null],[0.2997057398692108,0.6746016729415606,null]],"37046635288313572":[[0.4384492372973748,0.5972199522659719,null],[0.2531607838456425,0.5649843379775997,null]],"37230045973275479":[],"3860523807571425":[[0.7899790842887342,0.7467318447050739,null]],"3901990523265321":[],"4122786137987976":[],"427909170007735":[],"4426398164172326":[[0.2843445354672993,0.6607335479640698,null]],"4478163572471700":[],"5026082804197254":[],"740730485632754":[[0.7488230429840264,0.6900426739053614,null]],"749536038146240":[],"755126063369619":[],"790116040739014":[],"817455584410792":[],"820647700656490":[],"833524132783867":[],"838276004852617":[],"841136762387293":[],"866540066042247":[],"873103842125428":[],"881246267480927":[],"897403239302195":[],"917879701312494":[],"934293439634924":[],"945894851803282":[],"986609046276593":[],"986995307528492":[],"991739220159007":[],"992737779816905":[],"993081170303646":[],"994057612515361":[],"997653559315299":[]},"model":"Qwen/Qwen3-VL-32B-Instruct","n_panos":125,"n_uncached":0,"signature":{"coord_space":"norm1000","max_edge":2048,"max_new_tokens":1024,"model_id":"Qwen/Qwen3-VL-32B-Instruct","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list. Respond with JSON only: a list of {\"bbox_2d\": [x1, y1, x2, y2], \"label\": \"curb ramp\"}.","provider":"qwen","source_max_edge":4096,"tile":true,"views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}}
+{"city":"budapest_district5","detections":{"1003827842477344":[],"1007938414942666":[],"1019095247256685":[],"1027077646394579":[],"1037039761997906":[[0.753661650183105,0.5716294489663263,null],[0.8690434934396101,0.5921698323046172,null],[0.09097323783957839,0.6033745303075791,null]],"1037329802140159":[],"1037999235592907":[],"1038280075426836":[],"1042570498111893":[],"1043138107118713":[],"1046671274052950":[],"1137646927290075":[],"1170370375008478":[],"1188947286474652":[],"1216426636963136":[],"1234836825326131":[],"1263700272280191":[],"1279768990703921":[],"1290858892857927":[],"1320252873424067":[],"1321969539605547":[],"1328862755794452":[],"1330282068787409":[],"1338136654317995":[],"1358450323019655":[],"1382388957125864":[],"1402190408446458":[],"1402259001719568":[],"1415048523727047":[[0.6320770814797556,0.7170321397183919,null],[0.88077008125954,0.6010319929714012,null],[0.9297388777498468,0.6420170415471849,null],[0.06714662158324852,0.6606906380480676,null]],"1479510223377234":[[0.9457378830965617,0.6280053252900943,null],[0.29968810543055785,0.6572907027671258,null]],"1479649270603872":[],"1485983376024324":[[0.7698335990760192,0.6409366512101438,null]],"1493807495453961":[],"1503933797776035":[],"1507586557510184":[],"1526153925847560":[],"1526644485012786":[],"1526903691607662":[],"1531415762060842":[],"1531940155140947":[],"1533099125112846":[],"1547490957023264":[],"1548551026941868":[],"1550619426690125":[],"1554553479649915":[],"1560573605574046":[],"1579465080398346":[],"1621880408877798":[],"1628684669262976":[],"1642079667427179":[],"1644929466791193":[],"1665066454545157":[],"1665856994688538":[],"1679632769964162":[],"1688417845615915":[],"1708279679842703":[[0.9462482876138417,0.6519850461086084,null]],"1711104372904021":[],"1726281301363646":[],"1736795824019170":[],"1743265983784166":[[0.39343845166225333,0.6300363753399092,null]],"1743535723318388":[],"1743866716581524":[],"1762265868113165":[],"1769629404195835":[],"1779415216757575":[[0.8963378474565633,0.6469744991149612,null]],"1872865880061015":[],"1875441803044127":[],"1892190831489561":[],"1920855128875530":[],"1933608350737216":[],"1980881726130460":[[0.5582762343643185,0.5965172794337936,null],[0.18235903528396297,0.7148587194472441,null]],"2002394670457045":[[0.9616835102845476,0.7058177644912155,null]],"2039237576963899":[],"2055080831885811":[[0.7653392159053829,0.7122298586128143,null],[0.832893695338722,0.7547468732372489,null]],"2064297630816443":[[0.059470857403150224,0.5851085348855025,null]],"2080767922502123":[],"2088155352056225":[],"2185625018892068":[],"2216048919129703":[],"2221274008612179":[],"2243369796465669":[],"2392893427787498":[],"2511246802645139":[],"2641788216158646":[],"27177228765230843":[],"27260772950175386":[],"28215663224697246":[],"2962988900572350":[],"3015816262094291":[],"3140904756109373":[[0.847669357233338,0.5785735518060254,null]],"3296688647144785":[[0.2796097390360186,0.6693030721503049,null],[0.3173010350203889,0.6567633261001978,null]],"335801701340862":[],"3688080354666548":[[0.6726836988211788,0.5502044715604599,null],[0.5884391232626168,0.5791743706402792,null],[0.2997057398692108,0.6746016729415606,null]],"37046635288313572":[[0.4384492372973748,0.5972199522659719,null],[0.2531607838456425,0.5649843379775997,null]],"37230045973275479":[],"3860523807571425":[[0.7899790842887342,0.7467318447050739,null]],"3901990523265321":[],"4122786137987976":[],"427909170007735":[],"4426398164172326":[[0.2843445354672993,0.6607335479640698,null]],"4478163572471700":[],"5026082804197254":[],"740730485632754":[[0.7488230429840264,0.6900426739053614,null]],"749536038146240":[],"755126063369619":[],"790116040739014":[],"817455584410792":[],"820647700656490":[],"833524132783867":[],"838276004852617":[],"841136762387293":[],"866540066042247":[],"873103842125428":[],"881246267480927":[],"897403239302195":[],"917879701312494":[],"934293439634924":[],"945894851803282":[],"986609046276593":[],"986995307528492":[],"991739220159007":[],"992737779816905":[],"993081170303646":[],"994057612515361":[],"997653559315299":[]},"model":"Qwen/Qwen3-VL-32B-Instruct","n_panos":125,"n_uncached":0,"published_as":"Qwen/Qwen3-VL-32B-Instruct","signature":{"coord_space":"norm1000","max_edge":2048,"max_new_tokens":1024,"model_id":"Qwen/Qwen3-VL-32B-Instruct","prompt":"Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list. Respond with JSON only: a list of {\"bbox_2d\": [x1, y1, x2, y2], \"label\": \"curb ramp\"}.","provider":"qwen","source_max_edge":4096,"tile":true,"views":[[0.0,-30.0,90.0,90.0,1024,1024],[60.0,-30.0,90.0,90.0,1024,1024],[120.0,-30.0,90.0,90.0,1024,1024],[180.0,-30.0,90.0,90.0,1024,1024],[240.0,-30.0,90.0,90.0,1024,1024],[300.0,-30.0,90.0,90.0,1024,1024]]}}