From d704db996dfc26abf2da0706189e4d454928dd26 Mon Sep 17 00:00:00 2001 From: Jon Froehlich Date: Tue, 18 Aug 2026 06:41:13 -0700 Subject: [PATCH 1/7] One challenger registry, and freeze the witness pool the #46 pass was made under (#122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CHALLENGERS`, `SPARSE` and `DENSE` were three tuples kept in step by hand, and `silent_witness.py` iterated the roster directly with no way to point it anywhere else. That made adding a model to the benchmark quietly expensive in the one place nobody would look: a finished human pass. `rampnet/roster.py` is now the single table. It carries each model's spec, label, density class, the date it joined, and whether it is scored in the roster tables; `CHALLENGERS`, `SPARSE`, `DENSE` and the per-provider defaults all derive from it. The load-bearing part is `WITNESS_POOL_46`, written out literally rather than derived, and it is `silent_witness.py`'s new `--models` default. The hazard is not hypothetical — measured here: frozen pool (7 models) 59 unwitnessed, sparse-union excess +39.0 plus gemini-3.7-flash 58 unwitnessed, sparse-union excess +35.6 The one item that drops out is morgantown:356422300574178 at (0.09899, 0.54838). It is one of the 50 crops already tagged, and it carries one of only two `definition` verdicts in the whole pass. Adding an already-published leg to the roster would have orphaned it, changed the gallery's manifest digest, and moved the published lower bound from 0.0092 to 0.0088 recall points, with nothing raising a hand. Now a new challenger moves the comparison tables and leaves the human pass alone by construction. Also: - `silent_witness.py` takes `--models`, and derives its sparse/dense split from the pool it was given. An unregistered or unmeasured model now raises instead of being silently filed as dense, which is what `spec in SPARSE else "DENSE"` did. - The "union of the 5 sparse models" / "2 dense detectors" strings read their counts off the pool instead of having them baked in. - Both `silent_witness.json` and `fp_taxonomy.json` record the pool they ran over, so a verdict file can be matched to the pool that produced its items — the same reasoning as the detector signature inside each published detections file. - gemini-3.7-flash is registered as published-but-off-roster (#120). Its density is measured, not assumed: 1.90 boxes/pano over 2,109 panos, against gemini-3.6-flash's 2.34 and OWLv2's 72.77. Both committed artifacts regenerate with the `models` block as the only diff: every number is unchanged, and `unwitnessed` is identical in content and order. Co-Authored-By: Claude Opus 5 (1M context) --- analysis_out/fp_taxonomy.json | 35 ++++ analysis_out/silent_witness.json | 32 ++++ rampnet/roster.py | 246 +++++++++++++++++++++++++++++ scripts/analysis/fp_taxonomy.py | 57 +++---- scripts/analysis/silent_witness.py | 65 +++++--- 5 files changed, 379 insertions(+), 56 deletions(-) create mode 100644 rampnet/roster.py diff --git a/analysis_out/fp_taxonomy.json b/analysis_out/fp_taxonomy.json index bacd66aa..b1f6b05f 100644 --- a/analysis_out/fp_taxonomy.json +++ b/analysis_out/fp_taxonomy.json @@ -10,6 +10,41 @@ "paterson", "gainesville" ], + "models": { + "pool": null, + "specs": [ + "rampnet", + "gemini:gemini-3.6-flash", + "gemini:gemini-3.1-pro-preview", + "qwen:Qwen/Qwen3-VL-8B-Instruct", + "qwen:Qwen/Qwen3-VL-32B-Instruct", + "molmo:allenai/Molmo2-8B", + "owlv2", + "gdino" + ], + "labels": [ + "rampnet", + "gemini-3.6-flash", + "gemini-3.1-pro-preview", + "Qwen/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-32B-Instruct", + "allenai/Molmo2-8B", + "google/owlv2-large-patch14-ensemble", + "IDEA-Research/grounding-dino-base" + ], + "sparse": [ + "rampnet", + "gemini:gemini-3.6-flash", + "gemini:gemini-3.1-pro-preview", + "qwen:Qwen/Qwen3-VL-8B-Instruct", + "qwen:Qwen/Qwen3-VL-32B-Instruct", + "molmo:allenai/Molmo2-8B" + ], + "dense": [ + "owlv2", + "gdino" + ] + }, "per_model": { "rampnet @0.30": { "n_fp": 181, diff --git a/analysis_out/silent_witness.json b/analysis_out/silent_witness.json index af72774f..8d1dab7a 100644 --- a/analysis_out/silent_witness.json +++ b/analysis_out/silent_witness.json @@ -2,6 +2,38 @@ "bucket": "silent", "threshold": 0.3, "n": 128, + "models": { + "pool": "WITNESS_POOL_46", + "specs": [ + "gemini:gemini-3.6-flash", + "gemini:gemini-3.1-pro-preview", + "qwen:Qwen/Qwen3-VL-8B-Instruct", + "qwen:Qwen/Qwen3-VL-32B-Instruct", + "molmo:allenai/Molmo2-8B", + "owlv2", + "gdino" + ], + "labels": [ + "gemini-3.6-flash", + "gemini-3.1-pro-preview", + "Qwen/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-32B-Instruct", + "allenai/Molmo2-8B", + "google/owlv2-large-patch14-ensemble", + "IDEA-Research/grounding-dino-base" + ], + "sparse": [ + "gemini:gemini-3.6-flash", + "gemini:gemini-3.1-pro-preview", + "qwen:Qwen/Qwen3-VL-8B-Instruct", + "qwen:Qwen/Qwen3-VL-32B-Instruct", + "molmo:allenai/Molmo2-8B" + ], + "dense": [ + "owlv2", + "gdino" + ] + }, "per_model": { "gemini-3.6-flash": { "witnessed": 33, diff --git a/rampnet/roster.py b/rampnet/roster.py new file mode 100644 index 00000000..5a7e9ec2 --- /dev/null +++ b/rampnet/roster.py @@ -0,0 +1,246 @@ +"""The challenger roster: who is in the model comparison, since when, and how dense. + +One table, ``ROSTER``, replaces three tuples that used to be kept in step by hand — +``CHALLENGERS``, ``SPARSE`` and ``DENSE`` — plus the per-provider default model ids +that were copied into four argument parsers. Adding a model to the benchmark is now +one entry here. + +**Why this lives in the package rather than in an analysis script.** Both +``scripts/analysis/*`` and ``scripts/model_comparison/*`` need it, it must import +without torch so a fresh clone can score published detections on a laptop, and the +CPU-only test suite has to cover it. It follows the splits registry in +``scripts/analysis/miss_decomposition.py``, which eight scripts already import. + +**The one thing in here that must not move: ``WITNESS_POOL_46``.** It is written out +literally rather than derived from ``ROSTER`` precisely so that adding a challenger +cannot touch it. See the comment on it. + +Three properties of an entry are worth stating because they are easy to get wrong: + +* ``density`` is **evidence, not configuration** — it comes from the measured + boxes-per-panorama in ``docs/model_comparison.md`` (sparse models emit 1-4, the + open-vocabulary detectors 55-88). A new arm's density is unknown until it has been + run, so it is ``None``, and ``density_of`` raises rather than guessing. The old code + silently treated an unclassified model as dense. +* ``standing`` separates *scored in the roster tables* from *published but not scored*. + A leg can be run, verified and committed long before its write-up lands + (``gemini-3.7-flash``, #120); that is an omission of a write-up, not of a run, and + the distinction is data here rather than prose in a doc. +* ``label`` is the resolved model id used for result-table rows and for + ``benchmark/model_detections/