From 072f6c259352faeb98e0e2b092a360d50cc20561 Mon Sep 17 00:00:00 2001 From: gavinbee <29419542+gavinbee@users.noreply.github.com> Date: Sat, 30 May 2026 00:13:53 -0400 Subject: [PATCH] Constrain vision output with a JSON Schema so per-field confidences survive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vision call used format="json", which only guarantees valid JSON, not our nested {value, confidence} shape. Smaller models (qwen2.5vl:3b) collapse each field to a bare scalar, so _parse_field defaulted every confidence to 0.5 — silently destroying the signal that drives low-confidence review. Build a JSON Schema from the canonical src/schema constants and pass it to Ollama's format= so decoding is constrained to the object shape. Also detect an all-flat-scalar page and warn instead of masking it. Document that confidence quality is model-dependent (3b unreliable, 7b is the floor). Verified: 7b output unchanged (no regression); 3b now emits the object shape. Closes #45 Co-Authored-By: Claude Opus 4.7 --- docs/models.md | 13 ++++ src/vision_extract.py | 118 ++++++++++++++++++++++++++++++++++- tests/test_vision_extract.py | 97 +++++++++++++++++++++++++++- 3 files changed, 223 insertions(+), 5 deletions(-) diff --git a/docs/models.md b/docs/models.md index 6b5494a..e995c2b 100644 --- a/docs/models.md +++ b/docs/models.md @@ -40,6 +40,19 @@ python main.py scan.pdf --vision-model qwen2.5vl:32b # force the large tier `--edit-model` similarly overrides the text model. +## Per-field confidence and the model floor + +Every extracted value carries a model-reported `confidence` in `[0, 1]` (see [output-schema.md](output-schema.md)). To make the model emit those reliably, the vision call constrains decoding with a **JSON Schema** (passed to Ollama's `format=`), built from the canonical field list in `src/schema.py`. Without that constraint, smaller models flatten each field to a bare scalar and the confidence is lost — the parser then falls back to a neutral `0.5` and logs a warning (background: gh #45). + +Confidence quality is **model-dependent**, and the schema can't fix that — it only guarantees the *shape*, not honest *numbers*: + +| Model | Confidence behavior | +|---|---| +| `qwen2.5vl:3b` | Unreliable. Tends to stamp a single constant value on every field — no usable signal. Treat 3B output as "values only," not confidence-scored. | +| `qwen2.5vl:7b` _(default)_ | Coarse but useful. In practice a two-level signal — high (~0.9) on what it read confidently, lower (~0.6) on genuinely ambiguous cells (e.g. a crossed-out `successful`). Low values land on the rows worth a human's review. | + +**Practical implication:** the confidence-driven features (low-confidence surfacing in `--interactive`, the `row_confidence` composite) are meaningful from the **7B tier up**. On the 3B tier, prefer `--review-all` over relying on confidence thresholds. + ## Why this family, not something else Reasoning is in [`design/0001-initial-design.md`](design/0001-initial-design.md#models--concrete-pinning). Short version: Qwen2.5-VL is currently the strongest open vision-language family on OCRBench v2 / DocVQA under 15 B params, has good handwriting performance, and the same model family covers both the vision and text-edit roles so users only deal with one ecosystem. diff --git a/src/vision_extract.py b/src/vision_extract.py index 65d51ca..5c80d25 100644 --- a/src/vision_extract.py +++ b/src/vision_extract.py @@ -12,8 +12,10 @@ detected template's ``vision_prompt_addendum`` plus the source filename (so the model can resolve ``session_number`` from the form text, the filename, or both). -* **The model call** — ``client.generate(..., format="json")`` at - temperature 0, with the page PNG as the image. +* **The model call** — ``client.generate(..., format=)`` + at temperature 0, with the page PNG as the image. The schema (built + from ``src.schema``) constrains decoding to the ``{value, confidence}`` + object shape so smaller models can't drop the per-field confidence. * **Parsing + validation** — turn the model's JSON into typed ``FieldValue`` objects, coercing ``successful`` to bool/null and capturing per-field confidence. One retry on a structural parse @@ -145,6 +147,77 @@ def field(extra: str = "") -> str: }}""" +def _field_schema(value_type: str, *, extra: Optional[dict] = None) -> dict: + """A JSON-Schema object for one ``{value, confidence}`` field. + + ``value`` is the given type or null; ``confidence`` is a number. ``extra`` + adds further properties (e.g. ``source`` / ``rationale``). + """ + props: dict[str, Any] = { + "value": {"type": [value_type, "null"]}, + "confidence": {"type": "number"}, + } + if extra: + props.update(extra) + return { + "type": "object", + "properties": props, + "required": ["value", "confidence"], + } + + +def _build_json_schema() -> dict: + """JSON Schema for the page-extraction response. + + Passed to Ollama's ``format=`` so decoding is constrained to the + ``{value, confidence}`` object shape. Without this, smaller models + (e.g. ``qwen2.5vl:3b``) collapse each field to a bare scalar — which + our parser then has to default to ``_DEFAULT_CONFIDENCE``, silently + losing the per-field confidence signal (see gh #45). Built from the + canonical ``src.schema`` constants so it can't drift from the prompt + skeleton or the parser. + """ + meet_props = {k: _field_schema("string") for k in s.MEET_FIELDS} + + session_props: dict[str, Any] = { + k: _field_schema("string") + for k in (s.COMPETITION_COORDINATOR, s.CC_LEVEL, s.DATE_SESSION) + } + session_props[s.SESSION_NUMBER] = _field_schema( + "integer", + extra={"source": { + "type": "string", + "enum": ["form", "filename", "form+filename", "unknown"], + }}, + ) + + row_props = { + k: _field_schema("string") + for k in (s.OFFICIAL_NAME, s.CLUB, s.POSITION, s.LANE_NUMBER, + s.TIMES_WORKED_POSITION, s.MENTOR, s.LEVEL) + } + row_props[s.SUCCESSFUL] = _field_schema( + "boolean", extra={"rationale": {"type": "string"}}, + ) + + return { + "type": "object", + "properties": { + "meet": {"type": "object", "properties": meet_props}, + "session": {"type": "object", "properties": session_props}, + "rows": { + "type": "array", + "items": {"type": "object", "properties": row_props}, + }, + }, + "required": ["meet", "session", "rows"], + } + + +# Built once at import — the canonical field list is static. +_VISION_FORMAT = _build_json_schema() + + _BASE_INSTRUCTIONS = """\ You are extracting structured data from one page of a swimming On-Deck \ Evaluation form. The page image is attached. @@ -421,7 +494,7 @@ def _generate( model=model, prompt=prompt, images=[png_bytes], - format="json", + format=_VISION_FORMAT, options={"temperature": 0}, ) except ollama.ResponseError as exc: @@ -471,8 +544,47 @@ def _is_structural(parsed: Any) -> bool: # --------------------------------------------------------------------------- +def _response_is_flat(raw: dict) -> bool: + """True if the model returned bare scalars instead of ``{value, ...}`` + objects for every field it filled in. + + This is the failure mode behind gh #45: a model that ignores the + requested object shape gives us values with no confidence, which the + parser then has to default. We detect it so the caller can warn rather + than silently emit a page of ``_DEFAULT_CONFIDENCE``. A response is + "flat" only if it carried at least one value and *none* of them used + the object form. + """ + saw_value = False + sections: list[dict] = [] + for key in ("meet", "session"): + section = raw.get(key) + if isinstance(section, dict): + sections.append(section) + for row in raw.get("rows") or []: + if isinstance(row, dict): + sections.append(row) + for section in sections: + for value in section.values(): + if value is None: + continue + if isinstance(value, dict): + return False # at least one proper object → not flat + saw_value = True + return saw_value + + def _parse_response(raw: dict, page_number: int) -> PageExtraction: """Turn one page's raw model JSON into a ``PageExtraction``.""" + if _response_is_flat(raw): + log.warning( + "Page %d: vision model returned flat values with no per-field " + "confidence — defaulting all confidences to %.2f. This usually " + "means the model ignored the requested JSON shape (common on " + "smaller models like qwen2.5vl:3b); a larger model gives real " + "confidences. See gh #45.", + page_number, _DEFAULT_CONFIDENCE, + ) result = PageExtraction(page_number=page_number) meet_obj = raw.get("meet") or {} diff --git a/tests/test_vision_extract.py b/tests/test_vision_extract.py index f58f052..6d9ccc5 100644 --- a/tests/test_vision_extract.py +++ b/tests/test_vision_extract.py @@ -211,6 +211,93 @@ def test_bare_scalar_field_gets_default_confidence(self): assert fv.confidence == 0.5 +# --------------------------------------------------------------------------- +# JSON Schema constraining the response shape (gh #45) +# --------------------------------------------------------------------------- + + +class TestJsonSchema: + def setup_method(self): + self.schema = vision_extract._build_json_schema() + + def test_top_level_sections(self): + assert self.schema["type"] == "object" + assert set(self.schema["properties"]) == {"meet", "session", "rows"} + + def test_built_from_canonical_field_names(self): + meet = self.schema["properties"]["meet"]["properties"] + assert set(meet) == set(s.MEET_FIELDS) + row = self.schema["properties"]["rows"]["items"]["properties"] + for fld in s.ROW_FIELDS: + assert fld in row + + def test_each_field_requires_value_and_confidence(self): + name = self.schema["properties"]["rows"]["items"]["properties"][s.OFFICIAL_NAME] + assert name["required"] == ["value", "confidence"] + assert name["properties"]["confidence"]["type"] == "number" + # value is nullable so blank fields are representable. + assert "null" in name["properties"]["value"]["type"] + + def test_session_number_is_integer_with_source_enum(self): + sn = self.schema["properties"]["session"]["properties"][s.SESSION_NUMBER] + assert "integer" in sn["properties"]["value"]["type"] + assert set(sn["properties"]["source"]["enum"]) == { + "form", "filename", "form+filename", "unknown", + } + + def test_successful_is_boolean_with_rationale(self): + suc = self.schema["properties"]["rows"]["items"]["properties"][s.SUCCESSFUL] + assert "boolean" in suc["properties"]["value"]["type"] + assert "rationale" in suc["properties"] + + +# --------------------------------------------------------------------------- +# Flat-response detection + warning (gh #45) +# --------------------------------------------------------------------------- + + +class TestFlatResponseWarning: + def test_detects_all_flat_response(self): + raw = { + "meet": {"competition_name": "X", "host_club": "Y"}, + "rows": [{"official_name": "Allison Hill", "club": "HEX"}], + } + assert vision_extract._response_is_flat(raw) is True + + def test_object_shaped_response_not_flat(self): + assert vision_extract._response_is_flat(_good_response(n_rows=1)) is False + + def test_empty_response_not_flat(self): + # No values at all isn't "flat" — there's nothing being masked. + assert vision_extract._response_is_flat({"rows": []}) is False + + def test_mixed_response_not_flat(self): + # One proper object is enough to show the model understood the shape. + raw = { + "meet": {"competition_name": {"value": "X", "confidence": 0.9}, + "host_club": "Y"}, + "rows": [], + } + assert vision_extract._response_is_flat(raw) is False + + def test_warns_when_parsing_flat_page(self, caplog): + raw = { + "meet": {"competition_name": "X"}, + "rows": [{"official_name": "Allison Hill"}], + } + with caplog.at_level("WARNING", logger="src.vision_extract"): + vision_extract._parse_response(raw, page_number=2) + assert any( + "flat values" in r.message and "gh #45" in r.message + for r in caplog.records + ) + + def test_no_warning_on_object_shaped_page(self, caplog): + with caplog.at_level("WARNING", logger="src.vision_extract"): + vision_extract._parse_response(_good_response(n_rows=1), page_number=1) + assert not any("flat values" in r.message for r in caplog.records) + + # --------------------------------------------------------------------------- # Model call + retry # --------------------------------------------------------------------------- @@ -225,13 +312,19 @@ def test_single_call_when_valid(self): assert client.generate.call_count == 1 assert len(page.rows) == 1 - def test_passes_image_and_json_format(self): + def test_passes_image_and_schema_format(self): client = _client_returning(_good_response(n_rows=1)) extract_page(b"PNGBYTES", ONTARIO, "x.pdf", 1, client=client, model="qwen2.5vl:7b") _, kwargs = client.generate.call_args assert kwargs["model"] == "qwen2.5vl:7b" assert kwargs["images"] == [b"PNGBYTES"] - assert kwargs["format"] == "json" + # We constrain decoding with a JSON Schema (not the loose "json" + # string) so the model must emit {value, confidence} objects — see + # gh #45. The schema is the dict built from the canonical fields. + fmt = kwargs["format"] + assert isinstance(fmt, dict) + assert fmt["type"] == "object" + assert set(fmt["properties"]) == {"meet", "session", "rows"} assert kwargs["options"]["temperature"] == 0 def test_retries_once_on_bad_json_then_succeeds(self):