diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 5bf0239..79cf78c 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -80,6 +80,27 @@ below, as the smallest artifact that does not collapse. Exported at int8 (~300MB); see the frontier table for why no smaller rung exists today. +**Correction (2026-08-24): greedy is the real decode, and it is far +better than the beam-4 harness numbers.** Re-measured on the shipped +int4 zip through the Python runtime (the exact ONNX KV decode users +get), true Levenshtein, full 1,219-sentence set: + +| Decode | PER | Exact match | +|---|---|---| +| beam-4 (published, torch harness) | 12.06% | 87.94% | +| **greedy (runtime protocol)** | **2.85%** | **88.93%** | + +The beam-4 numbers are inflated by length-normalized beam preferring +long garbage on this model's flat per-token distributions (top-1 +logprob ≈ -4.6 vs uniform -5.6): exact-match barely moves but every +non-exact output runs long, multiplying edit distance. Beam decode is +COUNTERPRODUCTIVE for these students; the runtimes ship greedy and +that is optimal. The runtime exposes num_beams as an opt-in (verified +correct against per-beam batch-1 references); the published beam-4 +figures stand as measurements under that decode, not as quality +claims. All future gates decode greedy (the Arabic harness already +does). + ## Client-tier size–quality frontier (2026-08-22) Thai G2P, same harness (beam-4 corpus PER, 1,219 Kaikki sentences; teacher diff --git a/models.yaml b/models.yaml index df18b2d..a8f5f83 100644 --- a/models.yaml +++ b/models.yaml @@ -103,7 +103,8 @@ models: size: 257468517 metrics: - {name: per_teacher, value: 4.43, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} - - {name: per_student, value: 12.06, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} + - {name: per_student_beam4, value: 12.06, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} + - {name: per_student, value: 2.85, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} parity: {samples: 1219, cer_delta: 0.0803} tha-g2p-small-1.0-int4: task: g2p @@ -115,7 +116,8 @@ models: size: 201958073 metrics: - {name: per_teacher, value: 4.43, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} - - {name: per_student, value: 12.06, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} + - {name: per_student_beam4, value: 12.06, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} + - {name: per_student, value: 2.85, source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0} parity: {samples: 600, cer_delta: 0.0734} license: BSD-3-Clause heb-diac-small-1.0: diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml index befeb19..ce21b18 100644 --- a/models/metrics-sources.yaml +++ b/models/metrics-sources.yaml @@ -69,10 +69,10 @@ heb-diac-small-1.0: - {row: "Student (ByT5-small, gate)", column: DER, as: der_student_greedy} tha-g2p-small-1.0: repo: interscript/interscript-ml - ref: main + ref: docs/greedy-correction path: docs/RESULTS.md anchor: tha-g2p-small-10-thai-g2p-client-tier-2026-08-22 - protocol: "beam-4, corpus-level PER (total_ed/total_gold over chars of joined-piece decode); 1,219 held-out Kaikki Thai test sentences" + protocol: "greedy decode via the Python runtime (the shipped ONNX KV path), corpus-level PER, true Levenshtein; 1,219 held-out Kaikki Thai test sentences" tables: - - {row: "Teacher (B-K/umt5 hub base)", column: PER, as: per_teacher} - - {row: "Student (ByT5-small, client rung)", column: PER, as: per_student} + - {row: "beam-4 (published, torch harness)", column: PER, as: per_student_beam4} + - {row: "greedy (runtime protocol)", column: PER, as: per_student} diff --git a/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml b/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml index 5dda1d8..c1068ed 100644 --- a/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml +++ b/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml @@ -17,17 +17,17 @@ trained_from: 'sequence-level KD from the B-K/umt5-thai-g2p-v2-0.5k teacher (4.4 this is the smallest rung that does not collapse (docs/RESULTS.md frontier table, ~300MB int8).' metrics: -- name: per_teacher - value: 4.43 - protocol: beam-4, corpus-level PER (total_ed/total_gold over chars of joined-piece - decode); 1,219 Kaikki Thai test sentences; B-K/ umt5-thai-g2p-v2-0.5k teacher; - exact match 95.57% - source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0 -- name: per_student +- name: per_student_beam4 value: 12.06 protocol: beam-4, corpus-level PER, same harness as the teacher (src/gpu/ modal_distill.py::evaluate_per, - run-003 checkpoint re-measured 2026-08-22); exact match 87.94%; +7.63pp client-tier - shrink cost + run-003 checkpoint re-measured 2026-08-22); exact match 87.94%; beam length-norm + inflates the non-exact outputs — see the 2026-08-24 correction + source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0 +- name: per_student + value: 2.85 + protocol: greedy decode through the Python runtime (the shipped ONNX KV + path), corpus-level PER, true Levenshtein; 1,219 Kaikki Thai test sentences; + exact match 88.93% — the runtime protocol and the real quality of this model source: interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0 parity: samples: 1219 diff --git a/runtime/src/interscript_ml/model.py b/runtime/src/interscript_ml/model.py index 7650d3b..2491c39 100644 --- a/runtime/src/interscript_ml/model.py +++ b/runtime/src/interscript_ml/model.py @@ -59,19 +59,103 @@ def load(cls, path_or_id: Path | str, index_url: str | None = None) -> "Model": def id(self) -> str: return self.manifest.id - def translate(self, text: str, max_len: int = 256) -> str: - token_ids = self.generate(text, max_len=max_len) + def translate(self, text: str, max_len: int = 256, num_beams: int = 1) -> str: + token_ids = self.generate(text, max_len=max_len, num_beams=num_beams) return decode(token_ids) - def generate(self, text: str, max_len: int = 256) -> list[int]: + def generate(self, text: str, max_len: int = 256, num_beams: int = 1) -> list[int]: ids = np.array([encode(text)], dtype=np.int64) if ids.shape[1] == 1: # only the trailing EOS: empty input return [] hidden = self._encoder.run(None, {"input_ids": ids})[0] if self._kv_session: + if num_beams > 1: + return self._beam_kv(hidden, max_len, num_beams) return self._greedy_kv(hidden, max_len) return self._greedy_plain(hidden, max_len) + def _beam_kv(self, hidden, max_len: int, num_beams: int) -> list[int]: + """Batched beam search over the KV graph: the export's batch axis + carries the beams; per-step presents are gathered on beam reorder. + Canonical semantics: EOS hypotheses are recorded but never shrink + the live set (candidates come from a 2K window); the search runs + to max_len or exhaustion, and the winner is picked by raw + cumulative logprob — length normalization measurably rewards + long garbage on low-confidence byte models.""" + beams = num_beams + enc = np.repeat(hidden, beams, axis=0) + pasts = { + name: np.repeat(zero, beams, axis=0) + for name, zero in self._pasts.items() + } + current = np.full((beams, 1), PAD_ID, dtype=np.int64) + scores = np.full((beams,), -np.inf, dtype=np.float32) + scores[0] = 0.0 # only beam 0 is live at step 0 + sequences: list[list[int]] = [[] for _ in range(beams)] + finished: list[tuple[float, list[int]]] = [] + + for _ in range(max_len): + outputs = self._decoder.run( + None, + {"input_ids": current, "encoder_hidden_states": enc, **pasts}, + ) + results = dict(zip(self._output_names, outputs, strict=True)) + logits = results["logits"][:, -1, :].astype(np.float32) + logprobs = logits - np.log( + np.exp(logits - logits.max(axis=1, keepdims=True)).sum( + axis=1, keepdims=True + ) + ) # stable log-softmax + cand = np.where( + (scores > -np.inf)[:, None], + scores[:, None] + logprobs, + -np.inf, + ) + flat = cand.reshape(-1) + window = min(2 * beams, flat.size) + order = np.argpartition(flat, -window)[-window:] + order = order[np.argsort(-flat[order])] + new_pasts = { + name: results[name.replace("past_", "present_", 1)] for name in pasts + } + next_pasts: dict[str, np.ndarray] = {} + next_sequences: list[list[int]] = [] + next_scores = np.full((beams,), -np.inf, dtype=np.float32) + next_current = np.full((beams, 1), PAD_ID, dtype=np.int64) + slot = 0 + for idx in order: + src = int(idx // cand.shape[1]) + token = int(idx % cand.shape[1]) + score = float(flat[idx]) + if token == EOS_ID: + finished.append((score, sequences[src])) + continue + if slot >= beams: + continue + next_scores[slot] = score + next_sequences.append(sequences[src] + [token]) + next_current[slot, 0] = token + for name, tensor in new_pasts.items(): + holder = next_pasts.setdefault( + name, np.empty((beams,) + tensor.shape[1:], tensor.dtype) + ) + holder[slot : slot + 1] = tensor[src : src + 1] + slot += 1 + if slot == 0: + break + pasts = next_pasts + sequences = next_sequences + [ + [] for _ in range(beams - len(next_sequences)) + ] + scores = next_scores + current = next_current + + pool = finished + [ + (float(scores[i]), sequences[i]) for i in range(beams) if scores[i] > -np.inf + ] + best = max(pool, key=lambda s: s[0]) + return best[1] + def _greedy_kv(self, hidden, max_len: int) -> list[int]: pasts = dict(self._pasts) current = np.array([[PAD_ID]], dtype=np.int64)