From d7257043912f5658188c695ce6135fa2e6e7f7ac Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 10:47:36 +0800 Subject: [PATCH 1/7] fix(parity): derive zip version from metadata id (was hardcoded -1.0-) --- src/gpu/modal_export.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/gpu/modal_export.py b/src/gpu/modal_export.py index 0b5dc23..e3c79ac 100644 --- a/src/gpu/modal_export.py +++ b/src/gpu/modal_export.py @@ -21,6 +21,8 @@ from pathlib import Path +import re + import modal REPO_ROOT = Path(__file__).resolve().parent.parent.parent @@ -248,9 +250,11 @@ def parity_model(model_id: str, precisions: list[str], limit: int = 0) -> dict[s reference = reference_decode(model, [src for src, _ in pairs], max_len=128) out_dir = Path("/outputs/imf") / model_id + meta_path = Path("/root/interscript-ml", spec["metadata"]) + mid = re.search(r"^id:\s*(\S+)", meta_path.read_text(encoding="utf-8"), re.M).group(1) reports: dict[str, str] = {} for precision in precisions: - zip_path = out_dir / f"{model_id}-1.0-{precision}.zip" + zip_path = out_dir / f"{mid}-{precision}.zip" report = run_parity(model, zip_path, pairs, max_len=128, reference=reference) reports[precision] = ( f"samples={report.samples} cer_ref={report.cer_reference}pp " From 7905d5717087de9b09788bb7f22f10a6f2b040a6 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 11:30:19 +0800 Subject: [PATCH 2/7] feat(distill): heb-diac-small-s46 spec (re-distill client tier from s46) --- src/gpu/modal_distill.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 136fa33..8da9ea2 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -217,6 +217,15 @@ "val": "hebrew-v4/val.jsonl", "out": "rababa_hebrew_distill_small/run-001", }, + "heb-diac-small-s46": { + # re-distill the client tier from the s46 teacher (greedy 16.44 + # vs s43's 29.0) — the 1.0 student tracked its teacher's greedy + "teacher": "rababa_hebrew/run-s46-phonikud-plus/run-002-gold-ft/best", + "student_init": "google/byt5-small", + "train": "hebrew-v4/train.jsonl", + "val": "hebrew-v4/val.jsonl", + "out": "rababa_hebrew_distill_small/run-002-s46", + }, } app = modal.App("interscript-ml-distill", image=IMAGE) From 89024e5787f6025744eb4097bf633129dea63c81 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 12:04:44 +0800 Subject: [PATCH 3/7] feat(imf): int4 export tier (MatMulNBits blockwise 4-bit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen3.8-style quantization-ladder move for the client tier: int4 halves int8 again (~246MB -> ~130MB expected for ByT5-small) at some quality cost, measured by the parity gate (int4 limit 3.0pp). Client crystals only — the Ruby gem's bundled ORT cannot execute MatMulNBits; int4 zips must fail loudly on old runtimes, never fall back silently. --- docs/imf-v1.md | 2 +- src/imf/export.py | 24 ++++++++++++++++++++++++ src/imf/parity.py | 2 +- src/imf/schema.py | 4 ++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/imf-v1.md b/docs/imf-v1.md index 75fbeeb..8c7a72a 100644 --- a/docs/imf-v1.md +++ b/docs/imf-v1.md @@ -51,7 +51,7 @@ model.zip | `license` | str | non-empty (strict gate) | | `trained_from` | str | repo + run/checkpoint id | | `metrics` | list | `{name, value, protocol, source}`; `source` must be a `RESULTS.md#anchor` (strict gate) | -| `parity` | map? | `{samples, cer_delta}`; strict gate: samples >= 500, cer_delta <= 0.2pp fp32 / 1.0pp fp16 / 2.0pp int8 | +| `parity` | map? | `{samples, cer_delta}`; strict gate: samples >= 500, cer_delta <= 0.2pp fp32 / 1.0pp fp16 / 2.0pp int8 / 3.0pp int4 | | `sha256` | map | every `*.onnx` member -> hex digest; no dangling entries | The `id` does not encode precision: `khm-latn-1.0-fp16.zip` and diff --git a/src/imf/export.py b/src/imf/export.py index 35ad91d..12bf100 100644 --- a/src/imf/export.py +++ b/src/imf/export.py @@ -275,6 +275,28 @@ def quantize_int8(src: Path | str, dst: Path | str) -> Path: return Path(dst) +def quantize_int4(src: Path | str, dst: Path | str, block_size: int = 64) -> Path: + """fp32 -> 4-bit blockwise MatMul (MatMulNBits, com.microsoft domain). + + Halves int8 again at some quality cost; meant for the browser/edge + client tier. NOTE: old runtimes (the Ruby gem's bundled ORT) cannot + execute MatMulNBits — int4 zips are client-crystal territory and + their loaders should fail loudly rather than silently fall back. + """ + import onnx + from onnxruntime.quantization.matmul_nbits_quantizer import ( + MatMulNBitsQuantizer, + ) + + model = onnx.load(str(src)) + quant = MatMulNBitsQuantizer( + model=model, block_size=block_size, is_symmetric=True + ) + quant.process() + quant.model.save_model_to_file(str(dst)) + return Path(dst) + + def onnx_greedy_plain(encoder_sess, decoder_sess, text: str, max_len: int = 256) -> list[int]: """Greedy decode over ONNX sessions (plain decoder). Self-check helper. @@ -380,6 +402,8 @@ def export_zips( dst.write_bytes(src.read_bytes()) elif precision == "int8": quantize_int8(graphs[name], dst) + elif precision == "int4": + quantize_int4(graphs[name], dst) else: raise ValueError(f"unknown precision {precision!r}") meta = replace(metadata, precision=precision) diff --git a/src/imf/parity.py b/src/imf/parity.py index d490167..b0567c2 100644 --- a/src/imf/parity.py +++ b/src/imf/parity.py @@ -1,5 +1,5 @@ """WO03 parity gate: ONNX greedy vs the torch reference, precision-aware -CER-delta limits (0.2pp fp32, 1.0pp fp16, 2.0pp int8). +CER-delta limits (0.2pp fp32, 1.0pp fp16, 2.0pp int8, 3.0pp int4). The reference is the transformers decoder loop itself (the exact math the export wraps) rather than ``model.generate`` — generate's behavior is diff --git a/src/imf/schema.py b/src/imf/schema.py index b5d2723..7d9fd1f 100644 --- a/src/imf/schema.py +++ b/src/imf/schema.py @@ -18,7 +18,7 @@ TASKS = frozenset({"g2p", "diacritization", "translit"}) DECODERS = frozenset({"plain", "kv"}) -PRECISIONS = frozenset({"fp32", "fp16", "int8"}) +PRECISIONS = frozenset({"fp32", "fp16", "int8", "int4"}) # The Ruby onnxruntime gem bundles an old ORT that cannot load opset > 14. # Opset is pinned to 14 and validated against the actual graphs on load. @@ -71,7 +71,7 @@ class Parity: # Quantization widens the torch-vs-ONNX gap: measured deltas on khm # were ~0.43pp (fp16) and ~0.84pp (int8) against the 0.2pp fp32 bar, # so the gate is keyed on the declared precision. - MAX_CER_DELTA_BY_PRECISION = {"fp32": 0.2, "fp16": 1.0, "int8": 2.0} + MAX_CER_DELTA_BY_PRECISION = {"fp32": 0.2, "fp16": 1.0, "int8": 2.0, "int4": 3.0} MIN_SAMPLES = 500 @classmethod From e2d2b17e86f859cf4b117d6f9e7699c9e293c886 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 12:05:16 +0800 Subject: [PATCH 4/7] fix(eval): evaluate_der resolves the student via out_volume, not teacher_volume ara-diac-tiny writes its checkpoints to the secryst volume; the eval hardcoded the teacher's volume and from_pretrained fell through to the hub (repo_type error). --- src/gpu/modal_distill.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 8da9ea2..a878091 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -1076,7 +1076,10 @@ def evaluate_der(spec_id: str, window: int = 1400, limit: int = 0) -> dict: } teacher_path = (spec["teacher"] if spec.get("teacher_is_hub") else str(Path(vol_map[spec.get("teacher_volume", "rababa")]) / spec["teacher"])) - student_path = Path(vol_map[spec.get("teacher_volume", "rababa")]) / spec["out"] / "best" + student_path = ( + Path(vol_map[spec.get("out_volume", spec.get("teacher_volume", "rababa"))]) + / spec["out"] / "best" + ) tok = AutoTokenizer.from_pretrained("google/byt5-small") teacher = AutoModelForSeq2SeqLM.from_pretrained(teacher_path).to("cuda").eval() From 4ecdf9c251695a741b6898a1c6153e4f198ed073 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 12:08:45 +0800 Subject: [PATCH 5/7] fix(eval): reset generation max_length for windowed DER eval Custom students inherit T5's default max_length=20; 1400-byte windows clamp max_new_tokens to zero and generate raises. --- src/gpu/modal_distill.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index a878091..c625d5b 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -1084,6 +1084,10 @@ def evaluate_der(spec_id: str, window: int = 1400, limit: int = 0) -> dict: tok = AutoTokenizer.from_pretrained("google/byt5-small") teacher = AutoModelForSeq2SeqLM.from_pretrained(teacher_path).to("cuda").eval() student = AutoModelForSeq2SeqLM.from_pretrained(str(student_path)).to("cuda").eval() + # custom students carry T5's default max_length=20; windowed inputs + # run to 1400 bytes, clamping max_new_tokens to zero + for m in (teacher, student): + m.generation_config.max_length = 100_000 diac = re.compile("[ً-ٰٟۖ-ۭ]") table = pq.read_table("/opt/rababa/data/sadeed-diac-25/train.parquet") From 190da03e7d2194598f26cf1de00b7c99aa6d0b7b Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 12:15:16 +0800 Subject: [PATCH 6/7] =?UTF-8?q?fix(eval):=20eval=5Fder=20passed=20limit=20?= =?UTF-8?q?into=20the=20window=20slot=20=E2=80=94=20window=3D0=20killed=20?= =?UTF-8?q?generate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gpu/modal_distill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index c625d5b..c86c86e 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -1428,4 +1428,4 @@ def mk(spec: str = "tha-g2p-tiny-mk", epochs: int = 3) -> None: @app.local_entrypoint() def eval_der(spec: str = "ara-diac-small", limit: int = 0) -> None: - print(evaluate_der.remote(spec, limit)) + print(evaluate_der.remote(spec, limit=limit)) From 90ef82f526bc4c21efa6a713c0b8d0b4e3703686 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 13:11:34 +0800 Subject: [PATCH 7/7] fix(eval): prettytable in the distill image (sadeed evaluator dep) --- src/gpu/modal_distill.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index c86c86e..0a7c6a8 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -35,6 +35,7 @@ "numpy>=1.26", # arabic gate harness: Misraj evaluator + SadeedDiac-25 parquet "pyarabic", + "prettytable", "pandas", "pyarrow", )