From 1a943556e66170e2f1691b8754da01de9de9800f Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 23 Aug 2026 19:57:53 +0800 Subject: [PATCH 1/6] feat: IMF inference endpoint on Modal (parity-verified ONNX kv decode, X-API-Key auth, /infer + /health) --- src/api/inference.py | 141 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/api/inference.py diff --git a/src/api/inference.py b/src/api/inference.py new file mode 100644 index 0000000..cf2df7b --- /dev/null +++ b/src/api/inference.py @@ -0,0 +1,141 @@ +"""IMF v1 inference endpoint for api.interscript.org (Modal, CPU). + +Serves the shipped models from the secryst-models volume using the +exact ONNX kv decode the WO03 parity gate verified. Cold start loads +the fp32 zip (~30-60s); sessions are cached per container. + + modal deploy src/api/inference.py + +Auth: X-API-Key header must match the `api-inference-key` secret. +""" + +from pathlib import Path + +import modal + +models_volume = modal.Volume.from_name("secryst-models") + +image = ( + modal.Image.debian_slim(python_version="3.11") + .pip_install("onnxruntime==1.23.2", "pyyaml>=6.0", "fastapi>=0.115") + .add_local_dir(str(Path(__file__).resolve().parent.parent), "/root/interscript-ml", copy=True) + .workdir("/root/interscript-ml") + .env({"IMAGE_REV": "5"}) +) + +app = modal.App("interscript-inference", image=image) + +MAX_INPUT_BYTES = 4000 +MAX_OUTPUT_TOKENS = 8192 +ALLOWED_TASKS = ("diacritization", "g2p") + +_sessions: dict[str, tuple] = {} + + +def _zip_path(model_id: str) -> Path: + family = model_id.rsplit("-", 1)[0] + p = Path("/v/imf") / family / f"{model_id}-fp32.zip" + if not p.exists(): + raise KeyError(model_id) + return p + + +def _get_sessions(model_id: str) -> tuple: + import sys + + sys.path.insert(0, "/root/interscript-ml/src") + from imf.parity import _sessions_from_zip # the parity-verified loader + + if model_id not in _sessions: + _sessions[model_id] = _sessions_from_zip(_zip_path(model_id)) + return _sessions[model_id] + + +def _metadata(model_id: str) -> dict: + import zipfile + + import yaml + + with zipfile.ZipFile(_zip_path(model_id)) as zf: + return yaml.safe_load(zf.read("metadata.yaml")) + + +def _decode(tokens: list) -> str: + # ByT5 token ids are byte+3, with trailing EOS (id 1) + return bytes(t - 3 for t in tokens if t >= 3).decode("utf-8", "replace") + + +def make_api(): + import os + from fastapi import FastAPI, HTTPException, Request + + from pydantic import BaseModel + + api = FastAPI(title="Interscript inference", version="1.0.0") + + @api.post("/infer") + async def infer(request: Request) -> dict: + import sys + + key = request.headers.get("x-api-key", "") + if not key or key != os.environ.get("API_INFERENCE_KEY"): + raise HTTPException(401, "invalid or missing X-API-Key") + try: + body = await request.json() + except Exception: + raise HTTPException(400, "body must be JSON {model, input}") from None + if not isinstance(body, dict) or not body.get("model") or not body.get("input"): + raise HTTPException(400, "body must be {model, input}") + + return await _run_infer(body) + + async def _run_infer(body): + import sys + + models_volume.reload() + try: + meta = _metadata(body["model"]) + except KeyError: + raise HTTPException(404, f"unknown model {body['model']}") from None + if meta.get("task") not in ALLOWED_TASKS: + raise HTTPException(400, f"model {body['model']} task {meta.get('task')} is not served") + + if len(body["input"].encode("utf-8")) > MAX_INPUT_BYTES: + raise HTTPException(413, f"input exceeds {MAX_INPUT_BYTES} bytes") + + sys.path.insert(0, "/root/interscript-ml/src") + from imf.export import onnx_greedy_kv + + enc, kv = _get_sessions(body["model"]) + max_len = min(MAX_OUTPUT_TOKENS, 3 * len(body["input"].encode("utf-8")) + 256) + output = _decode(onnx_greedy_kv(enc, kv, body["input"], max_len)) + return { + "model": body["model"], + "task": meta["task"], + "source_script": meta.get("source_script"), + "input": body["input"], + "output": output, + } + + @api.get("/health") + def health() -> dict: + import glob + + models_volume.reload() + return {"ok": True, "models": len(glob.glob("/v/imf/*/*-fp32.zip"))} + + return api + + +@app.function( + cpu=4, + memory=8 * 1024, + timeout=10 * 60, + volumes={"/v": models_volume}, + secrets=[modal.Secret.from_name("api-inference-key")], + # cold starts (~30-60s model load) instead of a 24/7 warm container + # — money discipline; bump once traffic justifies it +) +@modal.asgi_app() +def web(): + return make_api() From 88fab76f2bc1ff2f712a5efc9e55cb5440d7d9b5 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 06:28:22 +0800 Subject: [PATCH 2/6] =?UTF-8?q?fix(distill):=20byt5=20decode=5Fjoined=20pr?= =?UTF-8?q?oduced=20mojibake=20=E2=80=94=20branch=20to=20batch=5Fdecode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit convert_ids_to_tokens on a byte-level vocab returns each byte token as a raw char; joining them double-encodes the text. This poisoned every Arabic label generated under 5.14 (both students trained on mojibake targets — their identical 83.08 DER was the bare-text constant, not a capacity verdict). Byte-exactness asserted locally; umt5 path unchanged. ara-diac-small relabels onto teacher_labels_v2.jsonl. --- src/gpu/modal_distill.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index c753c5c..76a11d4 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -116,6 +116,9 @@ "max_len": 1450, "label_beams": "1", "out": "rababa_arabic_distill_small/run-002", + # v2: every label generated before the byt5 decode_joined fix is + # mojibake (double-encoded); relabel from scratch on the new file + "labels_file": "teacher_labels_v2.jsonl", "mode": "sequence", "note": "r6 canonical (2.5793 DER); gate <= 3.07 windowed DER-CE", }, @@ -233,11 +236,24 @@ def decode_joined(tok, ids) -> str: - """Correct decode for umt5 teachers: 5.x batch_decode inserts spurious - spaces between sentencepiece pieces; pieces must join directly (the - targets are unspaced IPA strings).""" + """Decode teacher generations correctly per tokenizer family. + + umt5/sentencepiece: 5.x batch_decode inserts spurious spaces between + pieces; joining convert_ids_to_tokens is correct. + + byt5/byte-level: convert_ids_to_tokens returns each byte token as a + RAW CHARACTER (latin-1 view) — joining them produces mojibake. This + poisoned every Arabic label generated under 5.14 (both students + trained on double-encoded targets and scored the identical bare-text + DER). batch_decode is byte-exact here, so branch on it: if its + output round-trips the same token ids it is authoritative. + """ skip = {tok.pad_token, tok.eos_token, tok.bos_token} - return "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip) + joined = "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip) + if joined and all(ord(c) < 256 for c in joined): + # byte-level vocab decoded to raw chars — use the byte-exact path + return tok.batch_decode([ids], skip_special_tokens=True)[0] + return joined @app.function( From 0904479d9e59e64d26ca926982fc310dbe0c3137 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 06:29:12 +0800 Subject: [PATCH 3/6] =?UTF-8?q?docs:=20requalify=20ara-tiny=20verdict=20?= =?UTF-8?q?=E2=80=94=20labels=20were=20mojibake=20(confounded)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/RESULTS.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 897a532..5bf0239 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -121,11 +121,13 @@ reproducing its documented tier on this replication. | Teacher (r6, run-006-morph) | 1.32% | | **Student (33M from-scratch)** | **83.08%** — REJECTED | -Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude -with the same collapse signature as the Thai tiny tier (train loss -converges, test generalization absent). Verdict: sub-100M from-scratch -byte students do not generalize for diacritization any more than for -G2P — a pretrained backbone is non-negotiable. The Arabic client tier -therefore ships at the ByT5-small rung (ara-diac-small) or parks until -byte-level pretraining exists. The 30MB tier is closed as a negative -result across both task families. +Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude. + +**RETRACTION (2026-08-24):** this verdict is CONFOUNDED — every Arabic +label generated before the byt5 `decode_joined` fix was mojibake +(double-encoded targets); both Arabic students trained on corrupted +labels, and their identical DER scores are the bare-text constant, not +a capacity result. The numbers stand as measured but the capacity +conclusion for Arabic is UNPROVEN pending a clean-label re-run. The +Thai tiny verdict is unaffected (umt5/sentencepiece labels were +byte-exact); the pretrained-backbone law rests on Thai evidence. From 0adb949b87c7fc5103f5cbd03666c63700ded92c Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 08:35:15 +0800 Subject: [PATCH 4/6] lint: import order in api/inference (inherited from main) --- src/api/inference.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/api/inference.py b/src/api/inference.py index cf2df7b..c936359 100644 --- a/src/api/inference.py +++ b/src/api/inference.py @@ -67,15 +67,13 @@ def _decode(tokens: list) -> str: def make_api(): import os - from fastapi import FastAPI, HTTPException, Request - from pydantic import BaseModel + from fastapi import FastAPI, HTTPException, Request api = FastAPI(title="Interscript inference", version="1.0.0") @api.post("/infer") async def infer(request: Request) -> dict: - import sys key = request.headers.get("x-api-key", "") if not key or key != os.environ.get("API_INFERENCE_KEY"): From 332c5362b82fafe9616ad1b65e1c3b901a83576d Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 09:04:41 +0800 Subject: [PATCH 5/6] security(api): strict id check before zip path build (CWE-22 traversal); constant-time key compare --- src/api/inference.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/api/inference.py b/src/api/inference.py index c936359..1d3c1e0 100644 --- a/src/api/inference.py +++ b/src/api/inference.py @@ -9,6 +9,8 @@ Auth: X-API-Key header must match the `api-inference-key` secret. """ +import hmac +import re from pathlib import Path import modal @@ -32,7 +34,14 @@ _sessions: dict[str, tuple] = {} +_ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*-\d+\.\d+$") + + def _zip_path(model_id: str) -> Path: + # model_id arrives from the request body — a strict id check before + # any path construction (CodeQL: path traversal, CWE-22) + if not _ID_RE.match(model_id): + raise KeyError(model_id) family = model_id.rsplit("-", 1)[0] p = Path("/v/imf") / family / f"{model_id}-fp32.zip" if not p.exists(): @@ -76,7 +85,7 @@ def make_api(): async def infer(request: Request) -> dict: key = request.headers.get("x-api-key", "") - if not key or key != os.environ.get("API_INFERENCE_KEY"): + if not key or not hmac.compare_digest(key, os.environ.get("API_INFERENCE_KEY") or ""): raise HTTPException(401, "invalid or missing X-API-Key") try: body = await request.json() From 3af08fb5a12609d88c3b0a10d1a66570e2341b5f Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 09:35:25 +0800 Subject: [PATCH 6/6] =?UTF-8?q?security(api):=20resolve=20zips=20from=20th?= =?UTF-8?q?e=20volume=20listing=20only=20=E2=80=94=20user=20input=20never?= =?UTF-8?q?=20enters=20path=20construction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/inference.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/api/inference.py b/src/api/inference.py index 1d3c1e0..f30e9af 100644 --- a/src/api/inference.py +++ b/src/api/inference.py @@ -10,7 +10,6 @@ """ import hmac -import re from pathlib import Path import modal @@ -34,19 +33,17 @@ _sessions: dict[str, tuple] = {} -_ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*-\d+\.\d+$") - - def _zip_path(model_id: str) -> Path: - # model_id arrives from the request body — a strict id check before - # any path construction (CodeQL: path traversal, CWE-22) - if not _ID_RE.match(model_id): - raise KeyError(model_id) - family = model_id.rsplit("-", 1)[0] - p = Path("/v/imf") / family / f"{model_id}-fp32.zip" - if not p.exists(): - raise KeyError(model_id) - return p + # model_id arrives from the request body — paths are built ONLY from + # the server's own volume listing; the user value is used solely in + # an equality comparison, never in path construction (CWE-22) + import glob + + wanted = f"{model_id}-fp32.zip" + for z in glob.glob("/v/imf/*/*-fp32.zip"): + if z.rsplit("/", 1)[1] == wanted: + return Path(z) + raise KeyError(model_id) def _get_sessions(model_id: str) -> tuple: