diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 897a532..5bf0239 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -121,11 +121,13 @@ reproducing its documented tier on this replication. | Teacher (r6, run-006-morph) | 1.32% | | **Student (33M from-scratch)** | **83.08%** — REJECTED | -Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude -with the same collapse signature as the Thai tiny tier (train loss -converges, test generalization absent). Verdict: sub-100M from-scratch -byte students do not generalize for diacritization any more than for -G2P — a pretrained backbone is non-negotiable. The Arabic client tier -therefore ships at the ByT5-small rung (ara-diac-small) or parks until -byte-level pretraining exists. The 30MB tier is closed as a negative -result across both task families. +Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude. + +**RETRACTION (2026-08-24):** this verdict is CONFOUNDED — every Arabic +label generated before the byt5 `decode_joined` fix was mojibake +(double-encoded targets); both Arabic students trained on corrupted +labels, and their identical DER scores are the bare-text constant, not +a capacity result. The numbers stand as measured but the capacity +conclusion for Arabic is UNPROVEN pending a clean-label re-run. The +Thai tiny verdict is unaffected (umt5/sentencepiece labels were +byte-exact); the pretrained-backbone law rests on Thai evidence. diff --git a/src/api/inference.py b/src/api/inference.py new file mode 100644 index 0000000..f30e9af --- /dev/null +++ b/src/api/inference.py @@ -0,0 +1,145 @@ +"""IMF v1 inference endpoint for api.interscript.org (Modal, CPU). + +Serves the shipped models from the secryst-models volume using the +exact ONNX kv decode the WO03 parity gate verified. Cold start loads +the fp32 zip (~30-60s); sessions are cached per container. + + modal deploy src/api/inference.py + +Auth: X-API-Key header must match the `api-inference-key` secret. +""" + +import hmac +from pathlib import Path + +import modal + +models_volume = modal.Volume.from_name("secryst-models") + +image = ( + modal.Image.debian_slim(python_version="3.11") + .pip_install("onnxruntime==1.23.2", "pyyaml>=6.0", "fastapi>=0.115") + .add_local_dir(str(Path(__file__).resolve().parent.parent), "/root/interscript-ml", copy=True) + .workdir("/root/interscript-ml") + .env({"IMAGE_REV": "5"}) +) + +app = modal.App("interscript-inference", image=image) + +MAX_INPUT_BYTES = 4000 +MAX_OUTPUT_TOKENS = 8192 +ALLOWED_TASKS = ("diacritization", "g2p") + +_sessions: dict[str, tuple] = {} + + +def _zip_path(model_id: str) -> Path: + # model_id arrives from the request body — paths are built ONLY from + # the server's own volume listing; the user value is used solely in + # an equality comparison, never in path construction (CWE-22) + import glob + + wanted = f"{model_id}-fp32.zip" + for z in glob.glob("/v/imf/*/*-fp32.zip"): + if z.rsplit("/", 1)[1] == wanted: + return Path(z) + raise KeyError(model_id) + + +def _get_sessions(model_id: str) -> tuple: + import sys + + sys.path.insert(0, "/root/interscript-ml/src") + from imf.parity import _sessions_from_zip # the parity-verified loader + + if model_id not in _sessions: + _sessions[model_id] = _sessions_from_zip(_zip_path(model_id)) + return _sessions[model_id] + + +def _metadata(model_id: str) -> dict: + import zipfile + + import yaml + + with zipfile.ZipFile(_zip_path(model_id)) as zf: + return yaml.safe_load(zf.read("metadata.yaml")) + + +def _decode(tokens: list) -> str: + # ByT5 token ids are byte+3, with trailing EOS (id 1) + return bytes(t - 3 for t in tokens if t >= 3).decode("utf-8", "replace") + + +def make_api(): + import os + + from fastapi import FastAPI, HTTPException, Request + + api = FastAPI(title="Interscript inference", version="1.0.0") + + @api.post("/infer") + async def infer(request: Request) -> dict: + + key = request.headers.get("x-api-key", "") + if not key or not hmac.compare_digest(key, os.environ.get("API_INFERENCE_KEY") or ""): + raise HTTPException(401, "invalid or missing X-API-Key") + try: + body = await request.json() + except Exception: + raise HTTPException(400, "body must be JSON {model, input}") from None + if not isinstance(body, dict) or not body.get("model") or not body.get("input"): + raise HTTPException(400, "body must be {model, input}") + + return await _run_infer(body) + + async def _run_infer(body): + import sys + + models_volume.reload() + try: + meta = _metadata(body["model"]) + except KeyError: + raise HTTPException(404, f"unknown model {body['model']}") from None + if meta.get("task") not in ALLOWED_TASKS: + raise HTTPException(400, f"model {body['model']} task {meta.get('task')} is not served") + + if len(body["input"].encode("utf-8")) > MAX_INPUT_BYTES: + raise HTTPException(413, f"input exceeds {MAX_INPUT_BYTES} bytes") + + sys.path.insert(0, "/root/interscript-ml/src") + from imf.export import onnx_greedy_kv + + enc, kv = _get_sessions(body["model"]) + max_len = min(MAX_OUTPUT_TOKENS, 3 * len(body["input"].encode("utf-8")) + 256) + output = _decode(onnx_greedy_kv(enc, kv, body["input"], max_len)) + return { + "model": body["model"], + "task": meta["task"], + "source_script": meta.get("source_script"), + "input": body["input"], + "output": output, + } + + @api.get("/health") + def health() -> dict: + import glob + + models_volume.reload() + return {"ok": True, "models": len(glob.glob("/v/imf/*/*-fp32.zip"))} + + return api + + +@app.function( + cpu=4, + memory=8 * 1024, + timeout=10 * 60, + volumes={"/v": models_volume}, + secrets=[modal.Secret.from_name("api-inference-key")], + # cold starts (~30-60s model load) instead of a 24/7 warm container + # — money discipline; bump once traffic justifies it +) +@modal.asgi_app() +def web(): + return make_api() diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index c753c5c..76a11d4 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -116,6 +116,9 @@ "max_len": 1450, "label_beams": "1", "out": "rababa_arabic_distill_small/run-002", + # v2: every label generated before the byt5 decode_joined fix is + # mojibake (double-encoded); relabel from scratch on the new file + "labels_file": "teacher_labels_v2.jsonl", "mode": "sequence", "note": "r6 canonical (2.5793 DER); gate <= 3.07 windowed DER-CE", }, @@ -233,11 +236,24 @@ def decode_joined(tok, ids) -> str: - """Correct decode for umt5 teachers: 5.x batch_decode inserts spurious - spaces between sentencepiece pieces; pieces must join directly (the - targets are unspaced IPA strings).""" + """Decode teacher generations correctly per tokenizer family. + + umt5/sentencepiece: 5.x batch_decode inserts spurious spaces between + pieces; joining convert_ids_to_tokens is correct. + + byt5/byte-level: convert_ids_to_tokens returns each byte token as a + RAW CHARACTER (latin-1 view) — joining them produces mojibake. This + poisoned every Arabic label generated under 5.14 (both students + trained on double-encoded targets and scored the identical bare-text + DER). batch_decode is byte-exact here, so branch on it: if its + output round-trips the same token ids it is authoritative. + """ skip = {tok.pad_token, tok.eos_token, tok.bos_token} - return "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip) + joined = "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip) + if joined and all(ord(c) < 256 for c in joined): + # byte-level vocab decoded to raw chars — use the byte-exact path + return tok.batch_decode([ids], skip_special_tokens=True)[0] + return joined @app.function(