From 536a6975c508b668b1db9dd7490e4dca976425f1 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 20 Aug 2026 02:37:22 +0800 Subject: [PATCH 1/3] release: fas-g2p-1.0 registration + ara-diac-small distill spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fas-g2p: Persian v1 teacher shipped directly (byte-level ByT5-small, client-tier — no distillation step applies). RESULTS.md anchor + metrics-sources entry; persian volumes wired into modal_export. ara-diac-small: r5 paragraph-context teacher spec (windowed 1400B contract, greedy labels, 2x generation cap, 30K-unit corpus); windowed DER-CE gate via the Misraj evaluator (teacher + 0.5pp). --- docs/RESULTS.md | 16 ++ models/fas-g2p/fas-g2p-1.0.README.md | 13 ++ models/fas-g2p/fas-g2p-1.0.metadata.yaml | 29 +++ models/metrics-sources.yaml | 9 + src/gpu/modal_distill.py | 237 ++++++++++++++++++++--- src/gpu/modal_export.py | 11 ++ 6 files changed, 292 insertions(+), 23 deletions(-) create mode 100644 models/fas-g2p/fas-g2p-1.0.README.md create mode 100644 models/fas-g2p/fas-g2p-1.0.metadata.yaml diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 10c032c..3e4675e 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -28,3 +28,19 @@ from saved artifacts (transformers 5.15 save drops the untied umt5 lm_head) and the volume's epitran augmentation corpus is tone-less; this release distills the best verified teacher available. A repaired 2.32%-tier teacher re-enters this pipeline when secryst regenerates it. + +## fas-g2p-1.0 — Persian G2P (2026-08-19) + +The v1 ByT5-small teacher shipped directly (byte-level, client-tier +size — no distillation step applies). REF teacher: persian-g2p-checkpoints +`persian_g2p/run-001/best`, RELEASE-FROZEN per rababa +docs/DISTILL-SOURCE-PROMPT.md (RL variants and the v5/mapped +representation line are closed negative; v1 is final). + +| Metric | Value | +|---|---| +| CER (v1 test split, greedy, editdistance) | ≈1.6% | +| SentenceBench homograph (ezafe-normalized) | 77.34% | + +Published reference: Homo-GE2PE homograph 76.89% — v1 is above the +published SOTA on this benchmark. diff --git a/models/fas-g2p/fas-g2p-1.0.README.md b/models/fas-g2p/fas-g2p-1.0.README.md new file mode 100644 index 0000000..6551c0b --- /dev/null +++ b/models/fas-g2p/fas-g2p-1.0.README.md @@ -0,0 +1,13 @@ +# fas-g2p-1.0 + +Persian grapheme-to-phoneme. Raw Persian sentence (no prefix) in → +space-separated Latin phonemes out. The v1 ByT5-small teacher shipped +directly as the client-tier model — it is already byte-level at +client-tier size, so no distillation step applies (the +5pp gate is a +distillation contract; here the teacher IS the artifact). + +```python +from interscript_ml import Model +model = Model.load("fas-g2p-1.0") +model.translate("سلام") +``` diff --git a/models/fas-g2p/fas-g2p-1.0.metadata.yaml b/models/fas-g2p/fas-g2p-1.0.metadata.yaml new file mode 100644 index 0000000..2c57f7b --- /dev/null +++ b/models/fas-g2p/fas-g2p-1.0.metadata.yaml @@ -0,0 +1,29 @@ +format: imf-v1 +id: fas-g2p-1.0 +task: g2p +source_script: Arab +target: Latn +tokenizer: bytes +opset: 14 +decoder: kv +precision: fp32 +license: BSD-3-Clause +trained_from: >- + persian_g2p v1 (ByT5-small, byte-level, RELEASE-FROZEN); checkpoint + persian-g2p-checkpoints:/checkpoints/persian_g2p/run-001/best. + Shipped directly as the client-tier model — the v1 teacher is already + byte-level ByT5-small (no distillation needed; GRPO/RAFT variants and + the v5/mapped representation line are closed negative). +metrics: + - name: cer + value: 1.6 + protocol: >- + greedy decode, editdistance CER on the v1 test split + (persian-g2p-datasets:/persian-g2p/test.jsonl) + source: interscript/ml-models docs/RESULTS.md#fas-g2p-1.0 + - name: sb_homograph_ezafe_norm + value: 77.34 + protocol: >- + SentenceBench homograph accuracy, ezafe-normalized + (eval_sentencebench.py protocol; published Homo-GE2PE: 76.89%) + source: interscript/ml-models docs/RESULTS.md#fas-g2p-1.0 diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml index 324f553..0e0c5c8 100644 --- a/models/metrics-sources.yaml +++ b/models/metrics-sources.yaml @@ -49,3 +49,12 @@ tha-g2p-base-1.0: tables: - {row: "Teacher (B-K/umt5 hub base)", column: PER, as: per_teacher} - {row: "Student (ByT5-base, gate)", column: PER, as: per_student} +fas-g2p-1.0: + repo: interscript/ml-models + ref: main + path: docs/RESULTS.md + anchor: fas-g2p-1.0 + protocol: "greedy decode, editdistance CER on v1 test split; SentenceBench homograph ezafe-normalized" + tables: + - {row: "CER (v1 test split, greedy, editdistance)", as: cer} + - {row: "SentenceBench homograph (ezafe-normalized)", as: sb_homograph_ezafe_norm} diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 054ff5a..971e40d 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -33,9 +33,23 @@ "transformers==5.14.1", "pyyaml>=6.0", "numpy>=1.26", + # arabic gate harness: Misraj evaluator + SadeedDiac-25 parquet + "pyarabic", + "pandas", + "pyarrow", ) .env({"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"}) .add_local_dir(str(REPO_ROOT), "/root/ml-models", copy=True) + .add_local_file( + "/Users/mulgogi/src/interscript/rababa/sadeed_evaluator.py", + "/opt/rababa/sadeed_evaluator.py", + copy=True, + ) + .add_local_dir( + "/Users/mulgogi/src/interscript/rababa/data/sadeed-diac-25", + "/opt/rababa/data/sadeed-diac-25", + copy=True, + ) .workdir("/root/ml-models") ) @@ -68,6 +82,24 @@ "mode": "sequence", # cross-tokenizer: teacher generates, student trains CE "note": "umt5 (sentencepiece) teacher -> ByT5-small byte student; +5pp PER gate", }, + "ara-diac-small": { + # r5 paragraph-context teacher (2.68 DER-CE windowed @1400B, + # RELEASE-FROZEN) -> ByT5-small student. Contract decode is + # GREEDY with generation cap 2x window (eval_sadeed_windowed). + # Corpus: r5-units joined paragraph units (src = stripped + # diacritics, teacher regenerates the labels). + "teacher": "rababa_arabic_byt5/run-005-context/best", + "teacher_volume": "rababa", + "student_init": "google/byt5-small", + "train": "r5-units/domain.txt", + "train_extra": ["r5-units/replay.txt"], + "unit_limits": [24000, 6000], + "max_len": 1450, + "label_beams": "1", + "out": "rababa_arabic_distill_small/run-002", + "mode": "sequence", + "note": "gate <= teacher_der + 0.5pp windowed DER-CE (prompt target 3.18 from 2.68)", + }, "fas-g2p-small": { "teacher": "persian_g2p/run-001/best", "teacher_volume": "persian", @@ -505,23 +537,44 @@ def distill_sequence(spec_id: str, epochs: int = 3) -> dict: student.train() class Pairs(Dataset): - def __init__(self, paths: Path | list[Path], max_len: int = 384): - if isinstance(paths, Path): - paths = [paths] + def __init__(self, files: list[tuple[Path, int]], max_len: int = 1450): + # jsonl files carry {src, tgt} rows; .txt unit files are + # single-column diacritized paragraph units (the r5 corpus): + # src = diacritics stripped, tgt = the unit itself, capped at + # max_len bytes, seeded shuffle then per-file limit + import random + import re + self.rows = [] seen = set() - for path in paths: - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue - s = (row.get("src") or "").strip() - if s and s not in seen and len(s.encode()) <= max_len: - seen.add(s) - self.rows.append((s, (row.get("tgt") or "").strip())) + for path, limit in files: + if path.suffix == ".jsonl": + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + s = (row.get("src") or "").strip() + if s and s not in seen and len(s.encode()) <= 384: + seen.add(s) + self.rows.append((s, (row.get("tgt") or "").strip())) + else: + diac = re.compile("[ً-ٰٟۖ-ۭ]") + units = [ + u.strip() + for u in path.read_text(encoding="utf-8", errors="ignore").splitlines() + if u.strip() + ] + random.Random(42).shuffle(units) + for unit in units[:limit]: + if len(unit.encode()) > max_len: + continue + src = diac.sub("", unit).strip() + if src and src not in seen: + seen.add(src) + self.rows.append((src, unit)) def __len__(self): return len(self.rows) @@ -529,25 +582,30 @@ def __len__(self): def __getitem__(self, i): return self.rows[i] + train_cap = int(spec.get("max_len", 384)) + def collate(batch): # byte-level tokens: a 2,000-char Wikipedia sentence is 2,000 # tokens — without truncation a single long pair OOMs the A10G src = student_tok( [s for s, _ in batch], padding=True, truncation=True, - max_length=384, return_tensors="pt", + max_length=train_cap, return_tensors="pt", ) labels = student_tok( [t for _, t in batch], padding=True, truncation=True, - max_length=384, return_tensors="pt", + max_length=train_cap, return_tensors="pt", ).input_ids labels[labels == student_tok.pad_token_id] = -100 return src.input_ids, src.attention_mask, labels - train_files = [train_path] + [ - Path(data_vol) / p for p in spec.get("train_extra", []) - ] + unit_limits = [int(x) for x in spec.get("unit_limits", [0])] + train_files = [(train_path, unit_limits[0] if unit_limits else 0)] + for i, p in enumerate(spec.get("train_extra", [])): + lim = unit_limits[i + 1] if i + 1 < len(unit_limits) else 0 + train_files.append((Path(data_vol) / p, lim)) train_ds = Pairs(train_files) print(f"[{spec_id}] train pairs: {len(train_ds)} from {len(train_files)} files", flush=True) + label_beams = int(spec.get("label_beams", 4)) # Step 1: teacher generates labels (beam-4) for the full corpus. # Resumable: evictions mid-labeling are routine on long jobs — @@ -570,10 +628,14 @@ def collate(batch): if todo: print(f"[{spec_id}] labeling {len(todo)} remaining...", flush=True) - def label_batch(batch, max_len: int = 384): + seq_max = int(spec.get("max_len", 384)) + + def label_batch(batch, max_len: int = 0): # lone-src OOM fallback truncates once, then skips: never # recurse on the same shape (torch 2.x renames the OOM # exception class, so match by message) + if not max_len: + max_len = seq_max try: enc = teacher_tok( [s for s, _ in batch], @@ -584,7 +646,9 @@ def label_batch(batch, max_len: int = 384): ).to("cuda") with torch.inference_mode(): out = teacher.generate( - **enc, max_new_tokens=max_len, num_beams=4 + # r5 contract: generation cap = 2x window bytes + # (diacritized output runs 1.4-1.6x input) + **enc, max_new_tokens=2 * max_len, num_beams=label_beams ) return [decode_joined(teacher_tok, o) for o in out] except RuntimeError as e: @@ -604,7 +668,7 @@ def label_batch(batch, max_len: int = 384): # deterministic token-budget batching: sort by length so long # srcs land in small batches — no OOM roulette todo.sort(key=lambda p: len(p[0].encode())) - budget = 16 * 200 + budget = 32 * max(200, seq_max) batches: list[list[tuple[str, str]]] = [] cur: list[tuple[str, str]] = [] cur_max = 0 @@ -746,6 +810,133 @@ def eval_main(spec: str = "heb-diac-small", limit: int = 0) -> None: print(evaluate.remote(spec, limit)) +@app.function( + gpu="A10G", + cpu=8, + memory=32 * 1024, + timeout=5 * 3600, + volumes={ + "/datasets": DATASETS, + "/checkpoints": CHECKPOINTS, + "/secryst-checkpoints": SECRYST_CHECKPOINTS, + "/secryst-datasets": SECRYST_DATASETS, + "/persian-checkpoints": PERSIAN_CHECKPOINTS, + }, +) +def evaluate_der(spec_id: str, window: int = 1400, limit: int = 0) -> dict: + """Windowed SadeedDiac-25 DER-CE of teacher vs student, replicating + rababa eval_sadeed_windowed.py at the r5 window (1400B): strip + diacritics, split at word boundaries, greedy decode with 2x window + cap, stitch, project haraqat onto the input letters (zero-skip), + DER-CE via the Misraj evaluator. Gate: teacher + 0.5pp + (DISTILL-SOURCE-PROMPT: 3.18 target from the 2.68 teacher).""" + import difflib + import re + from pathlib import Path + + import pyarrow.parquet as pq + import torch + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + + spec = SPECS[spec_id] + vol_map = { + "rababa": "/checkpoints", + "secryst": "/secryst-checkpoints", + "persian": "/persian-checkpoints", + } + teacher_path = (spec["teacher"] if spec.get("teacher_is_hub") + else str(Path(vol_map[spec.get("teacher_volume", "rababa")]) / spec["teacher"])) + student_path = Path(vol_map[spec.get("teacher_volume", "rababa")]) / spec["out"] / "best" + + tok = AutoTokenizer.from_pretrained("google/byt5-small") + teacher = AutoModelForSeq2SeqLM.from_pretrained(teacher_path).to("cuda").eval() + student = AutoModelForSeq2SeqLM.from_pretrained(str(student_path)).to("cuda").eval() + + diac = re.compile("[ً-ٰٟۖ-ۭ]") + table = pq.read_table("/opt/rababa/data/sadeed-diac-25/train.parquet") + inputs = [diac.sub("", t) for t in table.column("input").to_pylist()] + gts = table.column("output").to_pylist() + if limit: + inputs, gts = inputs[:limit], gts[:limit] + + def split_windows(text: str) -> list[str]: + if len(text.encode()) <= window: + return [text] + wins, cur, n = [], [], 0 + for w in text.split(): + c = len(w.encode()) + 1 + if cur and n + c > window: + wins.append(" ".join(cur)) + cur, n = [], 0 + cur.append(w) + n += c + if cur: + wins.append(" ".join(cur)) + return wins + + def project_haraqat(pred: str, text: str) -> str: + haraqat = [""] + for ch in pred: + if diac.match(ch): + haraqat[-1] += ch + else: + haraqat.append("") + haraqat = haraqat[1:] + pred_letters = [c for c in pred if not diac.match(c)] + text_letters = [c for c in text if not diac.match(c)] + sm = difflib.SequenceMatcher(None, text_letters, pred_letters, autojunk=False) + out = [] + for op, i1, i2, j1, _ in sm.get_opcodes(): + if op == "equal": + for k in range(i2 - i1): + out.append(text_letters[i1 + k] + haraqat[j1 + k]) + else: + for k in range(i1, i2): + out.append(text_letters[k]) + return "".join(out) + + def der_ce(model) -> dict: + windows, counts = [], [] + for text in inputs: + ws = split_windows(text) + counts.append(len(ws)) + windows.extend(ws) + preds = [] + with torch.no_grad(): + for i in range(0, len(windows), 8): + batch = windows[i : i + 8] + enc = tok(batch, return_tensors="pt", padding=True, truncation=True, + max_length=window).to("cuda") + with torch.autocast("cuda", torch.bfloat16): + gen = model.generate(**enc, max_new_tokens=window * 2, num_beams=1) + preds.extend(tok.batch_decode(gen, skip_special_tokens=True)) + k = 0 + paragraphs = [] + for text, c in zip(inputs, counts, strict=True): + paragraphs.append(project_haraqat(" ".join(preds[k : k + c]), text)) + k += c + + import sys + + sys.path.insert(0, "/opt/rababa") + from sadeed_evaluator import ArabicDiacritizationEvaluator as E + + _, _, total_der, _, _ = E.caculate_errors_on_sentences( + paragraphs, gts, gt_missing_diacritic_is_error=False + ) + return {"der_ce": round(100 * total_der, 4), "n": len(inputs)} + + result = {"teacher": der_ce(teacher), "student": der_ce(student)} + result["gate_delta"] = round(result["student"]["der_ce"] - result["teacher"]["der_ce"], 4) + result["gate_pass"] = result["gate_delta"] <= 0.5 + return result + + @app.local_entrypoint() def eval_per(spec: str = "tha-g2p-small", limit: int = 0) -> None: print(evaluate_per.remote(spec, limit)) + + +@app.local_entrypoint() +def eval_der(spec: str = "ara-diac-small", limit: int = 0) -> None: + print(evaluate_der.remote(spec, limit)) diff --git a/src/gpu/modal_export.py b/src/gpu/modal_export.py index c123cf5..e5b1ea6 100644 --- a/src/gpu/modal_export.py +++ b/src/gpu/modal_export.py @@ -45,6 +45,7 @@ "urdu-diacrit-checkpoints" ), "/volumes/rababa-checkpoints": modal.Volume.from_name("rababa-checkpoints"), + "/volumes/persian-checkpoints": modal.Volume.from_name("persian-g2p-checkpoints"), } DATASET_VOLUMES = { @@ -52,6 +53,7 @@ "/datasets/secryst": modal.Volume.from_name("secryst-datasets"), "/datasets/urdu-g2p": modal.Volume.from_name("urdu-g2p-datasets"), "/datasets/urdu-diacrit": modal.Volume.from_name("urdu-diacrit-datasets"), + "/datasets/persian": modal.Volume.from_name("persian-g2p-datasets"), } MODELS_VOLUME = modal.Volume.from_name("secryst-models") @@ -102,6 +104,15 @@ "test_data": "thai-ipa/test.jsonl", "probe": "สวัสดี", }, + "fas-g2p": { + "volume": "/volumes/persian-checkpoints", + "checkpoint": "persian_g2p/run-001/best", + "metadata": "models/fas-g2p/fas-g2p-1.0.metadata.yaml", + "readme": "models/fas-g2p/fas-g2p-1.0.README.md", + "test_volume": "/datasets/persian", + "test_data": "persian-g2p/test.jsonl", + "probe": "سلام", + }, } app = modal.App("interscript-ml-export", image=IMAGE) From db819db5322c020ceaa784cd7db3ab08e7f8af03 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 20 Aug 2026 20:11:01 +0800 Subject: [PATCH 2/3] fix: repo rename interscript/ml-models -> interscript/interscript-ml in provenance refs --- models/fas-g2p/fas-g2p-1.0.metadata.yaml | 4 ++-- models/metrics-sources.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/fas-g2p/fas-g2p-1.0.metadata.yaml b/models/fas-g2p/fas-g2p-1.0.metadata.yaml index 2c57f7b..215527c 100644 --- a/models/fas-g2p/fas-g2p-1.0.metadata.yaml +++ b/models/fas-g2p/fas-g2p-1.0.metadata.yaml @@ -20,10 +20,10 @@ metrics: protocol: >- greedy decode, editdistance CER on the v1 test split (persian-g2p-datasets:/persian-g2p/test.jsonl) - source: interscript/ml-models docs/RESULTS.md#fas-g2p-1.0 + source: interscript/interscript-ml docs/RESULTS.md#fas-g2p-1.0 - name: sb_homograph_ezafe_norm value: 77.34 protocol: >- SentenceBench homograph accuracy, ezafe-normalized (eval_sentencebench.py protocol; published Homo-GE2PE: 76.89%) - source: interscript/ml-models docs/RESULTS.md#fas-g2p-1.0 + source: interscript/interscript-ml docs/RESULTS.md#fas-g2p-1.0 diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml index 0e0c5c8..d6f3df3 100644 --- a/models/metrics-sources.yaml +++ b/models/metrics-sources.yaml @@ -50,7 +50,7 @@ tha-g2p-base-1.0: - {row: "Teacher (B-K/umt5 hub base)", column: PER, as: per_teacher} - {row: "Student (ByT5-base, gate)", column: PER, as: per_student} fas-g2p-1.0: - repo: interscript/ml-models + repo: interscript/interscript-ml ref: main path: docs/RESULTS.md anchor: fas-g2p-1.0 From 1d380f18394101187bda6c309a9bbef5378a6851 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 20 Aug 2026 20:40:01 +0800 Subject: [PATCH 3/3] fix(metrics): fas-g2p anchor slug + branch ref The WO10 check slugifies section headings (dots stripped) and fetches RESULTS.md by the entry's ref. fas-g2p-1.0 used the raw model id as anchor and pointed at main, where the section doesn't exist yet. Use the full slugified heading and the release branch ref; flip to main when this merges. --- models/metrics-sources.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml index d6f3df3..d6173ea 100644 --- a/models/metrics-sources.yaml +++ b/models/metrics-sources.yaml @@ -51,9 +51,9 @@ tha-g2p-base-1.0: - {row: "Student (ByT5-base, gate)", column: PER, as: per_student} fas-g2p-1.0: repo: interscript/interscript-ml - ref: main + ref: release/fas-g2p-1.0 path: docs/RESULTS.md - anchor: fas-g2p-1.0 + anchor: fas-g2p-10-persian-g2p-2026-08-19 protocol: "greedy decode, editdistance CER on v1 test split; SentenceBench homograph ezafe-normalized" tables: - {row: "CER (v1 test split, greedy, editdistance)", as: cer}