From 9d81d958a6dcfeac44cc8bf3a7cb3f23f190d663 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 1 Sep 2026 10:57:15 +0200 Subject: [PATCH] eval: s46 on the Dicta ACL 2020 test corpora (modern-text surface) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three public-domain sets (Modern/HebrewWiki, Poetry, Rabbinic) with the same greedy seq2seq DER harness as the Nakdimon row — per-corpus DERs, resumable, results to dicta_eval.json. Corpora uploaded to rababa-datasets:dicta-test/. This is the Hebrew table's disclosed modern-text gap being measured rather than skipped. --- eval_hebrew_dicta.py | 130 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 eval_hebrew_dicta.py diff --git a/eval_hebrew_dicta.py b/eval_hebrew_dicta.py new file mode 100644 index 0000000..5524cef --- /dev/null +++ b/eval_hebrew_dicta.py @@ -0,0 +1,130 @@ +"""s46 on the Dicta diacritization test corpora (2026-09-01). + +The modern-text surface TODO.training-work/07: three public-domain +test sets (Modern/HebrewWiki, Poetry, Rabbinic) from Dicta's ACL 2020 +demo, evaluated with the same greedy seq2seq DER harness that produced +s46's Nakdimon number (16.44 greedy). Resumable per example, per +corpus. Results land in dicta_eval.json beside the checkpoint. + +Usage: + modal run --detach eval_hebrew_dicta.py +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import modal + +checkpoints_volume = modal.Volume.from_name("rababa-checkpoints", create_if_missing=True) +datasets_volume = modal.Volume.from_name("rababa-datasets", create_if_missing=True) + +RUN = "rababa_hebrew/run-s46-phonikud-plus" +_NIKUD_MARKS = set("ְֱֲֳִֵֶַָֹֺֻּֽֿׁׂ־") + +CORPORA = { + "modern_wiki": "dicta-test/ModernTestCorpus-HebrewWiki1.txt", + "poetry": "dicta-test/PoetryTestCorpus-Poetry1.txt", + "rabbinic": "dicta-test/RabbinicTestCorpus-BetYosef1.txt", +} + +image = ( + modal.Image.debian_slim(python_version="3.11") + .apt_install("build-essential", "git", "curl") + .pip_install("torch==2.5.1", "transformers==4.46.3", "numpy>=1.26,<3", "tqdm>=4.66") + .add_local_dir("src", "/opt/rababa/src", copy=True) + .workdir("/opt/rababa") + .env({"PYTHONPATH": "/opt/rababa/src", "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"}) +) + +app = modal.App("rababa-hebrew-dicta", image=image) + + +@app.function( + gpu="A10G", + timeout=4 * 60 * 60, + volumes={"/checkpoints": checkpoints_volume, "/datasets": datasets_volume}, +) +def evaluate() -> dict: + import torch + from transformers import T5ForConditionalGeneration, ByT5Tokenizer + from rababa.evaluate import seq2seq_der + + checkpoints_volume.reload() + datasets_volume.reload() + + ckpt = str(Path("/checkpoints") / RUN / "run-002-gold-ft" / "best") + model = T5ForConditionalGeneration.from_pretrained(ckpt).to("cuda") + tokenizer = ByT5Tokenizer.from_pretrained(ckpt) + model.eval() + + results = {} + for name, rel in CORPORA.items(): + path = Path("/datasets") / rel + examples = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + undiacritized = "".join(c for c in line if c not in _NIKUD_MARKS).strip() + if 2 <= len(undiacritized) <= 512: + examples.append((undiacritized, line)) + print(f"[{name}] {len(examples)} examples", flush=True) + + prog = Path("/checkpoints") / RUN / f"dicta_progress_{name}.jsonl" + saved: dict[int, tuple[float, int]] = {} + if prog.exists(): + for l in prog.read_text(encoding="utf-8").splitlines(): + if l.strip(): + row = json.loads(l) + saved[row["i"]] = (row["der"], row["n"]) + + missing = [i for i in range(len(examples)) if i not in saved] + n_new = 0 + with torch.no_grad(), prog.open("a", encoding="utf-8") as out: + for bi in range(0, len(missing), 16): + idxs = missing[bi : bi + 16] + batch = [examples[i][0] for i in idxs] + enc = tokenizer(batch, return_tensors="pt", padding=True, + truncation=True, max_length=512).to("cuda") + gen = model.generate(**enc, max_new_tokens=512, num_beams=1) + preds = tokenizer.batch_decode(gen, skip_special_tokens=True) + for i, pred in zip(idxs, preds): + der, n = seq2seq_der(pred.strip(), examples[i][1]) + saved[i] = (der, n) + out.write(json.dumps({"i": i, "der": der, "n": n}) + "\n") + n_new += len(idxs) + if n_new % 320 == 0: + out.flush() + checkpoints_volume.commit() + tw = sum(d * n for d, n in saved.values()) + tc = sum(n for _, n in saved.values()) + print(f"[{name}] {len(saved)}/{len(examples)} " + f"DER={tw/max(1,tc):.4f} (committed)", flush=True) + checkpoints_volume.commit() + + total_wrong = sum(d * n for d, n in saved.values()) + total_positions = sum(n for _, n in saved.values()) + der = total_wrong / max(1, total_positions) + results[name] = { + "der_greedy": round(der, 4), "n": len(saved), "positions": total_positions, + } + print(f"=== {name}: s46 greedy DER {der:.4f} ({len(saved)} examples) ===", + flush=True) + + result = { + "checkpoint": ckpt, + "nakdimon_greedy": 0.1644, + "corpora": results, + "source": "Dicta ACL 2020 test corpora (public domain)", + } + (Path("/checkpoints") / RUN / "dicta_eval.json").write_text( + json.dumps(result, indent=2), encoding="utf-8") + checkpoints_volume.commit() + return result + + +@app.local_entrypoint() +def main(): + print(evaluate.remote())