diff --git a/docs/RESULTS.md b/docs/RESULTS.md new file mode 100644 index 0000000..10c032c --- /dev/null +++ b/docs/RESULTS.md @@ -0,0 +1,30 @@ +# ml-models results + +Evaluated results for models produced in this repository. Each section +anchor is the provenance target referenced by model metadata +(`models/*/` metadata.yaml) and `models/metrics-sources.yaml`. + +## tha-g2p-base-1.0 — Thai G2P distillation (2026-08-19) + +Sequence-level KD: B-K/umt5-thai-g2p-v2-0.5k teacher -> ByT5-base +student over 48,757 beam-4 teacher-generated labels (Kaikki + epitran +Wikipedia corpus, deduplicated, degenerate outputs filtered). Harness: +beam-4, corpus-level PER (total_ed / total_gold over characters of +joined-piece decode), 1,219 held-out Kaikki Thai test sentences +(`src/gpu/modal_distill.py::evaluate_per`). + +| Model | PER | Exact match | +|---|---|---| +| Teacher (B-K/umt5 hub base) | 4.43% | 95.57% | +| **Student (ByT5-base, gate)** | **9.19%** | 90.81% | + +Distillation cost: +4.76pp, inside the +5pp budget +(docs/DISTILL-SOURCE-PROMPT.md). ByT5-small ablations for reference: +12.63% on 23K labels, 12.06% on 48.7K labels (capacity-limited, both +rejected by the gate). + +Context: the secryst-published 2.32% umt5 teacher is unrecoverable +from saved artifacts (transformers 5.15 save drops the untied umt5 +lm_head) and the volume's epitran augmentation corpus is tone-less; +this release distills the best verified teacher available. A repaired +2.32%-tier teacher re-enters this pipeline when secryst regenerates it. diff --git a/models.yaml b/models.yaml index 04d3c7d..1f45a2a 100644 --- a/models.yaml +++ b/models.yaml @@ -10,8 +10,8 @@ # verify sha256, atomically rename into place # 4. load the zip (IMF v1: member sha256 verification on load) # -# Overrides: INTERSCRIPT_ML_INDEX (URL or path to an index like this one), -# INTERSCRIPT_ML_CACHE (cache directory; default ~/.cache/interscript). +# Overrides: SECRYST_INDEX (URL or path to an index like this one), +# SECRYST_CACHE (cache directory; default ~/.cache/secryst). # # Adding a model: it must have passed the WO03 gate (strict validation, # parity written into the zip) before an entry ships here. diff --git a/models/metrics-sources.yaml b/models/metrics-sources.yaml index 1b2260d..56564c9 100644 --- a/models/metrics-sources.yaml +++ b/models/metrics-sources.yaml @@ -40,3 +40,12 @@ heb-diac-1.0: - row: "s43 (production)" as: der_beam4 protocol: "beam=4 standard decode (reference quality; beam search is not in v1 runtimes); Nakdimon test split, 5,095 examples; ByT5-base s43" +tha-g2p-base-1.0: + repo: interscript/ml-models + ref: main + path: docs/RESULTS.md + anchor: tha-g2p-base-1.0 + protocol: "beam-4, corpus-level PER (total_ed/total_gold over chars of joined-piece decode); 1,219 held-out Kaikki Thai test sentences" + tables: + - {row: "Teacher (B-K/umt5 hub base)", column: PER, as: per_teacher} + - {row: "Student (ByT5-base, gate)", column: PER, as: per_student} diff --git a/models/tha-g2p-base/tha-g2p-base-1.0.metadata.yaml b/models/tha-g2p-base/tha-g2p-base-1.0.metadata.yaml index 96c4130..fb631e5 100644 --- a/models/tha-g2p-base/tha-g2p-base-1.0.metadata.yaml +++ b/models/tha-g2p-base/tha-g2p-base-1.0.metadata.yaml @@ -23,10 +23,10 @@ metrics: beam-4, corpus-level PER (total_ed/total_gold over chars of joined-piece decode); 1,219 Kaikki Thai test sentences; B-K/umt5-thai-g2p-v2-0.5k teacher - source: interscript/ml-models src/gpu/modal_distill.py::evaluate_per + source: interscript/ml-models docs/RESULTS.md#tha-g2p-base-1.0 - name: per_student value: 9.19 protocol: >- beam-4, corpus-level PER, same harness as the teacher; gate +4.76pp <= +5pp (docs/DISTILL-SOURCE-PROMPT.md); exact match 90.81% - source: interscript/ml-models release tha-g2p-base-1.0 + source: interscript/ml-models docs/RESULTS.md#tha-g2p-base-1.0 diff --git a/npm/models/package.json b/npm/models/package.json index 96f07c7..f4b8769 100644 --- a/npm/models/package.json +++ b/npm/models/package.json @@ -1,7 +1,7 @@ { - "name": "@interscript/models", + "name": "@secryst/models", "version": "0.0.1", - "description": "Manifest of available Interscript ML models + their current versions, URLs, and checksums.", + "description": "Manifest of available secryst (interscript-ml contract) models + their current versions, URLs, and checksums.", "license": "MIT", "main": "manifest.json", "files": [ diff --git a/runtime/README.md b/runtime/README.md index 7ef41cf..2415c4a 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -1,9 +1,14 @@ -# interscript-ml (Python runtime) +# secryst (Python crystal) -The reference Python runtime for **IMF v1** model zips — the phonological -layer of Interscript. The Ruby (secryst gem) and TypeScript -(@interscript/ml) runtimes are diffed against this one on shared golden -sets. +The Python crystal — reference implementation of **IMF v1** model zips +and the `models.yaml` index (the **interscript-ml** contract), the +phonological layer of Interscript. The Ruby (`secryst` gem) and +TypeScript (`@secryst/ml`) crystals are diffed against this one on +shared golden sets. This crystal owns golden generation and numerical +adjudication for the family. + +Home repo: https://github.com/secryst/secryst-py (this copy in +ml-models/runtime is the frozen origin; the package now lives there). ```python from interscript_ml import Model @@ -25,14 +30,14 @@ model = Model.load("khm-latn-1.0.zip") # or: a local zip path directly - Dynamic fetch per the `models.yaml` contract (shared with the Ruby and TypeScript runtimes): resolve id -> channel URL, download to temp, verify whole-file sha256 against the index, atomically install into - `~/.cache/interscript/models//`. Overrides: - `INTERSCRIPT_ML_INDEX` (URL or path), `INTERSCRIPT_ML_CACHE`. + `~/.cache/secryst/models//`. Overrides: + `SECRYST_INDEX` (URL or path), `SECRYST_CACHE`. Install: `pip install ./runtime` (from the ml-models checkout) or `pip install -e "./runtime[dev]"` for development. Tests: `python -m pytest runtime/tests` — tiny-graph zips, no torch -needed. The end-to-end golden test runs when `INTERSCRIPT_ML_E2E_ZIP` +needed. The end-to-end golden test runs when `SECRYST_E2E_ZIP` points at a real zip (e.g. `models/khm-latn/khm-latn-1.0-fp32.zip`) and asserts byte-identical outputs against `golden/khm-latn-100.jsonl`. diff --git a/runtime/src/interscript_ml/registry.py b/runtime/src/interscript_ml/registry.py index 1833281..7c14d8c 100644 --- a/runtime/src/interscript_ml/registry.py +++ b/runtime/src/interscript_ml/registry.py @@ -21,8 +21,8 @@ DEFAULT_INDEX_URL = ( "https://raw.githubusercontent.com/interscript/ml-models/main/models.yaml" ) -ENV_INDEX = "INTERSCRIPT_ML_INDEX" -ENV_CACHE = "INTERSCRIPT_ML_CACHE" +ENV_INDEX = "SECRYST_INDEX" +ENV_CACHE = "SECRYST_CACHE" class RegistryError(ValueError): @@ -51,7 +51,7 @@ class IndexEntry: def cache_dir() -> Path: if os.environ.get(ENV_CACHE): return Path(os.environ[ENV_CACHE]) - return Path.home() / ".cache" / "interscript" + return Path.home() / ".cache" / "secryst" def load_index(index_url: str | None = None) -> dict[str, IndexEntry]: diff --git a/runtime/tests/test_model.py b/runtime/tests/test_model.py index bc9df4e..b72677d 100644 --- a/runtime/tests/test_model.py +++ b/runtime/tests/test_model.py @@ -2,7 +2,7 @@ Tiny-graph zips built with the onnx package (no torch, no training repo). The end-to-end golden test runs only when a real zip is provided -via INTERSCRIPT_ML_E2E_ZIP. +via SECRYST_E2E_ZIP. """ from __future__ import annotations @@ -147,9 +147,9 @@ def test_missing_graph_rejected(tmp_path: Path) -> None: def test_golden_set_e2e() -> None: """Run against a real zip: byte-identical outputs on the golden set.""" - zip_path = os.environ.get("INTERSCRIPT_ML_E2E_ZIP") + zip_path = os.environ.get("SECRYST_E2E_ZIP") if not zip_path: - pytest.skip("set INTERSCRIPT_ML_E2E_ZIP to a real IMF zip") + pytest.skip("set SECRYST_E2E_ZIP to a real IMF zip") golden = Path(__file__).resolve().parent.parent.parent / "golden" / "khm-latn-100.jsonl" if "khm" not in Path(zip_path).name: pytest.skip("golden file is khm-latn specific") diff --git a/runtime/tests/test_registry.py b/runtime/tests/test_registry.py index 2e9b996..ad6dbbe 100644 --- a/runtime/tests/test_registry.py +++ b/runtime/tests/test_registry.py @@ -38,7 +38,7 @@ def test_resolve_downloads_verifies_and_caches(tmp_path: Path) -> None: zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip") index = _index_file(tmp_path, zip_path) cache = tmp_path / "cache" - os.environ["INTERSCRIPT_ML_CACHE"] = str(cache) + os.environ["SECRYST_CACHE"] = str(cache) try: local = resolve("tiny-1.0", index_url=str(index)) assert local == cache / "models" / "tiny-1.0" / "tiny.zip" @@ -47,18 +47,18 @@ def test_resolve_downloads_verifies_and_caches(tmp_path: Path) -> None: zip_path.unlink() assert resolve("tiny-1.0", index_url=str(index)) == local finally: - os.environ.pop("INTERSCRIPT_ML_CACHE", None) + os.environ.pop("SECRYST_CACHE", None) def test_resolve_rejects_bad_download(tmp_path: Path) -> None: zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip") index = _index_file(tmp_path, zip_path, sha256="0" * 64) - os.environ["INTERSCRIPT_ML_CACHE"] = str(tmp_path / "cache") + os.environ["SECRYST_CACHE"] = str(tmp_path / "cache") try: with pytest.raises(RegistryError, match="sha256 mismatch"): resolve("tiny-1.0", index_url=str(index)) finally: - os.environ.pop("INTERSCRIPT_ML_CACHE", None) + os.environ.pop("SECRYST_CACHE", None) def test_resolve_unknown_id(tmp_path: Path) -> None: @@ -71,7 +71,7 @@ def test_resolve_unknown_id(tmp_path: Path) -> None: def test_model_load_by_id(tmp_path: Path) -> None: zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip") index = _index_file(tmp_path, zip_path) - os.environ["INTERSCRIPT_ML_CACHE"] = str(tmp_path / "cache") + os.environ["SECRYST_CACHE"] = str(tmp_path / "cache") try: from interscript_ml import Model @@ -79,7 +79,7 @@ def test_model_load_by_id(tmp_path: Path) -> None: assert model.id == "tiny-1.0" assert isinstance(model.translate("he", max_len=4), str) finally: - os.environ.pop("INTERSCRIPT_ML_CACHE", None) + os.environ.pop("SECRYST_CACHE", None) def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None: @@ -118,7 +118,7 @@ def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None: index_path = tmp_path / "models.yaml" index_path.write_text(yaml.safe_dump(index), encoding="utf-8") cache = tmp_path / "cache" - os.environ["INTERSCRIPT_ML_CACHE"] = str(cache) + os.environ["SECRYST_CACHE"] = str(cache) try: local = resolve("tiny-1.0", index_url=str(index_path)) assert local == cache / "models" / "tiny-1.0" / "tiny.zip" @@ -127,7 +127,7 @@ def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None: (channel / "tiny.zip.part-00").unlink() assert resolve("tiny-1.0", index_url=str(index_path)) == local finally: - os.environ.pop("INTERSCRIPT_ML_CACHE", None) + os.environ.pop("SECRYST_CACHE", None) def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None: @@ -162,9 +162,9 @@ def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None: } index_path = tmp_path / "models.yaml" index_path.write_text(yaml.safe_dump(index), encoding="utf-8") - os.environ["INTERSCRIPT_ML_CACHE"] = str(tmp_path / "cache") + os.environ["SECRYST_CACHE"] = str(tmp_path / "cache") try: with pytest.raises(RegistryError, match="part 0"): resolve("tiny-1.0", index_url=str(index_path)) finally: - os.environ.pop("INTERSCRIPT_ML_CACHE", None) + os.environ.pop("SECRYST_CACHE", None) diff --git a/scripts/publish_model.py b/scripts/publish_model.py index 72515ef..d270089 100644 --- a/scripts/publish_model.py +++ b/scripts/publish_model.py @@ -131,7 +131,7 @@ def release_notes(model_id: str, meta: dict, filename: str, size: int, "Runtimes reassemble split parts transparently and verify every sha256:", "", "```python", - "from interscript_ml import Model", + "from secryst import Model", f'model = Model.load("{model_id}")', "```", ]