diff --git a/docs/paper-assets/frontier.png b/docs/paper-assets/frontier.png index 333c23f..9fc5462 100644 Binary files a/docs/paper-assets/frontier.png and b/docs/paper-assets/frontier.png differ diff --git a/docs/paper-assets/subset-overstatement.png b/docs/paper-assets/subset-overstatement.png new file mode 100644 index 0000000..b18737e Binary files /dev/null and b/docs/paper-assets/subset-overstatement.png differ diff --git a/scripts/figures/frontier.py b/scripts/figures/frontier.py new file mode 100644 index 0000000..cab8720 --- /dev/null +++ b/scripts/figures/frontier.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Paper-B figure: the size-quality frontier + the subset-overstatement +pairs, from the durable verdicts (final_eval.json per run). Fetches the +small JSON files from Modal volumes; no GPU. + + python scripts/figures/frontier.py --out docs/paper-assets/ +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import tempfile +from pathlib import Path + +RUNS = [ + # (label, params_m, full_set_der, subset_der, volume, path) + ("1.0 AdamW/r6/3ep", 300, 8.2590, 3.658, "rababa-checkpoints", + "rababa_arabic_distill_small/run-002"), + ("lite Muon/6ep", 190, 5.784, 2.6495, "rababa-checkpoints", + "rababa_arabic_distill_small/run-009-layerdrop-6ep"), + ("2.0 Muon/r7/3ep", 300, 4.8218, None, "rababa-checkpoints", + "rababa_arabic_distill_small/run-006-r7-muon"), + ("2.1 Muon/r7/6ep", 300, 4.5701, 2.0062, "rababa-checkpoints", + "rababa_arabic_distill_small/run-007-r7-muon-6ep"), + ("teacher r7", 580, 2.2864, None, "rababa-checkpoints", None), + ("tiny-max (30M full levers)", 33, 73.9489, None, "rababa-checkpoints", + "rababa_arabic_distill_small/run-010-tiny-max"), +] + + +def fetch_eval(volume: str, path: str | None) -> dict | None: + if not path: + return None + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + out = f.name + r = subprocess.run( + ["modal", "volume", "get", volume, f"{path}/final_eval.json", out], + capture_output=True, text=True) + if r.returncode != 0: + return None + try: + return json.loads(Path(out).read_text()) + except Exception: + return None + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="docs/paper-assets") + args = ap.parse_args() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + # Figure 1: the frontier (params vs full-set DER) + fig, ax = plt.subplots(figsize=(6, 4)) + pts = [(r[1], r[2]) for r in RUNS if r[2] is not None] + labels = [r[0] for r in RUNS if r[2] is not None] + ax.plot([p[0] for p in pts], [p[1] for p in pts], "o-") + for (x, y), lab in zip(pts, labels): + ax.annotate(lab, (x, y), fontsize=7, xytext=(4, 4), + textcoords="offset points") + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlabel("parameters (M, log)") + ax.set_ylabel("full-set windowed DER-CE (log)") + ax.set_title("The client-tier size–quality frontier (all full-set, CI-bracketed)") + fig.tight_layout() + fig.savefig(out / "frontier.png", dpi=200) + + # Figure 2: subset overstatement pairs + pairs = [(r[0], r[3], r[2]) for r in RUNS if r[3] is not None] + fig2, ax2 = plt.subplots(figsize=(6, 4)) + idx = range(len(pairs)) + w = 0.35 + ax2.bar([i - w / 2 for i in idx], [p[1] for p in pairs], w, label="first-300 subset") + ax2.bar([i + w / 2 for i in idx], [p[2] for p in pairs], w, label="full 1,200") + ax2.set_xticks(list(idx)) + ax2.set_xticklabels([p[0] for p in pairs], fontsize=7, rotation=20) + ax2.set_ylabel("DER-CE") + ax2.set_title("Subset overstatement: 2–4× inflation across five instances") + ax2.legend() + fig2.tight_layout() + fig2.savefig(out / "subset-overstatement.png", dpi=200) + print(f"wrote {out}/frontier.png and {out}/subset-overstatement.png") + + +if __name__ == "__main__": + main()