From 11cdd5daae05f62de380d8cf142f56c50b96c897 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 31 Aug 2026 06:05:51 +0200 Subject: [PATCH] fix(eval): retry empty checkpoint rows instead of resuming them as done MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exhausted API retries checkpoint as empty-string predictions. Resuming treated them as completed rows, silently scoring them as all-error — the 2026-08-31 GLM-5.3-Flash run first read 15.96 DER because 140 such rows (produced by the pre-#62 400-ing payload) were resumed as 'done'; the true number was 8.57. Empty rows are now dropped at load and re-fetched, with a startup line disclosing how many. --- eval_sadeed_glm.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eval_sadeed_glm.py b/eval_sadeed_glm.py index 552d281..d9ff0a7 100644 --- a/eval_sadeed_glm.py +++ b/eval_sadeed_glm.py @@ -111,6 +111,15 @@ def main() -> None: for line in CKPT.read_text(encoding="utf-8").splitlines(): row = json.loads(line) done[row["idx"]] = row["pred"] + # exhausted retries checkpoint as empty strings; resuming them + # as "done" silently scores them as all-error rows (the 2026- + # 08-31 GLM-5.3-Flash run read 15.96 DER with 140 such rows) + empties = [i for i, pred in done.items() if not pred.strip()] + for i in empties: + del done[i] + if empties: + print(f"[glm] dropping {len(empties)} empty checkpoint rows " + "for retry", flush=True) todo = [i for i in range(len(inputs)) if i not in done] print(f"[glm] model={MODEL} effort={EFFORT or 'thinking-disabled'} " f"done={len(done)} todo={len(todo)}", flush=True)