From aab4ab5207b10651078db89a51282b15352cd00b Mon Sep 17 00:00:00 2001 From: Daniel Morris Date: Wed, 5 Aug 2026 22:07:00 +0100 Subject: [PATCH] persona-smoke: strip markdown fences, refresh baseline Every fenced generation died at ILO-L001 on the backtick, which is how the whole smoke set read failed once the API key was restored. Port _strip_fences from closed-loop-bench.py. Baseline re-recorded on post-#785/#786 main. Still weak - the -- out: regex only captures one line so multi-line expectations can never match, and Haiku is nondeterministic across attempts - but it reflects what main actually does today. Gate redesign tracked in ILO-534. --- bench/persona-smoke-baseline.json | 168 ++++++++++++++++++++++++++++++ scripts/persona-smoke.py | 19 +++- 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 bench/persona-smoke-baseline.json diff --git a/bench/persona-smoke-baseline.json b/bench/persona-smoke-baseline.json new file mode 100644 index 000000000..6ce5578b8 --- /dev/null +++ b/bench/persona-smoke-baseline.json @@ -0,0 +1,168 @@ +{ + "generated": "2026-08-05T21:05:51Z", + "personas": { + "schedule-arithmetic": { + "persona": "schedule-arithmetic", + "outcome": "partial", + "generation_tokens": 795, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-text" + ] + }, + "event-chronology": { + "persona": "event-chronology", + "outcome": "failed", + "generation_tokens": 859, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-text", + "ilo-builtins-io" + ] + }, + "linear-regression": { + "persona": "linear-regression", + "outcome": "working", + "generation_tokens": 218, + "attempts": 2, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-math", + "ilo-builtins-io" + ] + }, + "k-means": { + "persona": "k-means", + "outcome": "failed", + "generation_tokens": 1612, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-math" + ] + }, + "api-poller": { + "persona": "api-poller", + "outcome": "failed", + "generation_tokens": 324, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-io" + ] + }, + "batch-http-fetch": { + "persona": "batch-http-fetch", + "outcome": "failed", + "generation_tokens": 691, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-io" + ] + }, + "csv-pipeline": { + "persona": "csv-pipeline", + "outcome": "failed", + "generation_tokens": 278, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-io", + "ilo-builtins-text" + ] + }, + "log-scanner": { + "persona": "log-scanner", + "outcome": "failed", + "generation_tokens": 1311, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-io", + "ilo-builtins-text" + ] + }, + "config-shaper": { + "persona": "config-shaper", + "outcome": "failed", + "generation_tokens": 830, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-language-records", + "ilo-builtins-io" + ] + }, + "record-transform": { + "persona": "record-transform", + "outcome": "failed", + "generation_tokens": 335, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-language-records" + ] + }, + "cron-explainer": { + "persona": "cron-explainer", + "outcome": "failed", + "generation_tokens": 1044, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-text" + ] + }, + "text-mining": { + "persona": "text-mining", + "outcome": "failed", + "generation_tokens": 501, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-builtins-text" + ] + }, + "doc-discovery": { + "persona": "doc-discovery", + "outcome": "failed", + "generation_tokens": 486, + "attempts": 3, + "tool_use_count": 0, + "modules_loaded": [ + "ilo-language", + "ilo-tools", + "ilo-agent" + ] + } + }, + "module_tokens": { + "ilo-agent": 1511, + "ilo-builtins-core": 1069, + "ilo-builtins-io": 2469, + "ilo-builtins-math": 1409, + "ilo-builtins-text": 1763, + "ilo-edit-loop": 587, + "ilo-engines": 447, + "ilo-errors": 970, + "ilo-examples": 561, + "ilo-language-records": 185, + "ilo-language": 2272, + "ilo-tools": 519 + } +} \ No newline at end of file diff --git a/scripts/persona-smoke.py b/scripts/persona-smoke.py index 9f2248373..66c303674 100644 --- a/scripts/persona-smoke.py +++ b/scripts/persona-smoke.py @@ -219,11 +219,28 @@ def call_haiku(system: str, user: str, api_key: str) -> tuple[str, int]: with urllib.request.urlopen(req, timeout=60) as resp: body = json.loads(resp.read()) - text = body["content"][0]["text"] + text = _strip_fences(body["content"][0]["text"]) tokens = body["usage"]["output_tokens"] return text, tokens +def _strip_fences(text: str) -> str: + """Strip markdown code fences (```lang ... ```) from model output. + + Same helper as closed-loop-bench.py: despite "no markdown fences" in the + prompt, models wrap output often enough that every fenced program used to + die at ILO-L001 on the backtick - which is how the whole smoke set came + back `failed` when the baseline was re-recorded (ILO-534). + """ + stripped = text.strip() + if stripped.startswith("```"): + lines = stripped.split("\n") + start = 1 + end = len(lines) - 1 if lines[-1].strip() == "```" else len(lines) + return "\n".join(lines[start:end]) + return text + + # --------------------------------------------------------------------------- # Helper: run generated ilo code and determine outcome # ---------------------------------------------------------------------------