From 42ab7caef48aa04d92dab3d9a0ee0c17e8c6849c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 08:21:49 -0400 Subject: [PATCH] npu_serve_smoke: A4 separates "budget went to reasoning" from "the model said nothing" (engine#2111) DEMO-SMOKE.md wrote this refinement as a condition - *"once engine#2111's reasoning_tokens is in the build"* - and the field is now on main (completion_tokens_details.reasoning_tokens, absent where nothing was split). Before this, the content arm reported FAIL for both of these: empty content, finish_reason length, reasoning_tokens 8 the budget went into a block we strip -> SKIP empty content, finish_reason stop the model really said nothing -> FAIL Two different faults, one verdict, and the one that is not the model's was being charged to the model. The four cases now stay four answers, and each has an arm that can fail: * empty + length + reasoning_tokens > 0 -> SKIP, with the count in the reason * empty + length + reasoning_tokens == -1 -> FAIL, naming that -1 means "not counted", which is not zero * empty + stop -> FAIL (the model said nothing) * degenerate content -> FAIL (that is the model, not the budget) Verified by breaking one arm on purpose (it fails, and only it), and by re-judging the 10:54Z artefacts - still PASS, so the refinement does not turn a healthy run into a failure. --self-test: 16 judging cases, 4 engine#2111 content arms, 3 stand-in runs, rc 0. --- tools/npu_serve_smoke.py | 58 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py index 4ae057f56..7fde600e2 100755 --- a/tools/npu_serve_smoke.py +++ b/tools/npu_serve_smoke.py @@ -115,7 +115,39 @@ def judge(log_text, replies, floor=60.0, first_token_ms=600.0, fast_route=FAST_R bad.append("reply %d: one character repeated (%r)" % (i, c[:12])) if not replies: bad.append("no replies") - out.append(("content", not bad, "; ".join(bad) if bad else "%d replies, all with content" % len(replies))) + # engine#2111 (its own refinement, now that the field is in the build): an EMPTY content with + # finish_reason "length" and reasoning_tokens > 0 is not a degeneracy - the budget went into a thinking block + # the engine strips - so it is a SKIP with a reason, not a FAIL. `reasoning_tokens == -1` means the route could + # not count it (e.g. /v1/completions) and must NOT be read as zero: that stays a FAIL that says so. + # "No replies" keeps its own arm: an empty list is nothing readable, which is never a pass. + if not replies: + out.append(("content", False, "no replies")) + return out + bad, skipped = [], [] + for i, r in enumerate(replies): + c = content_of(r) + if c and not degenerate(c): + continue + choice = (r.get("choices") or [{}])[0] + rt = (((r.get("usage") or {}).get("completion_tokens_details")) or {}).get("reasoning_tokens") + if not c: + if (choice.get("finish_reason") or "").lower() == "length" and isinstance(rt, int) and rt > 0: + skipped.append("reply %d: budget spent in reasoning (reasoning_tokens %d, finish_reason length)" + % (i, rt)) + continue + bad.append("reply %d: empty content%s" % ( + i, " (reasoning_tokens -1: the route could not count it, which is not zero)" if rt == -1 else "")) + else: + bad.append("reply %d: content is one character repeated" % i) + if bad: + out.append(("content", False, "; ".join(bad))) + elif skipped and len(skipped) == len(replies): + out.append(("content", True, "SKIP: " + "; ".join(skipped))) + elif skipped: + out.append(("content", True, "%d replies with content, %d skipped: %s" + % (len(replies) - len(skipped), len(skipped), "; ".join(skipped)))) + else: + out.append(("content", True, "%d replies, all with content" % len(replies))) return out @@ -458,11 +490,33 @@ def note_is(name, verdicts, load1, want_hint, want_none=False): if rc != 2: fails.append("rejudge() on a directory with no reply_*.json returned %r, wanted 2" % rc) + + # engine#2111's refinement, now that completion_tokens_details.reasoning_tokens is in the build: four cases that + # must stay FOUR answers, because collapsing any pair of them is how an ambiguity becomes a wrong report. + def _r(content, fr="stop", rt="absent"): + r = {"choices": [{"finish_reason": fr, "message": {"content": content}}], + "timings": {"prompt_ms": 200.0, "predicted_ms": 100.0, "predicted_per_second": 80.0}, + "usage": {"completion_tokens": 8}} + if rt != "absent": + r["usage"]["completion_tokens_details"] = {"reasoning_tokens": rt} + return r + + def _content_ok(reply): + return dict((n, ok) for n, ok, _ in judge(fast_log, [reply]))["content"] + + if not _content_ok(_r("", "length", 8)): + fails.append("empty content + finish_reason length + reasoning_tokens 8 must be a SKIP, not a FAIL") + if _content_ok(_r("", "length", -1)): + fails.append("reasoning_tokens -1 must not pass: the route could not count it, which is not zero") + if _content_ok(_r("", "stop", 8)): + fails.append("empty content with finish_reason stop must FAIL: the model said nothing, not the budget") + if _content_ok(_r("//////", "stop")): + fails.append("a degenerate content must still FAIL: that is the model, not the budget") self_test_run(fails) for f in fails: print("FAIL " + f) print("npu_serve_smoke self-test: %d case(s) failed" % len(fails) if fails - else "npu_serve_smoke self-test: 16 judging cases, 3 runs against a stand-in serve and a TERM mid-request ok") + else "npu_serve_smoke self-test: 16 judging cases, 4 engine#2111 content arms, 3 runs against a stand-in serve and a TERM mid-request ok") return 1 if fails else 0