From 414115ea9f422f2e221ca9e594317201de679e87 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 00:58:54 -0400 Subject: [PATCH 1/3] hw-gate: an npuserve phase, so the gate can fail a serve that is on the slow route (refs #2106) scripts/hw-gate.sh --phases npuserve runs tools/npu_serve_smoke.py on HW_GATE_NPU_SERVE_BIN / HW_GATE_NPU_SERVE_MODEL under the same NPU claim as the npu phase and reports the four assertions in the summary row. PASS needs the tool's exit status and its RESULT line. Not in the default phases; skips with no NPU or no settings. The tool's self-test now also drives its start / ask / stop path against a stand-in serve (fast, slow, dead), and tests/hw_gate_npuserve_test.sh (ctest hw_gate_npuserve) runs the phase end to end against the same stand-in: 13 checks, no device, the fleet's queues untouched. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 4 ++ docs/hw-gate.md | 18 +++++++- scripts/hw-gate.sh | 37 ++++++++++++++++ tests/hw_gate_npuserve_test.sh | 74 +++++++++++++++++++++++++++++++ tools/npu_serve_smoke.py | 79 +++++++++++++++++++++++++++++++++- 5 files changed, 209 insertions(+), 3 deletions(-) create mode 100755 tests/hw_gate_npuserve_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a8719d47..1f39e33d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -385,6 +385,10 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # resolves outside the payload, and a runtime set shipped without the aqlprofile agent libhsa opens. Each # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) +# engine#2106: the hardware gate's `npuserve` phase, end to end against the stand-in serve tools/npu_serve_smoke.py +# writes: a fast-route serve passes the gate, a slow-route one fails it while its answers are fine, a dead one fails, +# and an unconfigured or NPU-less host skips. No device, no 1bit binary, and the fleet's queues are not touched. +add_test(NAME hw_gate_npuserve COMMAND ${CMAKE_SOURCE_DIR}/tests/hw_gate_npuserve_test.sh) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. diff --git a/docs/hw-gate.md b/docs/hw-gate.md index c8cb13604..2c709acdd 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -289,8 +289,22 @@ Measured on strixhalo, 2026-11-11, Qwen3-0.6B, one ticket each: | a build of main without the private route, model-generic directory | per-op | `FAIL`: `route`, `rate` (median 1.9 tok/s) and `first-token` (1,063 ms) fail; `content` passes | The second row is the point: its three answers were fine, and only the other three assertions say -the release would have shipped the slow route. The check is not wired into `scripts/hw-gate.sh` or the -release script yet; it is the instrument, and the release lane decides where it runs. +the release would have shipped the slow route. + +**In the gate: the `npuserve` phase.** `scripts/hw-gate.sh --phases npuserve` runs the check under the +same NPU claim as the `npu` phase and adds its row to the summary, with the four assertion lines as the +detail: + +```sh +HW_GATE_NPU_SERVE_BIN=/path/to/1bit HW_GATE_NPU_SERVE_MODEL=/path/to/Qwen3-0.6B scripts/hw-gate.sh --phases npuserve +``` + +`HW_GATE_NPU_SERVE_FLOOR` sets the tok/s floor (default 60) and `HW_GATE_NPU_SERVE_PORT` the port. The +row is `PASS` only when the tool exits 0 **and** printed `RESULT: PASS`, so a truncated log cannot pass. +Like `npu`, the phase is not in the default phases and skips when there is no NPU or the two settings +are absent; the release procedure has to name it and set them for it to gate anything. It has not been +added to the release script. `tests/hw_gate_npuserve_test.sh` drives the phase end to end against a +stand-in serve (fast, slow, a floor it misses, dead at start, unconfigured, no NPU) with no device. ## Measured diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 62f5c7f2d..f58f97b38 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -35,6 +35,9 @@ # a 64-position greedy build, the prompt prefill, run-to-run reproducibility and a # tok/s floor (HW_GATE_NPU_FLOOR). Auto-skips with no NPU or no checker configured; not # in the default phases (add it with --phases npu). +# npuserve the shipped binary's `1bit serve --device npu` on a small model: the fast route's own log line, a +# decode floor, the first token and a non-empty answer (tools/npu_serve_smoke.py; engine#2106, #2102). +# Auto-skips with no NPU or no HW_GATE_NPU_SERVE_BIN / HW_GATE_NPU_SERVE_MODEL; not in the default phases. # placement every graph the server scheduled for a gate model (load, warmup, the kat prompts, # the needle) is all on HRX: ggml-hrx's placement report (GGML_HRX_REPORT=2, engine #390) # names no node off HRX, as `1bit serve` now requires by default (docs/serve.md) @@ -659,6 +662,40 @@ if has npu; then fi fi +# ---- npuserve (the shipped binary's `serve --device npu`: on the fast route, at speed, with an answer) ---- +# The npu phase above checks the engine through a private checker and never starts `1bit serve`, so it cannot see a +# shipped binary whose serve quietly takes the slow per-op route: that serve still answers, at about 2 tok/s where the +# fast route gives 80 to 100 (engine#2102, #2106). tools/npu_serve_smoke.py asserts the route's own log line, the decode +# floor, the first token and a non-empty answer. Set HW_GATE_NPU_SERVE_BIN to the 1bit under test and +# HW_GATE_NPU_SERVE_MODEL to the small model directory; HW_GATE_NPU_SERVE_FLOOR is the tok/s floor (default 60). +# It runs under the same NPU claim as the npu phase and, like it, is not in the default phases. +if has npuserve; then + t0=$(date +%s) + smoke=$(cd "$(dirname "$0")/.." && pwd)/tools/npu_serve_smoke.py + if [ ! -e "${HW_GATE_NPU_DEV:-/dev/accel/accel0}" ]; then + result npuserve SKIP 0 "no NPU (/dev/accel/accel0) on this host" + elif [ -z "${HW_GATE_NPU_SERVE_BIN:-}" ] || [ -z "${HW_GATE_NPU_SERVE_MODEL:-}" ]; then + result npuserve SKIP 0 "no serve under test configured (HW_GATE_NPU_SERVE_BIN and HW_GATE_NPU_SERVE_MODEL)" + else + cool + log=$OUT/npuserve.log + hold_args=() + if [ "${HW_GATE_NPU_HOLD:-1}" != 0 ] && [ -n "$NPU_HOLD" ] && ! token_held_by_ancestor "$NPU_TOKEN"; then + hold_args=("$NPU_HOLD" -l hw-gate-npuserve -t "${HW_GATE_NPU_HOLD_MINUTES:-20}" -w "$LOCK_WAIT" --) + fi + "${hold_args[@]}" flock -w "$LOCK_WAIT" -E 75 "$LOCK" python3 "$smoke" --bin "$HW_GATE_NPU_SERVE_BIN" \ + --model "$HW_GATE_NPU_SERVE_MODEL" --out "$OUT/npuserve" --floor "${HW_GATE_NPU_SERVE_FLOOR:-60}" \ + ${HW_GATE_NPU_SERVE_PORT:+--port "$HW_GATE_NPU_SERVE_PORT"} > "$log" 2>&1 + rc=$? + st=FAIL + # the tool's own verdict line AND its exit status: a pass needs both, so a truncated log cannot pass + [ "$rc" = 0 ] && grep -q '^RESULT: PASS$' "$log" && st=PASS + [ "$rc" = 75 ] && infra=1 + detail=$(grep -E '^(route|rate|first-token|content|serve) ' "$log" | sed -E 's/ +/ /g' | tr '\n' ';' | sed 's/;$//; s/;/; /g') + result npuserve $st $(( $(date +%s) - t0 )) "${detail:-no assertion lines in the log (exit $rc)}" + fi +fi + # ---- summary --------------------------------------------------------------------------------- total=$(( $(date +%s) - START )) nfail=$(awk -F'\t' '$2 == "FAIL"' "$RESULTS" | wc -l) diff --git a/tests/hw_gate_npuserve_test.sh b/tests/hw_gate_npuserve_test.sh new file mode 100755 index 000000000..167ad2c44 --- /dev/null +++ b/tests/hw_gate_npuserve_test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# hw_gate_npuserve_test.sh: the hardware gate's `npuserve` phase (engine#2106), end to end, with no device. +# +# scripts/hw-gate.sh --phases npuserve is run against the stand-in serve that tools/npu_serve_smoke.py writes +# (--write-stand-in): a process that prints a route line and answers with the timings it is told to. Three runs: +# a fast-route stand-in at 90 tok/s -> the phase is PASS and the gate exits 0 +# a slow-route stand-in at 1.9 tok/s -> the phase is FAIL, names route and rate, and the gate exits 1 +# nothing configured -> the phase is SKIP and the gate exits 0 +# The gate is kept off the fleet's GPU and NPU queues (HW_GATE_GPU_HOLD=0, HW_GATE_TOKEN=none, HW_GATE_NPU_HOLD=0), its +# lock and output live in a directory made here, and HW_GATE_NPU_DEV points the "is there an NPU" probe at a file +# that exists, so the phase runs on a machine with none. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +work=$(mktemp -d "$PWD/hw_gate_npuserve_test.XXXXXX") || exit 1 # the working directory, not /tmp +trap 'rm -rf "$work"' EXIT +fail=0 +check() { if eval "$1"; then echo "ok $2"; else echo "FAIL $2"; fail=1; fi; } + +python3 "$root/tools/npu_serve_smoke.py" --write-stand-in "$work/fake1bit" || { echo "FAIL cannot write the stand-in"; exit 1; } +touch "$work/npu-present" + +gate() { # gate [VAR=value ...]: runs the phase, leaves its text in $work/.txt and its exit in $rc + local name=$1; shift + env HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_NPU_HOLD=0 HW_GATE_NPU_DEV="$work/npu-present" \ + HW_GATE_LOCK="$work/lock" HW_GATE_OUT="$work/out-$name" "$@" \ + timeout 120 "$root/scripts/hw-gate.sh" --phases npuserve > "$work/$name.txt" 2>&1 + rc=$? +} +port() { python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])'; } + +gate fast FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 0 ]' "a fast-route serve at 90 tok/s: the gate exits 0 (got $rc)" +check 'grep -q "^| npuserve | PASS |" "$work/fast.txt"' "... and the summary row is PASS" +check 'grep -q "rate ok median 90.0 tok/s" "$work/fast.txt"' "... with the measured rate in the row" + +gate slow FAKE_MODE=slow FAKE_TPS=1.9 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 1 ]' "a slow-route serve at 1.9 tok/s: the gate exits 1 (got $rc)" +check 'grep -q "^| npuserve | FAIL |" "$work/slow.txt"' "... and the summary row is FAIL" +check 'grep -q "route FAIL the serve log names the per-op forward" "$work/slow.txt"' "... naming the route" +check 'grep -q "rate FAIL median 1.9 tok/s" "$work/slow.txt"' "... and the rate" +check 'grep -q "content ok" "$work/slow.txt"' "... while the answers themselves were fine (the reason the phase exists)" + +# a fast route under a floor it does not reach: the floor is the gate's own setting +gate floor FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_FLOOR=120 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 1 ] && grep -q "rate FAIL median 90.0 tok/s (floor 120.0" "$work/floor.txt"' "HW_GATE_NPU_SERVE_FLOOR=120 fails a 90 tok/s serve on the rate alone" + +gate dead FAKE_MODE=dead HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 1 ] && grep -q "^| npuserve | FAIL |" "$work/dead.txt" && grep -q "never became ready" "$work/dead.txt"' "a serve that dies at start is FAIL, and the row says it never became ready" + +gate skip +check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/skip.txt"' "with no binary and model configured the phase is SKIP and the gate exits 0" + +rm -f "$work/npu-present" +gate nodev HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model +check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/nodev.txt" && grep -q "no NPU" "$work/nodev.txt"' "with no NPU the phase is SKIP even when configured" + +check '! pgrep -f "$work/fake1bit" > /dev/null' "no stand-in serve is left running" +[ "$fail" = 0 ] && echo "hw_gate_npuserve: all checks passed" +exit "$fail" diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py index cb5a9d2d0..8b6aa753a 100755 --- a/tools/npu_serve_smoke.py +++ b/tools/npu_serve_smoke.py @@ -188,6 +188,75 @@ def rejudge(args): return report(judge(log_text, replies, args.floor, args.first_token_ms, args.fast_route, args.slow_route)) +FAKE_SERVE = r'''#!/usr/bin/env python3 +# A stand-in for `1bit serve`, written by npu_serve_smoke.py --self-test: it prints a route line, answers /health and +# /v1/chat/completions with the timings FAKE_TPS gives, and exits on SIGTERM. FAKE_MODE=dead exits before listening. +import http.server, json, os, signal, sys +port = int(sys.argv[sys.argv.index("-p") + 1]) +mode, tps = os.environ.get("FAKE_MODE", "fast"), float(os.environ.get("FAKE_TPS", "90")) +if mode == "dead": + print("1bit serve: cannot open the device", flush=True); sys.exit(3) +print("1bit unified: on the private route dx: dense Qwen3 on the dx kernels" if mode == "fast" else + "1bit serve: a model-generic container: serving it through the per-op forward", flush=True) +class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): pass + def _send(self, obj): + b = json.dumps(obj).encode(); self.send_response(200); self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(b))); self.end_headers(); self.wfile.write(b) + def do_GET(self): self._send({"status": "ok"}) + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + self._send({"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Paris."}}], + "timings": {"prompt_ms": 200.0 if mode == "fast" else 1100.0, "predicted_ms": 1000.0 * 8 / tps, "predicted_per_second": tps}, + "usage": {"completion_tokens": 8}}) +srv = http.server.HTTPServer(("127.0.0.1", port), H) +def bye(*a): + if os.environ.get("FAKE_MARK"): open(os.environ["FAKE_MARK"], "w").write("terminated") + os._exit(0) +signal.signal(signal.SIGTERM, bye) +srv.serve_forever() +''' + + +def self_test_run(fails): + """run() end to end against the stand-in: it starts the serve, waits, asks, stops it, and returns the verdict.""" + import contextlib + import io + import shutil + import socket + import tempfile + d = tempfile.mkdtemp(prefix="npu_serve_smoke_selftest.", dir=os.getcwd()) # the working directory, not /tmp + try: + fake = os.path.join(d, "fake1bit") + with open(fake, "w") as f: + f.write(FAKE_SERVE) + os.chmod(fake, 0o755) + for mode, tps, want_exit, want_line in (("fast", "90", 0, "RESULT: PASS"), ("slow", "1.9", 1, "route FAIL"), + ("dead", "90", 1, "it never became ready")): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + mark = os.path.join(d, "mark." + mode) + os.environ.update(FAKE_MODE=mode, FAKE_TPS=tps, FAKE_MARK=mark) + args = argparse.Namespace(bin=fake, model="model-dir", out=os.path.join(d, mode), port=port, floor=60.0, first_token_ms=600.0, + fast_route=FAST_ROUTE, slow_route=SLOW_ROUTE, max_tokens=8, ready_seconds=5 if mode == "dead" else 30, + request_seconds=30) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = run(args) + text = buf.getvalue() + if rc != want_exit or want_line not in text: + fails.append("run() against a %s stand-in: exit %d, output %r" % (mode, rc, text[-300:])) + if mode != "dead" and not os.path.exists(mark): + fails.append("run() against a %s stand-in left the serve running (no SIGTERM seen)" % mode) + if mode == "fast" and not os.path.exists(os.path.join(d, mode, "reply_2.json")): + fails.append("run() did not keep the three replies") + finally: + for k in ("FAKE_MODE", "FAKE_TPS", "FAKE_MARK"): + os.environ.pop(k, None) + shutil.rmtree(d, ignore_errors=True) + + def self_test(): fails = [] @@ -245,9 +314,11 @@ def expect(name, verdicts, want): if not degenerate("//////") or degenerate("391") or degenerate("aa") or not degenerate("a a a a"): fails.append("degenerate(): wrong on a fixture") + self_test_run(fails) for f in fails: print("FAIL " + f) - print("npu_serve_smoke self-test: %d case(s) failed" % len(fails) if fails else "npu_serve_smoke self-test: 16 cases ok") + print("npu_serve_smoke self-test: %d case(s) failed" % len(fails) if fails + else "npu_serve_smoke self-test: 16 judging cases and 3 runs against a stand-in serve ok") return 1 if fails else 0 @@ -255,6 +326,7 @@ def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--self-test", action="store_true") p.add_argument("--judge", metavar="DIR") + p.add_argument("--write-stand-in", metavar="PATH", help="write the self-test's stand-in serve to PATH (for testing a caller)") p.add_argument("--bin") p.add_argument("--model") p.add_argument("--out", default="~/.cache/1bit/npu-serve-smoke") @@ -267,6 +339,11 @@ def main(): p.add_argument("--ready-seconds", type=int, default=120) p.add_argument("--request-seconds", type=int, default=120) a = p.parse_args() + if a.write_stand_in: + with open(a.write_stand_in, "w") as f: + f.write(FAKE_SERVE) + os.chmod(a.write_stand_in, 0o755) + return 0 if a.self_test: return self_test() if a.judge: From fe89793b71daea2c6e977c68fea497dee86ca63d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 01:31:31 -0400 Subject: [PATCH 2/3] hw-gate npuserve tests: hold the loopback-port lock, and write the stand-in owner-only (CI at 414115e) CI's build job failed on route_port_lock: hw_gate_npuserve starts a loopback server without RESOURCE_LOCK route_ports. Both it and npu_serve_smoke_self (which now starts one too) take the lock. CodeQL flagged the stand-in serve's 0755 as world readable; it is written 0700. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 3 +++ tools/npu_serve_smoke.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f39e33d5..4cd0fc90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -389,6 +389,9 @@ add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) # writes: a fast-route serve passes the gate, a slow-route one fails it while its answers are fine, a dead one fails, # and an unconfigured or NPU-less host skips. No device, no 1bit binary, and the fleet's queues are not touched. add_test(NAME hw_gate_npuserve COMMAND ${CMAKE_SOURCE_DIR}/tests/hw_gate_npuserve_test.sh) +# Both start a loopback serve on a port picked by binding port 0 and releasing it, so they take the lock every +# loopback-server test takes (tests/route_port_lock_test.py): two of them racing for one freed port is a flaky failure. +set_tests_properties(hw_gate_npuserve npu_serve_smoke_self PROPERTIES RESOURCE_LOCK route_ports) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py index 8b6aa753a..a580e1113 100755 --- a/tools/npu_serve_smoke.py +++ b/tools/npu_serve_smoke.py @@ -230,7 +230,7 @@ def self_test_run(fails): fake = os.path.join(d, "fake1bit") with open(fake, "w") as f: f.write(FAKE_SERVE) - os.chmod(fake, 0o755) + os.chmod(fake, 0o700) for mode, tps, want_exit, want_line in (("fast", "90", 0, "RESULT: PASS"), ("slow", "1.9", 1, "route FAIL"), ("dead", "90", 1, "it never became ready")): with socket.socket() as s: @@ -342,7 +342,7 @@ def main(): if a.write_stand_in: with open(a.write_stand_in, "w") as f: f.write(FAKE_SERVE) - os.chmod(a.write_stand_in, 0o755) + os.chmod(a.write_stand_in, 0o700) return 0 if a.self_test: return self_test() From 52f3a435a492498e5db6b5dd40e76285014c7d25 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 01:52:38 -0400 Subject: [PATCH 3/3] hw-gate npuserve: the serve cannot outlive a TERMed run; the gate passes its own floor, length and reply count (review of #2125) CHANGES at fe89793: when the ticket's hold ends, the TERM reaches the smoke script (a flock sits between them) and Python's default action skipped the finally that stops the serve, leaving 1bit serve running with no ticket. The script now turns TERM/HUP/INT into SystemExit (the serve is stopped, exit 128+signal) and starts the serve with a parent-death signal. Pinned twice: the tool's self-test TERMs a real run mid-request (without the fix: exit -15 and the serve left running), and hw_gate_npuserve_test.sh sends gpu-hold's kill sequence to the phase's flock. READY-with-asks at fe89793: the phase now passes --floor, --max-tokens and --replies itself (30 tok/s, 96, 5), so the rate assertion does not depend on a default in another file. The floor is 30 because single fast-route replies have been measured from 48.8 to 101 tok/s on this shared box and the slow route is 1.9. Docs say the phase is blocking. Co-Authored-By: Claude Fable 5.2 --- docs/hw-gate.md | 24 ++++++++++-- scripts/hw-gate.sh | 13 +++++-- tests/hw_gate_npuserve_test.sh | 27 ++++++++++++- tools/npu_serve_smoke.py | 71 ++++++++++++++++++++++++++++++---- 4 files changed, 120 insertions(+), 15 deletions(-) diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 2c709acdd..e9b70eb52 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -299,12 +299,30 @@ detail: HW_GATE_NPU_SERVE_BIN=/path/to/1bit HW_GATE_NPU_SERVE_MODEL=/path/to/Qwen3-0.6B scripts/hw-gate.sh --phases npuserve ``` -`HW_GATE_NPU_SERVE_FLOOR` sets the tok/s floor (default 60) and `HW_GATE_NPU_SERVE_PORT` the port. The -row is `PASS` only when the tool exits 0 **and** printed `RESULT: PASS`, so a truncated log cannot pass. +The three numbers that decide the rate assertion are all passed by the phase, so none is a default +hidden in another file: `HW_GATE_NPU_SERVE_FLOOR` (tok/s, default **30**), `HW_GATE_NPU_SERVE_MAX_TOKENS` +(answer length, default 96) and `HW_GATE_NPU_SERVE_REPLIES` (requests, default 5; the rate is their +median). `HW_GATE_NPU_SERVE_PORT` sets the port. The row is `PASS` only when the tool exits 0 **and** +printed `RESULT: PASS`, so a truncated log cannot pass, and it always carries the measured median rate +and first-token time. + +The gate's floor is 30, not the demo's 60, because the box is shared. On the fast route the release +candidate's single replies have been measured between 48.8 and 101 tok/s depending on load; the slow +route is 1.9. A floor of 30 is fifteen times above the failure it guards against and under every fast +reply seen, so "slow machine" does not read as "slow route". The `route` assertion does not depend on +load at all. When the phase is named, a `FAIL` fails the gate like any other phase: it is blocking, +with no retry built in; a failed row shows which assertions failed and the numbers, so a rate-only +failure with `route ok` is recognisable as load and can be re-run. + +If the NPU ticket ends while a request hangs, the ticket's TERM reaches the smoke script (a `flock` sits +between them), and the script stops the serve before it exits; the serve is also started so that the +kernel TERMs it if the script dies any other way. A `1bit serve` left holding the NPU after its ticket +is the failure this prevents. Like `npu`, the phase is not in the default phases and skips when there is no NPU or the two settings are absent; the release procedure has to name it and set them for it to gate anything. It has not been added to the release script. `tests/hw_gate_npuserve_test.sh` drives the phase end to end against a -stand-in serve (fast, slow, a floor it misses, dead at start, unconfigured, no NPU) with no device. +stand-in serve (fast, slow, a floor it misses, dead at start, unconfigured, no NPU, and the ticket's +kill sequence arriving mid-request) with no device. ## Measured diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index f58f97b38..f8509147a 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -667,8 +667,14 @@ fi # shipped binary whose serve quietly takes the slow per-op route: that serve still answers, at about 2 tok/s where the # fast route gives 80 to 100 (engine#2102, #2106). tools/npu_serve_smoke.py asserts the route's own log line, the decode # floor, the first token and a non-empty answer. Set HW_GATE_NPU_SERVE_BIN to the 1bit under test and -# HW_GATE_NPU_SERVE_MODEL to the small model directory; HW_GATE_NPU_SERVE_FLOOR is the tok/s floor (default 60). -# It runs under the same NPU claim as the npu phase and, like it, is not in the default phases. +# HW_GATE_NPU_SERVE_MODEL to the small model directory. The three numbers that decide the rate assertion are all +# passed from here, so none of them is a default in another file: HW_GATE_NPU_SERVE_FLOOR (tok/s, default 30), +# HW_GATE_NPU_SERVE_MAX_TOKENS (answer length, default 96) and HW_GATE_NPU_SERVE_REPLIES (requests, default 5; the +# rate is their median). The floor is 30 and not the demo's 60 because this box is shared: single replies of the +# release candidate on the fast route have been measured from 48.8 to 101 tok/s under load, and the slow route is +# 1.9, so 30 is 15 times above the failure it guards against and below every fast reply seen. The route assertion +# does not depend on load at all. It runs under the same NPU claim as the npu phase and, like it, is not in the +# default phases; when it is named, a FAIL fails the gate like any other phase. if has npuserve; then t0=$(date +%s) smoke=$(cd "$(dirname "$0")/.." && pwd)/tools/npu_serve_smoke.py @@ -684,7 +690,8 @@ if has npuserve; then hold_args=("$NPU_HOLD" -l hw-gate-npuserve -t "${HW_GATE_NPU_HOLD_MINUTES:-20}" -w "$LOCK_WAIT" --) fi "${hold_args[@]}" flock -w "$LOCK_WAIT" -E 75 "$LOCK" python3 "$smoke" --bin "$HW_GATE_NPU_SERVE_BIN" \ - --model "$HW_GATE_NPU_SERVE_MODEL" --out "$OUT/npuserve" --floor "${HW_GATE_NPU_SERVE_FLOOR:-60}" \ + --model "$HW_GATE_NPU_SERVE_MODEL" --out "$OUT/npuserve" --floor "${HW_GATE_NPU_SERVE_FLOOR:-30}" \ + --max-tokens "${HW_GATE_NPU_SERVE_MAX_TOKENS:-96}" --replies "${HW_GATE_NPU_SERVE_REPLIES:-5}" \ ${HW_GATE_NPU_SERVE_PORT:+--port "$HW_GATE_NPU_SERVE_PORT"} > "$log" 2>&1 rc=$? st=FAIL diff --git a/tests/hw_gate_npuserve_test.sh b/tests/hw_gate_npuserve_test.sh index 167ad2c44..72042e17d 100755 --- a/tests/hw_gate_npuserve_test.sh +++ b/tests/hw_gate_npuserve_test.sh @@ -27,7 +27,7 @@ set -u root=$(cd "$(dirname "$0")/.." && pwd) work=$(mktemp -d "$PWD/hw_gate_npuserve_test.XXXXXX") || exit 1 # the working directory, not /tmp -trap 'rm -rf "$work"' EXIT +trap 'pkill -f "$work/fake1bit" 2> /dev/null; rm -rf "$work"' EXIT fail=0 check() { if eval "$1"; then echo "ok $2"; else echo "FAIL $2"; fail=1; fi; } @@ -55,6 +55,12 @@ check 'grep -q "route FAIL the serve log names the per-op forward" "$work/slow.t check 'grep -q "rate FAIL median 1.9 tok/s" "$work/slow.txt"' "... and the rate" check 'grep -q "content ok" "$work/slow.txt"' "... while the answers themselves were fine (the reason the phase exists)" +# the gate's own numbers reach the tool: five replies by default, and the floor it states +check 'grep -q "each: 90.0, 90.0, 90.0, 90.0, 90.0" "$work/fast.txt"' "the gate asks for five replies by default" +check 'grep -q "(floor 30.0;" "$work/fast.txt"' "... against its own default floor of 30 tok/s" +gate replies FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_REPLIES=3 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check 'grep -q "each: 90.0, 90.0, 90.0)" "$work/replies.txt"' "HW_GATE_NPU_SERVE_REPLIES=3 sends three" + # a fast route under a floor it does not reach: the floor is the gate's own setting gate floor FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_FLOOR=120 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" check '[ "$rc" = 1 ] && grep -q "rate FAIL median 90.0 tok/s (floor 120.0" "$work/floor.txt"' "HW_GATE_NPU_SERVE_FLOOR=120 fails a 90 tok/s serve on the rate alone" @@ -69,6 +75,25 @@ rm -f "$work/npu-present" gate nodev HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/nodev.txt" && grep -q "no NPU" "$work/nodev.txt"' "with no NPU the phase is SKIP even when configured" +# The ticket ends while a request hangs (review of #2125). gpu-hold stops its job with `pkill -TERM -P ` and then +# TERMs the job, and in this phase the job is the flock, so the TERM lands on the smoke script, not on the serve. The +# serve must not outlive it: a real one would be holding the NPU with no ticket. +p=$(port) +env HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_NPU_HOLD=0 HW_GATE_NPU_DEV=/dev/null HW_GATE_LOCK="$work/lock" HW_GATE_OUT="$work/out-stall" \ + FAKE_MODE=stall FAKE_MARK="$work/mark.stall" HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$p" \ + "$root/scripts/hw-gate.sh" --phases npuserve > "$work/stall.txt" 2>&1 & +gatepid=$! +for _ in $(seq 300); do [ -e "$work/mark.stall.asked" ] && break; sleep 0.1; done +check '[ -e "$work/mark.stall.asked" ]' "a stalling serve is reached: the run is inside its first request" +flockpid=$(pgrep -f "flock .*$work/lock" | head -n 1) +check '[ -n "$flockpid" ]' "the phase runs the smoke under flock (pid ${flockpid:-none})" +if [ -n "$flockpid" ]; then pkill -TERM -P "$flockpid"; kill -TERM "$flockpid" 2> /dev/null; fi +for _ in $(seq 300); do pgrep -f "$work/fake1bit" > /dev/null || break; sleep 0.1; done +check '! pgrep -f "$work/fake1bit" > /dev/null' "when the ticket's kill sequence hits the flock, the serve is stopped too" +check '[ -e "$work/mark.stall" ]' "... by a SIGTERM it saw, not by being abandoned" +wait "$gatepid"; rc=$? +check '[ "$rc" = 1 ] && grep -q "^| npuserve | FAIL |" "$work/stall.txt"' "... and the gate reports the phase FAIL (exit $rc)" + check '! pgrep -f "$work/fake1bit" > /dev/null' "no stand-in serve is left running" [ "$fail" = 0 ] && echo "hw_gate_npuserve: all checks passed" exit "$fail" diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py index a580e1113..cd142527c 100755 --- a/tools/npu_serve_smoke.py +++ b/tools/npu_serve_smoke.py @@ -26,9 +26,11 @@ npu-hold -l npu-smoke -t 10 -- tools/npu_serve_smoke.py --bin <1bit> --model [--out ] -It starts the serve, waits for it, sends three prompts with thinking off, stops the serve, and prints one line per -assertion and `RESULT: PASS` or `RESULT: FAIL` (exit 0 / 1). It takes no device ticket itself: run it under npu-hold, -so the serve ends with the ticket. Logs go to --out (default ~/.cache/1bit/npu-serve-smoke), never to /tmp. +It starts the serve, waits for it, sends --replies requests (default 3) with thinking off, stops the serve, and prints +one line per assertion and `RESULT: PASS` or `RESULT: FAIL` (exit 0 / 1). It takes no device ticket itself: run it +under npu-hold. The serve ends with this script however the script ends: a TERM, HUP or INT stops the serve first +(exit 128 + signal), and the kernel TERMs the serve if the script is killed outright. Logs go to --out (default +~/.cache/1bit/npu-serve-smoke), never to /tmp. tools/npu_serve_smoke.py --self-test the judging on recorded shapes of each failure; no device, no binary tools/npu_serve_smoke.py --judge judge a finished run's serve.log and reply_*.json again @@ -125,14 +127,33 @@ def report(verdicts): return 0 if ok else 1 +def die_with_parent(): + """In the serve's process, before exec: ask the kernel to TERM it if this script dies, however it dies.""" + try: + import ctypes + ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGTERM) # PR_SET_PDEATHSIG + except Exception: + pass + if os.getppid() == 1: # the script was already gone when this ran + os._exit(1) + + def run(args): + # A ticket that ends TERMs this script, not the serve (the serve is this script's child, and in the gate a flock + # sits between the ticket and the script). Python's default action for SIGTERM ends the process without running + # `finally`, which is the only code that stops the serve -- so a hung run would leave `1bit serve` holding the NPU + # after its ticket (review of #2125). The handlers turn the signal into SystemExit, which unwinds through it. + def stop(signum, _frame): + sys.exit(128 + signum) + for sig in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + signal.signal(sig, stop) out = os.path.expanduser(args.out) os.makedirs(out, exist_ok=True) log_path = os.path.join(out, "serve.log") cmd = [args.bin, "serve", "-m", args.model, "--device", "npu", "--host", "127.0.0.1", "-p", str(args.port)] replies = [] with open(log_path, "wb") as log: - serve = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT) + serve = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, preexec_fn=die_with_parent) try: deadline = time.time() + args.ready_seconds ready = False @@ -145,7 +166,8 @@ def run(args): except OSError: time.sleep(0.5) if ready: - for i, prompt in enumerate(PROMPTS): + for i in range(args.replies): + prompt = PROMPTS[i % len(PROMPTS)] body = json.dumps({"model": "m", "max_tokens": args.max_tokens, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}, "messages": [{"role": "user", "content": prompt}]}).encode() @@ -196,7 +218,7 @@ def rejudge(args): mode, tps = os.environ.get("FAKE_MODE", "fast"), float(os.environ.get("FAKE_TPS", "90")) if mode == "dead": print("1bit serve: cannot open the device", flush=True); sys.exit(3) -print("1bit unified: on the private route dx: dense Qwen3 on the dx kernels" if mode == "fast" else +print("1bit unified: on the private route dx: dense Qwen3 on the dx kernels" if mode in ("fast", "stall") else "1bit serve: a model-generic container: serving it through the per-op forward", flush=True) class H(http.server.BaseHTTPRequestHandler): def log_message(self, *a): pass @@ -206,6 +228,9 @@ def _send(self, obj): def do_GET(self): self._send({"status": "ok"}) def do_POST(self): self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if mode == "stall": + open(os.environ["FAKE_MARK"] + ".asked", "w").write("asked") + import time; time.sleep(600) self._send({"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Paris."}}], "timings": {"prompt_ms": 200.0 if mode == "fast" else 1100.0, "predicted_ms": 1000.0 * 8 / tps, "predicted_per_second": tps}, "usage": {"completion_tokens": 8}}) @@ -240,7 +265,7 @@ def self_test_run(fails): os.environ.update(FAKE_MODE=mode, FAKE_TPS=tps, FAKE_MARK=mark) args = argparse.Namespace(bin=fake, model="model-dir", out=os.path.join(d, mode), port=port, floor=60.0, first_token_ms=600.0, fast_route=FAST_ROUTE, slow_route=SLOW_ROUTE, max_tokens=8, ready_seconds=5 if mode == "dead" else 30, - request_seconds=30) + request_seconds=30, replies=3) buf = io.StringIO() with contextlib.redirect_stdout(buf): rc = run(args) @@ -251,9 +276,38 @@ def self_test_run(fails): fails.append("run() against a %s stand-in left the serve running (no SIGTERM seen)" % mode) if mode == "fast" and not os.path.exists(os.path.join(d, mode, "reply_2.json")): fails.append("run() did not keep the three replies") + # The ticket-ends case: this script is TERMed while a request hangs. The serve must be stopped too, and the + # script must exit 128 + SIGTERM. Run as a real process, because the signal's default action is the bug. + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + mark = os.path.join(d, "mark.stall") + env = dict(os.environ, FAKE_MODE="stall", FAKE_TPS="90", FAKE_MARK=mark) + tool = subprocess.Popen([sys.executable, os.path.abspath(__file__), "--bin", fake, "--model", "model-dir", "--out", + os.path.join(d, "stall"), "--port", str(port), "--request-seconds", "300"], + env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.time() + 30 + while time.time() < deadline and not os.path.exists(mark + ".asked"): + time.sleep(0.1) + if not os.path.exists(mark + ".asked"): + fails.append("the stalling stand-in was never asked (the run did not reach its first request)") + tool.send_signal(signal.SIGTERM) + try: + rc = tool.wait(timeout=40) + except subprocess.TimeoutExpired: + tool.kill() + rc = None + if rc != 128 + signal.SIGTERM: + fails.append("a TERMed run exits %r, want %d" % (rc, 128 + signal.SIGTERM)) + deadline = time.time() + 10 + while time.time() < deadline and not os.path.exists(mark): + time.sleep(0.1) + if not os.path.exists(mark): + fails.append("a TERMed run left the serve running: the stand-in never saw its SIGTERM") finally: for k in ("FAKE_MODE", "FAKE_TPS", "FAKE_MARK"): os.environ.pop(k, None) + subprocess.run(["pkill", "-f", fake], check=False) # nothing of this test's may outlive it, whatever failed shutil.rmtree(d, ignore_errors=True) @@ -318,7 +372,7 @@ def expect(name, verdicts, want): for f in fails: print("FAIL " + f) print("npu_serve_smoke self-test: %d case(s) failed" % len(fails) if fails - else "npu_serve_smoke self-test: 16 judging cases and 3 runs against a stand-in serve ok") + else "npu_serve_smoke self-test: 16 judging cases, 3 runs against a stand-in serve and a TERM mid-request ok") return 1 if fails else 0 @@ -336,6 +390,7 @@ def main(): p.add_argument("--fast-route", default=FAST_ROUTE, help="the text the serve log prints on the fast route") p.add_argument("--slow-route", default=SLOW_ROUTE, help="the text the serve log prints on the slow route") p.add_argument("--max-tokens", type=int, default=96) + p.add_argument("--replies", type=int, default=3, help="how many requests to send (the three prompts, repeated); the rate is their median") p.add_argument("--ready-seconds", type=int, default=120) p.add_argument("--request-seconds", type=int, default=120) a = p.parse_args()