bong-water-water-bong · Nov 11, 2026 · Nov 11, 2026 · Nov 11, 2026 · Nov 11, 2026
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -385,6 +385,13 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s
 # resolves outside the payload, and a runtime set shipped without the aqlprofile agent libhsa opens. Each
 # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI.
 add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh)
+# engine#2106: the hardware gate's `npuserve` phase, end to end against the stand-in serve tools/npu_serve_smoke.py
+# writes: a fast-route serve passes the gate, a slow-route one fails it while its answers are fine, a dead one fails,
+# and an unconfigured or NPU-less host skips. No device, no 1bit binary, and the fleet's queues are not touched.
+add_test(NAME hw_gate_npuserve COMMAND ${CMAKE_SOURCE_DIR}/tests/hw_gate_npuserve_test.sh)
+# Both start a loopback serve on a port picked by binding port 0 and releasing it, so they take the lock every
+# loopback-server test takes (tests/route_port_lock_test.py): two of them racing for one freed port is a flaky failure.
+set_tests_properties(hw_gate_npuserve npu_serve_smoke_self PROPERTIES RESOURCE_LOCK route_ports)
 add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh)
 # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy.
 # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd.

diff --git a/docs/hw-gate.md b/docs/hw-gate.md
@@ -289,8 +289,40 @@ Measured on strixhalo, 2026-11-11, Qwen3-0.6B, one ticket each:
 | a build of main without the private route, model-generic directory | per-op | `FAIL`: `route`, `rate` (median 1.9 tok/s) and `first-token` (1,063 ms) fail; `content` passes |
 
 The second row is the point: its three answers were fine, and only the other three assertions say
-the release would have shipped the slow route. The check is not wired into `scripts/hw-gate.sh` or the
-release script yet; it is the instrument, and the release lane decides where it runs.
+the release would have shipped the slow route.
+
+**In the gate: the `npuserve` phase.** `scripts/hw-gate.sh --phases npuserve` runs the check under the
+same NPU claim as the `npu` phase and adds its row to the summary, with the four assertion lines as the
+detail:
+
+```sh
+HW_GATE_NPU_SERVE_BIN=/path/to/1bit HW_GATE_NPU_SERVE_MODEL=/path/to/Qwen3-0.6B scripts/hw-gate.sh --phases npuserve
+```
+
+The three numbers that decide the rate assertion are all passed by the phase, so none is a default
+hidden in another file: `HW_GATE_NPU_SERVE_FLOOR` (tok/s, default **30**), `HW_GATE_NPU_SERVE_MAX_TOKENS`
+(answer length, default 96) and `HW_GATE_NPU_SERVE_REPLIES` (requests, default 5; the rate is their
+median). `HW_GATE_NPU_SERVE_PORT` sets the port. The row is `PASS` only when the tool exits 0 **and**
+printed `RESULT: PASS`, so a truncated log cannot pass, and it always carries the measured median rate
+and first-token time.
+
+The gate's floor is 30, not the demo's 60, because the box is shared. On the fast route the release
+candidate's single replies have been measured between 48.8 and 101 tok/s depending on load; the slow
+route is 1.9. A floor of 30 is fifteen times above the failure it guards against and under every fast
+reply seen, so "slow machine" does not read as "slow route". The `route` assertion does not depend on
+load at all. When the phase is named, a `FAIL` fails the gate like any other phase: it is blocking,
+with no retry built in; a failed row shows which assertions failed and the numbers, so a rate-only
+failure with `route ok` is recognisable as load and can be re-run.
+
+If the NPU ticket ends while a request hangs, the ticket's TERM reaches the smoke script (a `flock` sits
+between them), and the script stops the serve before it exits; the serve is also started so that the
+kernel TERMs it if the script dies any other way. A `1bit serve` left holding the NPU after its ticket
+is the failure this prevents.
+Like `npu`, the phase is not in the default phases and skips when there is no NPU or the two settings
+are absent; the release procedure has to name it and set them for it to gate anything. It has not been
+added to the release script. `tests/hw_gate_npuserve_test.sh` drives the phase end to end against a
+stand-in serve (fast, slow, a floor it misses, dead at start, unconfigured, no NPU, and the ticket's
+kill sequence arriving mid-request) with no device.
 
 ## Measured
 

diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh
@@ -35,6 +35,9 @@
 #           a 64-position greedy build, the prompt prefill, run-to-run reproducibility and a
 #           tok/s floor (HW_GATE_NPU_FLOOR). Auto-skips with no NPU or no checker configured; not
 #           in the default phases (add it with --phases npu).
+#   npuserve  the shipped binary's `1bit serve --device npu` on a small model: the fast route's own log line, a
+#           decode floor, the first token and a non-empty answer (tools/npu_serve_smoke.py; engine#2106, #2102).
+#           Auto-skips with no NPU or no HW_GATE_NPU_SERVE_BIN / HW_GATE_NPU_SERVE_MODEL; not in the default phases.
 #   placement  every graph the server scheduled for a gate model (load, warmup, the kat prompts,
 #           the needle) is all on HRX: ggml-hrx's placement report (GGML_HRX_REPORT=2, engine #390)
 #           names no node off HRX, as `1bit serve` now requires by default (docs/serve.md)
@@ -659,6 +662,47 @@ if has npu; then
   fi
 fi
 
+# ---- npuserve (the shipped binary's `serve --device npu`: on the fast route, at speed, with an answer) ----
+# The npu phase above checks the engine through a private checker and never starts `1bit serve`, so it cannot see a
+# shipped binary whose serve quietly takes the slow per-op route: that serve still answers, at about 2 tok/s where the
+# fast route gives 80 to 100 (engine#2102, #2106). tools/npu_serve_smoke.py asserts the route's own log line, the decode
+# floor, the first token and a non-empty answer. Set HW_GATE_NPU_SERVE_BIN to the 1bit under test and
+# HW_GATE_NPU_SERVE_MODEL to the small model directory. The three numbers that decide the rate assertion are all
+# passed from here, so none of them is a default in another file: HW_GATE_NPU_SERVE_FLOOR (tok/s, default 30),
+# HW_GATE_NPU_SERVE_MAX_TOKENS (answer length, default 96) and HW_GATE_NPU_SERVE_REPLIES (requests, default 5; the
+# rate is their median). The floor is 30 and not the demo's 60 because this box is shared: single replies of the
+# release candidate on the fast route have been measured from 48.8 to 101 tok/s under load, and the slow route is
+# 1.9, so 30 is 15 times above the failure it guards against and below every fast reply seen. The route assertion
+# does not depend on load at all. It runs under the same NPU claim as the npu phase and, like it, is not in the
+# default phases; when it is named, a FAIL fails the gate like any other phase.
+if has npuserve; then
+  t0=$(date +%s)
+  smoke=$(cd "$(dirname "$0")/.." && pwd)/tools/npu_serve_smoke.py
+  if [ ! -e "${HW_GATE_NPU_DEV:-/dev/accel/accel0}" ]; then
+    result npuserve SKIP 0 "no NPU (/dev/accel/accel0) on this host"
+  elif [ -z "${HW_GATE_NPU_SERVE_BIN:-}" ] || [ -z "${HW_GATE_NPU_SERVE_MODEL:-}" ]; then
+    result npuserve SKIP 0 "no serve under test configured (HW_GATE_NPU_SERVE_BIN and HW_GATE_NPU_SERVE_MODEL)"
+  else
+    cool
+    log=$OUT/npuserve.log
+    hold_args=()
+    if [ "${HW_GATE_NPU_HOLD:-1}" != 0 ] && [ -n "$NPU_HOLD" ] && ! token_held_by_ancestor "$NPU_TOKEN"; then
+        hold_args=("$NPU_HOLD" -l hw-gate-npuserve -t "${HW_GATE_NPU_HOLD_MINUTES:-20}" -w "$LOCK_WAIT" --)
+    fi
+    "${hold_args[@]}" flock -w "$LOCK_WAIT" -E 75 "$LOCK" python3 "$smoke" --bin "$HW_GATE_NPU_SERVE_BIN" \
+        --model "$HW_GATE_NPU_SERVE_MODEL" --out "$OUT/npuserve" --floor "${HW_GATE_NPU_SERVE_FLOOR:-30}" \
+        --max-tokens "${HW_GATE_NPU_SERVE_MAX_TOKENS:-96}" --replies "${HW_GATE_NPU_SERVE_REPLIES:-5}" \
+        ${HW_GATE_NPU_SERVE_PORT:+--port "$HW_GATE_NPU_SERVE_PORT"} > "$log" 2>&1
+    rc=$?
+    st=FAIL
+    # the tool's own verdict line AND its exit status: a pass needs both, so a truncated log cannot pass
+    [ "$rc" = 0 ] && grep -q '^RESULT: PASS$' "$log" && st=PASS
+    [ "$rc" = 75 ] && infra=1
+    detail=$(grep -E '^(route|rate|first-token|content|serve) ' "$log" | sed -E 's/ +/ /g' | tr '\n' ';' | sed 's/;$//; s/;/; /g')
+    result npuserve $st $(( $(date +%s) - t0 )) "${detail:-no assertion lines in the log (exit $rc)}"
+  fi
+fi
+
 # ---- summary ---------------------------------------------------------------------------------
 total=$(( $(date +%s) - START ))
 nfail=$(awk -F'\t' '$2 == "FAIL"' "$RESULTS" | wc -l)

diff --git a/tests/hw_gate_npuserve_test.sh b/tests/hw_gate_npuserve_test.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# hw_gate_npuserve_test.sh: the hardware gate's `npuserve` phase (engine#2106), end to end, with no device.
+#
+# scripts/hw-gate.sh --phases npuserve is run against the stand-in serve that tools/npu_serve_smoke.py writes
+# (--write-stand-in): a process that prints a route line and answers with the timings it is told to. Three runs:
+#   a fast-route stand-in at 90 tok/s  -> the phase is PASS and the gate exits 0
+#   a slow-route stand-in at 1.9 tok/s -> the phase is FAIL, names route and rate, and the gate exits 1
+#   nothing configured                 -> the phase is SKIP and the gate exits 0
+# The gate is kept off the fleet's GPU and NPU queues (HW_GATE_GPU_HOLD=0, HW_GATE_TOKEN=none, HW_GATE_NPU_HOLD=0), its
+# lock and output live in a directory made here, and HW_GATE_NPU_DEV points the "is there an NPU" probe at a file
+# that exists, so the phase runs on a machine with none.
+set -u
+root=$(cd "$(dirname "$0")/.." && pwd)
+work=$(mktemp -d "$PWD/hw_gate_npuserve_test.XXXXXX") || exit 1      # the working directory, not /tmp
+trap 'pkill -f "$work/fake1bit" 2> /dev/null; rm -rf "$work"' EXIT
+fail=0
+check() { if eval "$1"; then echo "ok    $2"; else echo "FAIL  $2"; fail=1; fi; }
+
+python3 "$root/tools/npu_serve_smoke.py" --write-stand-in "$work/fake1bit" || { echo "FAIL  cannot write the stand-in"; exit 1; }
+touch "$work/npu-present"
+
+gate() {  # gate <name> [VAR=value ...]: runs the phase, leaves its text in $work/<name>.txt and its exit in $rc
+    local name=$1; shift
+    env HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_NPU_HOLD=0 HW_GATE_NPU_DEV="$work/npu-present" \
+        HW_GATE_LOCK="$work/lock" HW_GATE_OUT="$work/out-$name" "$@" \
+        timeout 120 "$root/scripts/hw-gate.sh" --phases npuserve > "$work/$name.txt" 2>&1
+    rc=$?
+}
+port() { python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])'; }
+
+gate fast FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)"
+check '[ "$rc" = 0 ]' "a fast-route serve at 90 tok/s: the gate exits 0 (got $rc)"
+check 'grep -q "^| npuserve | PASS |" "$work/fast.txt"' "... and the summary row is PASS"
+check 'grep -q "rate ok median 90.0 tok/s" "$work/fast.txt"' "... with the measured rate in the row"
+
+gate slow FAKE_MODE=slow FAKE_TPS=1.9 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)"
+check '[ "$rc" = 1 ]' "a slow-route serve at 1.9 tok/s: the gate exits 1 (got $rc)"
+check 'grep -q "^| npuserve | FAIL |" "$work/slow.txt"' "... and the summary row is FAIL"
+check 'grep -q "route FAIL the serve log names the per-op forward" "$work/slow.txt"' "... naming the route"
+check 'grep -q "rate FAIL median 1.9 tok/s" "$work/slow.txt"' "... and the rate"
+check 'grep -q "content ok" "$work/slow.txt"' "... while the answers themselves were fine (the reason the phase exists)"
+
+# the gate's own numbers reach the tool: five replies by default, and the floor it states
+check 'grep -q "each: 90.0, 90.0, 90.0, 90.0, 90.0" "$work/fast.txt"' "the gate asks for five replies by default"
+check 'grep -q "(floor 30.0;" "$work/fast.txt"' "... against its own default floor of 30 tok/s"
+gate replies FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_REPLIES=3 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)"
+check 'grep -q "each: 90.0, 90.0, 90.0)" "$work/replies.txt"' "HW_GATE_NPU_SERVE_REPLIES=3 sends three"
+
+# a fast route under a floor it does not reach: the floor is the gate's own setting
+gate floor FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_FLOOR=120 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)"
+check '[ "$rc" = 1 ] && grep -q "rate FAIL median 90.0 tok/s (floor 120.0" "$work/floor.txt"' "HW_GATE_NPU_SERVE_FLOOR=120 fails a 90 tok/s serve on the rate alone"
+
+gate dead FAKE_MODE=dead HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)"
+check '[ "$rc" = 1 ] && grep -q "^| npuserve | FAIL |" "$work/dead.txt" && grep -q "never became ready" "$work/dead.txt"' "a serve that dies at start is FAIL, and the row says it never became ready"
+
+gate skip
+check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/skip.txt"' "with no binary and model configured the phase is SKIP and the gate exits 0"
+
+rm -f "$work/npu-present"
+gate nodev HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model
+check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/nodev.txt" && grep -q "no NPU" "$work/nodev.txt"' "with no NPU the phase is SKIP even when configured"
+
+# The ticket ends while a request hangs (review of #2125). gpu-hold stops its job with `pkill -TERM -P <job>` and then
+# TERMs the job, and in this phase the job is the flock, so the TERM lands on the smoke script, not on the serve. The
+# serve must not outlive it: a real one would be holding the NPU with no ticket.
+p=$(port)
+env HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_NPU_HOLD=0 HW_GATE_NPU_DEV=/dev/null HW_GATE_LOCK="$work/lock" HW_GATE_OUT="$work/out-stall" \
+    FAKE_MODE=stall FAKE_MARK="$work/mark.stall" HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$p" \
+    "$root/scripts/hw-gate.sh" --phases npuserve > "$work/stall.txt" 2>&1 &
+gatepid=$!
+for _ in $(seq 300); do [ -e "$work/mark.stall.asked" ] && break; sleep 0.1; done
+check '[ -e "$work/mark.stall.asked" ]' "a stalling serve is reached: the run is inside its first request"
+flockpid=$(pgrep -f "flock .*$work/lock" | head -n 1)
+check '[ -n "$flockpid" ]' "the phase runs the smoke under flock (pid ${flockpid:-none})"
+if [ -n "$flockpid" ]; then pkill -TERM -P "$flockpid"; kill -TERM "$flockpid" 2> /dev/null; fi
+for _ in $(seq 300); do pgrep -f "$work/fake1bit" > /dev/null || break; sleep 0.1; done
+check '! pgrep -f "$work/fake1bit" > /dev/null' "when the ticket's kill sequence hits the flock, the serve is stopped too"
+check '[ -e "$work/mark.stall" ]' "... by a SIGTERM it saw, not by being abandoned"
+wait "$gatepid"; rc=$?
+check '[ "$rc" = 1 ] && grep -q "^| npuserve | FAIL |" "$work/stall.txt"' "... and the gate reports the phase FAIL (exit $rc)"
+
+check '! pgrep -f "$work/fake1bit" > /dev/null' "no stand-in serve is left running"
+[ "$fail" = 0 ] && echo "hw_gate_npuserve: all checks passed"
+exit "$fail"