diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a8719d47..4cd0fc90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -385,6 +385,13 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # resolves outside the payload, and a runtime set shipped without the aqlprofile agent libhsa opens. Each # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) +# engine#2106: the hardware gate's `npuserve` phase, end to end against the stand-in serve tools/npu_serve_smoke.py +# writes: a fast-route serve passes the gate, a slow-route one fails it while its answers are fine, a dead one fails, +# and an unconfigured or NPU-less host skips. No device, no 1bit binary, and the fleet's queues are not touched. +add_test(NAME hw_gate_npuserve COMMAND ${CMAKE_SOURCE_DIR}/tests/hw_gate_npuserve_test.sh) +# Both start a loopback serve on a port picked by binding port 0 and releasing it, so they take the lock every +# loopback-server test takes (tests/route_port_lock_test.py): two of them racing for one freed port is a flaky failure. +set_tests_properties(hw_gate_npuserve npu_serve_smoke_self PROPERTIES RESOURCE_LOCK route_ports) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. diff --git a/docs/hw-gate.md b/docs/hw-gate.md index c8cb13604..e9b70eb52 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -289,8 +289,40 @@ Measured on strixhalo, 2026-11-11, Qwen3-0.6B, one ticket each: | a build of main without the private route, model-generic directory | per-op | `FAIL`: `route`, `rate` (median 1.9 tok/s) and `first-token` (1,063 ms) fail; `content` passes | The second row is the point: its three answers were fine, and only the other three assertions say -the release would have shipped the slow route. The check is not wired into `scripts/hw-gate.sh` or the -release script yet; it is the instrument, and the release lane decides where it runs. +the release would have shipped the slow route. + +**In the gate: the `npuserve` phase.** `scripts/hw-gate.sh --phases npuserve` runs the check under the +same NPU claim as the `npu` phase and adds its row to the summary, with the four assertion lines as the +detail: + +```sh +HW_GATE_NPU_SERVE_BIN=/path/to/1bit HW_GATE_NPU_SERVE_MODEL=/path/to/Qwen3-0.6B scripts/hw-gate.sh --phases npuserve +``` + +The three numbers that decide the rate assertion are all passed by the phase, so none is a default +hidden in another file: `HW_GATE_NPU_SERVE_FLOOR` (tok/s, default **30**), `HW_GATE_NPU_SERVE_MAX_TOKENS` +(answer length, default 96) and `HW_GATE_NPU_SERVE_REPLIES` (requests, default 5; the rate is their +median). `HW_GATE_NPU_SERVE_PORT` sets the port. The row is `PASS` only when the tool exits 0 **and** +printed `RESULT: PASS`, so a truncated log cannot pass, and it always carries the measured median rate +and first-token time. + +The gate's floor is 30, not the demo's 60, because the box is shared. On the fast route the release +candidate's single replies have been measured between 48.8 and 101 tok/s depending on load; the slow +route is 1.9. A floor of 30 is fifteen times above the failure it guards against and under every fast +reply seen, so "slow machine" does not read as "slow route". The `route` assertion does not depend on +load at all. When the phase is named, a `FAIL` fails the gate like any other phase: it is blocking, +with no retry built in; a failed row shows which assertions failed and the numbers, so a rate-only +failure with `route ok` is recognisable as load and can be re-run. + +If the NPU ticket ends while a request hangs, the ticket's TERM reaches the smoke script (a `flock` sits +between them), and the script stops the serve before it exits; the serve is also started so that the +kernel TERMs it if the script dies any other way. A `1bit serve` left holding the NPU after its ticket +is the failure this prevents. +Like `npu`, the phase is not in the default phases and skips when there is no NPU or the two settings +are absent; the release procedure has to name it and set them for it to gate anything. It has not been +added to the release script. `tests/hw_gate_npuserve_test.sh` drives the phase end to end against a +stand-in serve (fast, slow, a floor it misses, dead at start, unconfigured, no NPU, and the ticket's +kill sequence arriving mid-request) with no device. ## Measured diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 62f5c7f2d..f8509147a 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -35,6 +35,9 @@ # a 64-position greedy build, the prompt prefill, run-to-run reproducibility and a # tok/s floor (HW_GATE_NPU_FLOOR). Auto-skips with no NPU or no checker configured; not # in the default phases (add it with --phases npu). +# npuserve the shipped binary's `1bit serve --device npu` on a small model: the fast route's own log line, a +# decode floor, the first token and a non-empty answer (tools/npu_serve_smoke.py; engine#2106, #2102). +# Auto-skips with no NPU or no HW_GATE_NPU_SERVE_BIN / HW_GATE_NPU_SERVE_MODEL; not in the default phases. # placement every graph the server scheduled for a gate model (load, warmup, the kat prompts, # the needle) is all on HRX: ggml-hrx's placement report (GGML_HRX_REPORT=2, engine #390) # names no node off HRX, as `1bit serve` now requires by default (docs/serve.md) @@ -659,6 +662,47 @@ if has npu; then fi fi +# ---- npuserve (the shipped binary's `serve --device npu`: on the fast route, at speed, with an answer) ---- +# The npu phase above checks the engine through a private checker and never starts `1bit serve`, so it cannot see a +# shipped binary whose serve quietly takes the slow per-op route: that serve still answers, at about 2 tok/s where the +# fast route gives 80 to 100 (engine#2102, #2106). tools/npu_serve_smoke.py asserts the route's own log line, the decode +# floor, the first token and a non-empty answer. Set HW_GATE_NPU_SERVE_BIN to the 1bit under test and +# HW_GATE_NPU_SERVE_MODEL to the small model directory. The three numbers that decide the rate assertion are all +# passed from here, so none of them is a default in another file: HW_GATE_NPU_SERVE_FLOOR (tok/s, default 30), +# HW_GATE_NPU_SERVE_MAX_TOKENS (answer length, default 96) and HW_GATE_NPU_SERVE_REPLIES (requests, default 5; the +# rate is their median). The floor is 30 and not the demo's 60 because this box is shared: single replies of the +# release candidate on the fast route have been measured from 48.8 to 101 tok/s under load, and the slow route is +# 1.9, so 30 is 15 times above the failure it guards against and below every fast reply seen. The route assertion +# does not depend on load at all. It runs under the same NPU claim as the npu phase and, like it, is not in the +# default phases; when it is named, a FAIL fails the gate like any other phase. +if has npuserve; then + t0=$(date +%s) + smoke=$(cd "$(dirname "$0")/.." && pwd)/tools/npu_serve_smoke.py + if [ ! -e "${HW_GATE_NPU_DEV:-/dev/accel/accel0}" ]; then + result npuserve SKIP 0 "no NPU (/dev/accel/accel0) on this host" + elif [ -z "${HW_GATE_NPU_SERVE_BIN:-}" ] || [ -z "${HW_GATE_NPU_SERVE_MODEL:-}" ]; then + result npuserve SKIP 0 "no serve under test configured (HW_GATE_NPU_SERVE_BIN and HW_GATE_NPU_SERVE_MODEL)" + else + cool + log=$OUT/npuserve.log + hold_args=() + if [ "${HW_GATE_NPU_HOLD:-1}" != 0 ] && [ -n "$NPU_HOLD" ] && ! token_held_by_ancestor "$NPU_TOKEN"; then + hold_args=("$NPU_HOLD" -l hw-gate-npuserve -t "${HW_GATE_NPU_HOLD_MINUTES:-20}" -w "$LOCK_WAIT" --) + fi + "${hold_args[@]}" flock -w "$LOCK_WAIT" -E 75 "$LOCK" python3 "$smoke" --bin "$HW_GATE_NPU_SERVE_BIN" \ + --model "$HW_GATE_NPU_SERVE_MODEL" --out "$OUT/npuserve" --floor "${HW_GATE_NPU_SERVE_FLOOR:-30}" \ + --max-tokens "${HW_GATE_NPU_SERVE_MAX_TOKENS:-96}" --replies "${HW_GATE_NPU_SERVE_REPLIES:-5}" \ + ${HW_GATE_NPU_SERVE_PORT:+--port "$HW_GATE_NPU_SERVE_PORT"} > "$log" 2>&1 + rc=$? + st=FAIL + # the tool's own verdict line AND its exit status: a pass needs both, so a truncated log cannot pass + [ "$rc" = 0 ] && grep -q '^RESULT: PASS$' "$log" && st=PASS + [ "$rc" = 75 ] && infra=1 + detail=$(grep -E '^(route|rate|first-token|content|serve) ' "$log" | sed -E 's/ +/ /g' | tr '\n' ';' | sed 's/;$//; s/;/; /g') + result npuserve $st $(( $(date +%s) - t0 )) "${detail:-no assertion lines in the log (exit $rc)}" + fi +fi + # ---- summary --------------------------------------------------------------------------------- total=$(( $(date +%s) - START )) nfail=$(awk -F'\t' '$2 == "FAIL"' "$RESULTS" | wc -l) diff --git a/tests/hw_gate_npuserve_test.sh b/tests/hw_gate_npuserve_test.sh new file mode 100755 index 000000000..72042e17d --- /dev/null +++ b/tests/hw_gate_npuserve_test.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# hw_gate_npuserve_test.sh: the hardware gate's `npuserve` phase (engine#2106), end to end, with no device. +# +# scripts/hw-gate.sh --phases npuserve is run against the stand-in serve that tools/npu_serve_smoke.py writes +# (--write-stand-in): a process that prints a route line and answers with the timings it is told to. Three runs: +# a fast-route stand-in at 90 tok/s -> the phase is PASS and the gate exits 0 +# a slow-route stand-in at 1.9 tok/s -> the phase is FAIL, names route and rate, and the gate exits 1 +# nothing configured -> the phase is SKIP and the gate exits 0 +# The gate is kept off the fleet's GPU and NPU queues (HW_GATE_GPU_HOLD=0, HW_GATE_TOKEN=none, HW_GATE_NPU_HOLD=0), its +# lock and output live in a directory made here, and HW_GATE_NPU_DEV points the "is there an NPU" probe at a file +# that exists, so the phase runs on a machine with none. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +work=$(mktemp -d "$PWD/hw_gate_npuserve_test.XXXXXX") || exit 1 # the working directory, not /tmp +trap 'pkill -f "$work/fake1bit" 2> /dev/null; rm -rf "$work"' EXIT +fail=0 +check() { if eval "$1"; then echo "ok $2"; else echo "FAIL $2"; fail=1; fi; } + +python3 "$root/tools/npu_serve_smoke.py" --write-stand-in "$work/fake1bit" || { echo "FAIL cannot write the stand-in"; exit 1; } +touch "$work/npu-present" + +gate() { # gate [VAR=value ...]: runs the phase, leaves its text in $work/.txt and its exit in $rc + local name=$1; shift + env HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_NPU_HOLD=0 HW_GATE_NPU_DEV="$work/npu-present" \ + HW_GATE_LOCK="$work/lock" HW_GATE_OUT="$work/out-$name" "$@" \ + timeout 120 "$root/scripts/hw-gate.sh" --phases npuserve > "$work/$name.txt" 2>&1 + rc=$? +} +port() { python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])'; } + +gate fast FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 0 ]' "a fast-route serve at 90 tok/s: the gate exits 0 (got $rc)" +check 'grep -q "^| npuserve | PASS |" "$work/fast.txt"' "... and the summary row is PASS" +check 'grep -q "rate ok median 90.0 tok/s" "$work/fast.txt"' "... with the measured rate in the row" + +gate slow FAKE_MODE=slow FAKE_TPS=1.9 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 1 ]' "a slow-route serve at 1.9 tok/s: the gate exits 1 (got $rc)" +check 'grep -q "^| npuserve | FAIL |" "$work/slow.txt"' "... and the summary row is FAIL" +check 'grep -q "route FAIL the serve log names the per-op forward" "$work/slow.txt"' "... naming the route" +check 'grep -q "rate FAIL median 1.9 tok/s" "$work/slow.txt"' "... and the rate" +check 'grep -q "content ok" "$work/slow.txt"' "... while the answers themselves were fine (the reason the phase exists)" + +# the gate's own numbers reach the tool: five replies by default, and the floor it states +check 'grep -q "each: 90.0, 90.0, 90.0, 90.0, 90.0" "$work/fast.txt"' "the gate asks for five replies by default" +check 'grep -q "(floor 30.0;" "$work/fast.txt"' "... against its own default floor of 30 tok/s" +gate replies FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_REPLIES=3 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check 'grep -q "each: 90.0, 90.0, 90.0)" "$work/replies.txt"' "HW_GATE_NPU_SERVE_REPLIES=3 sends three" + +# a fast route under a floor it does not reach: the floor is the gate's own setting +gate floor FAKE_MODE=fast FAKE_TPS=90 HW_GATE_NPU_SERVE_FLOOR=120 HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 1 ] && grep -q "rate FAIL median 90.0 tok/s (floor 120.0" "$work/floor.txt"' "HW_GATE_NPU_SERVE_FLOOR=120 fails a 90 tok/s serve on the rate alone" + +gate dead FAKE_MODE=dead HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$(port)" +check '[ "$rc" = 1 ] && grep -q "^| npuserve | FAIL |" "$work/dead.txt" && grep -q "never became ready" "$work/dead.txt"' "a serve that dies at start is FAIL, and the row says it never became ready" + +gate skip +check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/skip.txt"' "with no binary and model configured the phase is SKIP and the gate exits 0" + +rm -f "$work/npu-present" +gate nodev HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model +check '[ "$rc" = 0 ] && grep -q "^| npuserve | SKIP |" "$work/nodev.txt" && grep -q "no NPU" "$work/nodev.txt"' "with no NPU the phase is SKIP even when configured" + +# The ticket ends while a request hangs (review of #2125). gpu-hold stops its job with `pkill -TERM -P ` and then +# TERMs the job, and in this phase the job is the flock, so the TERM lands on the smoke script, not on the serve. The +# serve must not outlive it: a real one would be holding the NPU with no ticket. +p=$(port) +env HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_NPU_HOLD=0 HW_GATE_NPU_DEV=/dev/null HW_GATE_LOCK="$work/lock" HW_GATE_OUT="$work/out-stall" \ + FAKE_MODE=stall FAKE_MARK="$work/mark.stall" HW_GATE_NPU_SERVE_BIN="$work/fake1bit" HW_GATE_NPU_SERVE_MODEL=model HW_GATE_NPU_SERVE_PORT="$p" \ + "$root/scripts/hw-gate.sh" --phases npuserve > "$work/stall.txt" 2>&1 & +gatepid=$! +for _ in $(seq 300); do [ -e "$work/mark.stall.asked" ] && break; sleep 0.1; done +check '[ -e "$work/mark.stall.asked" ]' "a stalling serve is reached: the run is inside its first request" +flockpid=$(pgrep -f "flock .*$work/lock" | head -n 1) +check '[ -n "$flockpid" ]' "the phase runs the smoke under flock (pid ${flockpid:-none})" +if [ -n "$flockpid" ]; then pkill -TERM -P "$flockpid"; kill -TERM "$flockpid" 2> /dev/null; fi +for _ in $(seq 300); do pgrep -f "$work/fake1bit" > /dev/null || break; sleep 0.1; done +check '! pgrep -f "$work/fake1bit" > /dev/null' "when the ticket's kill sequence hits the flock, the serve is stopped too" +check '[ -e "$work/mark.stall" ]' "... by a SIGTERM it saw, not by being abandoned" +wait "$gatepid"; rc=$? +check '[ "$rc" = 1 ] && grep -q "^| npuserve | FAIL |" "$work/stall.txt"' "... and the gate reports the phase FAIL (exit $rc)" + +check '! pgrep -f "$work/fake1bit" > /dev/null' "no stand-in serve is left running" +[ "$fail" = 0 ] && echo "hw_gate_npuserve: all checks passed" +exit "$fail" diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py index cb5a9d2d0..cd142527c 100755 --- a/tools/npu_serve_smoke.py +++ b/tools/npu_serve_smoke.py @@ -26,9 +26,11 @@ npu-hold -l npu-smoke -t 10 -- tools/npu_serve_smoke.py --bin <1bit> --model [--out ] -It starts the serve, waits for it, sends three prompts with thinking off, stops the serve, and prints one line per -assertion and `RESULT: PASS` or `RESULT: FAIL` (exit 0 / 1). It takes no device ticket itself: run it under npu-hold, -so the serve ends with the ticket. Logs go to --out (default ~/.cache/1bit/npu-serve-smoke), never to /tmp. +It starts the serve, waits for it, sends --replies requests (default 3) with thinking off, stops the serve, and prints +one line per assertion and `RESULT: PASS` or `RESULT: FAIL` (exit 0 / 1). It takes no device ticket itself: run it +under npu-hold. The serve ends with this script however the script ends: a TERM, HUP or INT stops the serve first +(exit 128 + signal), and the kernel TERMs the serve if the script is killed outright. Logs go to --out (default +~/.cache/1bit/npu-serve-smoke), never to /tmp. tools/npu_serve_smoke.py --self-test the judging on recorded shapes of each failure; no device, no binary tools/npu_serve_smoke.py --judge judge a finished run's serve.log and reply_*.json again @@ -125,14 +127,33 @@ def report(verdicts): return 0 if ok else 1 +def die_with_parent(): + """In the serve's process, before exec: ask the kernel to TERM it if this script dies, however it dies.""" + try: + import ctypes + ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGTERM) # PR_SET_PDEATHSIG + except Exception: + pass + if os.getppid() == 1: # the script was already gone when this ran + os._exit(1) + + def run(args): + # A ticket that ends TERMs this script, not the serve (the serve is this script's child, and in the gate a flock + # sits between the ticket and the script). Python's default action for SIGTERM ends the process without running + # `finally`, which is the only code that stops the serve -- so a hung run would leave `1bit serve` holding the NPU + # after its ticket (review of #2125). The handlers turn the signal into SystemExit, which unwinds through it. + def stop(signum, _frame): + sys.exit(128 + signum) + for sig in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + signal.signal(sig, stop) out = os.path.expanduser(args.out) os.makedirs(out, exist_ok=True) log_path = os.path.join(out, "serve.log") cmd = [args.bin, "serve", "-m", args.model, "--device", "npu", "--host", "127.0.0.1", "-p", str(args.port)] replies = [] with open(log_path, "wb") as log: - serve = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT) + serve = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, preexec_fn=die_with_parent) try: deadline = time.time() + args.ready_seconds ready = False @@ -145,7 +166,8 @@ def run(args): except OSError: time.sleep(0.5) if ready: - for i, prompt in enumerate(PROMPTS): + for i in range(args.replies): + prompt = PROMPTS[i % len(PROMPTS)] body = json.dumps({"model": "m", "max_tokens": args.max_tokens, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}, "messages": [{"role": "user", "content": prompt}]}).encode() @@ -188,6 +210,107 @@ def rejudge(args): return report(judge(log_text, replies, args.floor, args.first_token_ms, args.fast_route, args.slow_route)) +FAKE_SERVE = r'''#!/usr/bin/env python3 +# A stand-in for `1bit serve`, written by npu_serve_smoke.py --self-test: it prints a route line, answers /health and +# /v1/chat/completions with the timings FAKE_TPS gives, and exits on SIGTERM. FAKE_MODE=dead exits before listening. +import http.server, json, os, signal, sys +port = int(sys.argv[sys.argv.index("-p") + 1]) +mode, tps = os.environ.get("FAKE_MODE", "fast"), float(os.environ.get("FAKE_TPS", "90")) +if mode == "dead": + print("1bit serve: cannot open the device", flush=True); sys.exit(3) +print("1bit unified: on the private route dx: dense Qwen3 on the dx kernels" if mode in ("fast", "stall") else + "1bit serve: a model-generic container: serving it through the per-op forward", flush=True) +class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): pass + def _send(self, obj): + b = json.dumps(obj).encode(); self.send_response(200); self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(b))); self.end_headers(); self.wfile.write(b) + def do_GET(self): self._send({"status": "ok"}) + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if mode == "stall": + open(os.environ["FAKE_MARK"] + ".asked", "w").write("asked") + import time; time.sleep(600) + self._send({"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Paris."}}], + "timings": {"prompt_ms": 200.0 if mode == "fast" else 1100.0, "predicted_ms": 1000.0 * 8 / tps, "predicted_per_second": tps}, + "usage": {"completion_tokens": 8}}) +srv = http.server.HTTPServer(("127.0.0.1", port), H) +def bye(*a): + if os.environ.get("FAKE_MARK"): open(os.environ["FAKE_MARK"], "w").write("terminated") + os._exit(0) +signal.signal(signal.SIGTERM, bye) +srv.serve_forever() +''' + + +def self_test_run(fails): + """run() end to end against the stand-in: it starts the serve, waits, asks, stops it, and returns the verdict.""" + import contextlib + import io + import shutil + import socket + import tempfile + d = tempfile.mkdtemp(prefix="npu_serve_smoke_selftest.", dir=os.getcwd()) # the working directory, not /tmp + try: + fake = os.path.join(d, "fake1bit") + with open(fake, "w") as f: + f.write(FAKE_SERVE) + os.chmod(fake, 0o700) + for mode, tps, want_exit, want_line in (("fast", "90", 0, "RESULT: PASS"), ("slow", "1.9", 1, "route FAIL"), + ("dead", "90", 1, "it never became ready")): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + mark = os.path.join(d, "mark." + mode) + os.environ.update(FAKE_MODE=mode, FAKE_TPS=tps, FAKE_MARK=mark) + args = argparse.Namespace(bin=fake, model="model-dir", out=os.path.join(d, mode), port=port, floor=60.0, first_token_ms=600.0, + fast_route=FAST_ROUTE, slow_route=SLOW_ROUTE, max_tokens=8, ready_seconds=5 if mode == "dead" else 30, + request_seconds=30, replies=3) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = run(args) + text = buf.getvalue() + if rc != want_exit or want_line not in text: + fails.append("run() against a %s stand-in: exit %d, output %r" % (mode, rc, text[-300:])) + if mode != "dead" and not os.path.exists(mark): + fails.append("run() against a %s stand-in left the serve running (no SIGTERM seen)" % mode) + if mode == "fast" and not os.path.exists(os.path.join(d, mode, "reply_2.json")): + fails.append("run() did not keep the three replies") + # The ticket-ends case: this script is TERMed while a request hangs. The serve must be stopped too, and the + # script must exit 128 + SIGTERM. Run as a real process, because the signal's default action is the bug. + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + mark = os.path.join(d, "mark.stall") + env = dict(os.environ, FAKE_MODE="stall", FAKE_TPS="90", FAKE_MARK=mark) + tool = subprocess.Popen([sys.executable, os.path.abspath(__file__), "--bin", fake, "--model", "model-dir", "--out", + os.path.join(d, "stall"), "--port", str(port), "--request-seconds", "300"], + env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.time() + 30 + while time.time() < deadline and not os.path.exists(mark + ".asked"): + time.sleep(0.1) + if not os.path.exists(mark + ".asked"): + fails.append("the stalling stand-in was never asked (the run did not reach its first request)") + tool.send_signal(signal.SIGTERM) + try: + rc = tool.wait(timeout=40) + except subprocess.TimeoutExpired: + tool.kill() + rc = None + if rc != 128 + signal.SIGTERM: + fails.append("a TERMed run exits %r, want %d" % (rc, 128 + signal.SIGTERM)) + deadline = time.time() + 10 + while time.time() < deadline and not os.path.exists(mark): + time.sleep(0.1) + if not os.path.exists(mark): + fails.append("a TERMed run left the serve running: the stand-in never saw its SIGTERM") + finally: + for k in ("FAKE_MODE", "FAKE_TPS", "FAKE_MARK"): + os.environ.pop(k, None) + subprocess.run(["pkill", "-f", fake], check=False) # nothing of this test's may outlive it, whatever failed + shutil.rmtree(d, ignore_errors=True) + + def self_test(): fails = [] @@ -245,9 +368,11 @@ def expect(name, verdicts, want): if not degenerate("//////") or degenerate("391") or degenerate("aa") or not degenerate("a a a a"): fails.append("degenerate(): wrong on a fixture") + self_test_run(fails) for f in fails: print("FAIL " + f) - print("npu_serve_smoke self-test: %d case(s) failed" % len(fails) if fails else "npu_serve_smoke self-test: 16 cases ok") + print("npu_serve_smoke self-test: %d case(s) failed" % len(fails) if fails + else "npu_serve_smoke self-test: 16 judging cases, 3 runs against a stand-in serve and a TERM mid-request ok") return 1 if fails else 0 @@ -255,6 +380,7 @@ def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--self-test", action="store_true") p.add_argument("--judge", metavar="DIR") + p.add_argument("--write-stand-in", metavar="PATH", help="write the self-test's stand-in serve to PATH (for testing a caller)") p.add_argument("--bin") p.add_argument("--model") p.add_argument("--out", default="~/.cache/1bit/npu-serve-smoke") @@ -264,9 +390,15 @@ def main(): p.add_argument("--fast-route", default=FAST_ROUTE, help="the text the serve log prints on the fast route") p.add_argument("--slow-route", default=SLOW_ROUTE, help="the text the serve log prints on the slow route") p.add_argument("--max-tokens", type=int, default=96) + p.add_argument("--replies", type=int, default=3, help="how many requests to send (the three prompts, repeated); the rate is their median") p.add_argument("--ready-seconds", type=int, default=120) p.add_argument("--request-seconds", type=int, default=120) a = p.parse_args() + if a.write_stand_in: + with open(a.write_stand_in, "w") as f: + f.write(FAKE_SERVE) + os.chmod(a.write_stand_in, 0o700) + return 0 if a.self_test: return self_test() if a.judge: