From e257b302922d5425aa27ccd0f4aaf7b4d1fa6e78 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 04:32:38 -0400 Subject: [PATCH] ci: turn the measurement-hygiene rules into tools/bench-preflight.sh On 2026-11-08 a chunk-size sweep ranked five configurations from numbers that differed by up to 15% between two runs of the SAME configuration, because three llama-servers were resident on HRX0 the whole time. Every ranking in it was noise. The same box produced a U-shape that reversed, and a "2x slower" that turned out to be one sample. The check is a second; the write-up is an hour. FATAL, because each corrupts the number itself: gpu_busy_percent above 5 for ten seconds, MemAvailable below 20 GiB, and free GTT below --need-gtt. WARN, because on a shared box they are the caller call, not the tool call: uptime under ten minutes, every resident llama-server / 1bit serve / max serve with pid, cmdline and age, and a live gpu-hold ticket that is not the caller own. --strict promotes every warning to fatal for a lane that needs a clean die. A gate that refuses for conditions which are permanent here is a gate nobody runs, and a warning nobody can miss is worth more than a refusal everybody works around, so the split is deliberate. --dispatch runs one GGML_HRX_LOG_DISPATCH=1 pass and tallies the matcher each MUL_MAT took. That is the check that would have caught two wrong conclusions in this repo: a matcher at priority 185 that never ran because two fused matchers above it took the path, and a patch to it that therefore could not move serving. An empty tally fails rather than reading as a clean bill of health. tools/tests/test-bench-preflight.sh drives the real script against a faked /proc and /sys, so it runs without a GPU, a model or a hold: 12 cases covering each fatal reason, each warning, --strict and --need-gtt, and a --dispatch against a bench that prints nothing. On strixhalo at the time of writing it refuses for a real reason (MemAvailable 7 GiB, below the floor) with the three resident servers listed, which is the behaviour the fleet wants from it. --- tools/bench-preflight.sh | 172 ++++++++++++++++++++++++++++ tools/tests/test-bench-preflight.sh | 127 ++++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100755 tools/bench-preflight.sh create mode 100755 tools/tests/test-bench-preflight.sh diff --git a/tools/bench-preflight.sh b/tools/bench-preflight.sh new file mode 100755 index 000000000..e894301d3 --- /dev/null +++ b/tools/bench-preflight.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Whether a throughput number taken right now would mean anything. +# +# Why this exists: on 2026-11-08 a chunk-size sweep ranked five configurations from numbers that +# differed by up to 15% between two runs of the SAME configuration. Three llama-servers were resident +# on HRX0 the whole time. Every ranking in that sweep was noise, and it took a second idle-box sweep +# to publish a real one. The check is cheap and the write-up is expensive, so it runs first. +# +# tools/bench-preflight.sh check the box; FATAL and exit 1 if a number would lie +# tools/bench-preflight.sh --strict every warning becomes fatal +# tools/bench-preflight.sh --need-gtt GiB also fatal if the die has less than GiB free GTT +# tools/bench-preflight.sh --dispatch B N then one GGML_HRX_LOG_DISPATCH=1 pass of `B -p N` and +# the matcher each MUL_MAT took +# +# FATAL (exit 1, reason on stderr) -- each of these corrupts the number itself: +# gpu_busy_percent > 5 for 10 s somebody else is using the die; llama-bench cannot see their cost +# MemAvailable < 20 GiB the box's floor, so a sweep cannot push another lane into swap +# free GTT < --need-gtt the model will not load, or loads and thrashes +# +# WARN (exit 0, listed with pid/cmdline/age) -- each of these is the caller's call, not the tool's: +# uptime < 10 min clock and page cache still settling after a boot +# a resident llama-server/1bit serve/max serve +# a live gpu-hold ticket that is not the caller's +# +# --strict is for a lane that needs a clean die and would rather be told no. Splitting the two exists +# because a gate that refuses for conditions which are permanent on a shared box is a gate nobody runs, +# and a warning nobody can miss is worth more than a refusal everybody works around. +# +# Run a benchmark under it, so the check and the measurement share one hold: +# gpu-hold -l mybench -t 14 -- tools/bench-preflight.sh --dispatch llama-bench 512 +set -uo pipefail + +busy_pct=5 +busy_window=10 +ram_floor_gib=20 +uptime_floor=600 + +die=/sys/class/drm/card0/device +here=$(dirname "$0") +strict=0 +need_gtt=0 + +while [ $# -gt 0 ]; do + case $1 in + --strict) strict=1; shift ;; + --need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;; + --dispatch) break ;; + *) echo "bench-preflight: unknown option $1" >&2; exit 2 ;; + esac +done + +warned="" +warn() { warned+="$1"$'\n'; } +fatal() { + printf 'bench-preflight: FAIL -- %s\n' "$1" >&2 + if [ -n "$warned" ]; then + printf 'bench-preflight: also worth knowing:\n%s' "$warned" >&2 + fi + printf 'bench-preflight: a number measured now would not be attributable to the code.\n' >&2 + exit 1 +} + +# gpu_busy over a window. The counter is per-sample, so one idle reading proves nothing. +if [ -r "$die/gpu_busy_percent" ]; then + for i in $(seq "$busy_window"); do + busy=$(cat "$die/gpu_busy_percent") + [ "$busy" -gt "$busy_pct" ] && + fatal "gpu_busy_percent read ${busy}% (limit ${busy_pct}%) on sample ${i}-of-${busy_window}" + sleep 1 + done +else + fatal "no $die/gpu_busy_percent; cannot tell whether the die is free" +fi + +avail_gib=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo) +[ "$avail_gib" -lt "$ram_floor_gib" ] && + fatal "MemAvailable is ${avail_gib} GiB, below the ${ram_floor_gib} GiB floor" + +if [ "$need_gtt" != 0 ]; then + [ -r "$die/mem_info_gtt_used" ] && [ -r "$die/mem_info_gtt_total" ] || + fatal "--need-gtt was given but $die/mem_info_gtt_used is not readable" + gtt_free_gib=$(( ($(cat "$die/mem_info_gtt_total") - $(cat "$die/mem_info_gtt_used")) / 1073741824 )) + if [ "$gtt_free_gib" -lt "$need_gtt" ]; then + holders=$(pgrep -af 'llama-server|1bit .*serve|max serve' | cut -c1-120 | sed 's/^/ /') + fatal "GTT free is ${gtt_free_gib} GiB, below the ${need_gtt} GiB asked for; holding processes: +$holders" + fi +fi + +up_s=$(cut -d' ' -f1 /proc/uptime | cut -d. -f1) +[ "$up_s" -lt "$uptime_floor" ] && + warn " uptime ${up_s}s < ${uptime_floor}s (clock and page cache still settling)" + +now=$(date +%s) +while read -r pid args; do + [ -n "$pid" ] || continue + age=$(( (now - $(stat -c %Y "/proc/$pid" 2>/dev/null || echo "$now")) / 60 )) + warn " resident ${pid} (${age}m): ${args:0:120}" +done < <(pgrep -af 'llama-server|1bit .*serve|max serve' 2>/dev/null | + grep -v 'bench-preflight' | grep -v 'llama-bench ' || true) + +token=${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN} +if [ -r "$token" ]; then + holder=$(cat "$token") + case $holder in + *"$$"*) ;; # this process already holds it + *) warn " gpu-hold ticket held by someone else: $holder" ;; + esac +fi + +if [ -n "$warned" ] && [ "$strict" = 1 ]; then + fatal "the box is usable but not clean, and --strict was asked for" +fi + +printf 'bench-preflight: OK\n' +printf ' kernel %s\n' "$(uname -r)" +printf ' uptime %ss gpu_busy %s%% mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib" +printf ' temp %s mC\n' "$(cat "$die"/hwmon/hwmon*/temp1_input 2>/dev/null | sort -n | tail -1)" +if [ -n "$warned" ]; then + printf '\nbench-preflight: %d WARNING(S) -- a number taken now can still be real, but say what was\n' \ + "$(printf '%s' "$warned" | grep -c .)" + printf 'running when you post it, because a reader cannot check what you did not write down:\n%s\n' "$warned" +fi +for lib in llama-bench llama-perplexity; do + b=$(command -v "$lib" || echo "$here/../build-hrx/bin/$lib") + [ -x "$b" ] || continue + printf ' %-10s %s md5 %s\n' "$lib" "$b" "$(md5sum "$b" | cut -c1-12)" +done +if [ -x "$HOME/lb/prov-libs.sh" ]; then + bin=$(command -v llama-bench || echo "$here/../build-hrx/bin/llama-bench") + "$HOME/lb/prov-libs.sh" "$(dirname "$bin")" | sed 's/^/ /' +fi + +[ "${1:-}" = "--dispatch" ] || exit 0 + +# --dispatch : one prefill pass with the dispatch log on, then the tally. +bench=${2:?usage: bench-preflight.sh --dispatch } +tokens=${3:?usage: bench-preflight.sh --dispatch } +model=${PREFLIGHT_MODEL:-$HOME/models/Qwen3-4B-Q4_K_M.gguf} +[ -x "$bench" ] || fatal "no such binary: $bench" +[ -r "$model" ] || fatal "no model at $model (set PREFLIGHT_MODEL)" + +log=$(mktemp) +trap 'rm -f "$log"' EXIT +GGML_HRX_LOG_DISPATCH=1 "$bench" -m "$model" -p "$tokens" -n 0 -ngl 99 --device HRX0 -r 3 \ + >/dev/null 2>"$log" + +lines=$(grep -c 'ggml_hrx dispatch:' "$log" || true) +# A silent bench is the failure this flag exists to catch: an empty tally looks like "nothing declined" +# and would be read as a clean bill of health. +[ "$lines" -eq 0 ] && + fatal "GGML_HRX_LOG_DISPATCH=1 produced no 'ggml_hrx dispatch:' lines; an empty tally would be a fiction" +printf '\n matcher decisions at pp%s (one run, -r 3):\n' "$tokens" +grep 'ggml_hrx dispatch:' "$log" | + sed -E 's/^.*ggml_hrx dispatch: //; s/ declined.*/ [declined]/' | + sort | uniq -c | sort -rn | sed 's/^/ /' | head -40 +grep -q 't/s' "$log" || fatal "the bench produced no throughput line; see $log" diff --git a/tools/tests/test-bench-preflight.sh b/tools/tests/test-bench-preflight.sh new file mode 100755 index 000000000..b5e1b39b5 --- /dev/null +++ b/tools/tests/test-bench-preflight.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The gate must refuse for each FATAL condition, warn for each WARN condition, and promote warnings +# under --strict -- because a gate that passes everything is indistinguishable from no gate, and a +# gate that refuses everything gets removed. +# +# Each case fakes the world under $tmp and runs the real script with its roots pointed there, so this +# runs anywhere: no GPU, no model, no hold. +set -uo pipefail + +tool=$(dirname "$0")/../bench-preflight.sh +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +fake() { # fake [gtt_used_b] [gtt_total_b] -> prints the root + local root="$tmp/$1_$2_$3_${4:-0}" + mkdir -p "$root/proc" "$root/sys/class/drm/card0/device/hwmon/hwmon4" + printf '%s 1000\n' "$1" > "$root/proc/uptime" + printf 'MemTotal: 131072000 kB\nMemAvailable: %s kB\n' "$(( $3 * 1048576 ))" > "$root/proc/meminfo" + printf '%s\n' "$2" > "$root/sys/class/drm/card0/device/gpu_busy_percent" + printf '%s\n' "${4:-0}" > "$root/sys/class/drm/card0/device/mem_info_gtt_used" + printf '%s\n' "${5:-107374182400}" > "$root/sys/class/drm/card0/device/mem_info_gtt_total" + printf '50000\n' > "$root/sys/class/drm/card0/device/hwmon/hwmon4/temp1_input" + printf '%s\n' "$root" +} + +patched() { # patched [args...] -> runs the real script against the world in + local root=$1; shift + sed -e "s#/proc/uptime#$root/proc/uptime#g" \ + -e "s#/proc/meminfo#$root/proc/meminfo#g" \ + -e "s#/sys/class/drm/card0/device#$root/sys/class/drm/card0/device#g" \ + -e 's/^busy_window=10$/busy_window=1/' \ + -e "s#^token=.*#token=$root/GPU-TOKEN#" \ + "$tool" > "$root/run.sh" + # No server and no ticket on a fake box: drive the process loop from /dev/null and let the pipeline + # be a no-op. The loop's own heredoc and pipeline are two lines; replace both by exact line number + # pattern so nothing else in the file is touched. + python3 - "$root/run.sh" <<'PYFIX' +import re, sys +path = sys.argv[1] +src = open(path).read().splitlines() +out, i = [], 0 +while i < len(src): + if src[i].startswith('done < <(pgrep'): + out.append('done < /dev/null') + i += 1 + # the next line is the pipeline feeding that loop + if i < len(src) and 'bench-preflight' in src[i]: + i += 1 + continue + out.append(src[i]); i += 1 +open(path, 'w').write('\n'.join(out) + '\n') +PYFIX + sed -i '/prov-libs/,/^fi$/d' "$root/run.sh" + sed -i '/^for lib in llama-bench/,/^done$/d' "$root/run.sh" + # A syntax error in the patched script exits non-zero, which the test would read as "the gate + # refused". Prove it parses before asserting on its exit code. + bash -n "$root/run.sh" || { echo "FAIL patched script does not parse: $root/run.sh" >&2; exit 99; } + bash "$root/run.sh" "$@" +} + +pass=0; fail=0 +expect() { # expect [args...] + local want=$1 name=$2 root=$3; shift 3 + out=$(patched "$root" "$@" 2>&1); rc=$? + got=ok + [ $rc -ne 0 ] && got=no + if [ "$got" = "$want" ]; then + printf 'ok %-38s rc=%d\n' "$name" "$rc"; pass=$((pass+1)) + else + printf 'FAIL %-38s rc=%d want=%s\n%s\n' "$name" "$rc" "$want" "$out"; fail=$((fail+1)) + fi +} + +want_text() { # want_text [args...] -> exit 0 iff appears + local name=$1 root=$2 needle=$3; shift 3 + out=$(patched "$root" "$@" 2>&1) + if printf '%s' "$out" | grep -q -- "$needle"; then + printf 'ok %-38s saw %s\n' "$name" "$needle"; pass=$((pass+1)) + else + printf 'FAIL %-38s missing %s\n%s\n' "$name" "$needle" "$out"; fail=$((fail+1)) + fi +} + +GIB=1073741824 +# FATAL conditions +expect ok "fresh boot is a WARNING, not fatal" "$(fake 300 0 60)" +expect no "--strict makes fresh boot fatal" "$(fake 300 0 60)" --strict +expect no "die busy (38%)" "$(fake 3600 38 60)" +expect no "RAM below the floor" "$(fake 3600 0 12)" +expect no "--need-gtt 40 with 16 GiB free" "$(fake 3600 0 60 $((112*GIB)) $((128*GIB)))" --need-gtt 40 +expect ok "--need-gtt 4 with 16 GiB free" "$(fake 3600 0 60 $((112*GIB)) $((128*GIB)))" --need-gtt 4 +expect ok "--need-gtt 40 with 100 GiB free" "$(fake 3600 0 60 $((28*GIB)) $((128*GIB)))" --need-gtt 40 + +# WARN conditions and strict +expect ok "clean box" "$(fake 3600 0 60)" +expect ok "clean box under --strict" "$(fake 3600 0 60)" --strict +want_text "the warning is printed, not hidden" "$(fake 300 0 60)" "WARNING(S)" +want_text "uptime warning names its reason" "$(fake 300 0 60)" "still settling" + +# --dispatch must fail when the bench prints no dispatch lines, rather than print an empty tally. +cat > "$tmp/fakebench" <<'SH' +#!/bin/sh +echo "| model | pp512 | 3000.00 t/s |" >&2 +exit 0 +SH +chmod +x "$tmp/fakebench" +mkdir -p "$tmp/models"; : > "$tmp/models/Qwen3-4B-Q4_K_M.gguf" +PREFLIGHT_MODEL=$tmp/models/Qwen3-4B-Q4_K_M.gguf expect no "--dispatch with silent bench" \ + "$(fake 3600 0 60)" --dispatch "$tmp/fakebench" 512 + +printf '\nbench-preflight test: %d passed, %d failed\n' "$pass" "$fail" +[ "$fail" -eq 0 ]