Skip to content

ci: tools/bench-preflight.sh - make the measurement-hygiene rules a check instead of a habit #1622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bong-water-water-bong merged 1 commit into from
Nov 9, 2026
Merged

ci: tools/bench-preflight.sh - make the measurement-hygiene rules a check instead of a habit #1622

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 172 additions & 0 deletions tools/bench-preflight.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Whether a throughput number taken right now would mean anything.
#
# Why this exists: on 2026-11-08 a chunk-size sweep ranked five configurations from numbers that
# differed by up to 15% between two runs of the SAME configuration. Three llama-servers were resident
# on HRX0 the whole time. Every ranking in that sweep was noise, and it took a second idle-box sweep
# to publish a real one. The check is cheap and the write-up is expensive, so it runs first.
#
# tools/bench-preflight.sh check the box; FATAL and exit 1 if a number would lie
# tools/bench-preflight.sh --strict every warning becomes fatal
# tools/bench-preflight.sh --need-gtt GiB also fatal if the die has less than GiB free GTT
# tools/bench-preflight.sh --dispatch B N then one GGML_HRX_LOG_DISPATCH=1 pass of `B -p N` and
# the matcher each MUL_MAT took
#
# FATAL (exit 1, reason on stderr) -- each of these corrupts the number itself:
# gpu_busy_percent > 5 for 10 s somebody else is using the die; llama-bench cannot see their cost
# MemAvailable < 20 GiB the box's floor, so a sweep cannot push another lane into swap
# free GTT < --need-gtt the model will not load, or loads and thrashes
#
# WARN (exit 0, listed with pid/cmdline/age) -- each of these is the caller's call, not the tool's:
# uptime < 10 min clock and page cache still settling after a boot
# a resident llama-server/1bit serve/max serve
# a live gpu-hold ticket that is not the caller's
#
# --strict is for a lane that needs a clean die and would rather be told no. Splitting the two exists
# because a gate that refuses for conditions which are permanent on a shared box is a gate nobody runs,
# and a warning nobody can miss is worth more than a refusal everybody works around.
#
# Run a benchmark under it, so the check and the measurement share one hold:
# gpu-hold -l mybench -t 14 -- tools/bench-preflight.sh --dispatch llama-bench 512
set -uo pipefail

busy_pct=5
busy_window=10
ram_floor_gib=20
uptime_floor=600

die=/sys/class/drm/card0/device
here=$(dirname "$0")
strict=0
need_gtt=0

while [ $# -gt 0 ]; do
case $1 in
--strict) strict=1; shift ;;
--need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;;
--dispatch) break ;;
*) echo "bench-preflight: unknown option $1" >&2; exit 2 ;;
esac
done

warned=""
warn() { warned+="$1"$'\n'; }
fatal() {
printf 'bench-preflight: FAIL -- %s\n' "$1" >&2
if [ -n "$warned" ]; then
printf 'bench-preflight: also worth knowing:\n%s' "$warned" >&2
fi
printf 'bench-preflight: a number measured now would not be attributable to the code.\n' >&2
exit 1
}

# gpu_busy over a window. The counter is per-sample, so one idle reading proves nothing.
if [ -r "$die/gpu_busy_percent" ]; then
for i in $(seq "$busy_window"); do
busy=$(cat "$die/gpu_busy_percent")
[ "$busy" -gt "$busy_pct" ] &&
fatal "gpu_busy_percent read ${busy}% (limit ${busy_pct}%) on sample ${i}-of-${busy_window}"
sleep 1
done
else
fatal "no $die/gpu_busy_percent; cannot tell whether the die is free"
fi

avail_gib=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo)
[ "$avail_gib" -lt "$ram_floor_gib" ] &&
fatal "MemAvailable is ${avail_gib} GiB, below the ${ram_floor_gib} GiB floor"

if [ "$need_gtt" != 0 ]; then
[ -r "$die/mem_info_gtt_used" ] && [ -r "$die/mem_info_gtt_total" ] ||
fatal "--need-gtt was given but $die/mem_info_gtt_used is not readable"
gtt_free_gib=$(( ($(cat "$die/mem_info_gtt_total") - $(cat "$die/mem_info_gtt_used")) / 1073741824 ))
if [ "$gtt_free_gib" -lt "$need_gtt" ]; then
holders=$(pgrep -af 'llama-server|1bit .*serve|max serve' | cut -c1-120 | sed 's/^/ /')
fatal "GTT free is ${gtt_free_gib} GiB, below the ${need_gtt} GiB asked for; holding processes:
$holders"
fi
fi

up_s=$(cut -d' ' -f1 /proc/uptime | cut -d. -f1)
[ "$up_s" -lt "$uptime_floor" ] &&
warn " uptime ${up_s}s < ${uptime_floor}s (clock and page cache still settling)"

now=$(date +%s)
while read -r pid args; do
[ -n "$pid" ] || continue
age=$(( (now - $(stat -c %Y "/proc/$pid" 2>/dev/null || echo "$now")) / 60 ))
warn " resident ${pid} (${age}m): ${args:0:120}"
done < <(pgrep -af 'llama-server|1bit .*serve|max serve' 2>/dev/null |
grep -v 'bench-preflight' | grep -v 'llama-bench ' || true)

token=${GPU_TOKEN_FILE:-$HOME/GPU-TOKEN}
if [ -r "$token" ]; then
holder=$(cat "$token")
case $holder in
*"$$"*) ;; # this process already holds it
*) warn " gpu-hold ticket held by someone else: $holder" ;;
esac
fi

if [ -n "$warned" ] && [ "$strict" = 1 ]; then
fatal "the box is usable but not clean, and --strict was asked for"
fi

printf 'bench-preflight: OK\n'
printf ' kernel %s\n' "$(uname -r)"
printf ' uptime %ss gpu_busy %s%% mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib"
printf ' temp %s mC\n' "$(cat "$die"/hwmon/hwmon*/temp1_input 2>/dev/null | sort -n | tail -1)"
if [ -n "$warned" ]; then
printf '\nbench-preflight: %d WARNING(S) -- a number taken now can still be real, but say what was\n' \
"$(printf '%s' "$warned" | grep -c .)"
printf 'running when you post it, because a reader cannot check what you did not write down:\n%s\n' "$warned"
fi
for lib in llama-bench llama-perplexity; do
b=$(command -v "$lib" || echo "$here/../build-hrx/bin/$lib")
[ -x "$b" ] || continue
printf ' %-10s %s md5 %s\n' "$lib" "$b" "$(md5sum "$b" | cut -c1-12)"
done
if [ -x "$HOME/lb/prov-libs.sh" ]; then
bin=$(command -v llama-bench || echo "$here/../build-hrx/bin/llama-bench")
"$HOME/lb/prov-libs.sh" "$(dirname "$bin")" | sed 's/^/ /'
fi

[ "${1:-}" = "--dispatch" ] || exit 0

# --dispatch <bench binary> <prompt tokens>: one prefill pass with the dispatch log on, then the tally.
bench=${2:?usage: bench-preflight.sh --dispatch <llama-bench> <prompt tokens>}
tokens=${3:?usage: bench-preflight.sh --dispatch <llama-bench> <prompt tokens>}
model=${PREFLIGHT_MODEL:-$HOME/models/Qwen3-4B-Q4_K_M.gguf}
[ -x "$bench" ] || fatal "no such binary: $bench"
[ -r "$model" ] || fatal "no model at $model (set PREFLIGHT_MODEL)"

log=$(mktemp)
trap 'rm -f "$log"' EXIT
GGML_HRX_LOG_DISPATCH=1 "$bench" -m "$model" -p "$tokens" -n 0 -ngl 99 --device HRX0 -r 3 \
>/dev/null 2>"$log"

lines=$(grep -c 'ggml_hrx dispatch:' "$log" || true)
# A silent bench is the failure this flag exists to catch: an empty tally looks like "nothing declined"
# and would be read as a clean bill of health.
[ "$lines" -eq 0 ] &&
fatal "GGML_HRX_LOG_DISPATCH=1 produced no 'ggml_hrx dispatch:' lines; an empty tally would be a fiction"
printf '\n matcher decisions at pp%s (one run, -r 3):\n' "$tokens"
grep 'ggml_hrx dispatch:' "$log" |
sed -E 's/^.*ggml_hrx dispatch: //; s/ declined.*/ [declined]/' |
sort | uniq -c | sort -rn | sed 's/^/ /' | head -40
grep -q 't/s' "$log" || fatal "the bench produced no throughput line; see $log"
127 changes: 127 additions & 0 deletions tools/tests/test-bench-preflight.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# The gate must refuse for each FATAL condition, warn for each WARN condition, and promote warnings
# under --strict -- because a gate that passes everything is indistinguishable from no gate, and a
# gate that refuses everything gets removed.
#
# Each case fakes the world under $tmp and runs the real script with its roots pointed there, so this
# runs anywhere: no GPU, no model, no hold.
set -uo pipefail

tool=$(dirname "$0")/../bench-preflight.sh
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT

fake() { # fake <uptime_s> <busy> <mem_avail_gib> [gtt_used_b] [gtt_total_b] -> prints the root
local root="$tmp/$1_$2_$3_${4:-0}"
mkdir -p "$root/proc" "$root/sys/class/drm/card0/device/hwmon/hwmon4"
printf '%s 1000\n' "$1" > "$root/proc/uptime"
printf 'MemTotal: 131072000 kB\nMemAvailable: %s kB\n' "$(( $3 * 1048576 ))" > "$root/proc/meminfo"
printf '%s\n' "$2" > "$root/sys/class/drm/card0/device/gpu_busy_percent"
printf '%s\n' "${4:-0}" > "$root/sys/class/drm/card0/device/mem_info_gtt_used"
printf '%s\n' "${5:-107374182400}" > "$root/sys/class/drm/card0/device/mem_info_gtt_total"
printf '50000\n' > "$root/sys/class/drm/card0/device/hwmon/hwmon4/temp1_input"
printf '%s\n' "$root"
}

patched() { # patched <root> [args...] -> runs the real script against the world in <root>
local root=$1; shift
sed -e "s#/proc/uptime#$root/proc/uptime#g" \
-e "s#/proc/meminfo#$root/proc/meminfo#g" \
-e "s#/sys/class/drm/card0/device#$root/sys/class/drm/card0/device#g" \
-e 's/^busy_window=10$/busy_window=1/' \
-e "s#^token=.*#token=$root/GPU-TOKEN#" \
"$tool" > "$root/run.sh"
# No server and no ticket on a fake box: drive the process loop from /dev/null and let the pipeline
# be a no-op. The loop's own heredoc and pipeline are two lines; replace both by exact line number
# pattern so nothing else in the file is touched.
python3 - "$root/run.sh" <<'PYFIX'
import re, sys
path = sys.argv[1]
src = open(path).read().splitlines()
out, i = [], 0
while i < len(src):
if src[i].startswith('done < <(pgrep'):
out.append('done < /dev/null')
i += 1
# the next line is the pipeline feeding that loop
if i < len(src) and 'bench-preflight' in src[i]:
i += 1
continue
out.append(src[i]); i += 1
open(path, 'w').write('\n'.join(out) + '\n')
PYFIX
sed -i '/prov-libs/,/^fi$/d' "$root/run.sh"
sed -i '/^for lib in llama-bench/,/^done$/d' "$root/run.sh"
# A syntax error in the patched script exits non-zero, which the test would read as "the gate
# refused". Prove it parses before asserting on its exit code.
bash -n "$root/run.sh" || { echo "FAIL patched script does not parse: $root/run.sh" >&2; exit 99; }
bash "$root/run.sh" "$@"
}

pass=0; fail=0
expect() { # expect <want: ok|no> <name> <root> [args...]
local want=$1 name=$2 root=$3; shift 3
out=$(patched "$root" "$@" 2>&1); rc=$?
got=ok
[ $rc -ne 0 ] && got=no
if [ "$got" = "$want" ]; then
printf 'ok %-38s rc=%d\n' "$name" "$rc"; pass=$((pass+1))
else
printf 'FAIL %-38s rc=%d want=%s\n%s\n' "$name" "$rc" "$want" "$out"; fail=$((fail+1))
fi
}

want_text() { # want_text <name> <root> <needle> [args...] -> exit 0 iff <needle> appears
local name=$1 root=$2 needle=$3; shift 3
out=$(patched "$root" "$@" 2>&1)
if printf '%s' "$out" | grep -q -- "$needle"; then
printf 'ok %-38s saw %s\n' "$name" "$needle"; pass=$((pass+1))
else
printf 'FAIL %-38s missing %s\n%s\n' "$name" "$needle" "$out"; fail=$((fail+1))
fi
}

GIB=1073741824
# FATAL conditions
expect ok "fresh boot is a WARNING, not fatal" "$(fake 300 0 60)"
expect no "--strict makes fresh boot fatal" "$(fake 300 0 60)" --strict
expect no "die busy (38%)" "$(fake 3600 38 60)"
expect no "RAM below the floor" "$(fake 3600 0 12)"
expect no "--need-gtt 40 with 16 GiB free" "$(fake 3600 0 60 $((112*GIB)) $((128*GIB)))" --need-gtt 40
expect ok "--need-gtt 4 with 16 GiB free" "$(fake 3600 0 60 $((112*GIB)) $((128*GIB)))" --need-gtt 4
expect ok "--need-gtt 40 with 100 GiB free" "$(fake 3600 0 60 $((28*GIB)) $((128*GIB)))" --need-gtt 40

# WARN conditions and strict
expect ok "clean box" "$(fake 3600 0 60)"
expect ok "clean box under --strict" "$(fake 3600 0 60)" --strict
want_text "the warning is printed, not hidden" "$(fake 300 0 60)" "WARNING(S)"
want_text "uptime warning names its reason" "$(fake 300 0 60)" "still settling"

# --dispatch must fail when the bench prints no dispatch lines, rather than print an empty tally.
cat > "$tmp/fakebench" <<'SH'
#!/bin/sh
echo "| model | pp512 | 3000.00 t/s |" >&2
exit 0
SH
chmod +x "$tmp/fakebench"
mkdir -p "$tmp/models"; : > "$tmp/models/Qwen3-4B-Q4_K_M.gguf"
PREFLIGHT_MODEL=$tmp/models/Qwen3-4B-Q4_K_M.gguf expect no "--dispatch with silent bench" \
"$(fake 3600 0 60)" --dispatch "$tmp/fakebench" 512

printf '\nbench-preflight test: %d passed, %d failed\n' "$pass" "$fail"
[ "$fail" -eq 0 ]
Loading