From 1cc4c2a713685419b91afba552f0aec9fb7b0e8b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 10:30:36 -0400 Subject: [PATCH] bench: one benchmark arm with its memory trace (engine#572), with #2049's review answered This lands #2049 (`tools/bench-arm.sh`, `bench-preflight.sh --watch`, ctest row bench_arm_watch) on current main, plus the three points of the CHANGES at 3e5b91a and the reason its CI failed. * An interrupted arm takes its bench with it. CMD runs in its own process group (set -m), and INT, TERM or HUP sends TERM to the whole group, then KILL after BENCH_ARM_GRACE_S (default 10 s); an EXIT trap does the same. Before, the arm had no trap and `--watch` reaches it by exec, so nothing could reap the bench: SIGTERM left it running, and SIGINT was deferred by bash and ignored. * `--watch` reaches the drop test. `bench-preflight.sh --drop-gib N --watch -- CMD` forwards it, and BENCH_DROP_GIB sets the default in both tools. Before, only --floor was forwarded, so drop was always 0 there. * The usage text says what --drop-gib cannot see: MemAvailable counts the arm's own footprint. * CI: "child status not propagated (got 1)" was the floor test, not the status. Arm 1 ran with the default 20 GiB floor, and a runner with less MemAvailable FAILs that first; it now passes --floor 0. Reproduced with BENCH_RAM_FLOOR_GIB=9999 on the old test. Test: test-bench-arm-watch.sh gains 4 arms (INT, TERM, a TERM-ignoring bench, BENCH_DROP_GIB) with a bounded wait so a deaf arm fails rather than hangs; test-bench-preflight.sh gains 4 --watch forwarding arms. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 5 ++ tools/bench-arm.sh | 115 ++++++++++++++++++++++++++++ tools/bench-preflight.sh | 23 +++++- tools/tests/test-bench-arm-watch.sh | 96 +++++++++++++++++++++++ tools/tests/test-bench-preflight.sh | 19 +++++ 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100755 tools/bench-arm.sh create mode 100755 tools/tests/test-bench-arm-watch.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 46b5f254b..28045432a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -418,6 +418,11 @@ add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --sel # GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered # here (the test fakes the box, so it needs no device and no ticket). add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-preflight.sh) +# engine#572: an arm's memory trace. --drop-gib exists because a floor relative to a reading taken before the arm is +# fragile -- other lanes move MemAvailable between the two -- while "fell by more than N GiB during this arm" is the +# contamination itself. Same reason the test uses --drop-gib rather than a floor. +add_test(NAME bench_arm_watch COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-arm-watch.sh + ${CMAKE_SOURCE_DIR}/tools/bench-arm.sh) # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. diff --git a/tools/bench-arm.sh b/tools/bench-arm.sh new file mode 100755 index 000000000..5c4d9bb81 --- /dev/null +++ b/tools/bench-arm.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run ONE benchmark arm under memory observation (engine#572). +# +# `tools/bench-preflight.sh` refuses to start below a MemAvailable floor, which is a statement about the moment it +# runs. On 2026-11-10 the second arm of a pp4096 pair started at 85 GiB available, another load arrived, MemAvailable +# fell to 1-3 GiB with swap full, and the arm timed out at 540 s: the number that came back (a timeout) and the arm +# that did not were both produced on a box nobody had described. This samples for the whole command and prints +# start/min/end, so an arm is either clean or visibly contaminated where its own number is printed. +# +# tools/bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD... +# run CMD, report memory, exit with CMD's status (or 1 if the arm was contaminated) +# +# Two independent contamination tests, because they catch different things: +# --floor GiB an ABSOLUTE floor (default 20, the fleet's box floor). What the preflight gates on at start. +# --drop-gib N a DROP during the arm: MemAvailable fell by more than N GiB between start and min. This is the one +# that describes contamination rather than the box's level, and it does not move when another lane's +# memory does -- the absolute reading in a pair can drift between two arms of the same run. +# +# BENCH_RAM_FLOOR_GIB and BENCH_DROP_GIB set the two defaults (20 and 0 = off); the flags override them. The env form is +# what reaches this through `bench-preflight.sh --watch`, which execs this tool. +# +# What --drop-gib cannot see: MemAvailable counts the arm's OWN footprint, so N has to be set above what the arm itself +# maps and allocates -- and an intruder smaller than that margin is indistinguishable from the arm. On a pp4096 arm of +# a large pack the two are the same order, so there the --floor test is the sharper instrument. +# +# An interrupted arm takes its bench with it: CMD runs in its own process group, and INT, TERM or HUP to this tool +# sends TERM to that whole group (KILL after BENCH_ARM_GRACE_S, default 10 s), so a bench that spawns workers does +# not outlive the arm that started it (the 11-08 orphan held GTT for 14 h). +set -uo pipefail + +floor=${BENCH_RAM_FLOOR_GIB:-20} +drop=${BENCH_DROP_GIB:-0} +grace=${BENCH_ARM_GRACE_S:-10} +while [ $# -gt 0 ]; do + case $1 in + --floor) floor=${2:?--floor needs a GiB value}; shift 2 ;; + --drop-gib) drop=${2:?--drop-gib needs a GiB value}; shift 2 ;; + --) shift; break ;; + *) break ;; + esac +done +[ $# -gt 0 ] || { echo "bench-arm: usage: bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD..." >&2; exit 2; } + +mem_avail() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } +swap_free() { awk '/SwapFree/ {printf "%d", $2/1048576}' /proc/meminfo; } + +child= +# Stop the child's whole process group: TERM, then KILL once the grace period is over. Safe to call twice. +stop_arm() { + [ -n "$child" ] || return 0 + kill -TERM -- "-$child" 2>/dev/null || return 0 + local i=0 + while kill -0 -- "-$child" 2>/dev/null && [ "$i" -lt $((grace * 10)) ]; do sleep 0.1; i=$((i + 1)); done + kill -KILL -- "-$child" 2>/dev/null + wait "$child" 2>/dev/null + child= +} +on_signal() { + echo "bench-arm: $1 received, stopping the arm's process group" >&2 + stop_arm; trap - EXIT; exit "$2" +} +trap 'on_signal INT 130' INT +trap 'on_signal TERM 143' TERM +trap 'on_signal HUP 129' HUP +trap 'stop_arm' EXIT + +start=$(mem_avail); min=$start; sw_start=$(swap_free); sw_min=$sw_start +# set -m puts the background job in its own process group (pgid = its pid), so stop_arm can signal the group. +set -m +"$@" & +child=$! +set +m +while kill -0 "$child" 2>/dev/null; do + m=$(mem_avail); s=$(swap_free) + [ "$m" -lt "$min" ] && min=$m + [ "$s" -lt "$sw_min" ] && sw_min=$s + sleep 0.25 +done +wait "$child"; rc=$? +child= +end=$(mem_avail) +# The end reading counts too: a short arm can fall and recover between samples, and the number produced at the end is +# the one a reader would quote. +[ "$end" -lt "$min" ] && min=$end +w_end=$(swap_free) +[ "$w_end" -lt "$sw_min" ] && sw_min=$w_end + +printf 'bench-arm: mem_avail start %s min %s end %s GiB; swap_free start %s min %s GiB\n' \ + "$start" "$min" "$end" "$sw_start" "$sw_min" +if [ "$min" -lt "$floor" ]; then + printf 'bench-arm: FAIL -- MemAvailable fell to %s GiB (floor %s) DURING the arm, so this number was not taken on a quiet box: say so, or discard it\n' \ + "$min" "$floor" >&2 + exit 1 +fi +if [ "$drop" -gt 0 ] && [ $((start - min)) -gt "$drop" ]; then + printf 'bench-arm: FAIL -- MemAvailable fell %s GiB during the arm (limit %s), so another load arrived mid-run: this number was not taken on a quiet box\n' \ + "$((start - min))" "$drop" >&2 + exit 1 +fi +exit "$rc" diff --git a/tools/bench-preflight.sh b/tools/bench-preflight.sh index 4cb295f73..efcd4893d 100755 --- a/tools/bench-preflight.sh +++ b/tools/bench-preflight.sh @@ -24,6 +24,9 @@ # tools/bench-preflight.sh check the box; FATAL and exit 1 if a number would lie # tools/bench-preflight.sh --strict every warning becomes fatal # tools/bench-preflight.sh --need-gtt GiB also fatal if the die has less than GiB free GTT +# tools/bench-preflight.sh [--drop-gib N] --watch -- CMD... +# gate, then run CMD under tools/bench-arm.sh: memory start/min/end, +# FAIL if MemAvailable fell below the floor or by more than N GiB # tools/bench-preflight.sh --dispatch B N then one GGML_HRX_LOG_DISPATCH=1 pass of `B -p N` and # the matcher each MUL_MAT took # @@ -47,18 +50,22 @@ set -uo pipefail busy_pct=5 busy_window=10 -ram_floor_gib=20 +ram_floor_gib=${BENCH_RAM_FLOOR_GIB:-20} uptime_floor=600 die=/sys/class/drm/card0/device here=$(dirname "$0") strict=0 need_gtt=0 +watch=0 +drop_gib=${BENCH_DROP_GIB:-0} while [ $# -gt 0 ]; do case $1 in --strict) strict=1; shift ;; --need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;; + --drop-gib) drop_gib=${2:?--drop-gib needs a GiB value}; shift 2 ;; + --watch) watch=1; shift; [ "${1:-}" = "--" ] && shift; break ;; --dispatch) break ;; *) echo "bench-preflight: unknown option $1" >&2; exit 2 ;; esac @@ -127,6 +134,20 @@ if [ -n "$warned" ] && [ "$strict" = 1 ]; then fatal "the box is usable but not clean, and --strict was asked for" fi +# --watch: the start gate says nothing about the rest of the run. engine#572's second arm started at 85 GiB +# available, another load arrived, MemAvailable fell to 1-3 GiB with swap full, and the arm timed out at 540 s +# -- so the number that came back (a timeout) and the number that did not (the arm) were both produced on a box +# nobody had described. Sample for the whole command and report start/min/end, so an arm is either clean or +# visibly contaminated where its number is printed. +if [ "$watch" = 1 ]; then + # engine#572: the start gate describes the wrong moment -- a mid-arm load can take MemAvailable from 85 GiB to + # 1-3 GiB and the arm times out, so the number is produced on a box nobody described. The sampler lives in its own + # tool so it can be tested on a busy box (this gate would refuse to reach it). + [ $# -gt 0 ] || { echo "bench-preflight: --watch needs a command after --" >&2; exit 2; } + # Both tests reach the arm: the floor, and the drop (--drop-gib N before --watch, or BENCH_DROP_GIB; 0 = off). + exec "$here/bench-arm.sh" --floor "$ram_floor_gib" --drop-gib "$drop_gib" -- "$@" +fi + printf 'bench-preflight: OK\n' printf ' kernel %s\n' "$(uname -r)" printf ' uptime %ss gpu_busy %s%% mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib" diff --git a/tools/tests/test-bench-arm-watch.sh b/tools/tests/test-bench-arm-watch.sh new file mode 100755 index 000000000..383a55330 --- /dev/null +++ b/tools/tests/test-bench-arm-watch.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# `bench-preflight.sh --watch` exists because a start gate describes the wrong moment (engine#572): the second arm of +# the pp4096 pair started at 85 GiB MemAvailable, another load arrived mid-arm, MemAvailable fell to 1-3 GiB with swap +# full, and the arm timed out -- so both the arm and its replacement number were produced on a box nobody had described. +# +# Three arms, each testing something a wrong implementation would get wrong: +# 1. a quiet child: the arm reports start/min/end and exits with the CHILD's status, not 0; +# 2. a child that really allocates: MemAvailable falls below a floor set just under the quiet value, so the watch +# FAILs -- the mid-arm detection, which is the whole point; +# 3. the same child with the floor left alone: no FAIL, because a drop that stays above the floor is not a +# contamination, and a gate that fires there would be ignored. +# 4-6. an interrupted arm takes its bench with it (INT, TERM; a child that ignores TERM is KILLed after the grace), +# grandchildren included, and the arm exits 128+signal; +# 7. BENCH_DROP_GIB reaches the drop test, which is how `bench-preflight.sh --watch` (an exec) sets it. +# +# Arm 1 passes --floor 0: its subject is the exit status, and with the default 20 GiB floor a CI runner with less +# MemAvailable than that FAILs the floor test first and reports 1 (the first CI run of this test, engine#2049). +# +# usage: test-bench-arm-watch.sh tools/bench-arm.sh +set -u +ba=${1:?usage: test-bench-arm-watch.sh path/to/bench-arm.sh} +[ -x "$ba" ] || { echo "not executable: $ba"; exit 1; } +fails=0 +ok() { echo "ok $*"; } +bad() { echo "FAIL $*"; fails=$((fails+1)); } + +avail=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo) + +# 1. quiet child, non-zero exit status must survive +out=$("$ba" --floor 0 -- sh -c 'exit 7' 2>&1); rc=$? +[ "$rc" = 7 ] && ok "a quiet arm exits with the child's status (7)" || bad "child status not propagated (got $rc)" +grep -q "mem_avail start " <<<"$out" && ok "and it reports mem start/min/end" || bad "no memory line: $out" + +# 2. a child that allocates: --drop-gib 1, so the drop is what trips it and the box's absolute level does not matter +# (a floor relative to a reading taken before the arm is fragile -- other lanes move MemAvailable between the two) +out=$("$ba" --drop-gib 1 --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$? +[ "$rc" = 1 ] && ok "a mid-arm drop below the floor FAILs the arm" || bad "mid-arm drop not caught (rc=$rc): $out" +grep -qi 'DURING the arm' <<<"$out" && ok "and says the drop happened during the arm" || bad "message does not say when: $out" + +# 3. the same allocation with the default floor: no FAIL (the box has room), child status preserved +out=$("$ba" --drop-gib 64 --floor 1 -- python3 -c 'x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096)' 2>&1); rc=$? +[ "$rc" = 0 ] && ok "a 3 GiB allocation under a 64 GiB drop limit is not a contamination" || bad "false positive (rc=$rc): $out" + +# 4-6. interrupt the arm. A non-interactive shell starts a background job with SIGINT ignored, and a signal ignored on +# entry cannot be trapped, so start the arm through a shim that restores the default first (as a terminal would). +alive() { pgrep -f "^sleep $1\$" >/dev/null; } +interrupt_case() { # interrupt_case + local sig=$1 want=$2 name=$3; shift 3 + BENCH_ARM_GRACE_S=1 python3 -c 'import os, signal, sys; signal.signal(signal.SIGINT, signal.SIG_DFL); os.execvp(sys.argv[1], sys.argv[1:])' \ + bash "$ba" --floor 0 -- "$@" >/dev/null 2>&1 & + local arm=$! i=0 + while ! alive 3172 && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done + alive 3172 || { bad "$name: the child never started"; kill -KILL "$arm" 2>/dev/null; return; } + kill "-$sig" "$arm" + # bounded: an arm with no handler can sit out the signal (bash defers it while a foreground `sleep` runs and then + # carries on), and an unbounded wait would hang the test instead of failing it + i=0; while kill -0 "$arm" 2>/dev/null && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done + if kill -0 "$arm" 2>/dev/null; then + bad "$name: the arm ignored SIG$sig"; kill -KILL "$arm"; wait "$arm" 2>/dev/null; pkill -KILL -f '^sleep 317[12]$' + return + fi + wait "$arm"; local rc=$? + i=0; while { alive 3171 || alive 3172; } && [ "$i" -lt 30 ]; do sleep 0.1; i=$((i + 1)); done + if alive 3171 || alive 3172; then + bad "$name: the bench outlived the arm (rc=$rc)"; pkill -KILL -f '^sleep 317[12]$' + elif [ "$rc" != "$want" ]; then + bad "$name: arm exited $rc, want $want" + else + ok "$name" + fi +} +interrupt_case INT 130 "SIGINT to the arm stops the bench and its worker (rc 130)" sh -c 'sleep 3171 & sleep 3172; wait' +interrupt_case TERM 143 "SIGTERM to the arm stops the bench and its worker (rc 143)" sh -c 'sleep 3171 & sleep 3172; wait' +interrupt_case TERM 143 "a bench that ignores TERM is KILLed after the grace" sh -c 'trap "" TERM; sleep 3171 & sleep 3172; wait' + +# 7. the env form of --drop-gib: what `bench-preflight.sh --watch` passes through its exec +out=$(BENCH_DROP_GIB=1 "$ba" --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$? +[ "$rc" = 1 ] && grep -q 'limit 1)' <<<"$out" && ok "BENCH_DROP_GIB=1 reaches the drop test" || bad "BENCH_DROP_GIB not honoured (rc=$rc): $out" + +if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi +echo "all checks passed" diff --git a/tools/tests/test-bench-preflight.sh b/tools/tests/test-bench-preflight.sh index d701fbb2d..31a360a48 100755 --- a/tools/tests/test-bench-preflight.sh +++ b/tools/tests/test-bench-preflight.sh @@ -320,5 +320,24 @@ orphan_case "SIGTERM to the preflight stops the bench" TERM orphan_case "SIGHUP to the preflight stops the bench" HUP orphan_case "a bench that ignores TERM is killed" TERM 1 +# --watch execs tools/bench-arm.sh, so what reaches the arm is only what --watch forwards: the floor AND the drop +# (engine#2049's review: only --floor was forwarded, so the drop test never ran on the documented route). A stand-in +# bench-arm.sh beside the patched script records its argv. +watch_case() { # watch_case [env-assignment] -- preflight args... + local name=$1 needle=$2 envset=$3 root; shift 3 + root=$(fake 3600 0 60) + printf '#!/bin/sh\nprintf "%%s " "$@" > "%s/arm-argv"\n' "$root" > "$root/bench-arm.sh"; chmod +x "$root/bench-arm.sh" + ( [ -z "$envset" ] || export "${envset?}"; patched "$root" "$@" ) >/dev/null 2>&1 + if grep -q -- "$needle" "$root/arm-argv" 2>/dev/null; then + printf 'ok %-38s argv has %s\n' "$name" "$needle"; pass=$((pass+1)) + else + printf 'FAIL %-38s argv: %s\n' "$name" "$(cat "$root/arm-argv" 2>/dev/null)"; fail=$((fail+1)) + fi +} +watch_case "--watch forwards --drop-gib" "--drop-gib 5 " "" --drop-gib 5 --watch -- true +watch_case "--watch forwards BENCH_DROP_GIB" "--drop-gib 7 " BENCH_DROP_GIB=7 --watch -- true +watch_case "--watch forwards the floor" "--floor 20 " "" --watch -- true +watch_case "and the command after --" "-- true " "" --watch -- true + printf '\nbench-preflight test: %d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ]