bench: one arm with its memory trace (#2049 + its review answered)

CMakeLists.txt

@@ -418,6 +418,11 @@ add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --sel
 # GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered
 # here (the test fakes the box, so it needs no device and no ticket).
 add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-preflight.sh)
+# engine#572: an arm's memory trace. --drop-gib exists because a floor relative to a reading taken before the arm is
+# fragile -- other lanes move MemAvailable between the two -- while "fell by more than N GiB during this arm" is the
+# contamination itself. Same reason the test uses --drop-gib rather than a floor.
+add_test(NAME bench_arm_watch COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-arm-watch.sh
+                                           ${CMAKE_SOURCE_DIR}/tools/bench-arm.sh)
 # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because
 # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic
 # traces with a known answer; no model, no device.

tools/bench-arm.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run ONE benchmark arm under memory observation (engine#572).
+#
+# `tools/bench-preflight.sh` refuses to start below a MemAvailable floor, which is a statement about the moment it
+# runs. On 2026-11-10 the second arm of a pp4096 pair started at 85 GiB available, another load arrived, MemAvailable
+# fell to 1-3 GiB with swap full, and the arm timed out at 540 s: the number that came back (a timeout) and the arm
+# that did not were both produced on a box nobody had described. This samples for the whole command and prints
+# start/min/end, so an arm is either clean or visibly contaminated where its own number is printed.
+#
+#   tools/bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD...
+#       run CMD, report memory, exit with CMD's status (or 1 if the arm was contaminated)
+#
+# Two independent contamination tests, because they catch different things:
+#   --floor GiB     an ABSOLUTE floor (default 20, the fleet's box floor). What the preflight gates on at start.
+#   --drop-gib N    a DROP during the arm: MemAvailable fell by more than N GiB between start and min. This is the one
+#                   that describes contamination rather than the box's level, and it does not move when another lane's
+#                   memory does -- the absolute reading in a pair can drift between two arms of the same run.
+#
+# BENCH_RAM_FLOOR_GIB and BENCH_DROP_GIB set the two defaults (20 and 0 = off); the flags override them. The env form is
+# what reaches this through `bench-preflight.sh --watch`, which execs this tool.
+#
+# What --drop-gib cannot see: MemAvailable counts the arm's OWN footprint, so N has to be set above what the arm itself
+# maps and allocates -- and an intruder smaller than that margin is indistinguishable from the arm. On a pp4096 arm of
+# a large pack the two are the same order, so there the --floor test is the sharper instrument.
+#
+# An interrupted arm takes its bench with it: CMD runs in its own process group, and INT, TERM or HUP to this tool
+# sends TERM to that whole group (KILL after BENCH_ARM_GRACE_S, default 10 s), so a bench that spawns workers does
+# not outlive the arm that started it (the 11-08 orphan held GTT for 14 h).
+set -uo pipefail
+
+floor=${BENCH_RAM_FLOOR_GIB:-20}
+drop=${BENCH_DROP_GIB:-0}
+grace=${BENCH_ARM_GRACE_S:-10}
+while [ $# -gt 0 ]; do
+    case $1 in
+        --floor) floor=${2:?--floor needs a GiB value}; shift 2 ;;
+        --drop-gib) drop=${2:?--drop-gib needs a GiB value}; shift 2 ;;
+        --) shift; break ;;
+        *) break ;;
+    esac
+done
+[ $# -gt 0 ] || { echo "bench-arm: usage: bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD..." >&2; exit 2; }
+
+mem_avail() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; }
+swap_free() { awk '/SwapFree/ {printf "%d", $2/1048576}' /proc/meminfo; }
+
+child=
+# Stop the child's whole process group: TERM, then KILL once the grace period is over. Safe to call twice.
+stop_arm() {
+    [ -n "$child" ] || return 0
+    kill -TERM -- "-$child" 2>/dev/null || return 0
+    local i=0
+    while kill -0 -- "-$child" 2>/dev/null && [ "$i" -lt $((grace * 10)) ]; do sleep 0.1; i=$((i + 1)); done
+    kill -KILL -- "-$child" 2>/dev/null
+    wait "$child" 2>/dev/null
+    child=
+}
+on_signal() {
+    echo "bench-arm: $1 received, stopping the arm's process group" >&2
+    stop_arm; trap - EXIT; exit "$2"
+}
+trap 'on_signal INT 130' INT
+trap 'on_signal TERM 143' TERM
+trap 'on_signal HUP 129' HUP
+trap 'stop_arm' EXIT
+
+start=$(mem_avail); min=$start; sw_start=$(swap_free); sw_min=$sw_start
+# set -m puts the background job in its own process group (pgid = its pid), so stop_arm can signal the group.
+set -m
+"$@" &
+child=$!
+set +m
+while kill -0 "$child" 2>/dev/null; do
+    m=$(mem_avail); s=$(swap_free)
+    [ "$m" -lt "$min" ] && min=$m
+    [ "$s" -lt "$sw_min" ] && sw_min=$s
+    sleep 0.25
+done
+wait "$child"; rc=$?
+child=
+end=$(mem_avail)
+# The end reading counts too: a short arm can fall and recover between samples, and the number produced at the end is
+# the one a reader would quote.
+[ "$end" -lt "$min" ] && min=$end
+w_end=$(swap_free)
+[ "$w_end" -lt "$sw_min" ] && sw_min=$w_end
+
+printf 'bench-arm: mem_avail start %s min %s end %s GiB; swap_free start %s min %s GiB\n' \
+       "$start" "$min" "$end" "$sw_start" "$sw_min"
+if [ "$min" -lt "$floor" ]; then
+    printf 'bench-arm: FAIL -- MemAvailable fell to %s GiB (floor %s) DURING the arm, so this number was not taken on a quiet box: say so, or discard it\n' \
+           "$min" "$floor" >&2
+    exit 1
+fi
+if [ "$drop" -gt 0 ] && [ $((start - min)) -gt "$drop" ]; then
+    printf 'bench-arm: FAIL -- MemAvailable fell %s GiB during the arm (limit %s), so another load arrived mid-run: this number was not taken on a quiet box\n' \
+           "$((start - min))" "$drop" >&2
+    exit 1
+fi
+exit "$rc"

tools/bench-preflight.sh

@@ -24,6 +24,9 @@
 #   tools/bench-preflight.sh                    check the box; FATAL and exit 1 if a number would lie
 #   tools/bench-preflight.sh --strict           every warning becomes fatal
 #   tools/bench-preflight.sh --need-gtt GiB     also fatal if the die has less than GiB free GTT
+#   tools/bench-preflight.sh [--drop-gib N] --watch -- CMD...
+#                                               gate, then run CMD under tools/bench-arm.sh: memory start/min/end,
+#                                               FAIL if MemAvailable fell below the floor or by more than N GiB
 #   tools/bench-preflight.sh --dispatch B N     then one GGML_HRX_LOG_DISPATCH=1 pass of `B -p N` and
 #                                               the matcher each MUL_MAT took
 #
@@ -47,18 +50,22 @@ set -uo pipefail
 
 busy_pct=5
 busy_window=10
-ram_floor_gib=20
+ram_floor_gib=${BENCH_RAM_FLOOR_GIB:-20}
 uptime_floor=600
 
 die=/sys/class/drm/card0/device
 here=$(dirname "$0")
 strict=0
 need_gtt=0
+watch=0
+drop_gib=${BENCH_DROP_GIB:-0}
 
 while [ $# -gt 0 ]; do
     case $1 in
         --strict) strict=1; shift ;;
         --need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;;
+        --drop-gib) drop_gib=${2:?--drop-gib needs a GiB value}; shift 2 ;;
+        --watch) watch=1; shift; [ "${1:-}" = "--" ] && shift; break ;;
         --dispatch) break ;;
         *) echo "bench-preflight: unknown option $1" >&2; exit 2 ;;
     esac
@@ -127,6 +134,20 @@ if [ -n "$warned" ] && [ "$strict" = 1 ]; then
     fatal "the box is usable but not clean, and --strict was asked for"
 fi
 
+# --watch: the start gate says nothing about the rest of the run. engine#572's second arm started at 85 GiB
+# available, another load arrived, MemAvailable fell to 1-3 GiB with swap full, and the arm timed out at 540 s
+# -- so the number that came back (a timeout) and the number that did not (the arm) were both produced on a box
+# nobody had described. Sample for the whole command and report start/min/end, so an arm is either clean or
+# visibly contaminated where its number is printed.
+if [ "$watch" = 1 ]; then
+    # engine#572: the start gate describes the wrong moment -- a mid-arm load can take MemAvailable from 85 GiB to
+    # 1-3 GiB and the arm times out, so the number is produced on a box nobody described. The sampler lives in its own
+    # tool so it can be tested on a busy box (this gate would refuse to reach it).
+    [ $# -gt 0 ] || { echo "bench-preflight: --watch needs a command after --" >&2; exit 2; }
+    # Both tests reach the arm: the floor, and the drop (--drop-gib N before --watch, or BENCH_DROP_GIB; 0 = off).
+    exec "$here/bench-arm.sh" --floor "$ram_floor_gib" --drop-gib "$drop_gib" -- "$@"
+fi
+
 printf 'bench-preflight: OK\n'
 printf '  kernel     %s\n' "$(uname -r)"
 printf '  uptime     %ss  gpu_busy %s%%  mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib"

tools/tests/test-bench-arm-watch.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# `bench-preflight.sh --watch` exists because a start gate describes the wrong moment (engine#572): the second arm of
+# the pp4096 pair started at 85 GiB MemAvailable, another load arrived mid-arm, MemAvailable fell to 1-3 GiB with swap
+# full, and the arm timed out -- so both the arm and its replacement number were produced on a box nobody had described.
+#
+# Three arms, each testing something a wrong implementation would get wrong:
+#   1. a quiet child: the arm reports start/min/end and exits with the CHILD's status, not 0;
+#   2. a child that really allocates: MemAvailable falls below a floor set just under the quiet value, so the watch
+#      FAILs -- the mid-arm detection, which is the whole point;
+#   3. the same child with the floor left alone: no FAIL, because a drop that stays above the floor is not a
+#      contamination, and a gate that fires there would be ignored.
+#   4-6. an interrupted arm takes its bench with it (INT, TERM; a child that ignores TERM is KILLed after the grace),
+#      grandchildren included, and the arm exits 128+signal;
+#   7. BENCH_DROP_GIB reaches the drop test, which is how `bench-preflight.sh --watch` (an exec) sets it.
+#
+# Arm 1 passes --floor 0: its subject is the exit status, and with the default 20 GiB floor a CI runner with less
+# MemAvailable than that FAILs the floor test first and reports 1 (the first CI run of this test, engine#2049).
+#
+# usage: test-bench-arm-watch.sh tools/bench-arm.sh
+set -u
+ba=${1:?usage: test-bench-arm-watch.sh path/to/bench-arm.sh}
+[ -x "$ba" ] || { echo "not executable: $ba"; exit 1; }
+fails=0
+ok() { echo "ok   $*"; }
+bad() { echo "FAIL $*"; fails=$((fails+1)); }
+
+avail=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo)
+
+# 1. quiet child, non-zero exit status must survive
+out=$("$ba" --floor 0 -- sh -c 'exit 7' 2>&1); rc=$?
+[ "$rc" = 7 ] && ok "a quiet arm exits with the child's status (7)" || bad "child status not propagated (got $rc)"
+grep -q "mem_avail start " <<<"$out" && ok "and it reports mem start/min/end" || bad "no memory line: $out"
+
+# 2. a child that allocates: --drop-gib 1, so the drop is what trips it and the box's absolute level does not matter
+# (a floor relative to a reading taken before the arm is fragile -- other lanes move MemAvailable between the two)
+out=$("$ba" --drop-gib 1 --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$?
+[ "$rc" = 1 ] && ok "a mid-arm drop below the floor FAILs the arm" || bad "mid-arm drop not caught (rc=$rc): $out"
+grep -qi 'DURING the arm' <<<"$out" && ok "and says the drop happened during the arm" || bad "message does not say when: $out"
+
+# 3. the same allocation with the default floor: no FAIL (the box has room), child status preserved
+out=$("$ba" --drop-gib 64 --floor 1 -- python3 -c 'x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096)' 2>&1); rc=$?
+[ "$rc" = 0 ] && ok "a 3 GiB allocation under a 64 GiB drop limit is not a contamination" || bad "false positive (rc=$rc): $out"
+
+# 4-6. interrupt the arm. A non-interactive shell starts a background job with SIGINT ignored, and a signal ignored on
+# entry cannot be trapped, so start the arm through a shim that restores the default first (as a terminal would).
+alive() { pgrep -f "^sleep $1\$" >/dev/null; }
+interrupt_case() {  # interrupt_case <signal> <want_rc> <name> <child command...>
+    local sig=$1 want=$2 name=$3; shift 3
+    BENCH_ARM_GRACE_S=1 python3 -c 'import os, signal, sys; signal.signal(signal.SIGINT, signal.SIG_DFL); os.execvp(sys.argv[1], sys.argv[1:])' \
+        bash "$ba" --floor 0 -- "$@" >/dev/null 2>&1 &
+    local arm=$! i=0
+    while ! alive 3172 && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done
+    alive 3172 || { bad "$name: the child never started"; kill -KILL "$arm" 2>/dev/null; return; }
+    kill "-$sig" "$arm"
+    # bounded: an arm with no handler can sit out the signal (bash defers it while a foreground `sleep` runs and then
+    # carries on), and an unbounded wait would hang the test instead of failing it
+    i=0; while kill -0 "$arm" 2>/dev/null && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done
+    if kill -0 "$arm" 2>/dev/null; then
+        bad "$name: the arm ignored SIG$sig"; kill -KILL "$arm"; wait "$arm" 2>/dev/null; pkill -KILL -f '^sleep 317[12]$'
+        return
+    fi
+    wait "$arm"; local rc=$?
+    i=0; while { alive 3171 || alive 3172; } && [ "$i" -lt 30 ]; do sleep 0.1; i=$((i + 1)); done
+    if alive 3171 || alive 3172; then
+        bad "$name: the bench outlived the arm (rc=$rc)"; pkill -KILL -f '^sleep 317[12]$'
+    elif [ "$rc" != "$want" ]; then
+        bad "$name: arm exited $rc, want $want"
+    else
+        ok "$name"
+    fi
+}
+interrupt_case INT 130 "SIGINT to the arm stops the bench and its worker (rc 130)" sh -c 'sleep 3171 & sleep 3172; wait'
+interrupt_case TERM 143 "SIGTERM to the arm stops the bench and its worker (rc 143)" sh -c 'sleep 3171 & sleep 3172; wait'
+interrupt_case TERM 143 "a bench that ignores TERM is KILLed after the grace" sh -c 'trap "" TERM; sleep 3171 & sleep 3172; wait'
+
+# 7. the env form of --drop-gib: what `bench-preflight.sh --watch` passes through its exec
+out=$(BENCH_DROP_GIB=1 "$ba" --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$?
+[ "$rc" = 1 ] && grep -q 'limit 1)' <<<"$out" && ok "BENCH_DROP_GIB=1 reaches the drop test" || bad "BENCH_DROP_GIB not honoured (rc=$rc): $out"
+
+if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi
+echo "all checks passed"

tools/tests/test-bench-preflight.sh

@@ -320,5 +320,24 @@ orphan_case "SIGTERM to the preflight stops the bench" TERM
 orphan_case "SIGHUP to the preflight stops the bench"  HUP
 orphan_case "a bench that ignores TERM is killed"      TERM 1
 
+# --watch execs tools/bench-arm.sh, so what reaches the arm is only what --watch forwards: the floor AND the drop
+# (engine#2049's review: only --floor was forwarded, so the drop test never ran on the documented route). A stand-in
+# bench-arm.sh beside the patched script records its argv.
+watch_case() {  # watch_case <name> <needle> [env-assignment] -- preflight args...
+    local name=$1 needle=$2 envset=$3 root; shift 3
+    root=$(fake 3600 0 60)
+    printf '#!/bin/sh\nprintf "%%s " "$@" > "%s/arm-argv"\n' "$root" > "$root/bench-arm.sh"; chmod +x "$root/bench-arm.sh"
+    ( [ -z "$envset" ] || export "${envset?}"; patched "$root" "$@" ) >/dev/null 2>&1
+    if grep -q -- "$needle" "$root/arm-argv" 2>/dev/null; then
+        printf 'ok   %-38s argv has %s\n' "$name" "$needle"; pass=$((pass+1))
+    else
+        printf 'FAIL %-38s argv: %s\n' "$name" "$(cat "$root/arm-argv" 2>/dev/null)"; fail=$((fail+1))
+    fi
+}
+watch_case "--watch forwards --drop-gib"           "--drop-gib 5 "  ""                 --drop-gib 5 --watch -- true
+watch_case "--watch forwards BENCH_DROP_GIB"       "--drop-gib 7 "  BENCH_DROP_GIB=7   --watch -- true
+watch_case "--watch forwards the floor"            "--floor 20 "    ""                 --watch -- true
+watch_case "and the command after --"              "-- true "       ""                 --watch -- true
+
 printf '\nbench-preflight test: %d passed, %d failed\n' "$pass" "$fail"
 [ "$fail" -eq 0 ]