bong-water-water-bong · Nov 10, 2026
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -341,6 +341,11 @@ add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --sel
 # GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered
 # here (the test fakes the box, so it needs no device and no ticket).
 add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-preflight.sh)
+# engine#572: an arm's memory trace. --drop-gib exists because a floor relative to a reading taken before the arm is
+# fragile -- other lanes move MemAvailable between the two -- while "fell by more than N GiB during this arm" is the
+# contamination itself. Same reason the test uses --drop-gib rather than a floor.
+add_test(NAME bench_arm_watch COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-arm-watch.sh
+                                           ${CMAKE_SOURCE_DIR}/tools/bench-arm.sh)
 # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because
 # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic
 # traces with a known answer; no model, no device.

diff --git a/tools/bench-arm.sh b/tools/bench-arm.sh
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run ONE benchmark arm under memory observation (engine#572).
+#
+# `tools/bench-preflight.sh` refuses to start below a MemAvailable floor, which is a statement about the moment it
+# runs. On 2026-11-10 the second arm of a pp4096 pair started at 85 GiB available, another load arrived, MemAvailable
+# fell to 1-3 GiB with swap full, and the arm timed out at 540 s: the number that came back (a timeout) and the arm
+# that did not were both produced on a box nobody had described. This samples for the whole command and prints
+# start/min/end, so an arm is either clean or visibly contaminated where its own number is printed.
+#
+#   tools/bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD...
+#       run CMD, report memory, exit with CMD's status (or 1 if the arm was contaminated)
+#
+# Two independent contamination tests, because they catch different things:
+#   --floor GiB     an ABSOLUTE floor (default 20, the fleet's box floor). What the preflight gates on at start.
+#   --drop-gib N    a DROP during the arm: MemAvailable fell by more than N GiB between start and min. This is the one
+#                   that describes contamination rather than the box's level, and it does not move when another lane's
+#                   memory does -- the absolute reading in a pair can drift between two arms of the same run.
+#
+# BENCH_RAM_FLOOR_GIB overrides the floor; 20 GiB is the fleet's box floor, the same one the preflight gates on.
+set -uo pipefail
+
+floor=${BENCH_RAM_FLOOR_GIB:-20}
+drop=0
+while [ $# -gt 0 ]; do
+    case $1 in
+        --floor) floor=${2:?--floor needs a GiB value}; shift 2 ;;
+        --drop-gib) drop=${2:?--drop-gib needs a GiB value}; shift 2 ;;
+        --) shift; break ;;
+        *) break ;;
+    esac
+done
+[ $# -gt 0 ] || { echo "bench-arm: usage: bench-arm.sh [--floor GiB] -- CMD..." >&2; exit 2; }
+
+mem_avail() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; }
+swap_free() { awk '/SwapFree/ {printf "%d", $2/1048576}' /proc/meminfo; }
+
+start=$(mem_avail); min=$start; sw_start=$(swap_free); sw_min=$sw_start
+"$@" &
+child=$!
+while kill -0 "$child" 2>/dev/null; do
+    m=$(mem_avail); s=$(swap_free)
+    [ "$m" -lt "$min" ] && min=$m
+    [ "$s" -lt "$sw_min" ] && sw_min=$s
+    sleep 0.25
+done
+wait "$child"; rc=$?
+end=$(mem_avail)
+# The end reading counts too: a short arm can fall and recover between samples, and the number produced at the end is
+# the one a reader would quote.
+[ "$end" -lt "$min" ] && min=$end
+w_end=$(swap_free)
+[ "$w_end" -lt "$sw_min" ] && sw_min=$w_end
+
+printf 'bench-arm: mem_avail start %s min %s end %s GiB; swap_free start %s min %s GiB\n' \
+       "$start" "$min" "$end" "$sw_start" "$sw_min"
+if [ "$min" -lt "$floor" ]; then
+    printf 'bench-arm: FAIL -- MemAvailable fell to %s GiB (floor %s) DURING the arm, so this number was not taken on a quiet box: say so, or discard it\n' \
+           "$min" "$floor" >&2
+    exit 1
+fi
+if [ "$drop" -gt 0 ] && [ $((start - min)) -gt "$drop" ]; then
+    printf 'bench-arm: FAIL -- MemAvailable fell %s GiB during the arm (limit %s), so another load arrived mid-run: this number was not taken on a quiet box\n' \
+           "$((start - min))" "$drop" >&2
+    exit 1
+fi
+exit "$rc"
diff --git a/tools/bench-preflight.sh b/tools/bench-preflight.sh
@@ -47,18 +47,20 @@ set -uo pipefail
 
 busy_pct=5
 busy_window=10
-ram_floor_gib=20
+ram_floor_gib=${BENCH_RAM_FLOOR_GIB:-20}
 uptime_floor=600
 
 die=/sys/class/drm/card0/device
 here=$(dirname "$0")
 strict=0
 need_gtt=0
+watch=0
 
 while [ $# -gt 0 ]; do
     case $1 in
         --strict) strict=1; shift ;;
         --need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;;
+        --watch) watch=1; shift; [ "${1:-}" = "--" ] && shift; break ;;
         --dispatch) break ;;
         *) echo "bench-preflight: unknown option $1" >&2; exit 2 ;;
     esac
@@ -127,6 +129,19 @@ if [ -n "$warned" ] && [ "$strict" = 1 ]; then
     fatal "the box is usable but not clean, and --strict was asked for"
 fi
 
+# --watch: the start gate says nothing about the rest of the run. engine#572's second arm started at 85 GiB
+# available, another load arrived, MemAvailable fell to 1-3 GiB with swap full, and the arm timed out at 540 s
+# -- so the number that came back (a timeout) and the number that did not (the arm) were both produced on a box
+# nobody had described. Sample for the whole command and report start/min/end, so an arm is either clean or
+# visibly contaminated where its number is printed.
+if [ "$watch" = 1 ]; then
+    # engine#572: the start gate describes the wrong moment -- a mid-arm load can take MemAvailable from 85 GiB to
+    # 1-3 GiB and the arm times out, so the number is produced on a box nobody described. The sampler lives in its own
+    # tool so it can be tested on a busy box (this gate would refuse to reach it).
+    [ $# -gt 0 ] || { echo "bench-preflight: --watch needs a command after --" >&2; exit 2; }
+    exec "$here/bench-arm.sh" --floor "$ram_floor_gib" -- "$@"
+fi
+
 printf 'bench-preflight: OK\n'
 printf '  kernel     %s\n' "$(uname -r)"
 printf '  uptime     %ss  gpu_busy %s%%  mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib"

diff --git a/tools/tests/test-bench-arm-watch.sh b/tools/tests/test-bench-arm-watch.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# `bench-preflight.sh --watch` exists because a start gate describes the wrong moment (engine#572): the second arm of
+# the pp4096 pair started at 85 GiB MemAvailable, another load arrived mid-arm, MemAvailable fell to 1-3 GiB with swap
+# full, and the arm timed out -- so both the arm and its replacement number were produced on a box nobody had described.
+#
+# Three arms, each testing something a wrong implementation would get wrong:
+#   1. a quiet child: the arm reports start/min/end and exits with the CHILD's status, not 0;
+#   2. a child that really allocates: MemAvailable falls below a floor set just under the quiet value, so the watch
+#      FAILs -- the mid-arm detection, which is the whole point;
+#   3. the same child with the floor left alone: no FAIL, because a drop that stays above the floor is not a
+#      contamination, and a gate that fires there would be ignored.
+#
+# usage: test-bench-arm-watch.sh tools/bench-arm.sh
+set -u
+ba=${1:?usage: test-bench-arm-watch.sh path/to/bench-arm.sh}
+[ -x "$ba" ] || { echo "not executable: $ba"; exit 1; }
+fails=0
+ok() { echo "ok   $*"; }
+bad() { echo "FAIL $*"; fails=$((fails+1)); }
+
+avail=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo)
+
+# 1. quiet child, non-zero exit status must survive
+out=$("$ba" -- sh -c 'exit 7' 2>&1); rc=$?
+[ "$rc" = 7 ] && ok "a quiet arm exits with the child's status (7)" || bad "child status not propagated (got $rc)"
+grep -q "mem_avail start " <<<"$out" && ok "and it reports mem start/min/end" || bad "no memory line: $out"
+
+# 2. a child that allocates: --drop-gib 1, so the drop is what trips it and the box's absolute level does not matter
+# (a floor relative to a reading taken before the arm is fragile -- other lanes move MemAvailable between the two)
+out=$("$ba" --drop-gib 1 --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$?
+[ "$rc" = 1 ] && ok "a mid-arm drop below the floor FAILs the arm" || bad "mid-arm drop not caught (rc=$rc): $out"
+grep -qi 'DURING the arm' <<<"$out" && ok "and says the drop happened during the arm" || bad "message does not say when: $out"
+
+# 3. the same allocation with the default floor: no FAIL (the box has room), child status preserved
+out=$("$ba" --drop-gib 64 --floor 1 -- python3 -c 'x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096)' 2>&1); rc=$?
+[ "$rc" = 0 ] && ok "a 3 GiB allocation under a 64 GiB drop limit is not a contamination" || bad "false positive (rc=$rc): $out"
+
+if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi
+echo "all checks passed"