From 3e5b91ae597c350c896993904e9f126b510d651b Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 18:31:48 -0400 Subject: [PATCH] bench: one benchmark arm, with its memory trace, so a contaminated number says so (engine#572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tools/bench-preflight.sh` refuses to start below a MemAvailable floor, and that is a statement about the moment it runs. On 2026-11-10 the second arm of a pp4096 pair started at **85 GiB available**, another load arrived, MemAvailable fell to **1-3 GiB with swap full**, and the arm timed out at 540 s: the number that came back (a timeout) and the arm that did not were both produced on a box nobody had described, and the only record of it is a comment. `tools/bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD...` runs one arm and prints bench-arm: mem_avail start 62 min 59 end 62 GiB; swap_free start 45 min 45 GiB then exits with the child's status, or 1 when the arm was contaminated. **Two contamination tests, because they catch different things:** * `--floor GiB` — an absolute floor (default 20, the fleet's box floor), the same gate the preflight applies at start; * `--drop-gib N` — MemAvailable fell by more than N GiB **between start and min**. This is the one that describes contamination rather than the box's level, and it does not move when another lane's memory does. I built the first version of the test around a floor-relative-to-a-reading and it was flaky within minutes for exactly that reason: the absolute level drifts between two arms of the same run. Sampling is 0.25 s and the **end reading counts toward the minimum**, because a short arm can fall and recover between samples while the end figure is the one a reader would quote. `tools/bench-preflight.sh --watch -- CMD` delegates here, so the die gate and the arm trace stay one implementation. Arms (5, in `tools/tests/test-bench-arm-watch.sh`, registered as ctest `bench_arm_watch`): a quiet arm exits with the **child's** status (7) rather than 0; it reports start/min/end; a child that allocates and *touches* 3 GiB trips `--drop-gib 1` and the message says the drop happened **during the arm**; and the same allocation under a 64 GiB limit is **not** a contamination, so the check cannot pass by always failing. Two fixture lessons worth keeping: an untouched `bytearray` proves nothing (overcommit — the first version of the allocation arm measured a 3 GiB allocation as a 0 GiB drop), and a child that touches and exits in 0.6 s is invisible at a 0.25 s interval, so the test holds the memory for 2 s rather than racing the sampler. Suite 196/196; copyright clean. Refs #572, #2005. --- CMakeLists.txt | 5 ++ tools/bench-arm.sh | 81 +++++++++++++++++++++++++++++ tools/bench-preflight.sh | 17 +++++- tools/tests/test-bench-arm-watch.sh | 54 +++++++++++++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100755 tools/bench-arm.sh create mode 100755 tools/tests/test-bench-arm-watch.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 86cb25cf9..5afe18636 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,11 @@ add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --sel # GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered # here (the test fakes the box, so it needs no device and no ticket). add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-preflight.sh) +# engine#572: an arm's memory trace. --drop-gib exists because a floor relative to a reading taken before the arm is +# fragile -- other lanes move MemAvailable between the two -- while "fell by more than N GiB during this arm" is the +# contamination itself. Same reason the test uses --drop-gib rather than a floor. +add_test(NAME bench_arm_watch COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-arm-watch.sh + ${CMAKE_SOURCE_DIR}/tools/bench-arm.sh) # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. diff --git a/tools/bench-arm.sh b/tools/bench-arm.sh new file mode 100755 index 000000000..8c0d203e2 --- /dev/null +++ b/tools/bench-arm.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run ONE benchmark arm under memory observation (engine#572). +# +# `tools/bench-preflight.sh` refuses to start below a MemAvailable floor, which is a statement about the moment it +# runs. On 2026-11-10 the second arm of a pp4096 pair started at 85 GiB available, another load arrived, MemAvailable +# fell to 1-3 GiB with swap full, and the arm timed out at 540 s: the number that came back (a timeout) and the arm +# that did not were both produced on a box nobody had described. This samples for the whole command and prints +# start/min/end, so an arm is either clean or visibly contaminated where its own number is printed. +# +# tools/bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD... +# run CMD, report memory, exit with CMD's status (or 1 if the arm was contaminated) +# +# Two independent contamination tests, because they catch different things: +# --floor GiB an ABSOLUTE floor (default 20, the fleet's box floor). What the preflight gates on at start. +# --drop-gib N a DROP during the arm: MemAvailable fell by more than N GiB between start and min. This is the one +# that describes contamination rather than the box's level, and it does not move when another lane's +# memory does -- the absolute reading in a pair can drift between two arms of the same run. +# +# BENCH_RAM_FLOOR_GIB overrides the floor; 20 GiB is the fleet's box floor, the same one the preflight gates on. +set -uo pipefail + +floor=${BENCH_RAM_FLOOR_GIB:-20} +drop=0 +while [ $# -gt 0 ]; do + case $1 in + --floor) floor=${2:?--floor needs a GiB value}; shift 2 ;; + --drop-gib) drop=${2:?--drop-gib needs a GiB value}; shift 2 ;; + --) shift; break ;; + *) break ;; + esac +done +[ $# -gt 0 ] || { echo "bench-arm: usage: bench-arm.sh [--floor GiB] -- CMD..." >&2; exit 2; } + +mem_avail() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } +swap_free() { awk '/SwapFree/ {printf "%d", $2/1048576}' /proc/meminfo; } + +start=$(mem_avail); min=$start; sw_start=$(swap_free); sw_min=$sw_start +"$@" & +child=$! +while kill -0 "$child" 2>/dev/null; do + m=$(mem_avail); s=$(swap_free) + [ "$m" -lt "$min" ] && min=$m + [ "$s" -lt "$sw_min" ] && sw_min=$s + sleep 0.25 +done +wait "$child"; rc=$? +end=$(mem_avail) +# The end reading counts too: a short arm can fall and recover between samples, and the number produced at the end is +# the one a reader would quote. +[ "$end" -lt "$min" ] && min=$end +w_end=$(swap_free) +[ "$w_end" -lt "$sw_min" ] && sw_min=$w_end + +printf 'bench-arm: mem_avail start %s min %s end %s GiB; swap_free start %s min %s GiB\n' \ + "$start" "$min" "$end" "$sw_start" "$sw_min" +if [ "$min" -lt "$floor" ]; then + printf 'bench-arm: FAIL -- MemAvailable fell to %s GiB (floor %s) DURING the arm, so this number was not taken on a quiet box: say so, or discard it\n' \ + "$min" "$floor" >&2 + exit 1 +fi +if [ "$drop" -gt 0 ] && [ $((start - min)) -gt "$drop" ]; then + printf 'bench-arm: FAIL -- MemAvailable fell %s GiB during the arm (limit %s), so another load arrived mid-run: this number was not taken on a quiet box\n' \ + "$((start - min))" "$drop" >&2 + exit 1 +fi +exit "$rc" diff --git a/tools/bench-preflight.sh b/tools/bench-preflight.sh index be3c23f18..f57910b2e 100755 --- a/tools/bench-preflight.sh +++ b/tools/bench-preflight.sh @@ -47,18 +47,20 @@ set -uo pipefail busy_pct=5 busy_window=10 -ram_floor_gib=20 +ram_floor_gib=${BENCH_RAM_FLOOR_GIB:-20} uptime_floor=600 die=/sys/class/drm/card0/device here=$(dirname "$0") strict=0 need_gtt=0 +watch=0 while [ $# -gt 0 ]; do case $1 in --strict) strict=1; shift ;; --need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;; + --watch) watch=1; shift; [ "${1:-}" = "--" ] && shift; break ;; --dispatch) break ;; *) echo "bench-preflight: unknown option $1" >&2; exit 2 ;; esac @@ -127,6 +129,19 @@ if [ -n "$warned" ] && [ "$strict" = 1 ]; then fatal "the box is usable but not clean, and --strict was asked for" fi +# --watch: the start gate says nothing about the rest of the run. engine#572's second arm started at 85 GiB +# available, another load arrived, MemAvailable fell to 1-3 GiB with swap full, and the arm timed out at 540 s +# -- so the number that came back (a timeout) and the number that did not (the arm) were both produced on a box +# nobody had described. Sample for the whole command and report start/min/end, so an arm is either clean or +# visibly contaminated where its number is printed. +if [ "$watch" = 1 ]; then + # engine#572: the start gate describes the wrong moment -- a mid-arm load can take MemAvailable from 85 GiB to + # 1-3 GiB and the arm times out, so the number is produced on a box nobody described. The sampler lives in its own + # tool so it can be tested on a busy box (this gate would refuse to reach it). + [ $# -gt 0 ] || { echo "bench-preflight: --watch needs a command after --" >&2; exit 2; } + exec "$here/bench-arm.sh" --floor "$ram_floor_gib" -- "$@" +fi + printf 'bench-preflight: OK\n' printf ' kernel %s\n' "$(uname -r)" printf ' uptime %ss gpu_busy %s%% mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib" diff --git a/tools/tests/test-bench-arm-watch.sh b/tools/tests/test-bench-arm-watch.sh new file mode 100755 index 000000000..bb92b3798 --- /dev/null +++ b/tools/tests/test-bench-arm-watch.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# `bench-preflight.sh --watch` exists because a start gate describes the wrong moment (engine#572): the second arm of +# the pp4096 pair started at 85 GiB MemAvailable, another load arrived mid-arm, MemAvailable fell to 1-3 GiB with swap +# full, and the arm timed out -- so both the arm and its replacement number were produced on a box nobody had described. +# +# Three arms, each testing something a wrong implementation would get wrong: +# 1. a quiet child: the arm reports start/min/end and exits with the CHILD's status, not 0; +# 2. a child that really allocates: MemAvailable falls below a floor set just under the quiet value, so the watch +# FAILs -- the mid-arm detection, which is the whole point; +# 3. the same child with the floor left alone: no FAIL, because a drop that stays above the floor is not a +# contamination, and a gate that fires there would be ignored. +# +# usage: test-bench-arm-watch.sh tools/bench-arm.sh +set -u +ba=${1:?usage: test-bench-arm-watch.sh path/to/bench-arm.sh} +[ -x "$ba" ] || { echo "not executable: $ba"; exit 1; } +fails=0 +ok() { echo "ok $*"; } +bad() { echo "FAIL $*"; fails=$((fails+1)); } + +avail=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo) + +# 1. quiet child, non-zero exit status must survive +out=$("$ba" -- sh -c 'exit 7' 2>&1); rc=$? +[ "$rc" = 7 ] && ok "a quiet arm exits with the child's status (7)" || bad "child status not propagated (got $rc)" +grep -q "mem_avail start " <<<"$out" && ok "and it reports mem start/min/end" || bad "no memory line: $out" + +# 2. a child that allocates: --drop-gib 1, so the drop is what trips it and the box's absolute level does not matter +# (a floor relative to a reading taken before the arm is fragile -- other lanes move MemAvailable between the two) +out=$("$ba" --drop-gib 1 --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$? +[ "$rc" = 1 ] && ok "a mid-arm drop below the floor FAILs the arm" || bad "mid-arm drop not caught (rc=$rc): $out" +grep -qi 'DURING the arm' <<<"$out" && ok "and says the drop happened during the arm" || bad "message does not say when: $out" + +# 3. the same allocation with the default floor: no FAIL (the box has room), child status preserved +out=$("$ba" --drop-gib 64 --floor 1 -- python3 -c 'x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096)' 2>&1); rc=$? +[ "$rc" = 0 ] && ok "a 3 GiB allocation under a 64 GiB drop limit is not a contamination" || bad "false positive (rc=$rc): $out" + +if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi +echo "all checks passed"