diff --git a/CMakeLists.txt b/CMakeLists.txt index 86cb25cf9..5afe18636 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,11 @@ add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --sel # GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered # here (the test fakes the box, so it needs no device and no ticket). add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-preflight.sh) +# engine#572: an arm's memory trace. --drop-gib exists because a floor relative to a reading taken before the arm is +# fragile -- other lanes move MemAvailable between the two -- while "fell by more than N GiB during this arm" is the +# contamination itself. Same reason the test uses --drop-gib rather than a floor. +add_test(NAME bench_arm_watch COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-arm-watch.sh + ${CMAKE_SOURCE_DIR}/tools/bench-arm.sh) # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. diff --git a/tools/bench-arm.sh b/tools/bench-arm.sh new file mode 100755 index 000000000..8c0d203e2 --- /dev/null +++ b/tools/bench-arm.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run ONE benchmark arm under memory observation (engine#572). +# +# `tools/bench-preflight.sh` refuses to start below a MemAvailable floor, which is a statement about the moment it +# runs. On 2026-11-10 the second arm of a pp4096 pair started at 85 GiB available, another load arrived, MemAvailable +# fell to 1-3 GiB with swap full, and the arm timed out at 540 s: the number that came back (a timeout) and the arm +# that did not were both produced on a box nobody had described. This samples for the whole command and prints +# start/min/end, so an arm is either clean or visibly contaminated where its own number is printed. +# +# tools/bench-arm.sh [--floor GiB] [--drop-gib N] -- CMD... +# run CMD, report memory, exit with CMD's status (or 1 if the arm was contaminated) +# +# Two independent contamination tests, because they catch different things: +# --floor GiB an ABSOLUTE floor (default 20, the fleet's box floor). What the preflight gates on at start. +# --drop-gib N a DROP during the arm: MemAvailable fell by more than N GiB between start and min. This is the one +# that describes contamination rather than the box's level, and it does not move when another lane's +# memory does -- the absolute reading in a pair can drift between two arms of the same run. +# +# BENCH_RAM_FLOOR_GIB overrides the floor; 20 GiB is the fleet's box floor, the same one the preflight gates on. +set -uo pipefail + +floor=${BENCH_RAM_FLOOR_GIB:-20} +drop=0 +while [ $# -gt 0 ]; do + case $1 in + --floor) floor=${2:?--floor needs a GiB value}; shift 2 ;; + --drop-gib) drop=${2:?--drop-gib needs a GiB value}; shift 2 ;; + --) shift; break ;; + *) break ;; + esac +done +[ $# -gt 0 ] || { echo "bench-arm: usage: bench-arm.sh [--floor GiB] -- CMD..." >&2; exit 2; } + +mem_avail() { awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo; } +swap_free() { awk '/SwapFree/ {printf "%d", $2/1048576}' /proc/meminfo; } + +start=$(mem_avail); min=$start; sw_start=$(swap_free); sw_min=$sw_start +"$@" & +child=$! +while kill -0 "$child" 2>/dev/null; do + m=$(mem_avail); s=$(swap_free) + [ "$m" -lt "$min" ] && min=$m + [ "$s" -lt "$sw_min" ] && sw_min=$s + sleep 0.25 +done +wait "$child"; rc=$? +end=$(mem_avail) +# The end reading counts too: a short arm can fall and recover between samples, and the number produced at the end is +# the one a reader would quote. +[ "$end" -lt "$min" ] && min=$end +w_end=$(swap_free) +[ "$w_end" -lt "$sw_min" ] && sw_min=$w_end + +printf 'bench-arm: mem_avail start %s min %s end %s GiB; swap_free start %s min %s GiB\n' \ + "$start" "$min" "$end" "$sw_start" "$sw_min" +if [ "$min" -lt "$floor" ]; then + printf 'bench-arm: FAIL -- MemAvailable fell to %s GiB (floor %s) DURING the arm, so this number was not taken on a quiet box: say so, or discard it\n' \ + "$min" "$floor" >&2 + exit 1 +fi +if [ "$drop" -gt 0 ] && [ $((start - min)) -gt "$drop" ]; then + printf 'bench-arm: FAIL -- MemAvailable fell %s GiB during the arm (limit %s), so another load arrived mid-run: this number was not taken on a quiet box\n' \ + "$((start - min))" "$drop" >&2 + exit 1 +fi +exit "$rc" diff --git a/tools/bench-preflight.sh b/tools/bench-preflight.sh index be3c23f18..f57910b2e 100755 --- a/tools/bench-preflight.sh +++ b/tools/bench-preflight.sh @@ -47,18 +47,20 @@ set -uo pipefail busy_pct=5 busy_window=10 -ram_floor_gib=20 +ram_floor_gib=${BENCH_RAM_FLOOR_GIB:-20} uptime_floor=600 die=/sys/class/drm/card0/device here=$(dirname "$0") strict=0 need_gtt=0 +watch=0 while [ $# -gt 0 ]; do case $1 in --strict) strict=1; shift ;; --need-gtt) need_gtt=${2:?--need-gtt needs a GiB value}; shift 2 ;; + --watch) watch=1; shift; [ "${1:-}" = "--" ] && shift; break ;; --dispatch) break ;; *) echo "bench-preflight: unknown option $1" >&2; exit 2 ;; esac @@ -127,6 +129,19 @@ if [ -n "$warned" ] && [ "$strict" = 1 ]; then fatal "the box is usable but not clean, and --strict was asked for" fi +# --watch: the start gate says nothing about the rest of the run. engine#572's second arm started at 85 GiB +# available, another load arrived, MemAvailable fell to 1-3 GiB with swap full, and the arm timed out at 540 s +# -- so the number that came back (a timeout) and the number that did not (the arm) were both produced on a box +# nobody had described. Sample for the whole command and report start/min/end, so an arm is either clean or +# visibly contaminated where its number is printed. +if [ "$watch" = 1 ]; then + # engine#572: the start gate describes the wrong moment -- a mid-arm load can take MemAvailable from 85 GiB to + # 1-3 GiB and the arm times out, so the number is produced on a box nobody described. The sampler lives in its own + # tool so it can be tested on a busy box (this gate would refuse to reach it). + [ $# -gt 0 ] || { echo "bench-preflight: --watch needs a command after --" >&2; exit 2; } + exec "$here/bench-arm.sh" --floor "$ram_floor_gib" -- "$@" +fi + printf 'bench-preflight: OK\n' printf ' kernel %s\n' "$(uname -r)" printf ' uptime %ss gpu_busy %s%% mem_avail %s GiB\n' "$up_s" "$busy" "$avail_gib" diff --git a/tools/tests/test-bench-arm-watch.sh b/tools/tests/test-bench-arm-watch.sh new file mode 100755 index 000000000..bb92b3798 --- /dev/null +++ b/tools/tests/test-bench-arm-watch.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# `bench-preflight.sh --watch` exists because a start gate describes the wrong moment (engine#572): the second arm of +# the pp4096 pair started at 85 GiB MemAvailable, another load arrived mid-arm, MemAvailable fell to 1-3 GiB with swap +# full, and the arm timed out -- so both the arm and its replacement number were produced on a box nobody had described. +# +# Three arms, each testing something a wrong implementation would get wrong: +# 1. a quiet child: the arm reports start/min/end and exits with the CHILD's status, not 0; +# 2. a child that really allocates: MemAvailable falls below a floor set just under the quiet value, so the watch +# FAILs -- the mid-arm detection, which is the whole point; +# 3. the same child with the floor left alone: no FAIL, because a drop that stays above the floor is not a +# contamination, and a gate that fires there would be ignored. +# +# usage: test-bench-arm-watch.sh tools/bench-arm.sh +set -u +ba=${1:?usage: test-bench-arm-watch.sh path/to/bench-arm.sh} +[ -x "$ba" ] || { echo "not executable: $ba"; exit 1; } +fails=0 +ok() { echo "ok $*"; } +bad() { echo "FAIL $*"; fails=$((fails+1)); } + +avail=$(awk '/MemAvailable/ {printf "%d", $2/1048576}' /proc/meminfo) + +# 1. quiet child, non-zero exit status must survive +out=$("$ba" -- sh -c 'exit 7' 2>&1); rc=$? +[ "$rc" = 7 ] && ok "a quiet arm exits with the child's status (7)" || bad "child status not propagated (got $rc)" +grep -q "mem_avail start " <<<"$out" && ok "and it reports mem start/min/end" || bad "no memory line: $out" + +# 2. a child that allocates: --drop-gib 1, so the drop is what trips it and the box's absolute level does not matter +# (a floor relative to a reading taken before the arm is fragile -- other lanes move MemAvailable between the two) +out=$("$ba" --drop-gib 1 --floor 1 -- python3 -c 'import time; x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096); time.sleep(2)' 2>&1); rc=$? +[ "$rc" = 1 ] && ok "a mid-arm drop below the floor FAILs the arm" || bad "mid-arm drop not caught (rc=$rc): $out" +grep -qi 'DURING the arm' <<<"$out" && ok "and says the drop happened during the arm" || bad "message does not say when: $out" + +# 3. the same allocation with the default floor: no FAIL (the box has room), child status preserved +out=$("$ba" --drop-gib 64 --floor 1 -- python3 -c 'x = bytearray(3 * 1024**3); x[::4096] = b"\x01" * (len(x) // 4096)' 2>&1); rc=$? +[ "$rc" = 0 ] && ok "a 3 GiB allocation under a 64 GiB drop limit is not a contamination" || bad "false positive (rc=$rc): $out" + +if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi +echo "all checks passed"