diff --git a/.github/workflows/hrx-build.yml b/.github/workflows/hrx-build.yml index 15d55c191..6478a59ae 100644 --- a/.github/workflows/hrx-build.yml +++ b/.github/workflows/hrx-build.yml @@ -39,6 +39,11 @@ # comments, blanks) configures with ONEBIT_HRX=ON and skips the compile: such a line creates no target and sets # no flag, but it is still evaluated at configure time, including inside if(ONEBIT_HRX) blocks ci.yml never # configures. tools/ci/hrx_build_scope.py decides; anything it does not recognise builds. +# Memory (engine#1504): the box is shared and its GTT is host RAM, and on 2026-11-08 both runners were OOM-killed, +# the second time 76 s after a restart that handed this job a compile. So the compile waits for MemAvailable to +# reach a floor (scripts/mem-wait.sh, up to 15 min, then the job fails and says why), and it runs under +# scripts/mem-guard.sh --term, which ends the whole thermal-run scope before the kernel's OOM killer picks +# someone else's process. # Fork PRs never reach the box: the `if:` requires the head repository to be this one. name: hrx-build @@ -111,13 +116,20 @@ jobs: if: steps.scope.outputs.build != 'true' run: echo "only test registration changed in CMakeLists.txt; the ONEBIT_HRX=ON configure above passed, the compile is skipped" | tee -a "$GITHUB_STEP_SUMMARY" + - name: Wait for memory (or refuse) + if: steps.scope.outputs.build == 'true' + timeout-minutes: 20 + # 24 GB free before a 4-core compile starts; it is stopped if the box drops under 12 GB (hw-gate's floor) + run: scripts/mem-wait.sh 24 15 + - name: Build llama_hrx and onebit if: steps.scope.outputs.build == 'true' env: CCACHE_BASEDIR: ${{ github.workspace }} run: | set -euo pipefail - ~/lb/thermal-run.sh cmake --build "$HRX_BUILD" --target onebit -j 4 + # the guard outside thermal-run: SIGTERM makes thermal-run end its whole scope (ninja and the compilers) + scripts/mem-guard.sh --term 12 ~/lb/thermal-run.sh cmake --build "$HRX_BUILD" --target onebit -j 4 # the build must have produced the HRX backend and the server onebit runs ls -l "$HRX_BUILD"/hrx/llama/bin/llama-server find "$HRX_BUILD"/hrx/llama -name 'libggml-hrx.so*' -print -quit | grep . diff --git a/docs/hw-gate.md b/docs/hw-gate.md index f7c06fcb7..9ad7bfeb2 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -63,6 +63,11 @@ hrx-system) through `cmake/hrx.cmake`, then links `onebit` with its HRX wiring. while every PR-Agent review queued behind it. It takes no GPU token and no NPU lock. Fork PRs never run it. It costs no GitHub Actions minutes. The token is read-only, and the build tree persists in `~/.cache/hrx-build/bench` on the box, so a later run rebuilds only what changed. +- **Memory.** The box is shared, and on Strix Halo GTT is host RAM. Before it compiles, the job waits for 24 GB of + `MemAvailable` (`scripts/mem-wait.sh`, polled once a minute for up to 15 minutes). If memory doesn't free up, the job + fails with the reading, and a re-run is the fix. The compile itself runs under `scripts/mem-guard.sh --term 12`, + which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. On 2026-11-08 both self-hosted + runners were OOM-killed, the second time 76 s after a restart handed this job a compile (engine#1504). - **What it is not.** It is a compile and link check of the public plumbing only. It does not build the private HIP add-on (the gate builds that) and does not dispatch anything. Green means the pinned pair builds into the engine. It does not replace the `hw-gate` label on a pin bump. diff --git a/scripts/mem-guard.sh b/scripts/mem-guard.sh index e6193f0ec..1a7e35e59 100755 --- a/scripts/mem-guard.sh +++ b/scripts/mem-guard.sh @@ -14,14 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# mem-guard.sh : run cmd and kill it if MemAvailable drops below +# mem-guard.sh [--term] : run cmd and kill it if MemAvailable drops below # the floor, before the kernel's OOM killer picks other processes (GPU allocations on Strix # Halo's unified memory escape cgroup limits, so MemoryMax alone does not protect the box). +# --term sends SIGTERM first and SIGKILL 10 s later. Use it when cmd is ~/lb/thermal-run.sh: a +# SIGKILL'd thermal-run cannot run its trap, so its scope (ninja and every compiler under it) keeps +# running and keeps the memory; on SIGTERM it kills the whole scope (engine#1504). +term=0; [ "$1" = --term ] && { term=1; shift; } floor=$1; shift "$@" & pid=$! while kill -0 $pid 2>/dev/null; do avail=$(awk '/MemAvailable/ {print int($2/1048576)}' /proc/meminfo) - if [ "$avail" -lt "$floor" ]; then echo "GUARD: MemAvailable ${avail} GB < ${floor} GB, killing $pid" >&2; kill -9 $pid; fi + if [ "$avail" -lt "$floor" ]; then + echo "GUARD: MemAvailable ${avail} GB < ${floor} GB, killing $pid" >&2 + if [ $term = 1 ]; then + kill -TERM $pid + for _ in $(seq 20); do kill -0 $pid 2>/dev/null || break; sleep 0.5; done + fi + kill -9 $pid 2>/dev/null + fi sleep 0.5 done wait $pid diff --git a/scripts/mem-wait.sh b/scripts/mem-wait.sh new file mode 100755 index 000000000..00ce6df3b --- /dev/null +++ b/scripts/mem-wait.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# mem-wait.sh : return 0 once MemAvailable is at least the floor; poll once a +# minute and give up with exit 75 after max_wait_min. A preflight for a heavy job on the shared box: starting a +# compile into an exhausted box is what turned one OOM into two on 2026-11-08 (engine#1504; both self-hosted +# runners were OOM-killed 76 s after a restart, with 0 swap free). Each poll prints what the box looks like, so a +# refusal says why. The floor is MemAvailable only: on this APU GTT is host RAM, and MemAvailable already counts it. +floor=${1:?usage: mem-wait.sh }; max=${2:?usage: mem-wait.sh } +meminfo() { awk -v k="$1" '$1 == k":" {print int($2/1048576)}' /proc/meminfo; } +for i in $(seq 0 "$max"); do + avail=$(meminfo MemAvailable) + line="MemAvailable ${avail} GB (floor ${floor} GB), swap free $(meminfo SwapFree) of $(meminfo SwapTotal) GB" + if [ "$avail" -ge "$floor" ]; then echo "mem-wait: ok: $line"; exit 0; fi + [ "$i" = "$max" ] && break + echo "mem-wait: waiting ($((i + 1)) of $max min): $line" + sleep 60 +done +echo "mem-wait: REFUSED after ${max} min: $line. The box is short of memory, so this job did not start its build;" \ + "re-run it once memory frees (engine#1504)." >&2 +exit 75