diff --git a/CMakeLists.txt b/CMakeLists.txt index fe0677d42..1bce95028 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,6 +300,7 @@ add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) # fleet state present -> the claim uses the fleet's queue and token; absent -> the old ~/GPU-* paths, so a box # that has not been set up is unchanged. No GPU: the claim holds the token for three seconds and runs `true`. add_test(NAME gpu_hold_queue COMMAND ${CMAKE_SOURCE_DIR}/tests/gpu_hold_queue_test.sh) +add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) # engine #713: the pinned HRX toolchain (config/hrx-toolchain.config, scripts/fetch-rocm-sdk.sh) -- the # pin file's refusals, the completeness check against a fabricated tree, and that cmake and # weekly.sh both use the pin instead of the box's /opt/rocm-therock. No network, no SDK, no GPU. diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 0550eedf4..c8d993d8e 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -272,6 +272,33 @@ together with the rest of #1504's acceptance. ## What a restart policy cannot do It cannot notice that a runner is *not* coming back (a version mismatch, a revoked token, a full disk under -`/var/lib/ghrunner`). That is [engine #1504](https://github.com/1bit-MONSTER/engine/issues/1504)'s item 3 and -still open: a cheap row that asserts both self-hosted runners are `online=busy=false` through the API the release -lanes already use, so a runner that is gone turns a check red instead of leaving it pending for ever. +`/var/lib/ghrunner`). That is [engine #1504](https://github.com/1bit-MONSTER/engine/issues/1504)'s item 3, and it is +`scripts/check-ci-runners.sh`: + +```sh +scripts/check-ci-runners.sh --list # what the API says, and the verdict +scripts/check-ci-runners.sh --require strixhalo-engine strixhalo-bench # the fleet, named +``` + +The sentence to keep in mind while reading any check list: **every check that needs this box reads as pending, +never red**. That is why the 2026-11-08 outage lasted 3h45m instead of being visible in a minute — a job whose +runner does not exist never runs, so GitHub never fails it, and a pending check looks exactly like a busy queue. +That is the person this row is for: whoever is staring at a green-looking list. + +It exits 0 when both named runners are `online`, 1 with a `::error::` line per offender, and **2 when the list +cannot be read at all** — an unchecked runner is exactly the state the row exists to end. It is also 2 when the list +is **truncated** (one page is 100 runners): past that, a runner that exists on page two would be reported missing, +which is a false outage, and a monitor that cannot tell has to say so rather than answer wrongly. `--require` is not decoration: a +runner that has been *deleted* satisfies "every registered runner is online", so the two names have to be asked for +explicitly. A repository with no self-hosted runners registered is not an outage, so a fresh box and a fork both +exit 0. + +`scripts/weekly.sh` runs it in the preflight, as a fatal row: a release whose post-release checks queue for ever is +worse than a release that waits an hour, and `WEEKLY_SKIP_RUNNER_CHECK=1` is the deliberate way to publish during an +outage. `tests/check_ci_runners_test.sh` (ctest `check_ci_runners`) covers the arms with a fake `gh`: both online, +one offline, an unrequired dead runner, a required runner that is not registered, nothing registered, an API failure, +and `--list`. + +It cannot be a ctest itself — it needs the network and a token, which the GitHub-hosted job does not have. A +*scheduled* red (a cron workflow that fails once an hour while a runner is down) is the complete answer and is not +here: a new workflow changes what the repository's token may do, which is an RFC under CONTRIBUTING.md. diff --git a/scripts/check-ci-runners.sh b/scripts/check-ci-runners.sh new file mode 100755 index 000000000..344989704 --- /dev/null +++ b/scripts/check-ci-runners.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# check-ci-runners.sh [--repo OWNER/REPO] [--require NAME ...] [--list] +# +# engine #1504 item 3: something that NOTICES when a self-hosted runner is gone. +# +# On 2026-11-08 both runners were OOM-killed and stayed dead for 3h45m, and every check that needs this box read +# as *pending* - never green, never red - which is indistinguishable from "queued behind a busy runner". A restart +# policy (scripts/install-ci-runner.sh) shortens that outage; nothing but a row like this makes it visible, because +# a job whose runner does not exist never runs and therefore never fails. +# +# It asks the same API the release lanes already use. Two deliberate rules: +# - a repository with NO self-hosted runners registered is not this check's business (a fresh box, a fork): +# it says so and exits 0. A runner that exists and is not `online` is an outage, and that fails; +# - names given with --require must BOTH exist and be online, because "both runners are online" is the claim, +# and a deleted runner silently satisfies an "every runner is online" reading. +# +# It cannot be a ctest: it needs the network and a token. scripts/weekly.sh's preflight runs it; docs/ci-runners.md +# says where else it belongs and why a scheduled red needs a workflow change (an RFC under CONTRIBUTING). +set -uo pipefail + +repo=1bit-MONSTER/engine +require=() +list=0 +while [ $# -gt 0 ]; do + case $1 in + --repo) repo=$2; shift 2 ;; + --require) shift; while [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; do require+=("$1"); shift; done ;; + --list) list=1; shift ;; + -h|--help) sed -n '16,33p' "$0"; exit 0 ;; + *) echo "check-ci-runners.sh: unknown option $1" >&2; exit 2 ;; + esac +done + +command -v gh >/dev/null || { echo "check-ci-runners.sh: no gh on PATH (the runner list comes from the GitHub API)" >&2; exit 2; } + +json=$(gh api "repos/$repo/actions/runners?per_page=100" 2>/dev/null) || { + echo "::error::check-ci-runners: cannot read the runner list of $repo (gh api failed): an unchecked runner is the state this row exists to end" >&2 + exit 2; } + +# namestatusbusy, one per registered runner -- or the word "truncated", which is not a verdict at all. +# The list is one page of 100: if the repository ever has more, a runner that EXISTS beyond page one is reported +# missing, which is a false outage and a page at 3 a.m. for a pager that cannot see page two. A monitor that +# cannot tell has to say so (exit 2), same rule as an unreadable list. +rows=$(printf '%s' "$json" | python3 -c ' +import json,sys +d=json.load(sys.stdin) +rs=d.get("runners",[]) +total=d.get("total_count",len(rs)) +if total>len(rs) or len(rs)>=100: + print("truncated\t%s\t%s" % (len(rs), total)) + raise SystemExit(0) +for r in rs: + print("%s\t%s\t%s" % (r.get("name","?"), r.get("status","?"), str(r.get("busy")).lower())) +') || { echo "check-ci-runners.sh: cannot parse the runner list" >&2; exit 2; } + +case ${rows:-} in + truncated*) + have=$(printf '%s' "$rows" | cut -f2); total=$(printf '%s' "$rows" | cut -f3) + echo "::error::check-ci-runners: the runner list is truncated ($have of $total registered, one page is 100), so a runner on the next page would read as missing: this row cannot answer" >&2 + exit 2 ;; +esac + +if [ -z "$rows" ]; then + echo "ok no self-hosted runners registered for $repo: nothing for this row to watch" + exit 0 +fi + +if [ "$list" = 1 ]; then + printf '%s\n' "$rows" | while IFS=$'\t' read -r name status busy; do + printf '%-22s %-8s busy=%s\n' "$name" "$status" "$busy" + done +fi + +bad=0 +# every registered runner has to be online: one that is offline is an outage, whatever it is called +while IFS=$'\t' read -r name status busy; do + printf '%s\n' ${require[@]+"${require[@]}"} | grep -qxF "$name" && continue # required: checked below, with its own line + [ "$status" = online ] && continue + echo "::error::runner $name is $status (busy=$busy): every check that needs this box now reads as pending, not red" + bad=$((bad + 1)) +done < <(printf '%s\n' "$rows") + +# ... and the names that must exist have to exist, then be online +for name in ${require[@]+"${require[@]}"}; do + line=$(printf '%s\n' "$rows" | awk -F'\t' -v n="$name" '$1 == n') + if [ -z "$line" ]; then + echo "::error::runner $name is not registered for $repo: the runner that takes these jobs is gone, so they queue for ever" + bad=$((bad + 1)) + continue + fi + status=$(printf '%s' "$line" | cut -f2) + busy=$(printf '%s' "$line" | cut -f3) + [ "$status" = online ] || { + echo "::error::runner $name is $status (busy=$busy): every check that needs this box now reads as pending, not red" + bad=$((bad + 1)) + } +done + +if [ "$bad" = 0 ]; then + echo "ok $(printf '%s\n' "$rows" | wc -l) self-hosted runner(s) online${require[*]:+, including: ${require[*]}}" + exit 0 +fi +exit 1 diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 45ea725a4..e3f6b076c 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -678,6 +678,39 @@ preflight() { { echo "preflight: cannot read the kernel image $vmlinuz (os/mini/mkimage.sh): set VMLINUZ to a readable copy"; bad=1; } ;; esac done + # engine#1504: a self-hosted runner that is dead makes every check that needs this box read as PENDING, never + # red (2026-11-08: both dead 3h45m behind 24 jobs), so the state that damages a release most is the one nothing + # reports. scripts/check-ci-runners.sh asks the API the release lanes already use; it exits 0 when no runners + # are registered, so a fresh box or a fork is unaffected. WEEKLY_SKIP_RUNNER_CHECK=1 releases during an outage. + if [ "${WEEKLY_SKIP_RUNNER_CHECK:-0}" != 1 ]; then + if runner_out=$("$(cd "$(dirname "$0")" && pwd)/check-ci-runners.sh" \ + --require strixhalo-engine strixhalo-bench 2>&1); then + say "preflight: $runner_out" + else + printf '%s\n' "$runner_out" | sed 's/^/preflight: /' + echo "preflight: a self-hosted runner is not online, so the checks after this release would queue for ever (WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" + bad=1 + fi + fi + # The same fact from the other side: scripts/ci-runner-health.sh reads systemd's view (#1546, merged) and says WHY + # a runner is down -- Result=oom-kill, a restart window -- which the API above cannot see, and it fires the moment + # a unit dies instead of after the heartbeat timeout. Neither was ever invoked from anywhere until this row, so + # running both here is what makes the two halves one check at release time. Its exit 2 means "no such units on this + # host", which is a fresh box, not a failure. + if [ -x "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" ]; then + if health_out=$("$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" 2>&1); then + printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" + else + health_rc=$? + if [ "$health_rc" = 2 ]; then + say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" + else + printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: a runner unit is not accepting jobs (scripts/ci-runner-health.sh): $health_rc" + bad=1 + fi + fi + fi return $bad } preflight || { say "preflight failed: nothing was run"; exit 1; } diff --git a/tests/check_ci_runners_test.sh b/tests/check_ci_runners_test.sh new file mode 100755 index 000000000..c74005e97 --- /dev/null +++ b/tests/check_ci_runners_test.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# check_ci_runners_test.sh: the row that notices a dead runner (engine #1504 item 3). +# +# The failure it guards against cannot be seen from this box: a job whose runner does not exist never runs, so +# GitHub never fails it - the check reads as *pending*, which looks exactly like a busy queue. On 2026-11-08 that +# state lasted 3h45m with 9 hrx-build and 15 pr-agent jobs behind it. +# +# Arms, each with a fake `gh` on PATH so no network and no token are involved: +# 1. both runners online and required -> 0, and it names them; +# 2. one runner offline -> 1, and the message names the dead one (not the healthy one); +# 3. a required runner not registered at all -> 1, and says it is gone rather than online; +# 4. no self-hosted runners registered -> 0: a fresh box or a fork is not an outage; +# 5. the API call fails -> 2: an unchecked runner is exactly the state this row exists to end. +set -uo pipefail +here=$(cd "$(dirname "$0")" && pwd) +script=$here/../scripts/check-ci-runners.sh +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" + +# a fake gh: `gh api ` prints $FAKE_RUNNERS, or fails when $FAKE_API_FAIL=1 +cat > "$tmp/bin/gh" <<'SH' +#!/usr/bin/env bash +[ "${FAKE_API_FAIL:-0}" = 1 ] && { echo "gh: HTTP 401" >&2; exit 1; } +printf '%s\n' "${FAKE_RUNNERS:-}" +SH +chmod +x "$tmp/bin/gh" + +run() { # run [args...] + local json=$1; shift + FAKE_RUNNERS=$json PATH=$tmp/bin:$PATH bash "$script" "$@" 2>&1 +} + +two_online='{"total_count":2,"runners":[{"name":"strixhalo-engine","status":"online","busy":false},{"name":"strixhalo-bench","status":"online","busy":true}]}' +one_dead='{"total_count":2,"runners":[{"name":"strixhalo-engine","status":"online","busy":false},{"name":"strixhalo-bench","status":"offline","busy":false}]}' +only_bench='{"total_count":1,"runners":[{"name":"strixhalo-bench","status":"online","busy":false}]}' +none='{"total_count":0,"runners":[]}' +req=(--require strixhalo-engine strixhalo-bench) + +fail=0 +check() { # check [must-contain] + local rc=$2 out=$3 want=${4:-} + if [ "$rc" != "$1" ]; then + echo "FAIL $5: expected rc=$1, got rc=$rc" + printf '%s\n' "$out" | sed 's/^/ | /' + fail=$((fail + 1)) + return + fi + if [ -n "$want" ] && ! printf '%s' "$out" | grep -qF "$want"; then + echo "FAIL $5: rc was right but the output does not say \"$want\"" + printf '%s\n' "$out" | sed 's/^/ | /' + fail=$((fail + 1)) + return + fi + echo "ok $5" +} + +# 1. both online +out=$(run "$two_online" "${req[@]}"); check 0 $? "$out" "2 self-hosted runner(s) online" "both online -> 0, and it counts them" + +# 2. one offline: the message must name the dead one, exactly once (a required-but-dead runner used to be reported +# by both the loop and the --require pass, so the count is what pins the skip against the array-expansion bug) +out=$(run "$one_dead" "${req[@]}"); check 1 $? "$out" "runner strixhalo-bench is offline" "one offline -> 1, naming the dead runner" +n=$(printf '%s\n' "$out" | grep -c 'strixhalo-bench is offline') +[ "$n" = 1 ] || { echo "FAIL the dead runner was reported $n times (expected 1)"; fail=$((fail + 1)); } +printf '%s' "$out" | grep -q 'runner strixhalo-engine is' && { echo "FAIL the healthy runner was blamed too"; fail=$((fail + 1)); } + +# 2b. an unrequired runner that is offline still fails: a dead runner is an outage whatever it is called +out=$(run "$one_dead"); check 1 $? "$out" "strixhalo-bench is offline" "an unrequired dead runner still fails" + +# 3. a required runner that is not registered +out=$(run "$only_bench" "${req[@]}"); check 1 $? "$out" "runner strixhalo-engine is not registered" "a required runner missing -> 1, 'not registered'" +printf '%s' "$out" | grep -q 'strixhalo-engine is online' && { echo "FAIL a missing runner was reported as online"; fail=$((fail + 1)); } + +# 4. no runners at all is not an outage +out=$(run "$none" "${req[@]}"); check 0 $? "$out" "no self-hosted runners registered" "no runners registered -> 0" + +# 5. the API fails: 2, never a silent ok +out=$(FAKE_API_FAIL=1 FAKE_RUNNERS=$none PATH=$tmp/bin:$PATH bash "$script" "${req[@]}" 2>&1) +check 2 $? "$out" "cannot read the runner list" "API failure -> 2, explicitly" + +# 5b. more runners than one page: the list cannot be trusted, so 2 -- NOT a "missing" verdict for a required name +long=$(python3 -c 'import json; print(json.dumps({"total_count":150,"runners":[{"name":"r%d"%i,"status":"online","busy":False} for i in range(100)]}))') +out=$(run "$long" "${req[@]}"); check 2 $? "$out" "runner list is truncated" "a truncated list -> 2, not 'strixhalo-engine is not registered'" +printf '%s' "$out" | grep -q 'is not registered' && { echo "FAIL a truncated list blamed a required name"; fail=$((fail + 1)); } +full=$(python3 -c 'import json; print(json.dumps({"total_count":100,"runners":[{"name":"r%d"%i,"status":"online","busy":False} for i in range(100)]}))') +out=$(run "$full"); check 2 $? "$out" "truncated" "exactly one page -> 2 (may be truncated), never a vacuous ok" + +# 6. --list prints the table without changing the verdict +out=$(run "$two_online" --list); check 0 $? "$out" "strixhalo-engine" "--list prints runners and still exits 0" + +if [ "$fail" != 0 ]; then + echo "check_ci_runners_test: $fail failure(s)" + exit 1 +fi +echo "check_ci_runners_test: all arms pass"