From ae5f4840afd1fe9a1916f81b5303f50ac4d21db5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 10:41:32 -0400 Subject: [PATCH 1/3] ci: a row that notices when a self-hosted runner is gone (engine#1504 item 3) A job whose runner does not exist never runs, so GitHub never fails it: every check that needs this box reads as *pending*, which is indistinguishable from a busy queue. 2026-11-08 was that state for 3h45m with 9 hrx-build and 15 pr-agent jobs behind it, and no restart policy can report it -- a restart policy only shortens the outage. scripts/check-ci-runners.sh asks the API the release lanes already use: 0 when the named runners are online, 1 with a `::error::` line per offender, 2 when the list cannot be read at all. `--require` is not decoration: a runner that was *deleted* satisfies "every registered runner is online", so the fleet's two names are asked for explicitly. No registered runners at all is not an outage, so a fresh box and a fork exit 0. scripts/weekly.sh runs it in the preflight as a fatal row -- a release whose post-release checks queue for ever is worse than a release that waits -- with WEEKLY_SKIP_RUNNER_CHECK=1 as the deliberate way to publish during one. tests/check_ci_runners_test.sh (ctest check_ci_runners) covers seven arms against a fake `gh`, no network, and it pins one reported bug: a required-but-dead runner was reported twice, because `"$require"` expands only element 0 (measured on the pre-fix script: 2 lines, after: 1). Live here: `ok 2 self-hosted runner(s) online, including: strixhalo-engine strixhalo-bench`. A scheduled red still wants a cron workflow, which is an RFC under CONTRIBUTING (workflow permissions), so it is documented, not added. --- CMakeLists.txt | 1 + docs/ci-runners.md | 26 ++++++++- scripts/check-ci-runners.sh | 103 ++++++++++++++++++++++++++++++++ scripts/weekly.sh | 14 +++++ tests/check_ci_runners_test.sh | 104 +++++++++++++++++++++++++++++++++ 5 files changed, 245 insertions(+), 3 deletions(-) create mode 100755 scripts/check-ci-runners.sh create mode 100755 tests/check_ci_runners_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 52b37d053..65beecf97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -296,6 +296,7 @@ add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) # fleet state present -> the claim uses the fleet's queue and token; absent -> the old ~/GPU-* paths, so a box # that has not been set up is unchanged. No GPU: the claim holds the token for three seconds and runs `true`. add_test(NAME gpu_hold_queue COMMAND ${CMAKE_SOURCE_DIR}/tests/gpu_hold_queue_test.sh) +add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) # engine #713: the pinned HRX toolchain (config/hrx-toolchain.config, scripts/fetch-rocm-sdk.sh) -- the # pin file's refusals, the completeness check against a fabricated tree, and that cmake and # weekly.sh both use the pin instead of the box's /opt/rocm-therock. No network, no SDK, no GPU. diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 0550eedf4..b4f526b2f 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -272,6 +272,26 @@ together with the rest of #1504's acceptance. ## What a restart policy cannot do It cannot notice that a runner is *not* coming back (a version mismatch, a revoked token, a full disk under -`/var/lib/ghrunner`). That is [engine #1504](https://github.com/1bit-MONSTER/engine/issues/1504)'s item 3 and -still open: a cheap row that asserts both self-hosted runners are `online=busy=false` through the API the release -lanes already use, so a runner that is gone turns a check red instead of leaving it pending for ever. +`/var/lib/ghrunner`). That is [engine #1504](https://github.com/1bit-MONSTER/engine/issues/1504)'s item 3, and it is +`scripts/check-ci-runners.sh`: + +```sh +scripts/check-ci-runners.sh --list # what the API says, and the verdict +scripts/check-ci-runners.sh --require strixhalo-engine strixhalo-bench # the fleet, named +``` + +It exits 0 when both named runners are `online`, 1 with a `::error::` line per offender, and 2 when the list cannot +be read at all — an unchecked runner is exactly the state the row exists to end. `--require` is not decoration: a +runner that has been *deleted* satisfies "every registered runner is online", so the two names have to be asked for +explicitly. A repository with no self-hosted runners registered is not an outage, so a fresh box and a fork both +exit 0. + +`scripts/weekly.sh` runs it in the preflight, as a fatal row: a release whose post-release checks queue for ever is +worse than a release that waits an hour, and `WEEKLY_SKIP_RUNNER_CHECK=1` is the deliberate way to publish during an +outage. `tests/check_ci_runners_test.sh` (ctest `check_ci_runners`) covers the arms with a fake `gh`: both online, +one offline, an unrequired dead runner, a required runner that is not registered, nothing registered, an API failure, +and `--list`. + +It cannot be a ctest itself — it needs the network and a token, which the GitHub-hosted job does not have. A +*scheduled* red (a cron workflow that fails once an hour while a runner is down) is the complete answer and is not +here: a new workflow changes what the repository's token may do, which is an RFC under CONTRIBUTING.md. diff --git a/scripts/check-ci-runners.sh b/scripts/check-ci-runners.sh new file mode 100755 index 000000000..cb3dca467 --- /dev/null +++ b/scripts/check-ci-runners.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# check-ci-runners.sh [--repo OWNER/REPO] [--require NAME ...] [--list] +# +# engine #1504 item 3: something that NOTICES when a self-hosted runner is gone. +# +# On 2026-11-08 both runners were OOM-killed and stayed dead for 3h45m, and every check that needs this box read +# as *pending* - never green, never red - which is indistinguishable from "queued behind a busy runner". A restart +# policy (scripts/install-ci-runner.sh) shortens that outage; nothing but a row like this makes it visible, because +# a job whose runner does not exist never runs and therefore never fails. +# +# It asks the same API the release lanes already use. Two deliberate rules: +# - a repository with NO self-hosted runners registered is not this check's business (a fresh box, a fork): +# it says so and exits 0. A runner that exists and is not `online` is an outage, and that fails; +# - names given with --require must BOTH exist and be online, because "both runners are online" is the claim, +# and a deleted runner silently satisfies an "every runner is online" reading. +# +# It cannot be a ctest: it needs the network and a token. scripts/weekly.sh's preflight runs it; docs/ci-runners.md +# says where else it belongs and why a scheduled red needs a workflow change (an RFC under CONTRIBUTING). +set -uo pipefail + +repo=1bit-MONSTER/engine +require=() +list=0 +while [ $# -gt 0 ]; do + case $1 in + --repo) repo=$2; shift 2 ;; + --require) shift; while [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; do require+=("$1"); shift; done ;; + --list) list=1; shift ;; + -h|--help) sed -n '16,33p' "$0"; exit 0 ;; + *) echo "check-ci-runners.sh: unknown option $1" >&2; exit 2 ;; + esac +done + +command -v gh >/dev/null || { echo "check-ci-runners.sh: no gh on PATH (the runner list comes from the GitHub API)" >&2; exit 2; } + +json=$(gh api "repos/$repo/actions/runners?per_page=100" 2>/dev/null) || { + echo "::error::check-ci-runners: cannot read the runner list of $repo (gh api failed): an unchecked runner is the state this row exists to end" >&2 + exit 2; } + +# namestatusbusy, one per registered runner +rows=$(printf '%s' "$json" | python3 -c ' +import json,sys +d=json.load(sys.stdin) +for r in d.get("runners",[]): + print("%s\t%s\t%s" % (r.get("name","?"), r.get("status","?"), str(r.get("busy")).lower())) +') || { echo "check-ci-runners.sh: cannot parse the runner list" >&2; exit 2; } + +if [ -z "$rows" ]; then + echo "ok no self-hosted runners registered for $repo: nothing for this row to watch" + exit 0 +fi + +if [ "$list" = 1 ]; then + printf '%s\n' "$rows" | while IFS=$'\t' read -r name status busy; do + printf '%-22s %-8s busy=%s\n' "$name" "$status" "$busy" + done +fi + +bad=0 +# every registered runner has to be online: one that is offline is an outage, whatever it is called +while IFS=$'\t' read -r name status busy; do + printf '%s\n' ${require[@]+"${require[@]}"} | grep -qxF "$name" && continue # required: checked below, with its own line + [ "$status" = online ] && continue + echo "::error::runner $name is $status (busy=$busy): every check that needs this box now reads as pending, not red" + bad=$((bad + 1)) +done < <(printf '%s\n' "$rows") + +# ... and the names that must exist have to exist, then be online +for name in ${require[@]+"${require[@]}"}; do + line=$(printf '%s\n' "$rows" | awk -F'\t' -v n="$name" '$1 == n') + if [ -z "$line" ]; then + echo "::error::runner $name is not registered for $repo: the runner that takes these jobs is gone, so they queue for ever" + bad=$((bad + 1)) + continue + fi + status=$(printf '%s' "$line" | cut -f2) + busy=$(printf '%s' "$line" | cut -f3) + [ "$status" = online ] || { + echo "::error::runner $name is $status (busy=$busy): every check that needs this box now reads as pending, not red" + bad=$((bad + 1)) + } +done + +if [ "$bad" = 0 ]; then + echo "ok $(printf '%s\n' "$rows" | wc -l) self-hosted runner(s) online${require[*]:+, including: ${require[*]}}" + exit 0 +fi +exit 1 diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 45ea725a4..d0257c693 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -678,6 +678,20 @@ preflight() { { echo "preflight: cannot read the kernel image $vmlinuz (os/mini/mkimage.sh): set VMLINUZ to a readable copy"; bad=1; } ;; esac done + # engine#1504: a self-hosted runner that is dead makes every check that needs this box read as PENDING, never + # red (2026-11-08: both dead 3h45m behind 24 jobs), so the state that damages a release most is the one nothing + # reports. scripts/check-ci-runners.sh asks the API the release lanes already use; it exits 0 when no runners + # are registered, so a fresh box or a fork is unaffected. WEEKLY_SKIP_RUNNER_CHECK=1 releases during an outage. + if [ "${WEEKLY_SKIP_RUNNER_CHECK:-0}" != 1 ]; then + if runner_out=$("$(cd "$(dirname "$0")" && pwd)/check-ci-runners.sh" \ + --require strixhalo-engine strixhalo-bench 2>&1); then + say "preflight: $runner_out" + else + printf '%s\n' "$runner_out" | sed 's/^/preflight: /' + echo "preflight: a self-hosted runner is not online, so the checks after this release would queue for ever (WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" + bad=1 + fi + fi return $bad } preflight || { say "preflight failed: nothing was run"; exit 1; } diff --git a/tests/check_ci_runners_test.sh b/tests/check_ci_runners_test.sh new file mode 100755 index 000000000..d48e3d788 --- /dev/null +++ b/tests/check_ci_runners_test.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# check_ci_runners_test.sh: the row that notices a dead runner (engine #1504 item 3). +# +# The failure it guards against cannot be seen from this box: a job whose runner does not exist never runs, so +# GitHub never fails it - the check reads as *pending*, which looks exactly like a busy queue. On 2026-11-08 that +# state lasted 3h45m with 9 hrx-build and 15 pr-agent jobs behind it. +# +# Arms, each with a fake `gh` on PATH so no network and no token are involved: +# 1. both runners online and required -> 0, and it names them; +# 2. one runner offline -> 1, and the message names the dead one (not the healthy one); +# 3. a required runner not registered at all -> 1, and says it is gone rather than online; +# 4. no self-hosted runners registered -> 0: a fresh box or a fork is not an outage; +# 5. the API call fails -> 2: an unchecked runner is exactly the state this row exists to end. +set -uo pipefail +here=$(cd "$(dirname "$0")" && pwd) +script=$here/../scripts/check-ci-runners.sh +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" + +# a fake gh: `gh api ` prints $FAKE_RUNNERS, or fails when $FAKE_API_FAIL=1 +cat > "$tmp/bin/gh" <<'SH' +#!/usr/bin/env bash +[ "${FAKE_API_FAIL:-0}" = 1 ] && { echo "gh: HTTP 401" >&2; exit 1; } +printf '%s\n' "${FAKE_RUNNERS:-}" +SH +chmod +x "$tmp/bin/gh" + +run() { # run [args...] + local json=$1; shift + FAKE_RUNNERS=$json PATH=$tmp/bin:$PATH bash "$script" "$@" 2>&1 +} + +two_online='{"total_count":2,"runners":[{"name":"strixhalo-engine","status":"online","busy":false},{"name":"strixhalo-bench","status":"online","busy":true}]}' +one_dead='{"total_count":2,"runners":[{"name":"strixhalo-engine","status":"online","busy":false},{"name":"strixhalo-bench","status":"offline","busy":false}]}' +only_bench='{"total_count":1,"runners":[{"name":"strixhalo-bench","status":"online","busy":false}]}' +none='{"total_count":0,"runners":[]}' +req=(--require strixhalo-engine strixhalo-bench) + +fail=0 +check() { # check [must-contain] + local rc=$2 out=$3 want=${4:-} + if [ "$rc" != "$1" ]; then + echo "FAIL $5: expected rc=$1, got rc=$rc" + printf '%s\n' "$out" | sed 's/^/ | /' + fail=$((fail + 1)) + return + fi + if [ -n "$want" ] && ! printf '%s' "$out" | grep -qF "$want"; then + echo "FAIL $5: rc was right but the output does not say \"$want\"" + printf '%s\n' "$out" | sed 's/^/ | /' + fail=$((fail + 1)) + return + fi + echo "ok $5" +} + +# 1. both online +out=$(run "$two_online" "${req[@]}"); check 0 $? "$out" "2 self-hosted runner(s) online" "both online -> 0, and it counts them" + +# 2. one offline: the message must name the dead one, exactly once (a required-but-dead runner used to be reported +# by both the loop and the --require pass, so the count is what pins the skip against the array-expansion bug) +out=$(run "$one_dead" "${req[@]}"); check 1 $? "$out" "runner strixhalo-bench is offline" "one offline -> 1, naming the dead runner" +n=$(printf '%s\n' "$out" | grep -c 'strixhalo-bench is offline') +[ "$n" = 1 ] || { echo "FAIL the dead runner was reported $n times (expected 1)"; fail=$((fail + 1)); } +printf '%s' "$out" | grep -q 'runner strixhalo-engine is' && { echo "FAIL the healthy runner was blamed too"; fail=$((fail + 1)); } + +# 2b. an unrequired runner that is offline still fails: a dead runner is an outage whatever it is called +out=$(run "$one_dead"); check 1 $? "$out" "strixhalo-bench is offline" "an unrequired dead runner still fails" + +# 3. a required runner that is not registered +out=$(run "$only_bench" "${req[@]}"); check 1 $? "$out" "runner strixhalo-engine is not registered" "a required runner missing -> 1, 'not registered'" +printf '%s' "$out" | grep -q 'strixhalo-engine is online' && { echo "FAIL a missing runner was reported as online"; fail=$((fail + 1)); } + +# 4. no runners at all is not an outage +out=$(run "$none" "${req[@]}"); check 0 $? "$out" "no self-hosted runners registered" "no runners registered -> 0" + +# 5. the API fails: 2, never a silent ok +out=$(FAKE_API_FAIL=1 FAKE_RUNNERS=$none PATH=$tmp/bin:$PATH bash "$script" "${req[@]}" 2>&1) +check 2 $? "$out" "cannot read the runner list" "API failure -> 2, explicitly" + +# 6. --list prints the table without changing the verdict +out=$(run "$two_online" --list); check 0 $? "$out" "strixhalo-engine" "--list prints runners and still exits 0" + +if [ "$fail" != 0 ]; then + echo "check_ci_runners_test: $fail failure(s)" + exit 1 +fi +echo "check_ci_runners_test: all arms pass" From f16f629156831d1c21c60fa30ae41a27fa2deb80 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 11:02:10 -0400 Subject: [PATCH 2/3] ci: a truncated runner list says so instead of reporting a required runner missing (review finding) The review is right that the list is one page of 100 and that this file should close the gap while it is small: past 100 runners a --require'd runner that exists on page two is reported *missing*, which is a false outage -- exit 1 and someone paged at 3 a.m. for a pager that cannot see the next page. Two signals, both honest: the API's own total_count above what came back, and exactly one full page (which may hide more). Either one is exit 2, the same "I cannot tell" rule as an unreadable list, and it must NOT be confused with 'strixhalo-engine is not registered' -- the test asserts both. Also carries the sentence the review asked the docs row to keep, since the person who needs it is the one staring at a check list that reads pending rather than red: that is why 2026-11-08 lasted 3h45m. --- docs/ci-runners.md | 11 +++++++++-- scripts/check-ci-runners.sh | 19 +++++++++++++++++-- tests/check_ci_runners_test.sh | 7 +++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/ci-runners.md b/docs/ci-runners.md index b4f526b2f..c8d993d8e 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -280,8 +280,15 @@ scripts/check-ci-runners.sh --list # what t scripts/check-ci-runners.sh --require strixhalo-engine strixhalo-bench # the fleet, named ``` -It exits 0 when both named runners are `online`, 1 with a `::error::` line per offender, and 2 when the list cannot -be read at all — an unchecked runner is exactly the state the row exists to end. `--require` is not decoration: a +The sentence to keep in mind while reading any check list: **every check that needs this box reads as pending, +never red**. That is why the 2026-11-08 outage lasted 3h45m instead of being visible in a minute — a job whose +runner does not exist never runs, so GitHub never fails it, and a pending check looks exactly like a busy queue. +That is the person this row is for: whoever is staring at a green-looking list. + +It exits 0 when both named runners are `online`, 1 with a `::error::` line per offender, and **2 when the list +cannot be read at all** — an unchecked runner is exactly the state the row exists to end. It is also 2 when the list +is **truncated** (one page is 100 runners): past that, a runner that exists on page two would be reported missing, +which is a false outage, and a monitor that cannot tell has to say so rather than answer wrongly. `--require` is not decoration: a runner that has been *deleted* satisfies "every registered runner is online", so the two names have to be asked for explicitly. A repository with no self-hosted runners registered is not an outage, so a fresh box and a fork both exit 0. diff --git a/scripts/check-ci-runners.sh b/scripts/check-ci-runners.sh index cb3dca467..344989704 100755 --- a/scripts/check-ci-runners.sh +++ b/scripts/check-ci-runners.sh @@ -52,14 +52,29 @@ json=$(gh api "repos/$repo/actions/runners?per_page=100" 2>/dev/null) || { echo "::error::check-ci-runners: cannot read the runner list of $repo (gh api failed): an unchecked runner is the state this row exists to end" >&2 exit 2; } -# namestatusbusy, one per registered runner +# namestatusbusy, one per registered runner -- or the word "truncated", which is not a verdict at all. +# The list is one page of 100: if the repository ever has more, a runner that EXISTS beyond page one is reported +# missing, which is a false outage and a page at 3 a.m. for a pager that cannot see page two. A monitor that +# cannot tell has to say so (exit 2), same rule as an unreadable list. rows=$(printf '%s' "$json" | python3 -c ' import json,sys d=json.load(sys.stdin) -for r in d.get("runners",[]): +rs=d.get("runners",[]) +total=d.get("total_count",len(rs)) +if total>len(rs) or len(rs)>=100: + print("truncated\t%s\t%s" % (len(rs), total)) + raise SystemExit(0) +for r in rs: print("%s\t%s\t%s" % (r.get("name","?"), r.get("status","?"), str(r.get("busy")).lower())) ') || { echo "check-ci-runners.sh: cannot parse the runner list" >&2; exit 2; } +case ${rows:-} in + truncated*) + have=$(printf '%s' "$rows" | cut -f2); total=$(printf '%s' "$rows" | cut -f3) + echo "::error::check-ci-runners: the runner list is truncated ($have of $total registered, one page is 100), so a runner on the next page would read as missing: this row cannot answer" >&2 + exit 2 ;; +esac + if [ -z "$rows" ]; then echo "ok no self-hosted runners registered for $repo: nothing for this row to watch" exit 0 diff --git a/tests/check_ci_runners_test.sh b/tests/check_ci_runners_test.sh index d48e3d788..c74005e97 100755 --- a/tests/check_ci_runners_test.sh +++ b/tests/check_ci_runners_test.sh @@ -94,6 +94,13 @@ out=$(run "$none" "${req[@]}"); check 0 $? "$out" "no self-hosted runners regist out=$(FAKE_API_FAIL=1 FAKE_RUNNERS=$none PATH=$tmp/bin:$PATH bash "$script" "${req[@]}" 2>&1) check 2 $? "$out" "cannot read the runner list" "API failure -> 2, explicitly" +# 5b. more runners than one page: the list cannot be trusted, so 2 -- NOT a "missing" verdict for a required name +long=$(python3 -c 'import json; print(json.dumps({"total_count":150,"runners":[{"name":"r%d"%i,"status":"online","busy":False} for i in range(100)]}))') +out=$(run "$long" "${req[@]}"); check 2 $? "$out" "runner list is truncated" "a truncated list -> 2, not 'strixhalo-engine is not registered'" +printf '%s' "$out" | grep -q 'is not registered' && { echo "FAIL a truncated list blamed a required name"; fail=$((fail + 1)); } +full=$(python3 -c 'import json; print(json.dumps({"total_count":100,"runners":[{"name":"r%d"%i,"status":"online","busy":False} for i in range(100)]}))') +out=$(run "$full"); check 2 $? "$out" "truncated" "exactly one page -> 2 (may be truncated), never a vacuous ok" + # 6. --list prints the table without changing the verdict out=$(run "$two_online" --list); check 0 $? "$out" "strixhalo-engine" "--list prints runners and still exits 0" From 064fab09d47bfa7a9b536af74c49063404d2b0e0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 13:54:06 -0400 Subject: [PATCH 3/3] ci: the preflight runs the merged systemd-side check beside the API row (engine#1504 item 3) scripts/ci-runner-health.sh landed in #1546 while this PR was open and, measured: nothing on main invokes it -- weekly.sh has no call. So the two halves of 'something that notices' were both unwired: the API row says whether a runner is REGISTERED and online (and cannot say why), the systemd view says Result=oom-kill / restart window (and cannot see a registration that was revoked). Running both here is what makes them one check at release time. Its exit 2 means 'no actions.runner.* units on this host' -- a fresh box, not a failure -- so that arm is a note. Measured on this box just now, the two arms: health rc=1 (stub: 'last result oom-kill') -> 'preflight failed: nothing was run' health rc=2 (stub: 'no units on this host') -> note, preflight continues and against the real script: 'preflight: runner units: 3 PASS in systemd's view' beside 'ok 2 self-hosted runner(s) online, including: strixhalo-engine strixhalo-bench'. --- scripts/weekly.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/weekly.sh b/scripts/weekly.sh index d0257c693..e3f6b076c 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -692,6 +692,25 @@ preflight() { bad=1 fi fi + # The same fact from the other side: scripts/ci-runner-health.sh reads systemd's view (#1546, merged) and says WHY + # a runner is down -- Result=oom-kill, a restart window -- which the API above cannot see, and it fires the moment + # a unit dies instead of after the heartbeat timeout. Neither was ever invoked from anywhere until this row, so + # running both here is what makes the two halves one check at release time. Its exit 2 means "no such units on this + # host", which is a fresh box, not a failure. + if [ -x "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" ]; then + if health_out=$("$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" 2>&1); then + printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" + else + health_rc=$? + if [ "$health_rc" = 2 ]; then + say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" + else + printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: a runner unit is not accepting jobs (scripts/ci-runner-health.sh): $health_rc" + bad=1 + fi + fi + fi return $bad } preflight || { say "preflight failed: nothing was run"; exit 1; }