From 5a9e8ce9ec2d1990cdb6ccf358a1b6a396e2cba9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:50:49 -0400 Subject: [PATCH] ci: a runner that is down now blocks a release, a restart policy that would fail later warns (engine#1504) `ci-runner-health.sh` set `rc=1` for both kinds of failure, so scripts/weekly.sh's preflight could not tell them apart and had to treat them the same. They are not the same: * a runner that is not accepting jobs is an OUTAGE NOW -- every check that needs the box reads as pending for ever, and that is the 11-08 shape the preflight exists for; * since #2082 the same script also fails a unit that is `active/running` but whose restart policy would leave it down after the next OOM kill. On this box the live units legitimately still carry those directives: switching them is #1504's open owner item (item 2), not a fault of the release being cut. So: exit 1 for "down now", exit **3** for "policy only", 0 for clean, 2 for "nothing to inspect" (unchanged). weekly.sh blocks on 1 and warns on 3 -- a release that will not start until the owner switches the live units is a worse failure than a loud warning that says exactly which directive is wrong. Verified with a stubbed systemctl (SYSTEMCTL=, so no root and no real units), all three arms: MODE=good (active/running, Restart=always, OOMPolicy=continue, MemoryMax set) -> rc=0 MODE=policy (same but Restart=no, OOMPolicy=stop, MemoryMax=infinity) -> rc=3 MODE=down (inactive/dead) -> rc=1 and `ci-runner-health.sh --self-test` still reports ok (15 cases). Not stacked on #2091: that branch's base predates #2082, so it has no policy block to split -- building there would have reproduced the stale-base trap. Both PRs touch this file (this one in the state/policy loop, #2091 in the --runs block), so whichever lands second wants a rebase, not a hand-merge. --- scripts/ci-runner-health.sh | 21 ++++++++++++++++----- scripts/weekly.sh | 7 +++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/ci-runner-health.sh b/scripts/ci-runner-health.sh index 20f47c874..2204398a9 100755 --- a/scripts/ci-runner-health.sh +++ b/scripts/ci-runner-health.sh @@ -30,7 +30,10 @@ # scripts/ci-runner-health.sh [...] # named units, e.g. for a specific runner # scripts/ci-runner-health.sh --self-test # the verdict logic, with systemctl stubbed # -# Exit: 0 when every runner is active/running with no failure result; 1 when any is not; 2 when there is nothing to inspect +# Exit: 0 when every runner is active/running with no failure result AND its restart policy would survive the next OOM +# kill; 1 when a runner is not accepting jobs (an outage now); 3 when ONLY a restart policy would leave it down after the +# next OOM kill (a latent one -- engine#1504, and scripts/weekly.sh warns rather than blocks on it); 2 when there is +# nothing to inspect # (no such units on this host, so a silent pass would be the defect). set -uo pipefail @@ -125,7 +128,13 @@ if [ ${#units[@]} -eq 0 ]; then exit 2 fi -rc=0 +# engine#1504: a caller has to be able to tell the two kinds of failure apart. A unit that is not accepting jobs is an +# immediate outage -- every check that needs the box reads as pending for ever -- while a policy that would not survive the +# next OOM kill is a latent one. Exit 1 for the first, **3** for policy only, 0 for clean: scripts/weekly.sh blocks a +# release on 1 and warns on 3, because switching the live units' directives is an owner action (#1504 item 2) and a +# release that will not start until then is a worse failure than a loud warning. +state_bad=0 +policy_bad=0 for u in "${units[@]}"; do line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts -p Restart -p OOMPolicy -p MemoryMax 2>/dev/null) st=$(printf '%s\n' "$line" | sed -n 's/^ActiveState=//p') @@ -138,13 +147,15 @@ for u in "${units[@]}"; do v=$(runner_verdict "$st" "$sub" "$res" "$nr") case "$v" in PASS*) printf 'PASS %-52s %s\n' "$u" "${v#PASS }" ;; - *) printf 'FAIL %-52s %s\n' "$u" "${v#FAIL }"; rc=1 ;; + *) printf 'FAIL %-52s %s\n' "$u" "${v#FAIL }"; state_bad=1 ;; esac # engine#1504: alive now is not the same as after the next OOM kill, so the policy is its own line. p=$(policy_verdict "$rst" "$oom" "$mmax") case "$p" in PASS*) printf 'PASS %-52s %s\n' "" "${p#PASS }" ;; - *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; rc=1 ;; + *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; policy_bad=1 ;; esac done -exit "$rc" +if [ "$state_bad" = 1 ]; then exit 1; fi +if [ "$policy_bad" = 1 ]; then exit 3; fi +exit 0 diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 468864481..ea67f688f 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -704,6 +704,13 @@ preflight() { health_rc=$? if [ "$health_rc" = 2 ]; then say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" + elif [ "$health_rc" = 3 ]; then + # engine#1504: policy only. Every runner is accepting jobs NOW; what fails is that the next OOM kill + # would stop one for good. Switching the live units' directives is #1504's open owner item on this + # box, so a release that will not start until then is a worse failure than a loud warning. The + # split is ci-runner-health.sh's exit codes (1 = down now, 3 = policy only). + printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: WARNING a runner would not come back after the next OOM kill (its restart policy) - not blocking this release, and #1504's owner item on this box" else printf '%s\n' "$health_out" | sed 's/^/preflight: /' # Two kinds of FAIL reach here (engine#1504): a unit that is down NOW, and since #2082 a unit that is up