diff --git a/scripts/ci-runner-health.sh b/scripts/ci-runner-health.sh index 20f47c874..2204398a9 100755 --- a/scripts/ci-runner-health.sh +++ b/scripts/ci-runner-health.sh @@ -30,7 +30,10 @@ # scripts/ci-runner-health.sh [...] # named units, e.g. for a specific runner # scripts/ci-runner-health.sh --self-test # the verdict logic, with systemctl stubbed # -# Exit: 0 when every runner is active/running with no failure result; 1 when any is not; 2 when there is nothing to inspect +# Exit: 0 when every runner is active/running with no failure result AND its restart policy would survive the next OOM +# kill; 1 when a runner is not accepting jobs (an outage now); 3 when ONLY a restart policy would leave it down after the +# next OOM kill (a latent one -- engine#1504, and scripts/weekly.sh warns rather than blocks on it); 2 when there is +# nothing to inspect # (no such units on this host, so a silent pass would be the defect). set -uo pipefail @@ -125,7 +128,13 @@ if [ ${#units[@]} -eq 0 ]; then exit 2 fi -rc=0 +# engine#1504: a caller has to be able to tell the two kinds of failure apart. A unit that is not accepting jobs is an +# immediate outage -- every check that needs the box reads as pending for ever -- while a policy that would not survive the +# next OOM kill is a latent one. Exit 1 for the first, **3** for policy only, 0 for clean: scripts/weekly.sh blocks a +# release on 1 and warns on 3, because switching the live units' directives is an owner action (#1504 item 2) and a +# release that will not start until then is a worse failure than a loud warning. +state_bad=0 +policy_bad=0 for u in "${units[@]}"; do line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts -p Restart -p OOMPolicy -p MemoryMax 2>/dev/null) st=$(printf '%s\n' "$line" | sed -n 's/^ActiveState=//p') @@ -138,13 +147,15 @@ for u in "${units[@]}"; do v=$(runner_verdict "$st" "$sub" "$res" "$nr") case "$v" in PASS*) printf 'PASS %-52s %s\n' "$u" "${v#PASS }" ;; - *) printf 'FAIL %-52s %s\n' "$u" "${v#FAIL }"; rc=1 ;; + *) printf 'FAIL %-52s %s\n' "$u" "${v#FAIL }"; state_bad=1 ;; esac # engine#1504: alive now is not the same as after the next OOM kill, so the policy is its own line. p=$(policy_verdict "$rst" "$oom" "$mmax") case "$p" in PASS*) printf 'PASS %-52s %s\n' "" "${p#PASS }" ;; - *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; rc=1 ;; + *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; policy_bad=1 ;; esac done -exit "$rc" +if [ "$state_bad" = 1 ]; then exit 1; fi +if [ "$policy_bad" = 1 ]; then exit 3; fi +exit 0 diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 468864481..ea67f688f 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -704,6 +704,13 @@ preflight() { health_rc=$? if [ "$health_rc" = 2 ]; then say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" + elif [ "$health_rc" = 3 ]; then + # engine#1504: policy only. Every runner is accepting jobs NOW; what fails is that the next OOM kill + # would stop one for good. Switching the live units' directives is #1504's open owner item on this + # box, so a release that will not start until then is a worse failure than a loud warning. The + # split is ci-runner-health.sh's exit codes (1 = down now, 3 = policy only). + printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: WARNING a runner would not come back after the next OOM kill (its restart policy) - not blocking this release, and #1504's owner item on this box" else printf '%s\n' "$health_out" | sed 's/^/preflight: /' # Two kinds of FAIL reach here (engine#1504): a unit that is down NOW, and since #2082 a unit that is up