From 4f49afa18a45ad7fa52e442e6a54be915a23f588 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:32:12 -0400 Subject: [PATCH] ci-runner-health: --runs, so a cancelled check is a number and not a shrug (engine#2077) `pr_agent` reads red on PRs because it is CANCELLED, not because it failed: the job carries `concurrency: { group: pr-agent-, cancel-in-progress: true }`, right for a fast check and wrong for a minutes-long 30B review on one self-hosted runner. Nothing in the tree counted that third state. Measured while writing this (`gh run list --workflow=pr-agent.yml --limit 40`): --event=pull_request cancelled=22 success=17 running=1 -> 55 % cancelled (all events) cancelled=12 success=9 skipped=9 -> 30 % cancelled, the denominator lesson The two differ because `skipped` runs from non-PR events dilute the share, which is why --event exists. `--runs` reads conclusions through `gh run list` (read-only) and reuses this script's contract: verdict line first, counts second, rc 0 PASS / 1 FAIL / 2 nothing readable. The summariser is a pure function over conclusions on stdin, so its arms need no GitHub - including "no runs read is not a pass" and "one cancelled run is already over 0 %", i.e. the threshold's floor is reachable. --max-cancelled-pct defaults to 25. Rebased twice onto a moving main; both conflicts kept BOTH sides: main's six policy_verdict() arms plus my six run arms, one count (22 - counted from the printed arms, not from memory: 20 chk/chkP assertions + 2 case arms). The fix for the cancellation itself is one word in .github/workflows/pr-agent.yml, which no lane token can push; this makes the size of the problem measurable and testable from a lane, and names the cause. --- scripts/ci-runner-health.sh | 72 ++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/scripts/ci-runner-health.sh b/scripts/ci-runner-health.sh index 0cd0fa2cf..fbe3c635d 100755 --- a/scripts/ci-runner-health.sh +++ b/scripts/ci-runner-health.sh @@ -29,6 +29,7 @@ # scripts/ci-runner-health.sh # every actions.runner.* unit # scripts/ci-runner-health.sh [...] # named units, e.g. for a specific runner # scripts/ci-runner-health.sh --self-test # the verdict logic, with systemctl stubbed +# scripts/ci-runner-health.sh --runs # the RUN-CONCLUSION mix: a cancelled check is neither # # Exit: 0 when every runner is active/running with a restart policy that survives an OOM kill; 1 when a runner is not # accepting jobs NOW; 2 when there is nothing to inspect (no such units on this host, so a silent pass would be the defect); @@ -94,6 +95,33 @@ runner_verdict() { esac } +# engine#2077: a `cancelled` run is a THIRD state, and it is the one a merge gate silently reads as neither +# green nor a verdict about the diff. Measured on 2026-11-11: `gh run list --workflow=pr-agent.yml --event +# pull_request --limit 40` was 23 cancelled / 17 success, because `concurrency.cancel-in-progress: true` cancels a +# minutes-long review whenever the next push arrives. So the count is a health number like any other: this +# summarises a `conclusion` per line and fails when the cancelled share is too large to call the workflow healthy. +# +# The threshold is a knob, not a law: `--max-cancelled-pct` (default 25) is where a workflow stops being a signal +# about the diff. 0 is legal and means "a cancelled run is already too many". +runs_summary() { # ; conclusions on stdin, one per line + awk -v maxpct="${1:-25}" ' + { n++; c[$0]++ } + END { + if (n == 0) { print "FAIL no runs read - an empty list is not a pass"; exit 2 } + out = "" + for (k in c) out = out sprintf("%s=%d ", k, c[k]) + canc = ("cancelled" in c) ? c["cancelled"] : 0 + pct = 100.0 * canc / n + # verdict first: a caller (and the self-test) reads the conclusion from line 1, and the counts follow + if (pct > maxpct) + printf "FAIL %d of %d runs cancelled (%.0f%%, over %d%%) - a cancelled check is neither green nor a verdict about the diff; that is a throughput or concurrency defect, not a judgement\n", canc, n, pct, maxpct + else + printf "PASS %d of %d runs cancelled (%.0f%%, within %d%%)\n", canc, n, pct, maxpct + printf "%s\n", out + exit (pct > maxpct) ? 1 : 0 + }' +} + self_test() { rc=0 chk() { # @@ -125,12 +153,54 @@ self_test() { *'would come back'*) printf 'ok a missing ceiling alone says the runner comes back\n' ;; *) printf 'FAIL a missing ceiling alone should not say the unit stops for good\n'; rc=1 ;; esac - [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (16 cases)" || echo "ci-runner-health self-test: FAIL" + # engine#2077: the run-conclusion arms. A cancelled run must never read as a pass, and an empty list must not + # either - the same rule as the unit arms above (nothing readable is not a pass). + chkP() { case "$2" in "$1"*) printf 'ok %s\n' "$(printf '%s' "$2" | cut -c1-78)" ;; + *) printf 'FAIL want %s, got: %s\n' "$1" "$2"; rc=1 ;; esac; } + chkP PASS "$(printf 'success\nsuccess\nsuccess\nsuccess\ncancelled\n' | runs_summary 25)" + chkP FAIL "$(printf 'cancelled\ncancelled\ncancelled\nsuccess\n' | runs_summary 25)" + chkP FAIL "$(printf 'cancelled\n' | runs_summary 25)" # 100% cancelled: the #2077 shape + chkP FAIL "$(printf '' | runs_summary 25)" # no runs read is not a pass + chkP PASS "$(printf 'cancelled\nsuccess\nsuccess\nsuccess\nsuccess\n' | runs_summary 25)" + chkP FAIL "$(printf 'cancelled\nsuccess\nsuccess\nsuccess\nsuccess\n' | runs_summary 0)" + [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (22 cases)" || echo "ci-runner-health self-test: FAIL" return $rc } [ "${1:-}" = --self-test ] && { self_test; exit $?; } +# engine#2077: `--runs [--workflow NAME] [--event EVENT] [--limit N] [--max-cancelled-pct P]` reads run conclusions, not units. +# Read-only: `gh run list` and nothing else. Exit 2 when there is nothing readable, so a lane that cannot reach the +# API cannot report a silent pass (the same rule as the unit path below). +if [ "${1:-}" = --runs ]; then + shift + wf=; lim=40; maxpct=25; ev= + while [ $# -gt 0 ]; do + case "$1" in + --workflow) wf="${2:-}"; shift 2 ;; + --workflow=*) wf="${1#*=}"; shift ;; + --limit) lim="${2:-40}"; shift 2 ;; + --limit=*) lim="${1#*=}"; shift ;; + --max-cancelled-pct) maxpct="${2:-25}"; shift 2 ;; + --max-cancelled-pct=*) maxpct="${1#*=}"; shift ;; + # engine#2077: the EVENT filter changes the denominator. Without it `skipped` runs from non-PR events + # dilute the cancelled share (measured: 10/40 with all events, 23/40 on pull_request), so a caller + # comparing two numbers must say which population it counted. + --event) ev="${2:-}"; shift 2 ;; + --event=*) ev="${1#*=}"; shift ;; + *) echo "usage: $0 --runs [--workflow NAME] [--event EVENT] [--limit N] [--max-cancelled-pct P]" >&2; exit 2 ;; + esac + done + if ! command -v gh >/dev/null; then echo "FAIL no gh on PATH - cannot read run conclusions"; exit 2; fi + args=(run list --limit "$lim" --json conclusion,workflowName,event) + [ -n "$wf" ] && args+=(--workflow "$wf") + [ -n "$ev" ] && args+=(--event "$ev") + json=$(gh "${args[@]}" 2>/dev/null) || { echo "FAIL gh run list failed - cannot read run conclusions"; exit 2; } + printf '%s' "$json" | python3 -c 'import json,sys;[print(r.get("conclusion") or "running") for r in json.load(sys.stdin)]' 2>/dev/null \ + | runs_summary "$maxpct" + exit $? +fi + SC=${SYSTEMCTL:-systemctl} units=("$@") if [ ${#units[@]} -eq 0 ]; then