From bc426368ea63154695d5e69ce90a590986922b2e Mon Sep 17 00:00:00 2001 From: Ryan Curran Date: Mon, 17 Aug 2026 13:35:29 -0400 Subject: [PATCH] tart_health_agent: collect the worker_exit_69 semaphore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to the hangar change that replaces the guest-vs-tart-run uptime ratio as the crash-loop signal. worker-runner.sh writes /opt/worker/worker_exit_69 the first time generic-worker exits 69 (the "cannot free enough disk" path) and deletes it on any other exit, logging "Worker recovered from previous exit code 69". So its presence means the last worker exit was the panic path and its age means how long that has been true — which is exactly the crash loop, with no heuristic involved. Emitted only when the file exists, so an absent semaphore arrives as None rather than 0; 0 would otherwise read as "just started looping". Verified end to end against macmini-m4-236 with --dry-run: slot 1 mac-c51932 worker_exit_69_age_s = None (healthy, no semaphore) slot 2 mac-962a94 worker_exit_69_age_s = 2566 (semaphore backdated 42m) and the test semaphore was removed afterwards. Co-Authored-By: Claude Opus 5 (1M context) --- orchestrator/orchestrator/tart_health_agent.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/orchestrator/orchestrator/tart_health_agent.py b/orchestrator/orchestrator/tart_health_agent.py index e8205ab..4d027b6 100644 --- a/orchestrator/orchestrator/tart_health_agent.py +++ b/orchestrator/orchestrator/tart_health_agent.py @@ -202,6 +202,12 @@ def _kv(text: str) -> dict[str, str]: echo "guest_disk_gib=$(df -g / | awk 'NR==2{print $4}')" echo "cfg_worker=$(sudo grep -m1 -oE 'mac-[0-9a-f]+' /opt/worker/generic-worker.conf.yaml 2>/dev/null)" echo "guest_up_s=$(( $(date -u +%s) - $(sysctl -n kern.boottime | sed -n 's/^{ sec = \([0-9]*\).*/\1/p') ))" +# worker-runner.sh writes this the first time generic-worker exits 69 and deletes it on +# any other exit, so its presence means "the last worker exit was the disk-panic path" +# and its age means how long that has been true. Emitted only when present, so an +# absent file reads as None rather than 0 (0 would mean "just started looping"). +S=/opt/worker/worker_exit_69 +[ -f "$S" ] && echo "exit69_age_s=$(( $(date -u +%s) - $(stat -f %m "$S") ))" """ # NOTE the ANCHOR in that sed. kern.boottime prints # { sec = 1786987025, usec = 7795 } Mon Aug 17 17:17:05 2026 @@ -343,6 +349,8 @@ def collect_host(host: str, tc: dict[str, dict[str, Any]], probe_guests: bool) - s["configured_worker_id"] = g["cfg_worker"] if g.get("guest_epoch", "").isdigit(): s["clock_skew_s"] = int(g["guest_epoch"]) - int(datetime.now(timezone.utc).timestamp()) + if g.get("exit69_age_s", "").lstrip("-").isdigit(): + s["worker_exit_69_age_s"] = int(g["exit69_age_s"]) if not g: s["guest_reachable"] = False elif ip is None: