From 1d43f9cb797836015aecb5aa4c6e14cb3124c1f0 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:08:13 -0800 Subject: [PATCH 01/23] fix: reset per-task supervision markers at away-session boundaries (#8318) * fix: clear per-task away-session markers at return and fresh entry The five .subsuper per-task families (stale, reported-stale, dialog, paused, pause-until-due) were episode state with no session scope: a first-seen-idle clock left behind when a session ended inside the 240s recheck window was read by the next away session's first housekeeping pass as its own, reporting a "stale persisted Ns (possible wedge)" whose duration never happened. fm_afk_clear_stale_artifacts now drops all five families with the delivery artifacts on a fresh entry, and fm_afk_launch_stop drops them at the return. The launch backup/rollback lists cover the same set so a failed entry still restores prior state. Accepted cost, recorded on the cleanup's header: a declared wait spanning two away sessions restarts its recheck window at re-entry. * no-mistakes(document): Correct away-session lifecycle documentation * no-mistakes(ci): Initialized pendingBashComponents in the shared Pi test host, fixing the CI renderer crash. Full suite failure was downstream of that lane. Focused behavioral checks, shell syntax, and diff checks pass locally; the newer CI Pi version was not available locally --- .agents/skills/afk/SKILL.md | 5 +- bin/fm-afk-launch.sh | 24 +++++- bin/fm-afk-start.sh | 46 ++++-------- docs/t3code-backend.md | 6 +- tests/fm-afk-launch.test.sh | 113 ++++++++++++++++++++++++++++- tests/fm-backend-t3code.test.sh | 27 ++++--- tests/fm-calm-pi-extension.test.sh | 1 + 7 files changed, 171 insertions(+), 51 deletions(-) diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index b411c0f2d83..c9c125a2604 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -235,8 +235,9 @@ The single-line format makes submission unambiguous across harnesses; the carrie ### Stale-artifact lifecycle Treat `state/.subsuper-escalations`, its `.since` sidecar, `state/.subsuper-inject-wedged`, and `state/.subsuper-unknown-acked` as session-scoped delivery artifacts, not as the durable work record. -Always enter through `bin/fm-afk-launch.sh`, which clears prior-session artifacts only for a fresh entry and preserves the current session's buffer on refresh. -Always exit through `bin/fm-afk-launch.sh stop`, which keeps `state/.afk` present through the daemon's shutdown flush, clears it, and archives the posture record last. +The per-task markers `state/.subsuper-stale-`, `.subsuper-reported-stale-`, `.subsuper-dialog-`, `.subsuper-paused-`, and `.subsuper-pause-until-due-` are the same kind of session-scoped episode state: a still-true condition is re-derived fresh by the next session, and a declared wait spanning two away sessions restarts its recheck window at re-entry. +Always enter through `bin/fm-afk-launch.sh`, which clears prior-session artifacts and markers only for a fresh entry and preserves the current session's buffer on refresh. +Always exit through `bin/fm-afk-launch.sh stop`, which keeps `state/.afk` present through the daemon's shutdown flush, clears it with the session's per-task markers, and archives the posture record last. `docs/herdr-backend.md` "Away-mode supervisor support" and `docs/t3code-backend.md` "Away mode" own the backend mechanisms, and `docs/verification/runtime-backends.md` "Away-mode transport" owns active evidence. ### Reliability properties diff --git a/bin/fm-afk-launch.sh b/bin/fm-afk-launch.sh index 9b4f831b994..4299e125552 100755 --- a/bin/fm-afk-launch.sh +++ b/bin/fm-afk-launch.sh @@ -633,11 +633,15 @@ fm_afk_launch_restore_backup() { # "$FM_AFK_LAUNCH_STATE/.subsuper-escalations.since" \ "$FM_AFK_LAUNCH_STATE/.subsuper-inject-wedged" \ "$FM_AFK_LAUNCH_STATE/.subsuper-unknown-acked" \ - "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-* || result=1 + "$FM_AFK_LAUNCH_STATE"/.subsuper-stale-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-dialog-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-paused-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-pause-until-due-* || result=1 if [ "$had_afk" -eq 1 ]; then cp "$backup/.afk" "$FM_AFK_LAUNCH_STATE/.afk" || result=1 fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-unknown-acked "$backup"/.subsuper-reported-stale-*; do + for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-unknown-acked "$backup"/.subsuper-stale-* "$backup"/.subsuper-reported-stale-* "$backup"/.subsuper-dialog-* "$backup"/.subsuper-paused-* "$backup"/.subsuper-pause-until-due-*; do artifact=${artifact##*/} if [ -e "$backup/$artifact" ]; then cp -p "$backup/$artifact" "$FM_AFK_LAUNCH_STATE/$artifact" || result=1 @@ -799,7 +803,7 @@ fm_afk_launch_start() { had_afk=1 cp "$FM_AFK_LAUNCH_STATE/.afk" "$backup/.afk" || { rm -rf "$backup"; return 1; } fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-unknown-acked "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-*; do + for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-unknown-acked "$FM_AFK_LAUNCH_STATE"/.subsuper-stale-* "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-* "$FM_AFK_LAUNCH_STATE"/.subsuper-dialog-* "$FM_AFK_LAUNCH_STATE"/.subsuper-paused-* "$FM_AFK_LAUNCH_STATE"/.subsuper-pause-until-due-*; do artifact=${artifact##*/} if [ -e "$FM_AFK_LAUNCH_STATE/$artifact" ]; then cp -p "$FM_AFK_LAUNCH_STATE/$artifact" "$backup/$artifact" || { rm -rf "$backup"; return 1; } @@ -858,7 +862,7 @@ fm_afk_launch_start_native() { had_afk=1 cp "$FM_AFK_LAUNCH_STATE/.afk" "$backup/.afk" || { rm -rf "$backup"; return 1; } fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-unknown-acked "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-*; do + for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-unknown-acked "$FM_AFK_LAUNCH_STATE"/.subsuper-stale-* "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-* "$FM_AFK_LAUNCH_STATE"/.subsuper-dialog-* "$FM_AFK_LAUNCH_STATE"/.subsuper-paused-* "$FM_AFK_LAUNCH_STATE"/.subsuper-pause-until-due-*; do artifact=${artifact##*/} if [ -e "$FM_AFK_LAUNCH_STATE/$artifact" ]; then cp -p "$FM_AFK_LAUNCH_STATE/$artifact" "$backup/$artifact" || { rm -rf "$backup"; return 1; } @@ -937,6 +941,18 @@ fm_afk_launch_stop() { fm_afk_launch_log "failed to clear away-mode flag" result=1 fi + # The session's per-task supervision markers are episode state that ends with + # it: left behind, the NEXT session's first housekeeping reads the stale/pause + # clocks as its own and reports a possible wedge whose duration never + # happened. A wait that still stands is re-declared and re-windowed fresh. + if ! rm -f "$FM_AFK_LAUNCH_STATE"/.subsuper-stale-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-reported-stale-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-dialog-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-paused-* \ + "$FM_AFK_LAUNCH_STATE"/.subsuper-pause-until-due-*; then + fm_afk_launch_log "failed to clear the session's per-task supervision markers" + result=1 + fi if [ "$result" -eq 0 ] && fm_afk_contract_present "$FM_AFK_LAUNCH_STATE"; then if archived=$("$FM_AFK_CONTRACT_CMD" archive); then fm_afk_launch_log "away-posture record archived at $archived" diff --git a/bin/fm-afk-start.sh b/bin/fm-afk-start.sh index 6b9d76c4ffc..f8a7a911d17 100755 --- a/bin/fm-afk-start.sh +++ b/bin/fm-afk-start.sh @@ -7,7 +7,7 @@ # FM_AFK_STATE_PREPARED=1, checks state/.supervise-daemon.lock, and: # - prints "afk: daemon already running pid=" then exits 0 when that # lock is held by a live daemon (a REFRESH: no stale-artifact clear); -# - otherwise clears any prior away session's stale escalation artifacts +# - otherwise clears prior-session delivery caches and per-task markers # (fm_afk_clear_stale_artifacts) for a direct, non-prepared start, then # execs bin/fm-supervise-daemon.sh in the foreground. A prepared start was # already cleared transactionally by bin/fm-afk-launch.sh. @@ -46,39 +46,22 @@ fm_afk_start_usage() { sed -n '2,14p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' } -# fm_afk_clear_stale_artifacts: on a FRESH away-session entry (the daemon is not -# already running), drop the previous away session's leftover escalation-delivery -# artifacts so they cannot surface as stale escalations under the new session. -# These are session-scoped by timing: a fresh entry owns a new supervision -# session and the new daemon has not produced anything yet, so anything present -# here belongs to a PRIOR session. This never drops a genuinely-pending -# escalation - the delivery buffer is a transient cache, and any condition still -# true (a crew still blocked, a check still firing) is re-derived and re-escalated -# fresh by the daemon's heartbeat catch-all scan and the durable -# state/.wake-queue replay (see docs/herdr-backend.md "Away-mode stale-artifact -# lifecycle" and bin/fm-supervise-daemon.sh's escalate_add/inject_wedge_alarm). -# NOT called on a refresh (daemon already alive), so the current session's own -# buffered escalations are preserved. +# Call only before a fresh daemon can produce state, never on a live refresh: +# timing is what proves these caches and episode clocks belong to a prior session. +# Durable wake-queue rows and classification positions must remain intact for +# replay and catch-all reconciliation by bin/fm-supervise-daemon.sh. +# .agents/skills/afk/SKILL.md "Stale-artifact lifecycle" owns the session contract. fm_afk_clear_stale_artifacts() { # - local state=$1 line win key - # Unknown-state warnings suppress repeats even after delivery. Re-arm only - # warnings still in the buffer being discarded, preserving delivered ones. - if [ -s "$state/.subsuper-escalations" ] && compgen -G "$state/.subsuper-reported-stale-*" >/dev/null; then - _fm_wake_require_classify || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in - stale\ persisted\ *s\ \(possible\ wedge\):\ *) - win=${line#* (possible wedge): } - key=$(window_to_task "$win" "$state" | tr ':/.' '___') - rm -f "$state/.subsuper-reported-stale-$key" || return 1 - ;; - esac - done < "$state/.subsuper-escalations" - fi + local state=$1 rm -f "$state/.subsuper-escalations" \ "$state/.subsuper-escalations.since" \ "$state/.subsuper-inject-wedged" \ - "$state/.subsuper-unknown-acked" 2>/dev/null + "$state/.subsuper-unknown-acked" \ + "$state"/.subsuper-stale-* \ + "$state"/.subsuper-reported-stale-* \ + "$state"/.subsuper-dialog-* \ + "$state"/.subsuper-paused-* \ + "$state"/.subsuper-pause-until-due-* 2>/dev/null } daemon_lock_owner() { @@ -182,8 +165,7 @@ fm_afk_start_main() { fm_lock_remove_path "$FM_AFK_LOCK" 2>/dev/null || true fi - # Fresh start: clear the previous away session's stale delivery artifacts - # before the new daemon can surface them (fix for the leaked-artifact defect). + # Clear prior-session state before the new daemon can observe or produce it. if [ "${FM_AFK_STATE_PREPARED:-0}" != 1 ]; then fm_afk_clear_stale_artifacts "$FM_AFK_STATE" fi diff --git a/docs/t3code-backend.md b/docs/t3code-backend.md index 040292762d3..55061cef4f1 100644 --- a/docs/t3code-backend.md +++ b/docs/t3code-backend.md @@ -227,9 +227,9 @@ Resolve ambiguity by explicitly setting both `FM_SUPERVISOR_BACKEND=t3code` and Injection is a `t3_thread_send` only into a settled supervisor thread, one the [thread status table](#restart-and-liveness-behavior) reads idle/alive with no pending request; any other row, including `blocked` and an unreadable thread, defers the digest exactly as a busy pane does on every other backend. An unknown native busy verdict defers new-turn delivery but does not prove that work resumed. Away housekeeping keeps an overdue stale alert pending and buffers one possible-wedge report while the verdict remains unknown, including a failed run or a transient thread-read failure. -Delivered warnings stay suppressed while the same unknown condition persists. -If a fresh daemon entry discards a queued, undelivered warning, it clears that warning's reported marker and preserves its stale timer, so the next housekeeping pass queues exactly one replacement. -Refreshing a live daemon preserves the queue; a failed start restores the previous queue and reported markers. +Within the same away session, delivered warnings stay suppressed while the same unknown condition persists. +The [stale-artifact lifecycle](../.agents/skills/afk/SKILL.md#stale-artifact-lifecycle) owns session-boundary cleanup and recheck-window resets. +Refreshing a live daemon preserves the queue and episode markers; a failed start restores the previous queue and episode markers. Confirmed busy activity, a missing thread, or an idle verdict after the unknown condition was reported clears the stale and reported markers without another report, so a later unknown condition can report again. Declared external-wait rechecks also survive native uncertainty; the worker's declaration still controls their cadence. `stale_window_is_busy` and `housekeeping` in `bin/fm-supervise-daemon.sh` own stale tracking; `fm_afk_clear_stale_artifacts` in `bin/fm-afk-start.sh` and the launcher in `bin/fm-afk-launch.sh` own queue discard and startup rollback. diff --git a/tests/fm-afk-launch.test.sh b/tests/fm-afk-launch.test.sh index d045a093a1e..6f08377e1d5 100755 --- a/tests/fm-afk-launch.test.sh +++ b/tests/fm-afk-launch.test.sh @@ -256,7 +256,8 @@ unit_stop_archives_the_record_last() { } # --------------------------------------------------------------------------- -# UNIT 1: fm_afk_clear_stale_artifacts removes exactly the four stale artifacts. +# UNIT 1: fm_afk_clear_stale_artifacts removes exactly the four delivery +# artifacts plus the prior session's five per-task marker families. # --------------------------------------------------------------------------- unit_clear_stale() { local st @@ -266,6 +267,12 @@ unit_clear_stale() { : > "$st/state/.subsuper-escalations.since" : > "$st/state/.subsuper-inject-wedged" : > "$st/state/.subsuper-unknown-acked" + : > "$st/state/.subsuper-stale-task1" + : > "$st/state/.subsuper-reported-stale-task1" + : > "$st/state/.subsuper-dialog-task1" + : > "$st/state/.subsuper-paused-task1" + : > "$st/state/.subsuper-pause-until-due-task1" + : > "$st/state/.subsuper-seen-status-task1" # classification position must be kept : > "$st/state/.wake-queue" # durable queue must be untouched # Source fm-afk-start.sh inside a child bash (it sets `set -eu` and would # otherwise leak that into this test shell) and call the clear helper. @@ -279,6 +286,20 @@ unit_clear_stale() { else fail "clear-stale: stale artifacts survived" fi + if ! compgen -G "$st/state/.subsuper-stale-*" >/dev/null \ + && ! compgen -G "$st/state/.subsuper-reported-stale-*" >/dev/null \ + && ! compgen -G "$st/state/.subsuper-dialog-*" >/dev/null \ + && ! compgen -G "$st/state/.subsuper-paused-*" >/dev/null \ + && ! compgen -G "$st/state/.subsuper-pause-until-due-*" >/dev/null; then + pass "clear-stale: removes the prior session's five per-task marker families" + else + fail "clear-stale: per-task session markers survived" + fi + if [ -e "$st/state/.subsuper-seen-status-task1" ]; then + pass "clear-stale: keeps the durable classification position" + else + fail "clear-stale: removed the classification position" + fi if [ -e "$st/state/.wake-queue" ]; then pass "clear-stale: leaves the durable wake-queue intact (no pending work dropped)" else @@ -287,6 +308,95 @@ unit_clear_stale() { rm -rf "$st" } +# --------------------------------------------------------------------------- +# The per-task away markers are session-scoped episode state: the return path +# (fm_afk_launch_stop) and the fresh-entry cleanup must both drop all five +# families, or the next session's first housekeeping reads a dead session's +# idle clock as its own and reports a wedge whose duration never happened. +# The last pass replays the implicit-state-machines audit's reproduction: a +# hand-written prior-session clock, an idle pane read, delivery stubbed off. +# --------------------------------------------------------------------------- +unit_per_task_markers_session_scoped() { + local st state now + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-marker-scope.XXXXXX") + state="$st/state" + mkdir -p "$state" + now=$(date +%s) + # One finished ship task, as the audit's scratch home: a terminal status and + # a meta that still maps its window. + printf 'done [at=%s]: PR https://example.test/o/r/pull/1 checks green\n' "$now" > "$state/demo.status" + printf 'window=%s\nbackend=tmux\nharness=claude\n' 'fmaudit-none:fm-demo' > "$state/demo.meta" + printf '0:fixture\n' > "$state/.subsuper-seen-status-demo" + printf '%s\n' "$now" > "$state/.subsuper-last-scan" + seed_markers() { + # Same path and content as the daemon's stale_marker_record: the epoch the + # pane was first seen idle, mid-episode when the session ended. + printf '%s\n' $(( now - 172801 )) > "$state/.subsuper-stale-demo" + : > "$state/.subsuper-reported-stale-demo" + : > "$state/.subsuper-dialog-demo" + : > "$state/.subsuper-paused-demo" + : > "$state/.subsuper-pause-until-due-demo" + } + markers_gone() { + ! compgen -G "$state/.subsuper-stale-*" >/dev/null \ + && ! compgen -G "$state/.subsuper-reported-stale-*" >/dev/null \ + && ! compgen -G "$state/.subsuper-dialog-*" >/dev/null \ + && ! compgen -G "$state/.subsuper-paused-*" >/dev/null \ + && ! compgen -G "$state/.subsuper-pause-until-due-*" >/dev/null + } + + # The return path drops the session's markers with the away flag. + seed_markers + : > "$state/.afk" + FM_HOME="$st" FM_STATE_OVERRIDE="$state" "$LAUNCH" stop >/dev/null 2>&1 + if markers_gone && [ ! -e "$state/.afk" ] && [ -e "$state/.subsuper-seen-status-demo" ]; then + pass "return: stop drops all five per-task marker families, keeps the classification position" + else + fail "return: per-task markers survived stop ($(compgen -G "$state/.subsuper-*" | tr '\n' ' '))" + fi + + # The fresh-entry cleanup drops them alongside the delivery artifacts. + seed_markers + FM_HOME="$st" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1"; fm_afk_clear_stale_artifacts "$2"' _ "$START" "$state" + if markers_gone; then + pass "entry: fresh-session cleanup drops all five per-task marker families" + else + fail "entry: per-task markers survived fresh-session cleanup ($(compgen -G "$state/.subsuper-*" | tr '\n' ' '))" + fi + + # The audit narrative end to end: session one's clock goes through stop and + # the next entry's cleanup, then the new daemon's first housekeeping pass. + # Reset to the audit's repro shape first: ONLY the stale clock, because + # housekeeping's pause migration retires the episode before the wedge check + # whenever a paused marker is also present. + rm -f "$state"/.subsuper-stale-* "$state"/.subsuper-reported-stale-* \ + "$state"/.subsuper-dialog-* "$state"/.subsuper-paused-* \ + "$state"/.subsuper-pause-until-due-* + printf '%s\n' $(( now - 172801 )) > "$state/.subsuper-stale-demo" + : > "$state/.afk" + FM_HOME="$st" FM_STATE_OVERRIDE="$state" "$LAUNCH" stop >/dev/null 2>&1 + : > "$state/.afk" # a fresh entry, two days later in the audit's telling + printf '%s\n' "$(date +%s)" > "$state/.subsuper-last-scan" + FM_HOME="$st" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1"; fm_afk_clear_stale_artifacts "$2"' _ "$START" "$state" + FM_HOME="$st" FM_STATE_OVERRIDE="$state" FM_DAEMON_PRIMARY_HARNESS=claude \ + bash -c ' + . "$1" + stale_window_is_busy() { return 1; } + escalate_flush() { return 0; } + inject_msg() { return 0; } + inject_wedge_alarm() { return 0; } + housekeeping "$2" + ' _ "$ROOT/bin/fm-supervise-daemon.sh" "$state" + if ! grep -q 'possible wedge' "$state/.subsuper-escalations" 2>/dev/null; then + pass "session scope: a dead session's idle clock cannot report 'possible wedge' in the next session" + else + fail "session scope: first housekeeping reported a wedge that never happened ($(cat "$state/.subsuper-escalations"))" + fi + rm -rf "$st" +} + unit_relative_paths_are_absolute_before_daemon_launch() { local root home state out status linked_home root=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-relative-home.XXXXXX") @@ -1787,6 +1897,7 @@ e2e_rex() { } unit_clear_stale +unit_per_task_markers_session_scoped unit_enter_records_the_posture_in_one_step_without_a_daemon unit_retired_two_step_entry_is_refused unit_pi_never_launches_the_daemon diff --git a/tests/fm-backend-t3code.test.sh b/tests/fm-backend-t3code.test.sh index 03d1cef8634..45ad94912c7 100755 --- a/tests/fm-backend-t3code.test.sh +++ b/tests/fm-backend-t3code.test.sh @@ -909,42 +909,51 @@ test_native_restart_rearms_undelivered_stale_warning() { housekeeping "$1" ' "$state" || exit 1 [ -s "$state/.subsuper-escalations" ] || exit 1 - # The native daemon has stopped without delivering its buffer. + # The native daemon has stopped without delivering its buffer. Stop keeps + # the delivery buffer for the return/catch-up path but ends the session's + # per-task episode state: the stale clock and its report marker are gone. "$ROOT/bin/fm-afk-launch.sh" stop || exit 1 [ -s "$state/.subsuper-escalations" ] || exit 1 - [ -f "$state/.subsuper-reported-stale-worker" ] || exit 1 - # A warning already delivered for another condition must stay suppressed. + [ ! -e "$state/.subsuper-reported-stale-worker" ] || exit 1 + [ ! -e "$marker" ] || exit 1 + # A warning already delivered for another condition is the same + # session-scoped episode state; the next entry clears it too. printf 900 > "$state/.subsuper-reported-stale-delivered" "$ROOT/bin/fm-afk-launch.sh" enter --words 'supervise pending work' >/dev/null || exit 1 first=$(cat "$state/.subsuper-escalations") - # Failed startup must restore both the queue and its suppression markers. + # Failed startup must restore both the queue and the markers it captured. bash -c ' . "$0/bin/fm-afk-launch.sh" fm_afk_launch_record_write() { return 1; } ! fm_afk_launch_main start-native ' "$ROOT" || exit 1 [ "$(cat "$state/.subsuper-escalations")" = "$first" ] || exit 1 - [ -f "$state/.subsuper-reported-stale-worker" ] || exit 1 + [ -f "$state/.subsuper-reported-stale-delivered" ] || exit 1 "$ROOT/bin/fm-afk-launch.sh" start-native || exit 1 assert_absent "$state/.subsuper-escalations" "fresh startup must discard the old queue" assert_absent "$state/.subsuper-reported-stale-worker" "discarding an undelivered warning must re-arm reporting" - assert_present "$state/.subsuper-reported-stale-delivered" "startup must preserve suppression of delivered warnings" - [ "$(cat "$marker")" = 1 ] || fail "restart must preserve the stale condition's age" + assert_absent "$state/.subsuper-reported-stale-delivered" "startup must drop the prior session's suppression markers" + [ ! -e "$marker" ] || fail "restart must drop the prior session's stale clock" out=$(t3_run 'fm_backend_t3code_busy_state thread-live') [ "$out" = unknown ] || fail "the restarted thread must remain unknown" + # The still-unknown thread is re-derived on THIS session's fresh clock: a + # new stale wake opens a new episode at 1000, so the recheck reports its + # own 100s, never the prior session's 999s. t3_run ' . "$0/bin/fm-supervise-daemon.sh" _now() { printf 1000; } LOG="$1/daemon.log" + handle_wake "stale: thread-live" "$1" housekeeping "$1" _now() { printf 1100; } housekeeping "$1" housekeeping "$1" ' "$state" || exit 1 + [ "$(cat "$marker")" = 1000 ] || fail "the new session must restart the stale clock" [ "$(wc -l < "$state/.subsuper-escalations" | tr -d "[:space:]")" = 1 ] \ || fail "persistent uncertainty must queue exactly one replacement warning" assert_contains "$(cat "$state/.subsuper-escalations")" \ - 'stale persisted 999s (possible wedge): thread-live' "restart must report the retained stale condition" + 'stale persisted 100s (possible wedge): thread-live' "restart must re-derive the stale condition on a fresh clock" # Once that replacement is delivered, uncertainty must not report again. t3_run ' . "$0/bin/fm-supervise-daemon.sh" @@ -959,7 +968,7 @@ test_native_restart_rearms_undelivered_stale_warning() { [ "$(wc -l < "$state/delivered.log" | tr -d "[:space:]")" = 1 ] || exit 1 ) || fail "$session restart lost or duplicated the stale warning" done - pass "native restart re-arms discarded T3 stale warnings exactly once and preserves delivered suppression" + pass "native restart clears session episode state and re-derives T3 stale warnings once on a fresh clock" } test_housekeeping_preserves_unknown_stale_recheck() { diff --git a/tests/fm-calm-pi-extension.test.sh b/tests/fm-calm-pi-extension.test.sh index 080ab2435af..eec9488aad2 100755 --- a/tests/fm-calm-pi-extension.test.sh +++ b/tests/fm-calm-pi-extension.test.sh @@ -587,6 +587,7 @@ function makeHost(session) { Object.assign(host, { // Pi reads its session through runtimeHost, which a session replacement swaps. runtimeHost: { session }, + pendingBashComponents: [], compactionQueuedMessages: [], statuses: [], warnings: [], From 9fbefb8ed17518530dab6da4dcda0e7dfe38e4c7 Mon Sep 17 00:00:00 2001 From: Arthur Burkart Date: Wed, 11 Nov 2026 02:57:33 -0500 Subject: [PATCH 02/23] fix: surface herdr lab server failures and validate socket path length (#8322) * fix: surface herdr lab server failures and refuse over-long socket paths The lab helper discarded the herdr server's output and waited the full timeout even after the server exited, so a startup failure such as a socket path over the sun_path limit looked like a plain timeout. Provision now keeps the server output in a log in the helper state directory, stops waiting as soon as the server process exits, and prints the log tail in the failure message. Name and prepare now refuse a session whose client socket path would exceed the 103-byte sun_path limit, naming the limit. * fix: count socket path bytes, check it in provision, and use the documented config default The sun_path limit is a byte limit, so the length is now counted under the C locale. Provision runs the check up front so the reuse path is covered. The config root default is the documented ~/.config/herdr, and an early server exit rechecks status once before failing. * no-mistakes(review): Cap server log at 64 KiB; drop config-dir override * no-mistakes(ci): I fixed both Greptile findings in `bin/fm-herdr-lab.sh`. I ran `tests/fm-herdr-lab.test.sh` and every test passed, including the new ones. I did not check that the new pipe-capture test fails against the old collector. **ci-1 (captured provision can hang).** - Invariant: the log collector must never hold the caller's stdout or stderr open. - The collector now runs as `{ head -c "$cap" > "$server_log"; cat; } >/dev/null 2>&1`. The server pid stays in `$!`. - The log cap is 4096 bytes. One shared value, `fm_herdr_lab_server_log_cap`, now sets both the stored log size and the tail shown in the failure message. - I added `test_captured_provision_returns_promptly`, which captures provision's output through a command-substitution pipe and checks it returns promptly. - I updated the cap test to 4096 bytes and the header comment to match. **ci-2 (XDG_CONFIG_HOME ignored).** - Invariant: the socket-length check must use the same config root herdr resolves. - `fm_herdr_lab_config_dir` now returns `$XDG_CONFIG_HOME/herdr` when `XDG_CONFIG_HOME` is set, and `$HOME/.config/herdr` otherwise. `name`, `prepare` and `provision` all go through `fm_herdr_lab_check_socket_path`, so one change covers all three. - I added tests for a long `HOME` with a short `XDG_CONFIG_HOME` (accepted) and a short `HOME` with a long `XDG_CONFIG_HOME` (refused). - The test file now unsets `XDG_CONFIG_HOME` at the top so the other tests don't depend on the CI environment. - I updated the header comment to match * test: keep the fake server alive so the captured-output test proves the fix The fake server exited right after reporting running, so the captured-output test passed without the fix. It now stays alive and is cleaned up afterwards, and the test fails on the old redirection. The collector now redirects with exec in a subshell, because a group redirect leaves saved copies of the caller's pipe open. --- bin/fm-herdr-lab.sh | 64 ++++++++++++++++++++++++-- tests/fm-herdr-lab.test.sh | 94 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/bin/fm-herdr-lab.sh b/bin/fm-herdr-lab.sh index d4ce86b30e4..23321d59e07 100755 --- a/bin/fm-herdr-lab.sh +++ b/bin/fm-herdr-lab.sh @@ -15,6 +15,17 @@ # Session names must begin with "fm-lab-" and can never be "default". # The name command sanitizes the label, caps it at 16 characters, and appends # process/random suffixes to keep generated socket paths short. +# A session whose /sessions//herdr-client.sock path +# would exceed the 103-byte sun_path limit is refused by name, prepare and +# provision. +# The herdr config root is $XDG_CONFIG_HOME/herdr when XDG_CONFIG_HOME is set, +# otherwise $HOME/.config/herdr, the same root herdr resolves. +# The length is counted in bytes, not characters. +# Provision keeps the lab server output in .server.log in the state +# directory, truncated on every provision and capped at 4096 bytes (later output +# is discarded), and shown only as a bounded tail. +# Provision also stops waiting as soon as the server process exits, and prints +# the log tail when the session never reports running. # Every Herdr call made here carries --session : trailing, or # immediately before the first -- delimiter so it stays a Herdr option instead # of becoming a passthrough argument such as an agent start argument. @@ -61,6 +72,22 @@ fm_herdr_lab_validate_name() { # return 1 } +fm_herdr_lab_sun_path_limit=103 + +fm_herdr_lab_config_dir() { + printf '%s/herdr' "${XDG_CONFIG_HOME:-${HOME:-}/.config}" +} + +fm_herdr_lab_check_socket_path() { # + local name=$1 path bytes + path="$(fm_herdr_lab_config_dir)/sessions/$name/herdr-client.sock" + bytes=$(printf '%s' "$path" | LC_ALL=C wc -c | tr -d ' ') + if [ "$bytes" -gt "$fm_herdr_lab_sun_path_limit" ]; then + fm_herdr_lab_error "session '$name' would bind a ${bytes}-byte socket path, over the $fm_herdr_lab_sun_path_limit-byte sun_path limit: $path" + return 1 + fi +} + fm_herdr_lab_state_dir() { printf '%s' "${FM_HERDR_LAB_STATE_DIR:-${TMPDIR:-/tmp}/fm-herdr-lab-${UID}}" } @@ -128,6 +155,7 @@ fm_herdr_lab_fleet_state() { # fm_herdr_lab_prepare() { # local name=$1 sessions state_dir tripwire fm_herdr_lab_validate_name "$name" || return 1 + fm_herdr_lab_check_socket_path "$name" || return 1 command -v herdr >/dev/null 2>&1 || { fm_herdr_lab_error "herdr is required"; return 1; } command -v jq >/dev/null 2>&1 || { fm_herdr_lab_error "jq is required"; return 1; } @@ -436,9 +464,23 @@ fm_herdr_lab_cancel_provision() { # wait "$pid" 2>/dev/null || true } +fm_herdr_lab_server_log_cap=4096 + +fm_herdr_lab_server_log_path() { # + printf '%s/%s.server.log' "$(fm_herdr_lab_state_dir)" "$1" +} + +fm_herdr_lab_server_log_tail() { # + local log=$1 + [ -s "$log" ] || return 0 + fm_herdr_lab_error "server output (last 20 lines of $log):" + tail -n 20 "$log" | tail -c "$fm_herdr_lab_server_log_cap" | sed 's/^/fm-herdr-lab: /' >&2 +} + fm_herdr_lab_provision() { # - local name=$1 sessions tripwire running attempt server_pid max_attempts timeout_seconds + local name=$1 sessions tripwire running attempt server_pid max_attempts timeout_seconds server_log fm_herdr_lab_validate_name "$name" || return 1 + fm_herdr_lab_check_socket_path "$name" || return 1 command -v herdr >/dev/null 2>&1 || { fm_herdr_lab_error "herdr is required"; return 1; } command -v jq >/dev/null 2>&1 || { fm_herdr_lab_error "jq is required"; return 1; } @@ -463,7 +505,10 @@ fm_herdr_lab_provision() { # else fm_herdr_lab_prepare "$name" || return 1 fi - fm_herdr_lab_raw "$name" server >/dev/null 2>&1 & + server_log=$(fm_herdr_lab_server_log_path "$name") + mkdir -p "$(dirname "$server_log")" + : > "$server_log" + fm_herdr_lab_raw "$name" server > >(exec >/dev/null 2>&1; head -c "$fm_herdr_lab_server_log_cap" > "$server_log"; cat) 2>&1 & server_pid=$! attempt=0 max_attempts=300 @@ -477,11 +522,21 @@ fm_herdr_lab_provision() { # } return 0 fi + if ! kill -0 "$server_pid" 2>/dev/null; then + wait "$server_pid" 2>/dev/null || true + sleep 0.2 + running=$(fm_herdr_lab_cli "$name" status --json 2>/dev/null | jq -r '.server.running // false' 2>/dev/null) || running=false + [ "$running" = true ] && continue + fm_herdr_lab_error "lab server for '$name' exited before reporting running" + fm_herdr_lab_server_log_tail "$server_log" + return 1 + fi sleep 0.2 attempt=$((attempt + 1)) done fm_herdr_lab_cancel_provision "$server_pid" fm_herdr_lab_error "lab session '$name' did not report running within $timeout_seconds seconds" + fm_herdr_lab_server_log_tail "$server_log" return 1 } @@ -568,7 +623,10 @@ fm_herdr_lab_name() { #