From 0b3e9955b9a721260b0cc62ea5c9ecfeb342d86a Mon Sep 17 00:00:00 2001 From: theBlanketBaron Date: Sat, 14 Nov 2026 13:02:29 -0500 Subject: [PATCH] RAL-191: the watchman's brief lists the jobs in flight From the issues log, 1.73: when the shell windows were closed every session's background jobs ended, and nothing said what had been running. rally brief --watchman gains in_flight, read on this machine and read-only: for each project's stalls.locks, each held lock's holder pid, state, command and cwd (observed) and the owner its file names (declared), free slots counted, each fair pool's queue by owner (RAL-188), and a summary; what can't be read is named in blind, never shown as nothing running. Co-Authored-By: Claude Opus 5.5 --- docs/brief.md | 10 +++ docs/roadmap.md | 8 ++ src/in_flight.rs | 226 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 22 ++++- tests/cli.rs | 110 +++++++++++++++++++++++ 6 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 src/in_flight.rs diff --git a/docs/brief.md b/docs/brief.md index a02d2a6..e22676d 100644 --- a/docs/brief.md +++ b/docs/brief.md @@ -36,6 +36,16 @@ rally's projects, with every list bounded and every note cut short (well under 2 `reads_short` says so with both numbers. When this CLI is ahead of the service, a hint says a build looks staged and waiting on a restart: a hint, not a fact. All of it is observed from its own source. Nothing reads the lamplighter's rollout files. It needs `gh` signed in, as the digest does, and says so when a read fails. +10. **In flight** (RAL-191): the jobs a closed window would end (issues log 1.73, when every session's background + jobs ended with the shell windows and nothing said what had been running). For each project's `stalls.locks` + in the projects list beside the pairing, every held lock: its holder's pid, state, command line and working + directory, from `/proc` (observed), and the owner its lock file names (declared, as `dsm-heavy` writes it). + Each fair pool's queue comes too, with holders and waiters by owner (RAL-188's reader, see docs/stalls.md). + Free slots are counted, not listed, and a `summary` says "3 locks held, 1 job waiting for a slot". It's read + on this machine, by the CLI, wherever the service is, since that's where the jobs run. Read-only. What it + can't read is in `blind`: an unreadable projects list, no project naming a lock, a lock that can't be read, a + queue that can't be seen. It never reads as "nothing running". A job that holds no lock isn't seen: a gate or + probe run outside `dsm-heavy` or a project's locks won't show. What the ledger doesn't hold is said, not guessed at: what's deployed, a batch not recorded, and whether a gate process is still running (an open batch past its time is a prompt to look, not a finding). diff --git a/docs/roadmap.md b/docs/roadmap.md index e3f395b..04a0031 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1487,3 +1487,11 @@ principles named were durable, accurate and reactive. Found in rally 0x7 by plus it with `rally::script::write`, which writes through a child process (`sh`, `cat`, `chmod 755`) so the test process never holds the file open for writing; no write-then-chmod-755 of a script is left in `src/` or `tests/`; with a test that writes and runs a script 200 times beside four forking threads and is never busy. +- [ ] **RAL-191: The watchman's brief lists the jobs in flight.** From the DiscoSim issues log, 1.73 ("the + watchman's handoff names every job in flight, so a restart can tell what died"), and the watchman + (2026-11-14). Done when: `rally brief --watchman` has an `in_flight` section, read on this machine and + read-only: for each project's `stalls.locks` in the projects list, each held lock with its holder's pid, + state, command and working directory (observed) and the owner its file names (declared), free slots counted, + and each fair pool's queue by owner (RAL-188), with a one-line summary; an unreadable projects list, no + project naming a lock, an unreadable lock or an unseen queue is named in `blind`, never shown as nothing + running; with unit tests and a CLI test with real `flock` holders (docs/brief.md). diff --git a/src/in_flight.rs b/src/in_flight.rs new file mode 100644 index 0000000..ddfa8f1 --- /dev/null +++ b/src/in_flight.rs @@ -0,0 +1,226 @@ +//! The jobs in flight, for the watchman's brief (RAL-191). +//! +//! From the DiscoSim rally's issues log, 1.73: on 2026-11-13 Josh closed every shell window by +//! accident, and every session's background jobs ended with them: a gate inside its quick suite, +//! the teams' probes, seed runs and renders. The sessions came back under the same nicknames, but +//! nothing said what had been running, so the restarted watchman had to rebuild it from the teams' +//! word. Its suggested fix: the watchman's handoff names every job in flight, so a restart can tell +//! what died. +//! +//! `rally brief --watchman` reads, for each project's `stalls.locks` in the projects list, every +//! lock the patterns name: who holds it (pid, state, command line and working directory, from +//! `/proc`, observed), with the owner its lock file names (declared, as dsm-heavy writes it), and +//! each fair pool's queue, holders and waiters by owner ([`crate::slot_queue`]). Read-only: +//! nothing is created, locked or signalled. What can't be read is a blind spot, named; an empty +//! list never stands for one it couldn't read. + +use std::path::PathBuf; + +use serde_json::{Value, json}; + +use crate::lock_holder::{Matched, Seen}; + +/// One lock as the brief shows it: held or not, and by what. +pub fn lock_row(lock: &str, seen: &Seen, owner: Option<&str>) -> Value { + if seen.holders.is_empty() { + return json!({"lock": lock, "held": false, "kind": "observed"}); + } + let holders: Vec = seen + .holders + .iter() + .map(|pid| { + let proc = seen.procs.get(pid); + let command = seen + .cmdlines + .get(pid) + .cloned() + .or_else(|| proc.map(|p| p.comm.clone())) + .unwrap_or_else(|| "unknown: its process couldn't be read".into()); + json!({ + "pid": pid, + "state": proc.map(|p| p.state.to_string()), + "command": command.chars().take(300).collect::(), + "cwd": seen.cwds.get(pid), + }) + }) + .collect(); + let mut row = json!({ + "lock": lock, "held": true, "holders": holders, "kind": "observed", + "owner": owner, "owner_kind": "declared: the first word of the lock file", + }); + if seen.matched == Matched::InodeOnly { + row["matched_by"] = json!( + "inode only: no line had the file's device, so a file elsewhere with the same inode \ + would match too" + ); + } + row +} + +/// The first word of a held lock file, where its holder wrote one (dsm-heavy does). +fn owner_of(lock: &std::path::Path) -> Option { + let text = std::fs::read_to_string(lock).ok()?; + let word = text.lines().next()?.split_whitespace().next()?; + Some(word.chars().take(40).collect()) +} + +/// The brief's `in_flight` section, from each project's lock patterns (or why the projects list +/// couldn't be read): every lock read, each fair pool's queue, and what couldn't be seen. +pub fn section(projects: Result)>, String>) -> Value { + let projects = match projects { + Ok(projects) => projects, + Err(why) => { + return json!({ + "projects": [], + "blind": [format!("the projects list: {why}, so no lock could be read")], + "note": NOTE, + }); + } + }; + let mut blind = Vec::new(); + let mut out = Vec::new(); + let mut held = 0; + let mut waiting = 0; + for (prefix, patterns) in &projects { + if patterns.is_empty() { + continue; + } + let mut locks = Vec::new(); + for lock in crate::lock_holder::expand(patterns) { + let shown = lock.display().to_string(); + match crate::lock_holder::read(&lock) { + Ok(seen) => { + let owner = (!seen.holders.is_empty()) + .then(|| owner_of(&lock)) + .flatten(); + held += usize::from(!seen.holders.is_empty()); + locks.push(lock_row(&shown, &seen, owner.as_deref())); + } + Err(why) => blind.push(format!("the lock {shown} ({prefix}): {why}")), + } + } + let (_, queues) = + crate::slot_queue::check_now(patterns, None, crate::slot_queue::DEFAULT_WAIT_AFTER_MS); + for queue in &queues { + if queue["checked"] == false { + blind.push(format!( + "the queue {} ({prefix}): {}", + queue["queue"].as_str().unwrap_or("?"), + queue["reason"].as_str().unwrap_or("can't see the queue") + )); + } + for owner in queue["waiting"].as_array().into_iter().flatten() { + waiting += owner["jobs"].as_u64().unwrap_or(0) as usize; + } + } + // Only the held locks are listed one by one: a pool's free slots are counted. + let free = locks.iter().filter(|l| l["held"] == false).count(); + let shown: Vec = locks.into_iter().filter(|l| l["held"] == true).collect(); + out.push(json!({"project": prefix, "held": shown, "free": free, "queues": queues})); + } + if out.is_empty() { + blind.push( + "no project's stalls in the projects list names a lock, so nothing here can say \ + what's running" + .to_owned(), + ); + } + json!({ + "projects": out, + "summary": format!( + "{held} lock{} held, {waiting} job{} waiting for a slot", + if held == 1 { "" } else { "s" }, + if waiting == 1 { "" } else { "s" } + ), + "blind": blind, + "note": NOTE, + }) +} + +const NOTE: &str = "The jobs a closed window would end (issues log 1.73), as /proc shows them now \ + (observed): each held lock's holder, and each fair pool's queue. Owners are what each job \ + wrote (declared). Only locks a project's stalls names are read; a job holding none isn't seen."; + +#[cfg(test)] +mod tests { + use super::*; + use crate::lock_holder::Proc; + use std::collections::BTreeMap; + + fn seen(holders: Vec) -> Seen { + Seen { + matched: Matched::DeviceAndInode, + holders, + procs: BTreeMap::from([( + 4242, + Proc { + pid: 4242, + ppid: 1, + state: 'S', + comm: "timeout".into(), + cpu_ticks: 0, + start_ticks: 0, + }, + )]), + cwds: BTreeMap::from([(4242, "/home/u/.cache/teamA/tree".to_owned())]), + cmdlines: BTreeMap::from([(4242, "timeout 2400 godot --headless".to_owned())]), + ..Seen::default() + } + } + + #[test] + fn a_held_lock_names_its_holders_pid_command_and_cwd_and_its_declared_owner() { + let row = lock_row("/x/heavy.2.lock", &seen(vec![4242]), Some("teamA")); + assert_eq!(row["held"], true); + assert_eq!( + row["holders"], + json!([{"pid": 4242, "state": "S", "command": "timeout 2400 godot --headless", + "cwd": "/home/u/.cache/teamA/tree"}]) + ); + assert_eq!(row["owner"], "teamA"); + assert_eq!(row["kind"], "observed"); + assert!(row["owner_kind"].as_str().unwrap().starts_with("declared")); + } + + #[test] + fn a_holder_whose_process_couldnt_be_read_says_so() { + let row = lock_row("/x/heavy.lock", &seen(vec![7]), None); + assert_eq!( + row["holders"][0]["command"], + "unknown: its process couldn't be read" + ); + assert!(row["owner"].is_null()); + assert_eq!( + lock_row("/x/heavy.lock", &seen(vec![]), None), + json!({"lock": "/x/heavy.lock", "held": false, "kind": "observed"}) + ); + } + + #[test] + fn what_cant_be_read_is_blind_never_nothing_running() { + let unread = section(Err("can't read /p/projects.json: denied".into())); + assert_eq!( + unread["blind"], + json!([ + "the projects list: can't read /p/projects.json: denied, so no lock could be read" + ]) + ); + let none = section(Ok(vec![("RAL".into(), vec![])])); + assert!( + none["blind"][0] + .as_str() + .unwrap() + .starts_with("no project's stalls in the projects list names a lock"), + "{none}" + ); + // A pattern matching nothing is kept as given, and its read fails: blind, by name. + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("none-*.lock"); + let gone = section(Ok(vec![("DSM".into(), vec![missing.clone()])])); + let said = gone["blind"][0].as_str().unwrap(); + assert!( + said.starts_with(&format!("the lock {} (DSM): ", missing.display())), + "{gone}" + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 31acb4c..766dd55 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,6 +36,7 @@ pub mod evidence; pub mod gate; pub mod gate_check; pub mod gate_log; +pub mod in_flight; pub mod inbox; pub mod init; pub mod ledger; diff --git a/src/main.rs b/src/main.rs index e97896c..14b8cc2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2331,6 +2331,23 @@ fn projects_text( } } +/// Each project's `stalls.locks` from the projects list beside the pairing (RAL-191), or why the +/// list couldn't be read. +fn lock_patterns(config: &Path) -> Result)>, String> { + let text = projects_text(None, config).map_err(|e| e.to_string())?; + let projects = projects_from(&text)?; + Ok(projects + .into_iter() + .map(|project| { + let locks = project + .stalls + .map(|stalls| stalls.locks.iter().map(PathBuf::from).collect()) + .unwrap_or_default(); + (project.prefix, locks) + }) + .collect()) +} + /// The `builds` section of the watchman's brief (RAL-167): the service's commit by its own word, /// this CLI's, main's tip, and what's merged but not live, all observed; never fatal to the brief. fn builds_section(location: Option<&Location>, config: &Path) -> Value { @@ -5309,7 +5326,10 @@ fn run(cli: Cli) -> Result> { rally::watchman_brief::from_service(reply_value(&reply)?)? }; let mut report = report; - report["builds"] = builds_section(location.as_ref(), &config_path(cli.config)?); + let config = config_path(cli.config)?; + report["builds"] = builds_section(location.as_ref(), &config); + // What a closed window would end (RAL-191): each project's locks and queues. + report["in_flight"] = rally::in_flight::section(lock_patterns(&config)); return Ok(report.to_string()); } let mut report = if let Some(location) = &location { diff --git a/tests/cli.rs b/tests/cli.rs index bca274a..e3d10e7 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2006,3 +2006,113 @@ fn wait_green_waits_for_every_check_the_base_had_and_says_how_it_ended() { let (code, _) = wait_green(d, &["not-a-pr"]); assert_eq!(code, 2); } + +/// RAL-191: `rally brief --watchman` lists the jobs a closed window would end: each lock a +/// project's stalls names, held, with its holder's pid, command and working directory (observed) +/// and the owner its file names (declared), and each fair pool's queue; and says what it couldn't +/// read rather than showing nothing running. Real `flock` holders, in a temporary folder. +#[cfg(target_os = "linux")] +#[test] +fn the_watchmans_brief_lists_the_jobs_in_flight() { + use std::time::{Duration, Instant}; + let db = Ledger::new(); + ok(&db, "silken-tanager", &["ready", "RAL-3"]); + let config = db.dir.path().join("client.json"); + let slots = db.dir.path().join("slots"); + let waiting = slots.join("heavy.waiting"); + std::fs::create_dir_all(&waiting).unwrap(); + let held = slots.join("heavy.1.lock"); + std::fs::write(&held, "teamA 2026-11-14T10:00:00 suite\n").unwrap(); + std::fs::write(slots.join("heavy.2.lock"), "").unwrap(); + let waiter = waiting.join("teamB.1.1"); + std::fs::write(&waiter, "teamB\n").unwrap(); + let mut holders: Vec = [&held, &waiter] + .iter() + .map(|path| { + Command::new("flock") + .arg(path) + .args(["sleep", "60"]) + .current_dir(&slots) + .spawn() + .unwrap() + }) + .collect(); + let projects = |stalls: &str| { + std::fs::write( + db.dir.path().join("projects.json"), + format!( + r#"[{{"name": "DiscoSim", "prefix": "DSM", "repo": "o/DiscoSim", "path": "r.md", + "tick_meaning": "done"{stalls}}}]"# + ), + ) + .unwrap(); + }; + let pattern = slots.join("heavy.*.lock").display().to_string(); + projects(&format!(r#", "stalls": {{"locks": ["{pattern}"]}}"#)); + let brief = || { + let out = rally( + &db, + "watch", + &["--config", config.to_str().unwrap(), "brief", "--watchman"], + ); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + serde_json::from_slice::(&out.stdout).unwrap()["in_flight"].clone() + }; + // Waits, up to 10 s, for flock to hold both. + let until = Instant::now() + Duration::from_secs(10); + let seen = loop { + let seen = brief(); + if seen["summary"] == "1 lock held, 1 job waiting for a slot" { + break seen; + } + assert!(Instant::now() < until, "not seen within 10 s: {seen}"); + std::thread::sleep(Duration::from_millis(200)); + }; + let dsm = &seen["projects"][0]; + assert_eq!(dsm["project"], "DSM", "{seen}"); + assert_eq!(dsm["free"], 1); + let row = &dsm["held"][0]; + assert_eq!(row["lock"], held.display().to_string()); + assert_eq!(row["owner"], "teamA"); + assert_eq!(row["kind"], "observed"); + let holder = &row["holders"][0]; + assert_eq!(holder["pid"], holders[0].id()); + assert_eq!( + holder["command"], + format!("flock {} sleep 60", held.display()) + ); + assert_eq!(holder["cwd"], slots.display().to_string()); + assert_eq!( + dsm["queues"][0]["waiting"], + serde_json::json!([{"owner": "teamB", "jobs": 1, "longest_wait_min": 0}]) + ); + assert_eq!(seen["blind"], serde_json::json!([]), "{seen}"); + for holder in &mut holders { + let _ = holder.kill(); + let _ = holder.wait(); + } + // No project names a lock: blind, never "nothing running". + projects(""); + let seen = brief(); + assert!( + seen["blind"][0] + .as_str() + .unwrap() + .starts_with("no project's stalls in the projects list names a lock"), + "{seen}" + ); + // The projects list can't be parsed: blind, saying why. + std::fs::write(db.dir.path().join("projects.json"), "not json").unwrap(); + let seen = brief(); + assert!( + seen["blind"][0] + .as_str() + .unwrap() + .starts_with("the projects list: "), + "{seen}" + ); +}