From 4a2ef7aaa53386d4cf961d5f60b298193694607a Mon Sep 17 00:00:00 2001 From: theBlanketBaron Date: Sun, 15 Nov 2026 18:07:02 -0500 Subject: [PATCH] RAL-194: one walk of every process's open files for a pass; sh's exit status ahead of a broken pipe (a) Each lock whose taker had exited walked every process's /proc//fd for itself. A lock_holder::Pass now shares one walk across all the locks of a stalls or in_flight pass. Each process the walk names is still checked afresh by device and inode, and a lock the walk found nothing for is looked for alone before it's called a blind spot. (b) script::write returned the failed send (Broken pipe) ahead of sh's exit status when sh couldn't open the file and the text was too long for the pipe. The exit status now comes first, with the failed send beside it. Co-Authored-By: Claude Fable 5.2 --- docs/roadmap.md | 9 +++ docs/stalls.md | 7 ++ src/in_flight.rs | 8 +- src/lock_holder.rs | 186 +++++++++++++++++++++++++++++++++++++++++++-- src/script.rs | 38 +++++++-- 5 files changed, 231 insertions(+), 17 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index fb43f93..6567f54 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1521,3 +1521,12 @@ principles named were durable, accurate and reactive. Found in rally 0x7 by plus spot naming how many processes' open files couldn't be read, never "no holder"; the brief's `in_flight` shows the same; with a test that the stopped and possibly-hung checks prompt for a holder found this way (docs/stalls.md). +- [ ] **RAL-194: One walk of every process's open files for a pass, and sh's exit status ahead of a broken pipe.** + Follow-ups to RAL-192 and RAL-189, from jaunty-harrier (2026-11-14). (a) Each lock whose taker had exited + walked every process's `/proc//fd` for itself, so a pass over dsm-heavy's sixteen slots walked them sixteen + times, and the possibly-hung check twice that. (b) `script::write` returned the failed send ("Broken pipe") + ahead of sh's exit status when sh couldn't open the file and the text was too long for the pipe. Done when: + `rally stalls` and the brief's `in_flight` walk every process's open files once for all the locks of a pass, + each lock still naming its own holders, with a test that three locks held by descriptor are read with one walk; + and `script::write` reports sh's exit status first, with the failed send beside it, with a test of a text too + long for the pipe (docs/stalls.md). diff --git a/docs/stalls.md b/docs/stalls.md index 5b29a45..3490448 100644 --- a/docs/stalls.md +++ b/docs/stalls.md @@ -127,6 +127,13 @@ user's, or a sandboxed one's). Under `--prompts-only` that fails the run (RAL-17 some processes' files couldn't be read, `unreadable_processes` says how many. Kernel threads open no files and aren't counted. Only processes this user can see are read. +Every process's open files are walked once for a pass, not once for each lock (RAL-194): the first lock of a pass +that needs it walks them for all the pass's locks, and the others look theirs up. A pass is one reading of +`rally stalls`' locks, or of the brief's `in_flight` over every project's; the possibly-hung check's second sample +is a pass of its own. Each process the walk names is still checked afresh by device and inode, and a lock the +walk found nothing for is looked for alone before it's called a blind spot, since its holder may be newer than +the walk. + ## A fair slot pool's queue (RAL-188) From the DiscoSim rally's issues log, 1.84: one team's 25 `dsm-heavy` jobs took most of the heavy slots, and diff --git a/src/in_flight.rs b/src/in_flight.rs index 13cb1e3..59754d4 100644 --- a/src/in_flight.rs +++ b/src/in_flight.rs @@ -88,6 +88,12 @@ pub fn section(projects: Result)>, String>) -> Value { let mut out = Vec::new(); let mut held = 0; let mut waiting = 0; + // One pass over every project's locks: each process's open files are walked once (RAL-194). + let every: Vec = projects + .iter() + .flat_map(|(_, patterns)| crate::lock_holder::expand(patterns)) + .collect(); + let mut pass = crate::lock_holder::Pass::over(&every); for (prefix, patterns) in &projects { if patterns.is_empty() { continue; @@ -95,7 +101,7 @@ pub fn section(projects: Result)>, String>) -> Value { let mut locks = Vec::new(); for lock in crate::lock_holder::expand(patterns) { let shown = lock.display().to_string(); - match crate::lock_holder::read(&lock) { + match crate::lock_holder::read_in(&mut pass, &lock) { Ok(seen) => { let owner = (!seen.holders.is_empty()) .then(|| owner_of(&lock)) diff --git a/src/lock_holder.rs b/src/lock_holder.rs index 5728f82..8d4f7bf 100644 --- a/src/lock_holder.rs +++ b/src/lock_holder.rs @@ -207,6 +207,90 @@ fn open_on(path: &Path, dev: u64, ino: u64, pids: impl Iterator) -> (found, unreadable) } +/// One pass over a set of locks (RAL-194): what's read of every process once and shared by each +/// lock read in it. Before, each lock whose taker had exited walked every process's +/// `/proc//fd` for itself, so a pass over dsm-heavy's sixteen slots walked them sixteen times. +/// Now the first such lock walks them once, for every lock of the pass, and the rest look theirs +/// up. A pass is one moment's reading: start a new one to read again later, as the possibly-hung +/// check's second sample does. +#[derive(Debug, Default)] +pub struct Pass { + /// The locks of this pass, canonical: the files the one walk looks for. + wanted: BTreeSet, + scan: Option, + /// How many times every process's open files were walked: one for a pass, however many + /// locks it reads. + pub scans: usize, +} + +/// One walk of every process's open files. +#[derive(Debug, Default)] +struct Scan { + /// The files it looked for. + looked_for: BTreeSet, + /// For each of them, the processes with a descriptor whose link names it. + open: BTreeMap>, + /// How many processes' open files couldn't be read. + unreadable: usize, +} + +impl Pass { + /// A pass over these locks. A lock read in the pass without being named here is still read + /// rightly, at the cost of another walk. + pub fn over(locks: &[std::path::PathBuf]) -> Pass { + Pass { + wanted: locks + .iter() + .map(|lock| std::fs::canonicalize(lock).unwrap_or_else(|_| lock.clone())) + .collect(), + ..Pass::default() + } + } + + /// The processes among `pids` whose open files name `path`, from this pass's one walk, with + /// how many processes' open files couldn't be read. + #[cfg(target_os = "linux")] + fn open_by_link(&mut self, path: &Path, pids: &[u32]) -> (Vec, usize) { + self.wanted.insert(path.to_path_buf()); + if !self + .scan + .as_ref() + .is_some_and(|scan| scan.looked_for.contains(path)) + { + let mut scan = Scan { + looked_for: self.wanted.clone(), + ..Scan::default() + }; + for &pid in pids { + let Ok(fds) = std::fs::read_dir(format!("/proc/{pid}/fd")) else { + // Gone since the listing, or another user's: only the second is a blind spot. + if Path::new(&format!("/proc/{pid}")).exists() { + scan.unreadable += 1; + } + continue; + }; + for fd in fds.flatten() { + if let Ok(target) = std::fs::read_link(fd.path()) + && scan.looked_for.contains(&target) + { + let holding = scan.open.entry(target).or_default(); + if !holding.contains(&pid) { + holding.push(pid); + } + } + } + } + self.scans += 1; + self.scan = Some(scan); + } + let scan = self.scan.as_ref().expect("just walked"); + ( + scan.open.get(path).cloned().unwrap_or_default(), + scan.unreadable, + ) + } +} + /// Of the processes holding a file open, those whose parent doesn't hold it too: the top of each /// tree, so a job and its children are one holder. fn topmost(holding: &[u32], procs: &BTreeMap) -> Vec { @@ -420,9 +504,15 @@ pub const BY_DESCRIPTOR: &str = "open descriptor: the process /proc/locks names had exited or didn't have the file open, so the holders are the topmost processes with an \ open descriptor on it, by device and inode"; -/// Reads one lock's holders, waiters and the processes on this machine, from `/proc`. -#[cfg(target_os = "linux")] +/// Reads one lock's holders, waiters and the processes on this machine, from `/proc`, in a pass +/// of its own. pub fn read(lock: &Path) -> Result { + read_in(&mut Pass::default(), lock) +} + +/// [`read`], as one lock of a [`Pass`], sharing the pass's walk of every process's open files. +#[cfg(target_os = "linux")] +pub fn read_in(pass: &mut Pass, lock: &Path) -> Result { use std::os::unix::fs::MetadataExt; let meta = std::fs::metadata(lock).map_err(|e| format!("can't read the lock file: {e}"))?; let dev = meta.dev(); @@ -465,12 +555,22 @@ pub fn read(lock: &Path) -> Result { let (kernel_open, _) = open_on(&canonical, dev, meta.ino(), seen.holders.iter().copied()); if seen.holders.iter().any(|pid| !kernel_open.contains(pid)) { // Kernel threads (kthreadd, pid 2, and its children) open no files and show no fd list. - let user_space = seen + let user_space: Vec = seen .procs .values() .filter(|proc| proc.pid != 2 && proc.ppid != 2) - .map(|proc| proc.pid); - let (holding, unreadable) = open_on(&canonical, dev, meta.ino(), user_space); + .map(|proc| proc.pid) + .collect(); + // The pass's one walk says which processes' descriptors name the file; each of those is + // then checked now, by device and inode (RAL-194). + let (named, unreadable) = pass.open_by_link(&canonical, &user_space); + let (mut holding, _) = open_on(&canonical, dev, meta.ino(), named.into_iter()); + let mut unreadable = unreadable; + if holding.is_empty() { + // The walk may be older than this lock's holder, when another lock of the pass + // made it first: before calling it a blind spot, look for this lock alone. + (holding, unreadable) = open_on(&canonical, dev, meta.ino(), user_space.into_iter()); + } seen.unreadable = unreadable; if holding.is_empty() { let gone: Vec = seen.holders.iter().map(u32::to_string).collect(); @@ -519,7 +619,7 @@ pub fn read(lock: &Path) -> Result { } #[cfg(not(target_os = "linux"))] -pub fn read(_lock: &Path) -> Result { +pub fn read_in(_pass: &mut Pass, _lock: &Path) -> Result { Err("a lock's holder can only be read on Linux".into()) } @@ -562,10 +662,15 @@ pub fn check_with( let mut prompts = Vec::new(); let mut seen_all = Vec::new(); let locks = expand(patterns); - let first: Vec> = locks.iter().map(|lock| read(lock)).collect(); + // Each sample is one pass: every process's open files are walked once for all its locks. + let sample = || -> Vec> { + let mut pass = Pass::over(&locks); + locks.iter().map(|lock| read_in(&mut pass, lock)).collect() + }; + let first = sample(); let second: Option>> = hung_check.map(|h| { std::thread::sleep(std::time::Duration::from_millis(h.sample_ms)); - locks.iter().map(|lock| read(lock)).collect() + sample() }); let ticks = rustix::param::clock_ticks_per_second(); for (index, (lock, first)) in locks.iter().zip(first).enumerate() { @@ -934,4 +1039,69 @@ mod tests { ); assert_eq!(seen[0]["holder"], "no holder found in /proc/locks"); } + + /// RAL-194: three locks whose takers exited, each held by a job with the inherited + /// descriptor, are read in one pass with one walk of every process's open files, and each + /// names its own job. + #[cfg(target_os = "linux")] + #[test] + fn a_pass_walks_every_processs_open_files_once_for_all_its_locks() { + use std::os::unix::process::CommandExt; + use std::process::Command; + let dir = tempfile::tempdir().unwrap(); + let mut jobs = Vec::new(); + let mut locks = Vec::new(); + let mut groups = Vec::new(); + for n in 1..=3 { + let lock = dir.path().join(format!("heavy.{n}.lock")); + let said = dir.path().join(format!("job.{n}.pid")); + // The shell takes the lock, starts a job that inherits the descriptor, and waits to + // be ended by the guard; a second shell inside is the taker, which exits. + let Ok(child) = Command::new("sh") + .args([ + "-c", + "sh -c 'exec 9>>\"$0\"; flock 9 || exit 1; sleep 60 /dev/null 2>&1 & \ + echo $! > \"$1.tmp\"; mv \"$1.tmp\" \"$1\"' \"$0\" \"$1\"; sleep 60", + ]) + .arg(&lock) + .arg(&said) + .process_group(0) + .spawn() + else { + eprintln!("no sh here; skipped"); + return; + }; + groups.push(Group(child)); + let mut job = None; + for _ in 0..200 { + if let Ok(text) = std::fs::read_to_string(&said) { + job = text.trim().parse::().ok(); + break; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + let Some(job) = job else { + eprintln!("no flock(1) here; skipped"); + return; + }; + jobs.push(job); + locks.push(lock); + } + let mut pass = Pass::over(&locks); + for (lock, job) in locks.iter().zip(&jobs) { + let seen = read_in(&mut pass, lock).unwrap(); + assert!(seen.by_descriptor, "{seen:?}"); + assert_eq!(seen.holders, [*job]); + assert_eq!(seen.kernel_holders.len(), 1); + } + assert_eq!(pass.scans, 1, "one walk for the three locks"); + // A lock the pass wasn't told of is still read rightly, with a walk of its own. + let mut unnamed = Pass::default(); + for (lock, job) in locks.iter().zip(&jobs).take(2) { + assert_eq!(read_in(&mut unnamed, lock).unwrap().holders, [*job]); + } + assert_eq!(unnamed.scans, 2); + // And one read alone does as before. + assert_eq!(read(&locks[2]).unwrap().holders, [jobs[2]]); + } } diff --git a/src/script.rs b/src/script.rs index c57b601..dbc936d 100644 --- a/src/script.rs +++ b/src/script.rs @@ -30,15 +30,20 @@ pub fn write(path: impl AsRef, text: impl AsRef<[u8]>) -> std::io::Result< .spawn()?; let sent = child.stdin.take().expect("piped").write_all(text.as_ref()); let status = child.wait()?; - sent?; - if status.success() { - Ok(()) - } else { - Err(std::io::Error::other(format!( - "couldn't write the script {}: sh exited with {status}", - path.display() - ))) + // sh's own failure first (RAL-194): when it can't open the file it exits without reading, and + // a text too long for the pipe then fails to send with "Broken pipe", which says nothing of + // why. The exit status does. + if !status.success() { + return Err(std::io::Error::other(format!( + "couldn't write the script {}: sh exited with {status}{}", + path.display(), + match &sent { + Err(error) => format!(" (and sending the text failed: {error})"), + Ok(()) => String::new(), + } + ))); } + sent } #[cfg(test)] @@ -73,6 +78,23 @@ mod tests { ); } + /// RAL-194: a text too long for the pipe, to where sh can't write: sh exits without reading, + /// the send breaks, and the error is still sh's exit status, with the broken pipe beside it. + #[test] + fn shs_exit_status_is_reported_ahead_of_a_broken_pipe() { + let dir = tempfile::tempdir().unwrap(); + let long = vec![b'#'; 4 * 1024 * 1024]; + let err = write(dir.path().join("no-such-dir/x"), &long).unwrap_err(); + let text = err.to_string(); + assert!(text.contains("couldn't write the script"), "{text}"); + assert!(text.contains("sh exited with exit status: "), "{text}"); + assert!(text.contains("sending the text failed"), "{text}"); + // A long text where it can be written is written whole. + let path = dir.path().join("long"); + write(&path, &long).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), long); + } + /// Written and run 200 times while four threads start programs without pause: never busy. /// With `std::fs::write` and a chmod in place of [`write`], this failed a few in every /// hundred on Murph.