From 2a2ffa65af8db2f89c05045054b5f516680e45f3 Mon Sep 17 00:00:00 2001 From: theBlanketBaron Date: Sat, 14 Nov 2026 12:45:29 -0500 Subject: [PATCH] RAL-189: test scripts written without "Text file busy" A test that wrote a stand-in program with std::fs::write and ran it could fail with ETXTBSY: another thread's fork, between fork and exec, held a copy of the write descriptor (hosted CI, levi::tests::output_held_open_after_levi_ exits_doesnt_outlast_the_deadline on PR #275). rally::script::write writes the script through a child (sh, cat, chmod 755), so the test process never holds it open for writing; all 14 write-then-chmod sites use it, with a test that writes and runs a script 200 times beside four forking threads. Co-Authored-By: Claude Opus 5.5 --- docs/roadmap.md | 9 ++++ src/levi.rs | 4 +- src/lib.rs | 1 + src/script.rs | 111 +++++++++++++++++++++++++++++++++++++++++++++++ tests/cli.rs | 16 ++----- tests/service.rs | 8 +--- tests/shims.rs | 15 ++----- tests/web.rs | 37 ++++------------ 8 files changed, 138 insertions(+), 63 deletions(-) create mode 100644 src/script.rs diff --git a/docs/roadmap.md b/docs/roadmap.md index 5ed3622..e3f395b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1478,3 +1478,12 @@ principles named were durable, accurate and reactive. Found in rally 0x7 by plus slots; a waiting directory that exists while `/proc/locks` can't be read, or a `--queue` that isn't there, is a blind spot ("can't see the queue"), never 0 waiting; with unit tests and a CLI test with real `flock` holders (docs/stalls.md). +- [ ] **RAL-189: Test scripts written without "Text file busy".** From hosted CI on PR #275 (2026-11-14): + `levi::tests::output_held_open_after_levi_exits_doesnt_outlast_the_deadline` failed with ETXTBSY (os error + 26) running its fake levi. A test wrote the script with `std::fs::write` and ran it, and another thread's + fork, between its fork and exec, held a copy of the write descriptor. On Murph, writing and running a script + 300 times beside four forking threads failed 6 and 7 times in two runs. (The board-ages browser flake seen + on the same PR is earnest-raccoon's own story.) Done when: every test that writes a script and runs it writes + it with `rally::script::write`, which writes through a child process (`sh`, `cat`, `chmod 755`) so the test + process never holds the file open for writing; no write-then-chmod-755 of a script is left in `src/` or + `tests/`; with a test that writes and runs a script 200 times beside four forking threads and is never busy. diff --git a/src/levi.rs b/src/levi.rs index 9140a9c..0d472c7 100644 --- a/src/levi.rs +++ b/src/levi.rs @@ -367,11 +367,9 @@ mod tests { /// A stand-in for levi: a shell script in its own directory. fn fake_levi(script: &str) -> (tempfile::TempDir, PathBuf) { - use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("levi"); - std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + crate::script::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); (dir, path) } diff --git a/src/lib.rs b/src/lib.rs index 4bb7ebd..31acb4c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,7 @@ pub mod retry; pub mod roadmaps; #[doc(hidden)] pub mod scratch_pg; +pub mod script; pub mod secrets; pub mod server; pub mod session; diff --git a/src/script.rs b/src/script.rs new file mode 100644 index 0000000..c57b601 --- /dev/null +++ b/src/script.rs @@ -0,0 +1,111 @@ +//! Writing a script that's run straight after, without "Text file busy" (RAL-189). +//! +//! A test that writes a stand-in program (a fake `gh`, `levi` or `coredumpctl`) with +//! `std::fs::write` and then runs it can fail with ETXTBSY ("Text file busy", os error 26). The +//! file is open for writing in the test process for a moment, and if another thread of the same +//! process forks to start a program in that moment, the child holds a copy of that descriptor +//! until it execs. Running a file that any process holds open for writing is refused. The +//! descriptor is close-on-exec, which narrows the window but can't shut it, since the copy lives +//! between the fork and the exec. On 2026-11-14 hosted CI's `levi::tests:: +//! output_held_open_after_levi_exits_doesnt_outlast_the_deadline` failed so; on Murph, writing and +//! running a script 300 times beside four threads starting `true` failed 6 and 7 times in two runs. +//! +//! So the script is written by a child process (`sh`, `cat` and `chmod`), and this process never +//! holds the file open at all: no thread's fork can copy a descriptor that isn't there. By the time +//! [`write`] returns, the child has exited and nothing holds the file open for writing. The same +//! 300 runs this way failed none. The text goes to the child on its standard input. + +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +/// Writes `text` to `path` as an executable (mode 755), through a child process, so that it can be +/// run straight after even while other threads start programs. +pub fn write(path: impl AsRef, text: impl AsRef<[u8]>) -> std::io::Result<()> { + let path = path.as_ref(); + let mut child = Command::new("sh") + .args(["-c", "cat > \"$1\" && chmod 755 \"$1\"", "sh"]) + .arg(path) + .stdin(Stdio::piped()) + .spawn()?; + let sent = child.stdin.take().expect("piped").write_all(text.as_ref()); + let status = child.wait()?; + sent?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::other(format!( + "couldn't write the script {}: sh exited with {status}", + path.display() + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_script_is_written_whole_and_can_be_run() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a script, with spaces"); + write(&path, "#!/bin/sh\nprintf '%s' \"$1\"\n").unwrap(); + let out = Command::new(&path).arg("ran").output().unwrap(); + assert_eq!(out.stdout, b"ran"); + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o755); + // Written again, it's replaced, not added to. + write(&path, "#!/bin/sh\necho again\n").unwrap(); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "#!/bin/sh\necho again\n" + ); + } + + #[test] + fn a_script_where_none_can_be_written_is_an_error() { + let dir = tempfile::tempdir().unwrap(); + let err = write(dir.path().join("no-such-dir/x"), "x").unwrap_err(); + assert!( + err.to_string().contains("couldn't write the script"), + "{err}" + ); + } + + /// Written and run 200 times while four threads start programs without pause: never busy. + /// With `std::fs::write` and a chmod in place of [`write`], this failed a few in every + /// hundred on Murph. + #[test] + fn a_script_written_beside_forking_threads_is_never_busy() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + let stop = Arc::new(AtomicBool::new(false)); + let forkers: Vec<_> = (0..4) + .map(|_| { + let stop = stop.clone(); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + let _ = Command::new("true").status(); + } + }) + }) + .collect(); + let dir = tempfile::tempdir().unwrap(); + let mut busy = 0; + for i in 0..200 { + let path = dir.path().join(format!("s{i}")); + write(&path, "#!/bin/sh\nexit 0\n").unwrap(); + match Command::new(&path).status() { + Err(e) if e.raw_os_error() == Some(26) => busy += 1, + Err(e) => panic!("{e}"), + Ok(status) => assert!(status.success()), + } + } + stop.store(true, Ordering::Relaxed); + for forker in forkers { + forker.join().unwrap(); + } + assert_eq!(busy, 0, "runs refused as Text file busy"); + } +} diff --git a/tests/cli.rs b/tests/cli.rs index 1bf3376..bca274a 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -34,11 +34,9 @@ impl Ledger { fn quiet_gh() -> &'static std::path::Path { static DIR: std::sync::OnceLock = std::sync::OnceLock::new(); DIR.get_or_init(|| { - use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let gh = dir.path().join("gh"); - std::fs::write(&gh, "#!/bin/sh\necho '[]'\n").unwrap(); - std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(&gh, "#!/bin/sh\necho '[]'\n").unwrap(); dir }) .path() @@ -1726,9 +1724,7 @@ fn the_digest_says_at_once_when_a_merge_left_the_base_red() { *\"run list\"*\"--commit {OTHER_COMMIT}\"*) echo '{run}' ;;\n *) echo \"unexpected: $*\" >&2; exit 1 ;;\nesac\n" ); let write_gh = |text: &str| { - use std::os::unix::fs::PermissionsExt; - std::fs::write(bin.join("gh"), text).unwrap(); - std::fs::set_permissions(bin.join("gh"), std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(bin.join("gh"), text).unwrap(); }; write_gh(&script); let state = db.dir.path().join("digest.json"); @@ -1824,9 +1820,7 @@ fn the_digest_names_a_verified_story_whose_pull_request_is_behind_its_base() { *\"compare/main...{COMMIT}\"*) echo 3 ;;\n *) echo \"unexpected: $*\" >&2; exit 1 ;;\nesac\n" ); { - use std::os::unix::fs::PermissionsExt; - std::fs::write(bin.join("gh"), script).unwrap(); - std::fs::set_permissions(bin.join("gh"), std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(bin.join("gh"), script).unwrap(); } // The watchman's last digest covered everything up to the submission. let state = db.dir.path().join("digest.json"); @@ -1884,7 +1878,6 @@ fn the_digest_names_a_verified_story_whose_pull_request_is_behind_its_base() { /// the base's tip as `TIP`, and for each commit `runs-.json` and `status-.json`, or, on /// its n-th read of that kind, `runs--.json` where there is one, so checks can progress. fn wait_green_gh(dir: &std::path::Path) -> std::path::PathBuf { - use std::os::unix::fs::PermissionsExt; let bin = dir.join("bin"); std::fs::create_dir_all(&bin).unwrap(); let script = format!( @@ -1910,8 +1903,7 @@ esac tip = "e".repeat(40), parent = "f".repeat(40) ); - std::fs::write(bin.join("gh"), script).unwrap(); - std::fs::set_permissions(bin.join("gh"), std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(bin.join("gh"), script).unwrap(); bin } diff --git a/tests/service.rs b/tests/service.rs index 22f97fc..b20f467 100644 --- a/tests/service.rs +++ b/tests/service.rs @@ -4216,7 +4216,6 @@ fn a_fair_pools_queue_is_read_and_a_long_unfair_wait_is_a_prompt() { #[cfg(target_os = "linux")] #[test] fn stalls_take_a_projects_own_locks_and_window_and_can_say_nothing() { - use std::os::unix::fs::PermissionsExt; let service = start(); let pairing = service.add("a1", false); let config = service.dir.path().join("client/config.json"); @@ -4236,12 +4235,7 @@ fn stalls_take_a_projects_own_locks_and_window_and_can_say_nothing() { .unwrap(); // A shim that reads an empty roster, so the run sees everything it needs (RAL-174). let shim = service.dir.path().join("empty-roster-shim"); - std::fs::write( - &shim, - "#!/bin/sh\ncase \"$1\" in\n roster) printf '%s' '{\"read_at\": 1, \"sessions\": []}' ;;\n *) exit 2 ;;\nesac\n", - ) - .unwrap(); - std::fs::set_permissions(&shim, std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(&shim, "#!/bin/sh\ncase \"$1\" in\n roster) printf '%s' '{\"read_at\": 1, \"sessions\": []}' ;;\n *) exit 2 ;;\nesac\n").unwrap(); let stalls_with = |roster_from: Option<&str>, path: Option<&str>, extra: &[&str]| { let mut command = rally(); command diff --git a/tests/shims.rs b/tests/shims.rs index c12298e..cdf73b4 100644 --- a/tests/shims.rs +++ b/tests/shims.rs @@ -56,7 +56,6 @@ impl Rig { /// Writes an executable fake named `name` that answers as `fake`, and returns its path. fn fake(&self, case: &str, name: &str, fake: Fake) -> String { - use std::os::unix::fs::PermissionsExt; // Each fake's session is its own, so two populated rosters have two sessions. let roster = json!({"read_at": 1, "sessions": [{ "session_id": format!("{case}-{name}"), @@ -87,15 +86,10 @@ impl Rig { std::fs::create_dir_all(&folder).unwrap(); let path = folder.join(name); let log = self.log(); - std::fs::write( - &path, - format!( + rally::script::write(&path, format!( "#!/bin/sh\necho \"{name} $1\" >> '{}'\ncase \"$1\" in\n roster) {on_roster} ;;\n tell) {on_tell} ;;\n *) exit 2 ;;\nesac\n", log.display() - ), - ) - .unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + )).unwrap(); path.display().to_string() } @@ -139,14 +133,11 @@ impl Rig { /// PATH with a fake `gh` first, serving a roadmap with RAL-1 on it, for `rally wake`. fn path_with_gh(&self) -> String { - use std::os::unix::fs::PermissionsExt; let bin = self.dir.path().join("bin"); if !bin.exists() { std::fs::create_dir(&bin).unwrap(); let gh = "#!/bin/sh\ncase \"$*\" in\n *commits/HEAD*) echo 0123456789abcdef0123456789abcdef01234567 ;;\n *contents*) printf '%s\\n' '# Roadmap' '' '- [ ] **RAL-1: Something to do.** Done when: done.' ;;\n *) echo \"unexpected: $*\" >&2; exit 1 ;;\nesac\n"; - std::fs::write(bin.join("gh"), gh).unwrap(); - std::fs::set_permissions(bin.join("gh"), std::fs::Permissions::from_mode(0o755)) - .unwrap(); + rally::script::write(bin.join("gh"), gh).unwrap(); } format!( "{}:{}", diff --git a/tests/web.rs b/tests/web.rs index 142d421..5f6cf30 100644 --- a/tests/web.rs +++ b/tests/web.rs @@ -106,10 +106,8 @@ fn start_after_full( std::fs::write(&projects, projects_json).unwrap(); let bin = dir.path().join("bin"); if let Some(script) = fake_gh { - use std::os::unix::fs::PermissionsExt; std::fs::create_dir(&bin).unwrap(); - std::fs::write(bin.join("gh"), script).unwrap(); - std::fs::set_permissions(bin.join("gh"), std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(bin.join("gh"), script).unwrap(); } let status = on(&db, dir.path()) .args(["--credential", "a1", "--as", "someone"]) @@ -1197,10 +1195,9 @@ fn run_browser_env( /// says a fake assertion on standard error and exits with `status`; after that it runs the real /// command it's given. The command line for the test, with the real browser after it. fn chrome_that_fails(dir: &std::path::Path, aborts: u32, status: u32, chrome: &str) -> String { - use std::os::unix::fs::PermissionsExt; let script = dir.join("fake-chrome"); let counter = dir.join("runs"); - std::fs::write( + rally::script::write( &script, format!( "#!/bin/sh\nn=$(cat {counter} 2>/dev/null || echo 0)\necho $((n + 1)) > {counter}\n\ @@ -1213,7 +1210,6 @@ fn chrome_that_fails(dir: &std::path::Path, aborts: u32, status: u32, chrome: &s ), ) .unwrap(); - std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); format!("{} {chrome}", script.display()) } @@ -1273,18 +1269,10 @@ fn a_segfault_at_start_is_tried_once_more_and_a_second_fails() { // RAL-183: a stand-in coredumpctl lists a core at the moment the stand-in Chrome crashed, as // systemd would, with a pid the run never held: it's said as this run's crash, by its time. let cores = dir.path().join("coredumpctl"); - std::fs::write( - &cores, - format!( + rally::script::write(&cores, format!( "#!/bin/sh\nt=$(cat {})\necho \"[{{\\\"time\\\":$t,\\\"pid\\\":999999,\\\"sig\\\":11,\\\"exe\\\":\\\"/app/extra/chrome\\\"}}]\"\n", dir.path().join("crashed_at").display() - ), - ) - .unwrap(); - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&cores, std::fs::Permissions::from_mode(0o755)).unwrap(); - } + )).unwrap(); let (ok, seen, stderr) = run_browser_env( &viewer, &once, @@ -1331,23 +1319,16 @@ fn a_segfault_at_start_is_tried_once_more_and_a_second_fails() { /// error, with each Chrome core coredumpctl lists since the run began; the result stands. #[test] fn a_crash_at_teardown_is_said_and_the_result_stands() { - use std::os::unix::fs::PermissionsExt; let Some(chrome) = browser() else { return }; let viewer = start(); let dir = tempfile::tempdir().unwrap(); // Runs the real browser, then exits as a launcher does whose Chrome segfaulted on closing. let wrapper = dir.path().join("crashes-on-close"); - std::fs::write(&wrapper, "#!/bin/sh\n\"$@\"\nexit 139\n").unwrap(); - std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(&wrapper, "#!/bin/sh\n\"$@\"\nexit 139\n").unwrap(); // A stand-in coredumpctl: one Chrome core of another process, and a core that isn't Chrome's. let cores = dir.path().join("coredumpctl"); - std::fs::write( - &cores, - "#!/bin/sh\necho '[{\"time\":1000000,\"pid\":999999,\"sig\":11,\"exe\":\"/app/extra/chrome\"},\ - {\"time\":1000000,\"pid\":999998,\"sig\":11,\"exe\":\"/usr/bin/node\"}]'\n", - ) - .unwrap(); - std::fs::set_permissions(&cores, std::fs::Permissions::from_mode(0o755)).unwrap(); + rally::script::write(&cores, "#!/bin/sh\necho '[{\"time\":1000000,\"pid\":999999,\"sig\":11,\"exe\":\"/app/extra/chrome\"},\ + {\"time\":1000000,\"pid\":999998,\"sig\":11,\"exe\":\"/usr/bin/node\"}]'\n").unwrap(); let command = format!("{} {chrome}", wrapper.display()); let (ok, seen, stderr) = run_browser_env( &viewer, @@ -2810,15 +2791,13 @@ fn the_activity_strip_shows_who_is_working_and_what_just_changed() { /// Writes an executable `name` in `dir` that prints `stdout` and exits with `code`, standing in /// for a shim's `roster`. fn fake_roster_command(dir: &std::path::Path, name: &str, stdout: &str, code: i32) -> String { - use std::os::unix::fs::PermissionsExt; let path = dir.join(name); let body = stdout.replace('\'', "'\\''"); - std::fs::write( + rally::script::write( &path, format!("#!/bin/sh\nprintf '%s' '{body}'\nexit {code}\n"), ) .unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); path.display().to_string() }