From a408fbbb862960ef78df36643f2073d2987ef972 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 2 Sep 2026 16:59:46 -0600 Subject: [PATCH 01/12] Rebuild the bookmark store as a two-slot locker A store now writes every M.2 slot and acks only when all of them hold the record. A load adopts only when the slots agree, restores from the survivor when one is missing (an M.2 swap), and otherwise discards, with one exception: every write carries a nonce drawn once per process, and slots that disagree under one nonce were torn by a single process, so the higher sequence number is that process's newest write. Adopting it is safe because a store that never returned had no effects. The locker is tenant-generic; the bookmark tenant maps a discard to a fresh identity rather than risk resuming a stale one. Co-Authored-By: Claude Mythos 5 --- server/src/bookmark.rs | 472 ++++++--------------------- server/src/lib.rs | 1 + server/src/locker.rs | 628 ++++++++++++++++++++++++++++++++++++ server/tests/distributed.rs | 21 +- 4 files changed, 740 insertions(+), 382 deletions(-) create mode 100644 server/src/locker.rs diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index 7ccd8a9..756c0e7 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -4,143 +4,91 @@ //! Durable gossip peer identity across restarts. //! -//! A rumors [`Bookmark`] records who a peer is and how far it has -//! advanced, so a restarted sled reclaims its old identity instead of -//! stranding it. Rumors owns the record format and decides when to load -//! and store. We supply raw byte storage obeying two constraints: stores -//! are atomic, and a load never returns a record older than the newest -//! store we reported `Ok` (stale records corrupt causality, whereas -//! lost records merely strand identities). +//! A rumors [`Bookmark`] records a peer's identity and how far it has +//! advanced, so that a restarted sled may reclaim its previous identity +//! instead of stranding it. The invariant is (as usual) that we must +//! never adopt stale data, because in this case it could lead to causality +//! violations (which are bad). //! -//! Storage is one small file per configured (M.2) slot. Loads read every -//! slot, and take the record with the highest sequence number; that slot -//! becomes the *home*. Stores go only to the home, since writing both -//! would either make the server dependent on the health of both or, done -//! merely best-effort, let a stale record load after a fresher disk dies, -//! violating the constraint above. The slots must never both be written -//! by live peers, and a record must never be restored from a backup. -//! A [`BookmarkSource`] hands out one handle per peer, with generation -//! numbers ensuring that a straggler from an abandoned universe can't -//! clobber its successor's record. - -use std::fs::Permissions; -use std::io::{self, Cursor, Write as _}; -use std::os::unix::fs::PermissionsExt as _; +//! The record format and when to load & store are dictated by rumors. +//! We use a [`Tenant`] of a [`Locker`] to store it on disk(s); +//! if a load fails or the slots disagree, we assume a new identity +//! rather than risk resuming with a stale one. Generation numbers +//! ensure that a straggler from an abandoned universe can't clobber +//! its successor's record. + +use std::io::{self, Cursor}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use atomicwrites::{AtomicFile, OverwriteBehavior}; -use camino::{Utf8Path, Utf8PathBuf}; use rumors::{Bookmark, BookmarkError, Serialized}; -use slog::{Logger, o, warn}; +use slog::{Discard, Logger, o, warn}; use thiserror::Error; -use tokio::fs::read; use tokio::io::AsyncWrite; -use tokio::sync::Mutex; -use tokio::task::spawn_blocking; - -/// The envelope magic. The payload includes its own magic. -/// A change to our envelope means a new magic. -const MAGIC: &[u8; 12] = b"SUSHBOOKMARK"; - -/// Envelope layout: magic, big-endian sequence number, digest of the -/// sequence number and record together, record. The digest keeps a -/// damaged sequence number from silently reordering the slots. -const SEQ_LEN: usize = 8; -const DIGEST_LEN: usize = 32; - -fn digest(seq: u64, record: &[u8]) -> [u8; DIGEST_LEN] { - let mut hasher = sush_common::hash::Hasher::new(); - hasher.update(&seq.to_be_bytes()); - hasher.update(record); - *hasher.finalize().as_bytes() -} + +use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; + +pub const BOOKMARK: TenantSpec = TenantSpec { + file: "sush-bookmark", + magic: b"SUSHBOOKMARK", +}; /// What a bookmark load or store failed at. #[derive(Debug, Error)] pub enum BookmarkIoError { - #[error("bookmark I/O failed on `{path}`: {error}")] - Io { - path: Utf8PathBuf, - #[source] - error: io::Error, - }, - #[error("every bookmark slot is corrupt (last: `{path}`)")] - Corrupt { path: Utf8PathBuf }, #[error("serializing the bookmark record failed: {0}")] Serialize(#[source] io::Error), - #[error("no bookmark slot is writable")] - NoSlot, + #[error("storing the bookmark failed: {0}")] + Store(#[source] StoreError), #[error("the bookmark was handed to a newer peer")] - Fenced, + Superseded, } /// This server's bookmark storage. Hands out one handle per peer, /// each superseding the last. #[derive(Clone, Debug)] pub struct BookmarkSource { - shared: Arc, + ratchet: Arc, } #[derive(Debug)] -struct SharedStore { +struct Ratchet { log: Logger, - /// Candidate record files, one per boot M.2. Empty means this - /// server persists no identity (the standalone server, tests). - slots: Vec, - /// The newest generation. A handle from an older one may load - /// but not store. + locker: Locker, + tenant: Tenant, generation: AtomicU64, - /// Serializes loads and stores across handles, so the newest - /// record on disk is always the newest store anyone `Ok`'d. - state: Mutex, - /// The newest sequence number committed to disk by this process, - /// under the lock every rename takes. Renames commit in sequence - /// order or not at all; see [`commit`]. - committed: std::sync::Mutex, -} - -#[derive(Debug, Default)] -struct StoreState { - /// The slot holding the newest record, once known. - home: Option, - /// The sequence number of the newest record. - seq: u64, } impl BookmarkSource { - /// A source persisting to `slots`, each on its own device. - /// - /// Construct exactly one source per slot set per process, and feed - /// that same source to both [`seed_gossip`](crate::seed_gossip) - /// and [`spawn_gossip`](crate::gossip::spawn_gossip): everything - /// serializing the store lives inside it. The caller creates the - /// parent directories, one per boot M.2, writable by this server's - /// user. The record files are created and owned here. - pub fn new(log: &Logger, slots: Vec) -> Self { + /// A source persisting to `locker`. Construct exactly one source + /// per locker per process, and feed that same source to both + /// [`seed_gossip`](crate::seed_gossip) and + /// [`spawn_gossip`](crate::gossip::spawn_gossip). + /// A handle is disabled when its source hands out a newer one. + /// No source can disable another source's handles, so a second + /// one would let an old peer overwrite its replacement's record. + pub fn new(log: &Logger, locker: &Locker) -> Self { Self { - shared: Arc::new(SharedStore { + ratchet: Arc::new(Ratchet { log: log.new(o!("component" => "bookmark")), - slots, + locker: locker.clone(), + tenant: locker.tenant(BOOKMARK), generation: AtomicU64::new(0), - state: Mutex::new(StoreState::default()), - committed: std::sync::Mutex::new(0), }), } } /// A source that loads and persists nothing. pub fn null() -> Self { - Self::new(&Logger::root(slog::Discard, o!()), Vec::new()) + Self::new(&Logger::root(Discard, o!()), &Locker::null()) } /// A ratcheting handle for the next peer. /// All earlier handles are superseded. pub fn next_handle(&self) -> SushBookmark { - let generation = self.shared.generation.fetch_add(1, Ordering::SeqCst) + 1; SushBookmark { - shared: self.shared.clone(), - generation, + ratchet: self.ratchet.clone(), + generation: self.ratchet.generation.fetch_add(1, Ordering::SeqCst) + 1, shed: false, } } @@ -149,49 +97,17 @@ impl BookmarkSource { /// gossiping after its real bookmark failed. pub fn shed_handle(&self) -> SushBookmark { SushBookmark { - shared: self.shared.clone(), + ratchet: self.ratchet.clone(), generation: 0, shed: true, } } - /// A probing handle: reads like the current peer's, but without - /// superseding anything. - fn probe_handle(&self) -> SushBookmark { - SushBookmark { - shared: self.shared.clone(), - generation: self.shared.generation.load(Ordering::SeqCst), - shed: false, - } - } - - /// Does the storage work? Reads every slot and proves at least one - /// writable by writing. - pub async fn probe(&self) -> Result<(), BookmarkIoError> { - if self.shared.slots.is_empty() { - return Ok(()); - } - let usable = match self.probe_handle().load().await { - Ok(_) => { - let mut writable = false; - for path in &self.shared.slots { - let probe = path.with_extension("probe"); - if tokio::fs::write(&probe, b"").await.is_ok() { - let _ = tokio::fs::remove_file(&probe).await; - writable = true; - break; - } - } - if writable { - Ok(()) - } else { - Err(BookmarkIoError::NoSlot) - } - } - Err(error) => Err(error), - }; + /// Does the storage work? + pub async fn probe(&self) -> Result<(), StoreError> { + let usable = self.ratchet.locker.probe().await; if let Err(error) = &usable { - warn!(self.shared.log, "no usable bookmark storage"; "error" => %error); + warn!(self.ratchet.log, "no usable bookmark storage"; "error" => %error); } usable } @@ -200,7 +116,7 @@ impl BookmarkSource { /// One peer's handle on the [`BookmarkSource`]. #[derive(Debug)] pub struct SushBookmark { - shared: Arc, + ratchet: Arc, generation: u64, shed: bool, } @@ -208,50 +124,10 @@ pub struct SushBookmark { impl SushBookmark { /// Has this bookmark been overtaken by events? fn obe(&self) -> bool { - self.generation < self.shared.generation.load(Ordering::SeqCst) - } - - /// Split a checksummed envelope into its sequence number and record. - fn parse(bytes: &[u8]) -> Option<(u64, Vec)> { - let payload = bytes.strip_prefix(MAGIC)?; - let (seq, rest) = payload.split_first_chunk::()?; - let (sum, record) = rest.split_first_chunk::()?; - let seq = u64::from_be_bytes(*seq); - (digest(seq, record) == *sum).then(|| (seq, record.to_vec())) - } - - /// Build the checksummed envelope around `record`. - fn envelope(seq: u64, record: &[u8]) -> Vec { - let mut bytes = Vec::with_capacity(MAGIC.len() + SEQ_LEN + DIGEST_LEN + record.len()); - bytes.extend_from_slice(MAGIC); - bytes.extend_from_slice(&seq.to_be_bytes()); - bytes.extend_from_slice(&digest(seq, record)); - bytes.extend_from_slice(record); - bytes + self.generation < self.ratchet.generation.load(Ordering::SeqCst) } } -/// Rename `envelope` into place iff `seq` is newer than everything -/// committed by this process. A store future dropped at its await -/// detaches the blocking write, whose rename would otherwise land on -/// top of the newer record that beat it. Runs on the blocking pool. -fn commit(shared: &SharedStore, path: &Utf8Path, seq: u64, envelope: &[u8]) -> io::Result<()> { - let mut committed = shared.committed.lock().unwrap(); - if seq <= *committed { - return Err(io::Error::other("superseded by a newer record")); - } - AtomicFile::new(path, OverwriteBehavior::AllowOverwrite) - .write(|file| { - file.set_permissions(Permissions::from_mode(0o600))?; - file.write_all(envelope) - }) - .map_err(|error| match error { - atomicwrites::Error::Internal(error) | atomicwrites::Error::User(error) => error, - })?; - *committed = seq; - Ok(()) -} - impl BookmarkError for SushBookmark { type Error = BookmarkIoError; } @@ -260,111 +136,39 @@ impl Bookmark for SushBookmark { type Reader = Cursor>; async fn load(&self) -> Result, Self::Error> { - if self.shed || self.shared.slots.is_empty() { + if self.shed { return Ok(None); } + let mut guard = self.ratchet.tenant.lock().await; if self.obe() { - return Err(BookmarkIoError::Fenced); + return Err(BookmarkIoError::Superseded); } - let mut state = self.shared.state.lock().await; - let mut newest: Option<(u64, usize, Vec)> = None; - let mut corrupt: Option<&Utf8Path> = None; - for (index, path) in self.shared.slots.iter().enumerate() { - let bytes = match read(path).await { - Ok(bytes) => bytes, - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) => { - return Err(BookmarkIoError::Io { - path: path.clone(), - error, - }); - } - }; - match Self::parse(&bytes) { - Some((seq, record)) => { - if newest.as_ref().is_none_or(|(newest, ..)| seq > *newest) { - newest = Some((seq, index, record)); - } - } - None => { - warn!( - self.shared.log, "skipping corrupt bookmark slot"; - "path" => %path, - ); - corrupt = Some(path); - } + match guard.load().await { + Verdict::Adopt(record) | Verdict::Restore(record) => Ok(Some(Cursor::new(record))), + Verdict::Empty => Ok(None), + Verdict::Discard(reason) => { + warn!(self.ratchet.log, "assuming a fresh identity"; "reason" => %reason); + Ok(None) } } - match newest { - Some((seq, home, record)) => { - // A reserved sequence number outranks a re-read of the - // disk. Regressing would let a cancelled write's - // straggler collide with a fresh reservation. - if seq >= state.seq { - state.home = Some(home); - state.seq = seq; - } - Ok(Some(Cursor::new(record))) - } - // A present-but-unreadable record is an error: - // rumors must not mistake it for a fresh start. - None => match corrupt { - Some(path) => Err(BookmarkIoError::Corrupt { path: path.into() }), - None => Ok(None), - }, - } } async fn store(&self, write: F) -> Result<(), Self::Error> where F: for<'a> FnOnce(&'a mut (dyn AsyncWrite + Unpin + Send)) -> Serialized<'a> + Send, { - if self.shed || self.shared.slots.is_empty() { + if self.shed { return Ok(()); } - let mut buf = Cursor::new(Vec::new()); write(&mut buf).await.map_err(BookmarkIoError::Serialize)?; let record = buf.into_inner(); - let mut state = self.shared.state.lock().await; + let mut guard = self.ratchet.tenant.lock().await; if self.obe() { - return Err(BookmarkIoError::Fenced); - } - let home = match state.home { - Some(home) => home, - None => self - .shared - .slots - .iter() - .position(|path| { - path.parent() - .is_some_and(|parent| parent.as_std_path().is_dir()) - }) - .ok_or(BookmarkIoError::NoSlot)?, - }; - let path = self.shared.slots[home].clone(); - - // Reserve the sequence number first, since a cancelled write - // may still land and must be outnumbered. - state.seq += 1; - let seq = state.seq; - let envelope = Self::envelope(seq, &record); - - let shared = self.shared.clone(); - let target = path.clone(); - let written = spawn_blocking(move || commit(&shared, &target, seq, &envelope)) - .await - .map_err(|join| io::Error::other(join.to_string())) - .and_then(|result| result); - - match written { - Ok(()) => { - state.home = Some(home); - Ok(()) - } - Err(error) => Err(BookmarkIoError::Io { path, error }), + return Err(BookmarkIoError::Superseded); } + guard.store(&record).await.map_err(BookmarkIoError::Store) } } @@ -372,7 +176,7 @@ impl Bookmark for SushBookmark { mod test { use super::*; - use std::fs::{create_dir, metadata, read, write}; + use std::fs::create_dir; use camino::Utf8PathBuf; use tempfile::TempDir; @@ -385,20 +189,24 @@ mod test { move |w| Box::pin(async move { w.write_all(bytes).await }) } - /// Two slot paths in separate directories, like two M.2s. + /// Two slot directories, like two M.2s. fn slots(dir: &TempDir) -> Vec { ["m2a", "m2b"] .iter() .map(|m2| { - let parent = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); - create_dir(&parent).unwrap(); - parent.join("bookmark") + let slot = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); + create_dir(&slot).unwrap(); + slot }) .collect() } - fn envelope(seq: u64, record: &[u8]) -> Vec { - SushBookmark::envelope(seq, record) + fn test_log() -> Logger { + Logger::root(Discard, o!()) + } + + fn source(slots: Vec) -> BookmarkSource { + BookmarkSource::new(&test_log(), &Locker::new(&test_log(), slots)) } async fn read_back(handle: &SushBookmark) -> Option> { @@ -408,157 +216,73 @@ mod test { Some(bytes) } - fn test_log() -> Logger { - Logger::root(slog::Discard, o!()) - } - - /// A stored record loads back verbatim, sequenced and private. + /// A stored record loads back verbatim. #[tokio::test] async fn round_trip() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - let source = BookmarkSource::new(&test_log(), slots.clone()); + let source = source(slots(&dir)); let handle = source.next_handle(); assert!(read_back(&handle).await.is_none()); handle.store(record(b"who we are")).await.unwrap(); assert_eq!(read_back(&handle).await.unwrap(), b"who we are"); - - let bytes = read(&slots[0]).unwrap(); - assert_eq!(bytes, envelope(1, b"who we are")); - let mode = metadata(&slots[0]).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o600); } - /// The newest record wins the load regardless of slot, and its - /// slot becomes the home every store then writes. + /// A discarded verdict is a fresh start, not an error. #[tokio::test] - async fn newest_slot_is_home() { + async fn discard_mints_a_fresh_identity() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); let slots = slots(&dir); - write(&slots[0], envelope(5, b"stale")).unwrap(); - write(&slots[1], envelope(9, b"fresh")).unwrap(); - - let source = BookmarkSource::new(&test_log(), slots.clone()); - let handle = source.next_handle(); - assert_eq!(read_back(&handle).await.unwrap(), b"fresh"); - - handle.store(record(b"fresher")).await.unwrap(); - assert_eq!(read(&slots[0]).unwrap(), envelope(5, b"stale")); - assert_eq!(read(&slots[1]).unwrap(), envelope(10, b"fresher")); + for (slot, bytes) in slots.iter().zip([b"one", b"two"]) { + let lone = source(vec![slot.clone()]); + lone.next_handle().store(record(bytes)).await.unwrap(); + } + assert!(read_back(&source(slots).next_handle()).await.is_none()); } - /// Minting a new handle fences the old one's stores. + /// A new handle disables the old one's loads and stores. #[tokio::test] - async fn stale_generations_cannot_store() { + async fn stale_generations_are_disabled() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let source = BookmarkSource::new(&test_log(), slots(&dir)); + let source = source(slots(&dir)); let old = source.next_handle(); old.store(record(b"before")).await.unwrap(); let new = source.next_handle(); assert!(matches!( old.store(record(b"after")).await, - Err(BookmarkIoError::Fenced) + Err(BookmarkIoError::Superseded) )); + assert!(matches!(old.load().await, Err(BookmarkIoError::Superseded))); assert_eq!(read_back(&new).await.unwrap(), b"before"); } - /// A corrupt slot is skipped when another is valid, and is an - /// error rather than absence when nothing valid remains. - #[tokio::test] - async fn corruption_is_never_absence() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - write(&slots[0], b"scribble").unwrap(); - write(&slots[1], envelope(3, b"good")).unwrap(); - - let source = BookmarkSource::new(&test_log(), slots.clone()); - assert_eq!(read_back(&source.next_handle()).await.unwrap(), b"good"); - - write(&slots[1], b"more scribble").unwrap(); - assert!(matches!( - source.next_handle().load().await, - Err(BookmarkIoError::Corrupt { .. }) - )); - } - - /// A damaged sequence number fails the digest rather than silently - /// reordering the slots. - #[tokio::test] - async fn a_flipped_sequence_number_is_corruption() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - let mut bytes = envelope(3, b"good"); - bytes[MAGIC.len()] ^= 0x80; - write(&slots[0], bytes).unwrap(); - - let source = BookmarkSource::new(&test_log(), slots.clone()); - assert!(matches!( - source.next_handle().load().await, - Err(BookmarkIoError::Corrupt { .. }) - )); - } - - /// A straggling write from a dropped store future cannot land on - /// top of a newer committed record. - #[tokio::test] - async fn stragglers_cannot_clobber_newer_commits() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - let source = BookmarkSource::new(&test_log(), slots.clone()); - - let newer = envelope(7, b"newer"); - let straggler = envelope(6, b"stale"); - commit(&source.shared, &slots[0], 7, &newer).unwrap(); - assert!(commit(&source.shared, &slots[0], 6, &straggler).is_err()); - assert_eq!(read(&slots[0]).unwrap(), newer); - } - - /// A superseded handle can no longer load: its view of home and - /// sequence state belongs to a dead peer. - #[tokio::test] - async fn stale_generations_cannot_load() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let source = BookmarkSource::new(&test_log(), slots(&dir)); - let old = source.next_handle(); - old.store(record(b"before")).await.unwrap(); - let _new = source.next_handle(); - assert!(matches!(old.load().await, Err(BookmarkIoError::Fenced))); - } - /// Probing proves writability by writing, not by guessing from /// directory metadata. #[tokio::test] async fn probe_rejects_unwritable_storage() { - let source = BookmarkSource::new( - &test_log(), - vec![Utf8PathBuf::from("/nonexistent/sush/bookmark")], - ); - assert!(matches!(source.probe().await, Err(BookmarkIoError::NoSlot))); + let unwritable = source(vec![Utf8PathBuf::from("/nonexistent/sush")]); + assert!(unwritable.probe().await.is_err()); let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let source = BookmarkSource::new(&test_log(), slots(&dir)); - source.probe().await.unwrap(); + source(slots(&dir)).probe().await.unwrap(); } - /// A slotless source and a shed handle persist nothing and never - /// fail, and a shed handle ignores even an existing record. + /// A null source and a shed handle persist nothing and never fail, + /// and a shed handle ignores even an existing record. #[tokio::test] - async fn none_and_shed_touch_nothing() { - let source = BookmarkSource::null(); - let handle = source.next_handle(); - assert!(read_back(&handle).await.is_none()); + async fn null_and_shed_touch_nothing() { + let null = BookmarkSource::null(); + let handle = null.next_handle(); handle.store(record(b"lost")).await.unwrap(); assert!(read_back(&handle).await.is_none()); let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - write(&slots[0], envelope(7, b"kept")).unwrap(); - let source = BookmarkSource::new(&test_log(), slots.clone()); + let source = source(slots(&dir)); + source.next_handle().store(record(b"kept")).await.unwrap(); let shed = source.shed_handle(); assert!(read_back(&shed).await.is_none()); shed.store(record(b"dropped")).await.unwrap(); - assert_eq!(read(&slots[0]).unwrap(), envelope(7, b"kept")); + assert_eq!(read_back(&source.next_handle()).await.unwrap(), b"kept"); } } diff --git a/server/src/lib.rs b/server/src/lib.rs index 2675827..bf910af 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -18,6 +18,7 @@ pub mod history; pub mod io; pub mod job; pub mod link; +pub mod locker; pub mod manager; pub mod messages; pub mod mux; diff --git a/server/src/locker.rs b/server/src/locker.rs new file mode 100644 index 0000000..c63dab7 --- /dev/null +++ b/server/src/locker.rs @@ -0,0 +1,628 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Durable per-sled state that is never stale. +//! +//! A [`Locker`] spans the boot M.2s. Each M.2 contributes one *slot* +//! directory, and each [`Tenant`] owns one file in every slot. A store +//! writes every slot and returns `Ok` only when all of them hold the +//! new record. A load applies the pair rules: +//! +//! | slot A | slot B | verdict | +//! |---------------|-------------------------|--------------------------| +//! | record R | record R | adopt R | +//! | record R | missing | restore R (an M.2 swap) | +//! | missing | missing | fresh | +//! | R, nonce N, n | S ≠ R, nonce N, seq > n | adopt S (a torn write) | +//! | record R | record S ≠ R | discard | +//! | corrupt | anything | discard | +//! | I/O error | anything | discard | +//! +//! Every write carries a nonce drawn once per [`Locker`]. Slots that +//! disagree under one nonce were torn by a single process, so the +//! higher sequence number is that process's newest write, and a +//! newer-than-acknowledged record is safe to adopt: a store that +//! never returned had no effects. +//! +//! Discarding is a verdict, not an error, since each tenant decides +//! what starting over means. +//! +//! All of this is necessary to ensure the basic constraint that +//! **we must never adopt stale data**. + +use std::collections::BTreeSet; +use std::fs::Permissions; +use std::io::{self, Write as _}; +use std::os::unix::fs::PermissionsExt as _; +use std::sync::{Arc, Mutex as SyncMutex}; + +use atomicwrites::{AtomicFile, OverwriteBehavior}; +use camino::Utf8PathBuf; +use futures::TryFutureExt as _; +use slog::{Logger, o, warn}; +use thiserror::Error; +use tokio::fs::{read, remove_file, write}; +use tokio::sync::{Mutex, MutexGuard}; +use tokio::task::spawn_blocking; + +use sush_common::authn::Nonce; +use sush_common::hash::Hasher; + +const MAGIC_LEN: usize = 12; +const NONCE_LEN: usize = 32; +const SEQ_LEN: usize = 8; +const DIGEST_LEN: usize = 32; + +fn digest(nonce: &[u8; NONCE_LEN], seq: u64, record: &[u8]) -> [u8; DIGEST_LEN] { + let mut hasher = Hasher::new(); + hasher.update(nonce); + hasher.update(&seq.to_be_bytes()); + hasher.update(record); + *hasher.finalize().as_bytes() +} + +/// A file name and an envelope magic. +#[derive(Clone, Copy, Debug)] +pub struct TenantSpec { + pub file: &'static str, + pub magic: &'static [u8; MAGIC_LEN], +} + +/// What a load found across the slots. +#[derive(Debug)] +pub enum Verdict { + /// Every slot holds this record. + Adopt(Vec), + /// A slot is missing, but the survivors agree on this record. + Restore(Vec), + /// No slot holds a record. + Empty, + /// The slots cannot be trusted. + Discard(Discard), +} + +#[derive(Debug, Error)] +pub enum Discard { + #[error("the slots disagree")] + Disagree, + #[error("`{path}` is corrupt")] + Corrupt { path: Utf8PathBuf }, + #[error("reading `{path}` failed: {error}")] + Io { + path: Utf8PathBuf, + #[source] + error: io::Error, + }, +} + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("storing `{path}` failed: {error}")] + Io { + path: Utf8PathBuf, + #[source] + error: io::Error, + }, + #[error("superseded by a newer record")] + Superseded, + #[error("the store task died: {0}")] + Task(String), +} + +/// This sled's durable state storage. +#[derive(Clone, Debug)] +pub struct Locker { + log: Logger, + slots: Arc>, + /// Stamped on every envelope this process writes; see the torn + /// write rule in the module doc. + nonce: Nonce, + /// Files already claimed by a tenant. A second tenant over one + /// file would have its own sequence state, silently defeating + /// the straggler guard. + claimed: Arc>>, +} + +impl Locker { + /// A locker spans `slots`. Empty means nothing persists + /// (the standalone server, tests). + pub fn new(log: &Logger, slots: Vec) -> Self { + Self { + log: log.new(o!("component" => "locker")), + slots: Arc::new(slots), + nonce: Nonce::random(), + claimed: Arc::new(SyncMutex::new(BTreeSet::new())), + } + } + + /// A locker that loads and persists nothing. + pub fn null() -> Self { + Self::new(&Logger::root(slog::Discard, o!()), Vec::new()) + } + + /// A tenant of this locker, described by a (constant) specification. + /// Each file supports one tenant; a duplicate claim panics. + pub fn tenant(&self, spec: TenantSpec) -> Tenant { + assert!( + self.claimed.lock().unwrap().insert(spec.file), + "tenant `{}` is already claimed", + spec.file, + ); + Tenant { + log: self.log.new(o!("tenant" => spec.file)), + spec, + paths: self.slots.iter().map(|slot| slot.join(spec.file)).collect(), + nonce: self.nonce.clone(), + reserved: Mutex::new(0), + committed: Arc::new(SyncMutex::new(0)), + } + } + + /// Prove that every slot is writable by writing to every slot. + pub async fn probe(&self) -> Result<(), StoreError> { + for slot in self.slots.iter() { + let probe = slot.join("probe"); + if let Err(error) = write(&probe, b"").and_then(|()| remove_file(&probe)).await { + warn!(self.log, "unusable slot"; "path" => %probe); + return Err(StoreError::Io { path: probe, error }); + } + } + Ok(()) + } +} + +/// One tenant's files across the slots, with a two-stage +/// reserve/commit sequence number. +#[derive(Debug)] +pub struct Tenant { + log: Logger, + spec: TenantSpec, + paths: Vec, + nonce: Nonce, + /// The newest sequence number reserved or observed by this process. + reserved: Mutex, + /// The newest sequence number written to all slots. + committed: Arc>, +} + +impl Tenant { + /// Serialize loads and stores. Admission checks belong under the + /// guard. + pub async fn lock(&self) -> Guard<'_> { + Guard { + tenant: self, + reserved: self.reserved.lock().await, + } + } + + pub async fn load(&self) -> Verdict { + self.lock().await.load().await + } + + pub async fn store(&self, record: &[u8]) -> Result<(), StoreError> { + self.lock().await.store(record).await + } + + fn parse(&self, bytes: &[u8]) -> Option<(Nonce, u64, Vec)> { + let payload = bytes.strip_prefix(self.spec.magic.as_slice())?; + let (nonce, rest) = payload.split_first_chunk::()?; + let (seq, rest) = rest.split_first_chunk::()?; + let (sum, record) = rest.split_first_chunk::()?; + let seq = u64::from_be_bytes(*seq); + (digest(nonce, seq, record) == *sum) + .then(|| (Nonce::from_be_bytes(*nonce), seq, record.to_vec())) + } + + fn envelope(&self, seq: u64, record: &[u8]) -> Vec { + let mut bytes = + Vec::with_capacity(MAGIC_LEN + NONCE_LEN + SEQ_LEN + DIGEST_LEN + record.len()); + bytes.extend_from_slice(self.spec.magic); + let nonce = self.nonce.to_be_bytes(); + bytes.extend_from_slice(&nonce); + bytes.extend_from_slice(&seq.to_be_bytes()); + bytes.extend_from_slice(&digest(&nonce, seq, record)); + bytes.extend_from_slice(record); + bytes + } +} + +pub struct Guard<'a> { + tenant: &'a Tenant, + reserved: MutexGuard<'a, u64>, +} + +impl Guard<'_> { + pub async fn load(&mut self) -> Verdict { + let tenant = self.tenant; + if tenant.paths.is_empty() { + return Verdict::Empty; + } + + let mut found: Vec<(Nonce, u64, Vec)> = Vec::new(); + for path in &tenant.paths { + let bytes = match read(path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + return self.discard(Discard::Io { + path: path.clone(), + error, + }); + } + }; + match tenant.parse(&bytes) { + Some(parsed) => found.push(parsed), + None => return self.discard(Discard::Corrupt { path: path.clone() }), + } + } + if found.is_empty() { + return Verdict::Empty; + } + let restored = found.len() < tenant.paths.len(); + if !found.iter().all(|(_, _, record)| *record == found[0].2) { + // Slots torn by one process resolve to its newest write; + // see the module doc. Anything else is a genuine + // disagreement. + let (nonce, seq, record) = found + .iter() + .max_by_key(|(_, seq, _)| *seq) + .cloned() + .expect("found is non-empty"); + let torn = found.iter().all(|(n, ..)| *n == nonce) + && found.iter().filter(|(_, s, _)| *s == seq).count() == 1; + if !torn { + return self.discard(Discard::Disagree); + } + warn!(tenant.log, "adopted the newest write of a torn pair"; "seq" => seq); + if seq > *self.reserved { + *self.reserved = seq; + } + if let Err(error) = self.store(&record).await { + warn!(tenant.log, "failed to repair the torn pair"; "error" => %error); + } + return Verdict::Adopt(record); + } + + let (_, seq, record) = found.swap_remove(0); + let newest = found.iter().fold(seq, |max, (_, seq, _)| max.max(*seq)); + + // A reserved sequence number outranks a re-read of the disk. + // Regressing would let a cancelled write's straggler collide + // with a fresh reservation. + if newest > *self.reserved { + *self.reserved = newest; + } + if restored { + warn!(tenant.log, "restored from a lone slot"); + // Repair now: the survivor must not stay lone until the + // next natural store. + if let Err(error) = self.store(&record).await { + warn!(tenant.log, "failed to repair the lone slot"; "error" => %error); + } + Verdict::Restore(record) + } else { + Verdict::Adopt(record) + } + } + + pub async fn store(&mut self, record: &[u8]) -> Result<(), StoreError> { + let tenant = self.tenant; + if tenant.paths.is_empty() { + return Ok(()); + } + + *self.reserved += 1; + let seq = *self.reserved; + let envelope = tenant.envelope(seq, record); + let paths = tenant.paths.clone(); + let committed = tenant.committed.clone(); + spawn_blocking(move || commit(&committed, &paths, seq, &envelope)) + .await + .map_err(|join| StoreError::Task(join.to_string()))? + } + + fn discard(&self, reason: Discard) -> Verdict { + warn!(self.tenant.log, "discarding stored state"; "reason" => %reason); + Verdict::Discard(reason) + } +} + +/// Rename `envelope` into every slot iff `seq` is newer than the +/// latest committed version. A cancelled store drops only the async +/// side of the write; the blocking side is already detached, runs +/// to completion regardless, and must not overwrite a newer record. +fn commit( + committed: &SyncMutex, + paths: &[Utf8PathBuf], + seq: u64, + envelope: &[u8], +) -> Result<(), StoreError> { + let mut committed = committed.lock().unwrap(); + if seq <= *committed { + return Err(StoreError::Superseded); + } + for path in paths { + AtomicFile::new(path, OverwriteBehavior::AllowOverwrite) + .write(|file| { + file.set_permissions(Permissions::from_mode(0o600))?; + file.write_all(envelope) + }) + .map_err(|error| StoreError::Io { + path: path.clone(), + error: match error { + atomicwrites::Error::Internal(error) | atomicwrites::Error::User(error) => { + error + } + }, + })?; + } + *committed = seq; + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + use std::fs::{create_dir, metadata, read, remove_file as remove, write}; + + use tempfile::TempDir; + + const SPEC: TenantSpec = TenantSpec { + file: "record", + magic: b"SUSHLOCKTEST", + }; + + /// Two slot directories, like two M.2s. + fn slots(dir: &TempDir) -> Vec { + ["m2a", "m2b"] + .iter() + .map(|m2| { + let slot = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); + create_dir(&slot).unwrap(); + slot + }) + .collect() + } + + fn test_log() -> Logger { + Logger::root(slog::Discard, o!()) + } + + fn locker(slots: Vec) -> Locker { + Locker::new(&test_log(), slots) + } + + fn files(slots: &[Utf8PathBuf]) -> Vec { + slots.iter().map(|slot| slot.join(SPEC.file)).collect() + } + + /// A store lands the same envelope in every slot, sequenced and + /// private, and loads back adopted. + #[tokio::test] + async fn round_trip_writes_every_slot() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + + assert!(matches!(tenant.load().await, Verdict::Empty)); + tenant.store(b"who we are").await.unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"who we are" + )); + + let expected = tenant.envelope(1, b"who we are"); + for path in files(&slots) { + assert_eq!(read(&path).unwrap(), expected); + let mode = metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + } + + /// A missing slot restores from the survivor, and the next store + /// repairs it. + #[tokio::test] + async fn lone_slot_restores_and_repairs() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + tenant.store(b"kept").await.unwrap(); + + let files = files(&slots); + remove(&files[0]).unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Restore(record) if record == b"kept" + )); + + tenant.store(b"repaired").await.unwrap(); + assert_eq!(read(&files[0]).unwrap(), read(&files[1]).unwrap()); + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"repaired" + )); + } + + /// Disagreeing slots are discarded, not arbitrated by sequence + /// number. + #[tokio::test] + async fn disagreement_discards() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let a = locker(vec![slots[0].clone()]).tenant(SPEC); + a.store(b"one world").await.unwrap(); + let b = locker(vec![slots[1].clone()]).tenant(SPEC); + b.store(b"junk").await.unwrap(); + b.store(b"another").await.unwrap(); + + let tenant = locker(slots).tenant(SPEC); + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Disagree) + )); + } + + /// Equal records adopt even when their sequence numbers differ, + /// and new stores outnumber the highest. + #[tokio::test] + async fn legacy_sequence_skew_is_benign() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let a = locker(vec![slots[0].clone()]).tenant(SPEC); + a.store(b"same").await.unwrap(); + let b = locker(vec![slots[1].clone()]).tenant(SPEC); + b.store(b"junk").await.unwrap(); + b.store(b"same").await.unwrap(); + + let tenant = locker(slots.clone()).tenant(SPEC); + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"same" + )); + tenant.store(b"next").await.unwrap(); + let expected = tenant.envelope(3, b"next"); + for path in files(&slots) { + assert_eq!(read(&path).unwrap(), expected); + } + } + + /// A corrupt slot is discarded even beside a valid one. + #[tokio::test] + async fn corruption_discards() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + tenant.store(b"good").await.unwrap(); + + write(slots[0].join(SPEC.file), b"scribble").unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Corrupt { .. }) + )); + } + + /// A damaged nonce or sequence number fails the digest rather + /// than parsing. + #[tokio::test] + async fn flipped_envelope_bytes_are_corruption() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + let path = slots[0].join(SPEC.file); + for offset in [MAGIC_LEN, MAGIC_LEN + NONCE_LEN] { + tenant.store(b"good").await.unwrap(); + let mut bytes = read(&path).unwrap(); + bytes[offset] ^= 0x80; + write(&path, bytes).unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Corrupt { .. }) + )); + } + } + + /// Slots torn by one process resolve to its newest write, both + /// in that process and in the next. + #[tokio::test] + async fn torn_pair_resolves_to_newest_write() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + tenant.store(b"old").await.unwrap(); + write(slots[0].join(SPEC.file), tenant.envelope(2, b"new")).unwrap(); + + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"new" + )); + + let next = locker(slots.clone()).tenant(SPEC); + assert!(matches!( + next.load().await, + Verdict::Adopt(record) if record == b"new" + )); + next.store(b"repaired").await.unwrap(); + assert_eq!( + read(slots[0].join(SPEC.file)).unwrap(), + read(slots[1].join(SPEC.file)).unwrap(), + ); + } + + /// Equal sequence numbers cannot be arbitrated, even in one life. + #[tokio::test] + async fn torn_pair_with_equal_sequence_numbers_discards() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + write(slots[0].join(SPEC.file), tenant.envelope(2, b"x")).unwrap(); + write(slots[1].join(SPEC.file), tenant.envelope(2, b"y")).unwrap(); + + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Disagree) + )); + } + + /// A store failing partway errs, and the survivor still loads: + /// the record was never acknowledged, so either version is sound. + #[tokio::test] + async fn partial_store_fails_loudly() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let good = slots(&dir).swap_remove(0); + let gone = Utf8PathBuf::from_path_buf(dir.path().join("gone")).unwrap(); + let tenant = locker(vec![good.clone(), gone]).tenant(SPEC); + + assert!(matches!( + tenant.store(b"half").await, + Err(StoreError::Io { .. }) + )); + assert!(matches!( + tenant.load().await, + Verdict::Restore(record) if record == b"half" + )); + } + + /// A straggling write from a dropped store future cannot land on + /// top of a newer committed record. + #[tokio::test] + async fn stragglers_cannot_clobber_newer_commits() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + let paths = files(&slots); + + let newer = tenant.envelope(7, b"newer"); + let straggler = tenant.envelope(6, b"stale"); + commit(&tenant.committed, &paths, 7, &newer).unwrap(); + assert!(matches!( + commit(&tenant.committed, &paths, 6, &straggler), + Err(StoreError::Superseded) + )); + assert_eq!(read(&paths[0]).unwrap(), newer); + } + + /// Probing proves every slot writable by writing. + #[tokio::test] + async fn probe_requires_every_slot() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let mut slots = slots(&dir); + locker(slots.clone()).probe().await.unwrap(); + + slots.push(Utf8PathBuf::from("/nonexistent/sush")); + assert!(matches!( + locker(slots).probe().await, + Err(StoreError::Io { .. }) + )); + } + + /// A null locker persists nothing and never fails. + #[tokio::test] + async fn null_touches_nothing() { + let tenant = Locker::null().tenant(SPEC); + tenant.store(b"lost").await.unwrap(); + assert!(matches!(tenant.load().await, Verdict::Empty)); + Locker::null().probe().await.unwrap(); + } +} diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index 9b54af2..594e316 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -24,9 +24,10 @@ use sush_common::jobs::{ use sush_common::keys::pem_cert_chain; use sush_common::targets::{Cubbies, SledHealth}; use sush_common::version::VersionInfo; -use sush_server::bookmark::BookmarkSource; +use sush_server::bookmark::{BOOKMARK, BookmarkSource}; use sush_server::executor::PathIsolation; use sush_server::gossip::spawn_gossip; +use sush_server::locker::Locker; use sush_server::messages::v0::{Event, JobEvent, Message}; use sush_server::output::JobOutputDir; use sush_server::state::GossipUniverse; @@ -560,24 +561,25 @@ async fn bookmarks_survive_restart() { // Sled 2 keeps its identity in a bookmark; joining records it. let bookmark_dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slot = Utf8PathBuf::from_path_buf(bookmark_dir.path().join("bookmark")).unwrap(); + let slot = Utf8PathBuf::from_path_buf(bookmark_dir.path().to_path_buf()).unwrap(); + let record = slot.join(BOOKMARK.file); let b_shutdown = CancellationToken::new(); let b = Sled::start_with_bookmarks( &log, &dir, 2, &root_pem, - BookmarkSource::new(&log, vec![slot.clone()]), + BookmarkSource::new(&log, &Locker::new(&log, vec![slot.clone()])), &b_shutdown, ) .await; a.peers.send(BTreeSet::from([b.addr])).unwrap(); b.peers.send(BTreeSet::from([a.addr])).unwrap(); eventually("the joining sled records its identity", 120, async || { - slot.as_std_path().exists() + record.as_std_path().exists() }) .await; - let before = std::fs::read(&slot).unwrap(); + let before = std::fs::read(&record).unwrap(); // The next incarnation reads the record back, rejoins, and // advances it, reclaiming the previous life's identity. @@ -593,7 +595,7 @@ async fn bookmarks_survive_restart() { &dir, 2, &root_pem, - BookmarkSource::new(&log, vec![slot.clone()]), + BookmarkSource::new(&log, &Locker::new(&log, vec![slot.clone()])), &shutdown, ) .await; @@ -606,7 +608,7 @@ async fn bookmarks_survive_restart() { eventually( "the record advances past the previous life", 120, - async || std::fs::read(&slot).unwrap() != before, + async || std::fs::read(&record).unwrap() != before, ) .await; @@ -649,7 +651,10 @@ async fn gossip_survives_bookmark_failure() { &dir, 2, &root_pem, - BookmarkSource::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush/bookmark")]), + BookmarkSource::new( + &log, + &Locker::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush")]), + ), &shutdown, ) .await; From 68734ef9dd78175bcc05058e26d4a904d124c178 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 2 Sep 2026 21:40:10 -0600 Subject: [PATCH 02/12] Wire the execution boundary into the executor and state machine Before each spawn, the launcher writes the boundary: the job this sled is about to run, its session, and the causal frontier of everything the sled had seen when it committed. When the job ends, its terminal status joins the record. Launches drain in release order, so the record never regresses, and a job whose record cannot be written is refused, with the refusal gossiped. At rejoin, the sled compares the recorded frontier with the join frontier. Covered means every request it had processed was witnessed: replay refuses old jobs, the session chain never releases a resubmitted one, and zombie reaping reports what the sled left running, so the session keeps working here. Not covered means a suffix of its history died with it: jobs may have run that no sled can name, and their signed artifacts could be resubmitted and run twice. The sled refuses jobs of the recorded session until a new session supersedes it, and adjudicates the recorded job itself: the recorded ending if the job got one, and interrupted if it did not. A recorded job still running across a universe swap is skipped, since its events land in the new universe when it finishes. A start event no longer displaces a terminal status, so an adjudicated interrupted survives a concurrent replayed start in either arrival order. State's relationship to its own past (join frontier, boundary, lost session, zombies) now lives in one struct, Past. The wiring layer owns storage: JobManager takes the Locker, and Seed::grow makes the seed together with the locker's one bookmark source, which spawn_gossip consumes as a pair. BookmarkSource::probe is gone; the seed probes the locker itself. Accepted for now: jobs in a lost suffix other than the recorded one get no verdict of their own, only a refusal when retried; and a crash between a job's end and the outcome write falls back to interrupted for that job. Co-Authored-By: Claude Mythos 5 --- server/Cargo.toml | 1 + server/src/bookmark.rs | 37 +--- server/src/boundary.rs | 393 ++++++++++++++++++++++++++++++++++++ server/src/executor.rs | 109 +++++++--- server/src/gossip.rs | 63 +++++- server/src/lib.rs | 1 + server/src/main.rs | 5 +- server/src/manager.rs | 8 + server/src/state.rs | 345 ++++++++++++++++++++++++++----- server/tests/distributed.rs | 353 +++++++++++++++++++++++++++----- server/tests/gossip.rs | 30 +-- tests/src/manager_tests.rs | 63 ++++-- tests/src/test_utils.rs | 11 +- 13 files changed, 1226 insertions(+), 193 deletions(-) create mode 100644 server/src/boundary.rs diff --git a/server/Cargo.toml b/server/Cargo.toml index ce03ab9..a4b1bd4 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -18,6 +18,7 @@ test-support = [] [dependencies] atomicwrites.workspace = true bytes.workspace = true +ciborium.workspace = true camino.workspace = true bytesize.workspace = true chrono.workspace = true diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index 756c0e7..bcbc36e 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -54,24 +54,21 @@ pub struct BookmarkSource { #[derive(Debug)] struct Ratchet { log: Logger, - locker: Locker, tenant: Tenant, generation: AtomicU64, } impl BookmarkSource { - /// A source persisting to `locker`. Construct exactly one source - /// per locker per process, and feed that same source to both - /// [`seed_gossip`](crate::seed_gossip) and - /// [`spawn_gossip`](crate::gossip::spawn_gossip). - /// A handle is disabled when its source hands out a newer one. - /// No source can disable another source's handles, so a second - /// one would let an old peer overwrite its replacement's record. + /// A source persisting to `locker`. + /// [`Seed::grow`](crate::gossip::Seed::grow) makes the one source + /// a locker gets per process. A handle is disabled when its source + /// hands out a newer one. No source can disable another source's + /// handles, so a second source would let an old peer overwrite its + /// replacement's record. pub fn new(log: &Logger, locker: &Locker) -> Self { Self { ratchet: Arc::new(Ratchet { log: log.new(o!("component" => "bookmark")), - locker: locker.clone(), tenant: locker.tenant(BOOKMARK), generation: AtomicU64::new(0), }), @@ -102,15 +99,6 @@ impl BookmarkSource { shed: true, } } - - /// Does the storage work? - pub async fn probe(&self) -> Result<(), StoreError> { - let usable = self.ratchet.locker.probe().await; - if let Err(error) = &usable { - warn!(self.ratchet.log, "no usable bookmark storage"; "error" => %error); - } - usable - } } /// One peer's handle on the [`BookmarkSource`]. @@ -230,7 +218,7 @@ mod test { /// A discarded verdict is a fresh start, not an error. #[tokio::test] - async fn discard_mints_a_fresh_identity() { + async fn discard_assumes_fresh_identity() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); let slots = slots(&dir); for (slot, bytes) in slots.iter().zip([b"one", b"two"]) { @@ -257,17 +245,6 @@ mod test { assert_eq!(read_back(&new).await.unwrap(), b"before"); } - /// Probing proves writability by writing, not by guessing from - /// directory metadata. - #[tokio::test] - async fn probe_rejects_unwritable_storage() { - let unwritable = source(vec![Utf8PathBuf::from("/nonexistent/sush")]); - assert!(unwritable.probe().await.is_err()); - - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - source(slots(&dir)).probe().await.unwrap(); - } - /// A null source and a shed handle persist nothing and never fail, /// and a shed handle ignores even an existing record. #[tokio::test] diff --git a/server/src/boundary.rs b/server/src/boundary.rs new file mode 100644 index 0000000..d7d4168 --- /dev/null +++ b/server/src/boundary.rs @@ -0,0 +1,393 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The boundary between jobs we executed and jobs we only heard about. +//! +//! The gossip frontier answers "what have I heard"; the boundary +//! answers "where was I when I last committed to running a job." One +//! record, overwritten in chain order before each spawn, carries the +//! job, its session, our causal frontier at the moment of commitment, +//! and the job's ending, once it has one. +//! +//! After a restart, compare the recorded frontier with the join +//! frontier. If the join frontier covers it, every request we had +//! processed was witnessed: replay refuses old jobs, the session +//! chain never releases a resubmitted one, and zombie reaping reports +//! what we left running. If not, a suffix of our history died with +//! us: jobs may have run here that no sled can name, and their signed +//! artifacts could be resubmitted and run again. The state machine +//! then refuses jobs of the recorded session until a new session +//! supersedes it, and adjudicates the recorded job itself: the +//! recorded ending if the job got one, and interrupted if it did not. +//! +//! A boundary that cannot be written means the job must not run. A +//! boundary that cannot be trusted means no job may run at all, since +//! we cannot tell what the previous life committed to. Recovery is an +//! M.2 swap or a clean slate. + +use std::sync::Mutex as SyncMutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use ciborium::{de::from_reader, ser::into_writer}; +use rumors::{Network, Version}; +use serde::{Deserialize, Serialize}; +use slog::{Logger, o, warn}; +use thiserror::Error; + +use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId}; + +use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; + +pub const BOUNDARY: TenantSpec = TenantSpec { + file: "sush-boundary", + magic: b"SUSHBOUNDARY", +}; + +/// The execution boundary: the last job this sled committed to +/// running, everything it had seen when it committed, and how far the +/// job got. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Boundary { + pub network: Network, + pub session: SessionId, + pub job: JobId, + #[serde(with = "version_bytes")] + pub frontier: Version, + pub outcome: JobOutcome, +} + +/// How far the boundary job got. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum JobOutcome { + /// Committed to run, with no ending recorded. After a crash, + /// interrupted is the truth. + Committed, + /// The job's terminal status. + Ended(JobStatus), +} + +impl Boundary { + /// Whether the rack already knows everything we knew at + /// commitment. Covered means no committed job can be lost. + /// Anything else means a suffix of our history died with us. + /// The comparison includes third-party traffic we had seen, so a + /// join through a lagging peer can look uncovered; the cost is a + /// session refused on this sled until a new one supersedes it. + pub fn covered_by(&self, network: Network, join_frontier: Option<&Version>) -> bool { + self.network == network && join_frontier.is_some_and(|frontier| self.frontier <= *frontier) + } +} + +mod version_bytes { + use rumors::Version; + use serde::de::Error as _; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(version: &Version, serializer: S) -> Result { + serializer.serialize_bytes(&version.encode()) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let bytes = >::deserialize(deserializer)?; + Version::decode(bytes.as_slice()).map_err(D::Error::custom) + } +} + +fn encode(boundary: &Boundary) -> Vec { + let mut bytes = Vec::new(); + into_writer(boundary, &mut bytes).expect("writing to a Vec cannot fail"); + bytes +} + +fn decode(record: &[u8]) -> Option { + from_reader(record).ok() +} + +#[derive(Debug, Error)] +pub enum BoundaryError { + #[error("the boundary store is untrusted")] + Untrusted, + #[error(transparent)] + Store(#[from] StoreError), +} + +/// Durable storage for the [`Boundary`]. +#[derive(Debug)] +pub struct BoundaryStore { + log: Logger, + tenant: Tenant, + /// The record, readable synchronously by the state machine. + boundary: SyncMutex>, + /// Untrusted until loaded, and forever if the load discards: + /// writing would launder the disagreement into false agreement. + untrusted: AtomicBool, + loaded: AtomicBool, +} + +impl BoundaryStore { + pub fn new(log: &Logger, locker: &Locker) -> Self { + Self { + log: log.new(o!("component" => "boundary")), + tenant: locker.tenant(BOUNDARY), + boundary: SyncMutex::new(None), + untrusted: AtomicBool::new(true), + loaded: AtomicBool::new(false), + } + } + + /// Load the stored record once, at startup, before any job runs. + /// A later load could regress the in-memory record below a + /// spawned job, so a second call panics. + pub async fn load(&self) { + assert!( + !self.loaded.swap(true, Ordering::SeqCst), + "the boundary store loads once, at startup", + ); + let boundary = match self.tenant.load().await { + Verdict::Adopt(record) | Verdict::Restore(record) => match decode(&record) { + Some(boundary) => Some(boundary), + None => { + warn!(self.log, "undecodable boundary record"); + return; + } + }, + Verdict::Empty => None, + Verdict::Discard(_) => return, + }; + *self.boundary.lock().unwrap() = boundary; + self.untrusted.store(false, Ordering::SeqCst); + } + + pub fn untrusted(&self) -> bool { + self.untrusted.load(Ordering::SeqCst) + } + + pub fn boundary(&self) -> Option { + self.boundary.lock().unwrap().clone() + } + + /// Record how the boundary job ended, so the next life can tell + /// the truth instead of guessing. A stop displaces an adjudicated + /// Interrupted, mirroring the status arms in the state machine; + /// nothing else is overwritten, and a record that has moved on to + /// a newer job ignores the old job's ending. + pub async fn record_outcome(&self, job_id: &JobId, outcome: &JobStatus) { + debug_assert!(outcome.is_terminal()); + if self.untrusted() { + return; + } + let mut guard = self.tenant.lock().await; + let updated = { + let recorded = self.boundary.lock().unwrap(); + let Some(boundary) = recorded.as_ref().filter(|b| b.job == *job_id) else { + return; + }; + let displaces = matches!( + (&boundary.outcome, outcome), + (JobOutcome::Committed, _) + | ( + JobOutcome::Ended(JobStatus::Error { + error: ProcessError::Interrupted, + .. + }), + JobStatus::Stopped { .. }, + ) + ); + if !displaces { + return; + } + Boundary { + outcome: JobOutcome::Ended(outcome.clone()), + ..boundary.clone() + } + }; + if let Err(error) = guard.store(&encode(&updated)).await { + warn!( + self.log, "failed to record the boundary job's outcome"; + "job_id" => %job_id, "error" => %error, + ); + return; + } + *self.boundary.lock().unwrap() = Some(updated); + } + + /// Commit to executing the job at `boundary`. On failure the + /// caller must not run the job. + pub async fn advance(&self, boundary: &Boundary) -> Result<(), BoundaryError> { + if self.untrusted() { + // Defense in depth: the state machine already refuses + // execution when the store is untrusted, so no launch + // reaches this arm. + return Err(BoundaryError::Untrusted); + } + let mut guard = self.tenant.lock().await; + guard.store(&encode(boundary)).await?; + *self.boundary.lock().unwrap() = Some(boundary.clone()); + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::fs::create_dir; + + use camino::Utf8PathBuf; + use slog::Discard; + use tempfile::TempDir; + + /// Two M.2 slots. + fn slots(dir: &TempDir) -> Vec { + ["m2a", "m2b"] + .iter() + .map(|m2| { + let slot = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); + create_dir(&slot).unwrap(); + slot + }) + .collect() + } + + fn test_log() -> Logger { + Logger::root(Discard, o!()) + } + + async fn store(slots: Vec) -> BoundaryStore { + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots)); + store.load().await; + store + } + + fn network(seed: u8) -> Network { + serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() + } + + fn boundary(seed: u8) -> Boundary { + Boundary { + network: network(seed), + session: SessionId::random(), + job: JobId::random(), + frontier: "(1, 1, (0, 0, 2))".parse().unwrap(), + outcome: JobOutcome::Committed, + } + } + + #[tokio::test] + async fn commitments_survive_restarts() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + let first = store(slots.clone()).await; + assert!(first.boundary().is_none()); + + let (a, b) = (boundary(1), boundary(2)); + first.advance(&a).await.unwrap(); + first.advance(&b).await.unwrap(); + + let next = store(slots).await; + assert!(!next.untrusted()); + let recorded = next.boundary().unwrap(); + assert_eq!(recorded.network, b.network); + assert_eq!(recorded.session, b.session); + assert_eq!(recorded.job, b.job); + assert_eq!(recorded.frontier, b.frontier); + assert!(matches!(recorded.outcome, JobOutcome::Committed)); + } + + /// The boundary job's recorded ending survives into the next life. + /// A stop displaces an adjudicated interrupted; nothing else does, and + /// an ending for a superseded job is ignored. + #[tokio::test] + async fn outcomes_survive_and_heal() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + let first = store(slots.clone()).await; + let b = boundary(1); + first.advance(&b).await.unwrap(); + + let interrupted = JobStatus::Error { + job_id: b.job, + time_error: chrono::Utc::now(), + error: ProcessError::Interrupted, + }; + let killed = JobStatus::Error { + job_id: b.job, + time_error: chrono::Utc::now(), + error: ProcessError::Killed(9), + }; + first.record_outcome(&JobId::random(), &killed).await; + assert!(matches!( + first.boundary().unwrap().outcome, + JobOutcome::Committed + )); + + first.record_outcome(&b.job, &interrupted).await; + first.record_outcome(&b.job, &killed).await; + let next = store(slots).await; + assert!(matches!( + next.boundary().unwrap().outcome, + JobOutcome::Ended(JobStatus::Error { + error: ProcessError::Interrupted, + .. + }) + )); + } + + #[tokio::test] + async fn coverage_requires_frontier_and_universe() { + let boundary = boundary(1); + let covered: Version = "(2, 2, (0, 0, 3))".parse().unwrap(); + let behind: Version = "(1, 0, (0, 0, 2))".parse().unwrap(); + + assert!(boundary.covered_by(network(1), Some(&boundary.frontier))); + assert!(boundary.covered_by(network(1), Some(&covered))); + assert!(!boundary.covered_by(network(1), Some(&behind))); + assert!(!boundary.covered_by(network(2), Some(&covered))); + assert!(!boundary.covered_by(network(1), None)); + } + + #[tokio::test] + async fn disagreement_is_untrusted_and_pins() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + for (slot, seed) in slots.iter().zip([1, 2]) { + let lone = store(vec![slot.clone()]).await; + lone.advance(&boundary(seed)).await.unwrap(); + } + + let untrusted = store(slots.clone()).await; + assert!(untrusted.untrusted()); + assert!(untrusted.boundary().is_none()); + assert!(matches!( + untrusted.advance(&boundary(3)).await, + Err(BoundaryError::Untrusted) + )); + + let reload = store(slots).await; + assert!(reload.untrusted()); + } + + #[tokio::test] + async fn undecodable_record_is_untrusted() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + let scratch = Locker::new(&test_log(), slots.clone()); + scratch.tenant(BOUNDARY).store(b"scribble").await.unwrap(); + + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots)); + store.load().await; + assert!(store.untrusted()); + } + + #[tokio::test] + async fn unloaded_is_untrusted() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots(&dir))); + assert!(store.untrusted()); + assert!(matches!( + store.advance(&boundary(1)).await, + Err(BoundaryError::Untrusted) + )); + } +} diff --git a/server/src/executor.rs b/server/src/executor.rs index bae1c1a..347d8b1 100644 --- a/server/src/executor.rs +++ b/server/src/executor.rs @@ -7,7 +7,7 @@ //! Start, stop, and watch job processes. Driven by the session state //! machine, but session agnostic. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::io; use std::mem::MaybeUninit; use std::os::fd::AsRawFd as _; @@ -35,6 +35,7 @@ use sush_common::jobs::{ JobId, JobMode, JobOutputStream, JobStartRequest, ProcessError, SignedJob, VerifiedJob, }; +use crate::boundary::{Boundary, BoundaryStore}; use crate::io::JobIo; use crate::job::{Job, SocketSender}; use crate::messages::v0::{Event, JobEvent}; @@ -50,21 +51,40 @@ pub const DEFAULT_TERM: &str = "vt100"; /// experience backpressure; if we do, something is wrong. const EVENTS_CHANNEL_CAPACITY: usize = 16; +/// The launcher performs one small fsync per job, so at human job +/// rates this never fills. A full queue refuses the job. +const LAUNCH_CHANNEL_CAPACITY: usize = 16; + pub struct Executor { log: Logger, events: Arc>>>, path_isolation: PathIsolation, output_dir: JobOutputDir, + launch: mpsc::Sender, shutdown: CancellationToken, stop: BTreeMap, } +/// One validated job on its way to the launcher. +struct Launch { + log: Logger, + boundary: Boundary, + events: mpsc::Sender, + output_dir: JobOutputDir, + request: VerifiedJob, + params: JobStartParams, + path_isolation: PathIsolation, + tx_attachment: watch::Sender>, + stop: CancellationToken, +} + /// Executor methods should be infallible; errors are reported via events. impl Executor { pub fn new( log: Logger, path_isolation: PathIsolation, output_dir: JobOutputDir, + boundary: Arc, shutdown: CancellationToken, ) -> (Self, impl Stream + Send + 'static) { let (tx_events, rx_events) = mpsc::channel(EVENTS_CHANNEL_CAPACITY); @@ -80,6 +100,25 @@ impl Executor { } }); + // The queue keeps boundary writes in release order. If each job + // advanced the boundary from its own task, two writes could land + // out of order, and the record would name an older job than one + // that already spawned. + let (launch, mut queued) = mpsc::channel::(LAUNCH_CHANNEL_CAPACITY); + spawn(async move { + while let Some(launch) = queued.recv().await { + if let Err(error) = boundary.advance(&launch.boundary).await { + let error = ProcessError::Io { + what: "recording the execution boundary".to_string(), + error: error.to_string(), + }; + send_error(&launch.log, &launch.boundary.job, &launch.events, error).await; + continue; + } + spawn(job_spawn(launch)); + } + }); + // Return the executor and event stream. ( Self { @@ -87,6 +126,7 @@ impl Executor { events, path_isolation, output_dir, + launch, shutdown, stop: BTreeMap::new(), }, @@ -94,16 +134,19 @@ impl Executor { ) } + /// Returns whether the job was queued for launch, so the caller + /// keeps attachment points only for jobs that can arrive. pub fn job_start( &mut self, certs: &mut Certificates, request: SignedJob, params: JobStartParams, tx_attachment: watch::Sender>, - ) { + boundary: Boundary, + ) -> bool { let Some(events) = self.events.read().unwrap().as_ref().cloned() else { // No more events ⇒ shutting down ⇒ no new jobs allowed. - return; + return false; }; // Validate the job request. @@ -116,7 +159,7 @@ impl Executor { spawn(async move { send_error(&log, &job_id, &events, $err).await; }); - return; + return false; }}; } if request.payload().command.starts_with('-') { @@ -134,16 +177,29 @@ impl Executor { let stop = self.shutdown.child_token(); self.stop.insert(job_id, stop.clone()); - spawn(job_spawn( - self.log.new(o!("job_id" => job_id)), + let refused = self.launch.try_send(Launch { + log: self.log.new(o!("job_id" => job_id)), + boundary, events, - self.output_dir.clone(), - verified_request, + output_dir: self.output_dir.clone(), + request: verified_request, params, - self.path_isolation, + path_isolation: self.path_isolation, tx_attachment, stop, - )); + }); + if let Err(error) = refused { + self.stop.remove(&job_id); + self.job_refused( + job_id, + ProcessError::Io { + what: "queueing the job for launch".to_string(), + error: error.to_string(), + }, + ); + return false; + } + true } pub fn job_stop(&mut self, job_id: &JobId) { @@ -156,6 +212,12 @@ impl Executor { let _ = self.stop.remove(job_id); } + /// Every job accepted for launch whose stop token is still held: + /// queued, spawning, or running. + pub fn in_flight(&self) -> BTreeSet { + self.stop.keys().copied().collect() + } + /// Announce a job that will never run here. pub fn job_refused(&self, job_id: JobId, error: ProcessError) { let Some(events) = self.events.read().unwrap().as_ref().cloned() else { @@ -173,20 +235,21 @@ impl Executor { } /// Spawn a process for a job and return an attachment point if it is -/// interactive. Assumes the job request has already been validated, -/// e.g., as by [`crate::JobManager::job_start`]. -#[allow(clippy::too_many_arguments)] -async fn job_spawn( - log: Logger, - events: mpsc::Sender, - output_dir: JobOutputDir, - request: VerifiedJob, - params: JobStartParams, - path_isolation: PathIsolation, - tx_attachment: watch::Sender>, - stop: CancellationToken, -) { +/// interactive. Assumes [`Executor::job_start`] validated the request +/// and the launcher committed its boundary. +async fn job_spawn(launch: Launch) { use JobOutputStream::*; + let Launch { + log, + boundary: _, + events, + output_dir, + request, + params, + path_isolation, + tx_attachment, + stop, + } = launch; let JobStartRequest { job_id, session_id: _, diff --git a/server/src/gossip.rs b/server/src/gossip.rs index bc64d4a..0ffe21b 100644 --- a/server/src/gossip.rs +++ b/server/src/gossip.rs @@ -45,6 +45,7 @@ use rumors::link::routed::Endpoint; use crate::bookmark::{BookmarkSource, SushBookmark}; use crate::link::{AttestedBaseboards, CorpusSource, SprocketsDial, SprocketsLink, Transport}; +use crate::locker::Locker; /// The attested baseboards of our live gossip peers. pub type LinkedBaseboards = watch::Receiver>; @@ -90,6 +91,55 @@ impl Universe { } } +/// A seeded network paired with the source persisting its identity. +/// [`Seed::grow`] is the only constructor and [`spawn_gossip`] consumes +/// the pair whole, so a seed can never gossip against a source other +/// than its own. +#[derive(Debug)] +pub struct Seed { + rumors: Rumors, + bookmarks: BookmarkSource, +} + +impl Seed { + /// Seed a fresh universe with this server as its only peer, over + /// `locker`'s storage, making the locker's one [`BookmarkSource`]. + /// + /// A pristine seed's bookmark touches no storage, and identities + /// recorded there are reclaimed only after a migration returns us + /// to their universe. Bad storage would abort every session at the + /// persist gate, before the seed could even learn to migrate. The + /// probe runs first, and a failed probe sheds the bookmark. + pub async fn grow(log: &Logger, locker: &Locker) -> Self + where + T: DeserializeOwned + Serialize + Send + Sync + 'static, + { + let bookmarks = BookmarkSource::new(log, locker); + let handle = match locker.probe().await { + Ok(()) => bookmarks.next_handle(), + Err(_) => bookmarks.shed_handle(), + }; + let rumors = match Peer::seed().bookmark(handle).await { + Ok(peer) => peer.into_rumors(), + Err(unbookmarked) => match unbookmarked.peer.bookmark(bookmarks.shed_handle()).await { + Ok(peer) => peer.into_rumors(), + Err(_) => unreachable!("a shed bookmark never touches storage"), + }, + }; + Self { rumors, bookmarks } + } + + pub fn rumors(&self) -> &Rumors { + &self.rumors + } + + /// The network alone, for a seed that will never gossip + /// (see [`isolated`]). + pub fn into_rumors(self) -> Rumors { + self.rumors + } +} + /// A single-peer universe that never changes. The standalone server uses /// this, as does a sled that cannot gossip. The receiver outlives its /// sender. @@ -132,8 +182,7 @@ pub async fn spawn_gossip( corpus: CorpusSource, listen_addr: SocketAddrV6, peers: watch::Receiver>, - seed: Rumors, - bookmarks: BookmarkSource, + seed: Seed, shutdown: CancellationToken, ) -> io::Result<(SocketAddrV6, watch::Receiver>, LinkedBaseboards)> where @@ -149,8 +198,7 @@ where ) .await?; let bound = transport.bound(); - let (universe, linked) = - spawn_gossip_manager(log, config, transport, peers, seed, bookmarks, shutdown); + let (universe, linked) = spawn_gossip_manager(log, config, transport, peers, seed, shutdown); Ok((bound, universe, linked)) } @@ -164,13 +212,16 @@ pub fn spawn_gossip_manager( config: GossipConfig, transport: Transport, peers: watch::Receiver>, - seed: Rumors, - bookmarks: BookmarkSource, + seed: Seed, shutdown: CancellationToken, ) -> (watch::Receiver>, LinkedBaseboards) where T: DeserializeOwned + Serialize + Send + Sync + 'static, { + let Seed { + rumors: seed, + bookmarks, + } = seed; let (publish, subscribe) = watch::channel(Universe::genesis(seed.clone())); let (linked, subscribe_linked) = watch::channel(BTreeSet::new()); let manager = Manager { diff --git a/server/src/lib.rs b/server/src/lib.rs index bf910af..89312df 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -11,6 +11,7 @@ extern crate function_name; compile_error!("`test-support` must not be enabled for an embedded server"); pub mod bookmark; +pub mod boundary; pub mod error; pub mod executor; pub mod gossip; diff --git a/server/src/main.rs b/server/src/main.rs index b76c43d..4528dfa 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -20,9 +20,9 @@ use x509_cert::der::DecodePem as _; use sush_api::sush_api_mod::api_description; use sush_common::targets::Cubbies; -use sush_server::bookmark::BookmarkSource; use sush_server::executor::PathIsolation; use sush_server::gossip::{isolated, lonely}; +use sush_server::locker::Locker; use sush_server::manager::JobManager; use sush_server::output::JobOutputDir; use sush_server::server::ApiServer; @@ -95,7 +95,7 @@ async fn main() -> Result<(), String> { }; // TODO: get/seed Rumors network - let gossip = isolated(seed_gossip(&BookmarkSource::null()).await); + let gossip = isolated(seed_gossip(&log, &Locker::null()).await.into_rumors()); #[cfg(feature = "test-support")] let roots = overridable_root_certs(&override_root_certs).await?; @@ -112,6 +112,7 @@ async fn main() -> Result<(), String> { cubbies, gossip, lonely(), + &Locker::null(), &roots, shutdown.clone(), ) diff --git a/server/src/manager.rs b/server/src/manager.rs index ccde5e8..fb83347 100644 --- a/server/src/manager.rs +++ b/server/src/manager.rs @@ -38,10 +38,12 @@ use sush_common::keys::{KeyError, KeyId, SshPublicKey}; use sush_common::targets::{Cubbies, SledHealth, SledVersion}; use sush_common::version::LONG_VERSION; +use crate::boundary::BoundaryStore; use crate::error::JobError; use crate::executor::PathIsolation; use crate::gossip::LinkedBaseboards; use crate::job::SocketSender; +use crate::locker::Locker; use crate::messages::v0::{CertRequest, IdentityRequest, JobRequest, Request, SessionRequest}; use crate::output::{JobOutputDir, JobOutputFileStream}; use crate::state::{GossipUniverse, MAX_CERTS, State, StateManager}; @@ -107,6 +109,7 @@ impl JobManager { cubbies: watch::Receiver, universe: watch::Receiver, linked: LinkedBaseboards, + locker: &Locker, roots: &[impl AsRef], shutdown: CancellationToken, ) -> Result { @@ -119,6 +122,7 @@ impl JobManager { cubbies, universe, linked, + locker, &roots, shutdown, ) @@ -134,6 +138,7 @@ impl JobManager { cubbies: watch::Receiver, universe: watch::Receiver, linked: LinkedBaseboards, + locker: &Locker, roots: &[Certificate], shutdown: CancellationToken, ) -> Result { @@ -141,6 +146,8 @@ impl JobManager { let (tx_req, rx_req) = mpsc::channel(16); let requests = ReceiverStream::new(rx_req); let session_sush_nonce = Arc::new(SyncMutex::new(SessionSushNonce::random())); + let boundary = Arc::new(BoundaryStore::new(&log, locker)); + boundary.load().await; let (rx_state, join_state) = StateManager::run( log.new(o!("component" => "state manager")), path_isolation, @@ -151,6 +158,7 @@ impl JobManager { universe, roots, session_sush_nonce.clone(), + boundary, shutdown, )?; Ok(Self { diff --git a/server/src/state.rs b/server/src/state.rs index 219732c..1291f1e 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -14,7 +14,7 @@ use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use futures::{FutureExt as _, Stream, StreamExt}; use lru::LruCache; -use rumors::{CausalMessages, Peer, Rumors, Version}; +use rumors::{CausalMessages, Network, Rumors, Version}; use sled_hardware_types::BaseboardId; use slog::{Logger, debug, error, info, o, warn}; use tokio::sync::watch; @@ -34,11 +34,13 @@ use sush_common::keys::{KeyError, KeyId, Signature, SshPublicKey}; use sush_common::targets::Cubbies; use sush_common::version::{VersionInfo, VersionMap}; -use crate::bookmark::{BookmarkSource, SushBookmark}; +use crate::bookmark::SushBookmark; +use crate::boundary::{Boundary, BoundaryStore, JobOutcome}; use crate::executor::{Executor, PathIsolation}; -use crate::gossip::Universe; +use crate::gossip::{Seed, Universe}; use crate::history::JobHistory; use crate::job::SocketSender; +use crate::locker::Locker; use crate::messages::v0::{ CertRequest, Error, Event, IdentityRequest, JobEvent, JobRequest, Message, Request, SessionRequest, @@ -49,6 +51,7 @@ use crate::output::JobOutputDir; pub type AttachmentPoints = BTreeMap>>; pub type Certificates = BTreeMap; pub type GossipNetwork = Rumors; +pub type GossipSeed = Seed; pub type GossipUniverse = Universe; pub type QueuedJobs = BTreeMap; pub type RunningJobs = BTreeMap<(JobId, BaseboardId), DateTime>; @@ -308,6 +311,8 @@ impl<'a> SessionGuard<'a> { history: &mut JobHistory, executor: &mut Executor, attachments: &mut AttachmentPoints, + past: &Past, + rumors: Option<&GossipNetwork>, ) { while let Some(QueuedJob { job: request, @@ -320,6 +325,31 @@ impl<'a> SessionGuard<'a> { if request.payload().runs_on(own_baseboard, cubbies) { if replayed { warn!(log, "not executing replayed job"; "job_id" => %job_id); + } else if past.boundary.untrusted() { + warn!( + log, "refusing job, the execution boundary is untrusted"; + "job_id" => %job_id, + ); + executor.job_refused( + job_id, + ProcessError::Io { + what: "consulting the execution boundary".to_string(), + error: "the store is untrusted; this sled needs service".to_string(), + }, + ); + } else if past.lost_session == Some(self.session_id()) { + warn!( + log, "refusing job for session with lost history"; + "job_id" => %job_id, + ); + executor.job_refused( + job_id, + ProcessError::Io { + what: "consulting the execution boundary".to_string(), + error: "a restart lost part of this session's history on this sled; start a new session" + .to_string(), + }, + ); } else if history .get_job_status(&job_id) .map(|status| { @@ -327,8 +357,21 @@ impl<'a> SessionGuard<'a> { }) .unwrap_or(true) { - executor.job_start(certs, request.clone(), params, tx_attachment); - attachments.insert(job_id, rx_attachment); + let Some(rumors) = rumors else { + warn!(log, "not starting a job while draining"; "job_id" => %job_id); + self.job_started(request); + continue; + }; + let boundary = Boundary { + network: rumors.network(), + session: self.session_id(), + job: job_id, + frontier: rumors.snapshot().latest().clone(), + outcome: JobOutcome::Committed, + }; + if executor.job_start(certs, request.clone(), params, tx_attachment, boundary) { + attachments.insert(job_id, rx_attachment); + } } } self.job_started(request); @@ -336,6 +379,50 @@ impl<'a> SessionGuard<'a> { } } +/// This incarnation's relationship to its own past: where it entered +/// the universe, what a previous life committed to, and what it left +/// running. +#[derive(Debug)] +pub struct Past { + /// The causal frontier we joined this universe at, if we joined + /// rather than seeded it. + join_frontier: Option, + /// The last job this sled committed to executing, durable across + /// restarts. + boundary: Arc, + /// The session a previous life of this sled lost history of. Its + /// jobs must not execute here; see [`Boundary::covered_by`]. + lost_session: Option, + /// Jobs whose replayed start events say a previous life ran them. + /// The end of such a job may replay later and remove it from the + /// set, so the survivors are known only when replay finishes. + zombies: BTreeSet, +} + +impl Past { + pub fn new( + join_frontier: Option, + boundary: Arc, + lost_session: Option, + ) -> Self { + Self { + join_frontier, + boundary, + lost_session, + zombies: BTreeSet::new(), + } + } + + /// Whether a message at `version` is live traffic rather than + /// replayed history. Only strict causal descendants of the join + /// frontier are live. + fn is_live(&self, version: &Version) -> bool { + self.join_frontier + .as_ref() + .is_none_or(|frontier| version > frontier) + } +} + #[derive(Debug)] pub struct State { /// The ID of the baseboard (sled) the server is running on. @@ -361,14 +448,8 @@ pub struct State { roots: Box<[KeyId]>, /// Baseboards by cubby number, as much of it as is known. cubbies: Cubbies, - /// The causal frontier we joined this universe at, if we joined - /// rather than seeded it. - join_frontier: Option, - /// Jobs whose start event on our own baseboard arrived as replayed - /// history. A previous life started them, and no executor of ours - /// will ever stop them. A terminal event clears its job, so - /// mid-replay entries are only suspects. - zombies: BTreeSet, + /// This incarnation's relationship to its own past. + past: Past, /// Message versions from newer builds, each warned about once. unknown_versions: BTreeSet, /// Build provenance by sled. @@ -387,7 +468,7 @@ impl State { own_baseboard: BaseboardId, root_certs: &[Certificate], session_sush_nonce: Arc>, - join_frontier: Option, + past: Past, ) -> Result { let certs = root_certs .iter() @@ -414,8 +495,7 @@ impl State { unknown_versions: Default::default(), identities: LruCache::new(MAX_REGISTERED_IDENTITIES), revoked_keys: LruCache::new(MAX_REVOKED_KEYS), - join_frontier, - zombies: Default::default(), + past, }; new.validate_certs(&roots); for root in &roots { @@ -463,7 +543,8 @@ impl State { /// Jobs a previous life of this server started and left running. pub fn zombies(&self) -> BTreeSet { - self.zombies + self.past + .zombies .iter() .filter(|job_id| { self.running @@ -473,13 +554,29 @@ impl State { .collect() } - /// Whether a message at `version` is live traffic rather than - /// replayed history. Only strict causal descendants of the join - /// frontier are live. fn is_live(&self, version: &Version) -> bool { - self.join_frontier - .as_ref() - .is_none_or(|frontier| version > frontier) + self.past.is_live(version) + } + + /// The boundary record carries the boundary job's ending, so the + /// next life can tell the truth instead of guessing. The write is + /// spawned because the state machine is synchronous, and losing + /// the race to a crash only falls back to interrupted. + fn record_boundary_outcome(&self, job_id: &JobId) { + let Some(status) = self + .history + .get_job_status(job_id) + .and_then(|map| map.get(&self.own_baseboard)) + .cloned() + else { + return; + }; + if !status.is_terminal() { + return; + } + let store = self.past.boundary.clone(); + let job_id = *job_id; + spawn(async move { store.record_outcome(&job_id, &status).await }); } pub fn get_job_status(&self, job_id: &JobId) -> Option<&JobStatusMap> { @@ -546,6 +643,7 @@ impl State { &mut self, log: &Logger, executor: &mut Executor, + rumors: Option<&GossipNetwork>, incoming_version: &Version, message: &Arc, ) -> Result<(), Error> { @@ -749,6 +847,8 @@ impl State { &mut self.history, executor, &mut self.attachments, + &self.past, + rumors, ); } else { info!( @@ -816,6 +916,8 @@ impl State { &mut self.history, executor, &mut self.attachments, + &self.past, + rumors, ); } _ => { @@ -901,12 +1003,26 @@ impl State { // Track the active set of known-running jobs anywhere in the rack. Event::Job(job_event) => match job_event { JobEvent::Start(job_id, when) => { + // A start never displaces a terminal status. + // After an identity change, an adjudicated + // Interrupted and a late replayed start are + // concurrent, and the verdict must win in + // both arrival orders. + if self + .history + .get_job_status(job_id) + .and_then(|status| status.get(baseboard_id)) + .is_some_and(|status| status.is_terminal()) + { + info!(log, "ignoring a start for a settled job"; "job_id" => %job_id); + return Ok(()); + } info!(log, "job started"; "job_id" => %job_id, "when" => %when); self.running.insert((*job_id, baseboard_id.clone()), *when); // A replayed start on our own baseboard is a // previous life's. Only these can be zombies. if *baseboard_id == self.own_baseboard && !self.is_live(incoming_version) { - self.zombies.insert(*job_id); + self.past.zombies.insert(*job_id); } self.history.set_job_status( job_id, @@ -924,7 +1040,7 @@ impl State { info!(log, "job stopped"; "job_id" => %job_id, "when" => %when, "result" => ?result); if baseboard_id == &self.own_baseboard { self.attachments.remove(job_id); - self.zombies.remove(job_id); + self.past.zombies.remove(job_id); } self.running.remove(&(*job_id, baseboard_id.clone())); self.history.transition_job_status( @@ -963,13 +1079,14 @@ impl State { ); if *baseboard_id == self.own_baseboard { executor.job_stopped(job_id); + self.record_boundary_outcome(job_id); } } JobEvent::Error(job_id, when, error) => { error!(log, "job error"; "job_id" => %job_id, "when" => %when, "error" => %error); if baseboard_id == &self.own_baseboard { self.attachments.remove(job_id); - self.zombies.remove(job_id); + self.past.zombies.remove(job_id); } self.running.remove(&(*job_id, baseboard_id.clone())); self.history.transition_job_status( @@ -995,6 +1112,7 @@ impl State { ); if *baseboard_id == self.own_baseboard { executor.job_stopped(job_id); + self.record_boundary_outcome(job_id); } } }, @@ -1039,7 +1157,7 @@ fn apply_message( message: &Arc, ) { tx_state.send_modify(|state| { - if let Err(error) = state.update(log, executor, version, message) { + if let Err(error) = state.update(log, executor, rumors, version, message) { error!(log, "state update failed"; "error" => ?error); // Re-gossiping a replayed message's error would grow the // set a little more on every rejoin by every sled. @@ -1112,30 +1230,88 @@ fn reap_zombies( } } -/// Create a fresh gossip network with this server as its only peer. +/// Adjudicate the boundary job if replay gave it no status on our +/// baseboard: announce its recorded ending, or interrupted when none +/// was recorded. No status means its fate never left this sled, and +/// no other sled can ever report it. A replayed start makes it a +/// zombie instead, and a replayed end settles it. Adjudicate only +/// after replay drains, like [`reap_zombies`], and once per universe. +/// Sleds that witnessed a real terminal event keep it; the ruling +/// convinces only sleds that knew nothing, so verdicts can differ +/// across sleds. +fn adjudicate_boundary( + log: &Logger, + tx_state: &watch::Sender, + rumors: Option<&GossipNetwork>, + own_baseboard: &BaseboardId, + snapshot: Option<&Boundary>, + survivors: &BTreeSet, + adjudicated: &mut Option, +) { + let Some(boundary) = snapshot else { + return; + }; + // One ruling per boundary: a sled that swaps universes again + // without running a job must not re-adjudicate the same job. + if *adjudicated == Some(boundary.job) { + return; + } + // A survivor's events land in the new universe when it finishes, + // so it needs no verdict now and may need one at a later swap. + if survivors.contains(&boundary.job) { + return; + } + *adjudicated = Some(boundary.job); + let witnessed = { + let state = tx_state.borrow(); + state + .get_job_status(&boundary.job) + .is_some_and(|status| status.get(own_baseboard).is_some()) + }; + if witnessed { + return; + } + let job_id = boundary.job; + // A stopped ending announces the start and stop pair, so the + // ordinary status transitions apply on every sled; a bare stop + // with no prior start would be dropped. + let events = match &boundary.outcome { + JobOutcome::Ended(JobStatus::Stopped { + time_started, + time_stopped, + result, + output, + .. + }) => vec![ + JobEvent::Start(job_id, *time_started), + JobEvent::Stop(job_id, *time_stopped, result.clone(), output.clone()), + ], + JobOutcome::Ended(JobStatus::Error { + time_error, error, .. + }) => vec![JobEvent::Error(job_id, *time_error, error.clone())], + JobOutcome::Ended(_) | JobOutcome::Committed => vec![JobEvent::Error( + job_id, + Utc::now(), + ProcessError::Interrupted, + )], + }; + if let Some(rumors) = rumors { + for event in events { + rumors.send(Message::Event(own_baseboard.clone(), Event::Job(event)).into()); + } + } + warn!(log, "adjudicated an unwitnessed job from a previous life"; "job_id" => %job_id); +} + +/// Grow a fresh gossip seed over sush's message type. /// /// A peer that seeds its own network has no one to gossip with, so jobs run /// only on the server that accepted them, and no server learns about any other /// server's sessions. This stands in for joining the rack's network over -/// sprockets on the bootstrap network. -/// -/// A pristine seed's bookmark touches no storage, and identities -/// recorded there are reclaimed only after a migration returns us to -/// their universe. Bad storage would abort every session at the -/// persist gate, before the seed could even learn to migrate. Probe -/// first and shed on failure. -pub async fn seed_gossip(bookmarks: &BookmarkSource) -> GossipNetwork { - let handle = match bookmarks.probe().await { - Ok(()) => bookmarks.next_handle(), - Err(_) => bookmarks.shed_handle(), - }; - match Peer::seed().bookmark(handle).await { - Ok(peer) => peer.into_rumors(), - Err(unbookmarked) => match unbookmarked.peer.bookmark(bookmarks.shed_handle()).await { - Ok(peer) => peer.into_rumors(), - Err(_) => unreachable!("a shed bookmark never touches storage"), - }, - } +/// sprockets on the bootstrap network. Storage semantics are +/// [`Seed::grow`]'s. +pub async fn seed_gossip(log: &Logger, locker: &Locker) -> GossipSeed { + Seed::grow(log, locker).await } #[derive(Debug)] @@ -1165,6 +1341,7 @@ impl StateManager { mut universe: watch::Receiver, roots: &[Certificate], session_sush_nonce: Arc>, + store: Arc, shutdown: CancellationToken, ) -> Result<(watch::Receiver, JoinHandle<()>), KeyError> where @@ -1184,12 +1361,41 @@ impl StateManager { } = universe.borrow_and_update().clone(); let mut causal_messages = initial.causal_messages(); + // The boundary's session counts as lost unless the universe + // already knows everything the boundary knew. See + // [`Boundary::covered_by`]. Decisions read a snapshot of the + // boundary, never the live store: the launcher advances the + // store concurrently, and a job committed after the snapshot + // is this incarnation's, not the past's. + let lost = { + let log = log.clone(); + move |snapshot: Option<&Boundary>, network: Network, frontier: Option<&Version>| { + snapshot.and_then(|boundary| { + let covered = boundary.covered_by(network, frontier); + debug!( + log, "boundary coverage"; + "covered" => covered, + "recorded_network" => ?boundary.network, + "network" => ?network, + "recorded_frontier" => ?boundary.frontier, + "join_frontier" => ?frontier, + ); + (!covered).then_some(boundary.session) + }) + } + }; + let boundary = store.boundary(); + // We report our current state through a watch channel. let mut initial_state = State::new( own_baseboard.clone(), roots, session_sush_nonce.clone(), - frontier.clone(), + Past::new( + frontier.clone(), + store.clone(), + lost(boundary.as_ref(), initial.network(), frontier.as_ref()), + ), )?; initial_state.cubbies = cubbies.borrow_and_update().clone(); let (tx_state, rx_state) = watch::channel(initial_state); @@ -1200,6 +1406,7 @@ impl StateManager { log.new(o!("component" => "executor")), path_isolation, output_dir, + store, shutdown.child_token(), ); @@ -1219,9 +1426,10 @@ impl StateManager { // history); `survivors` are jobs this incarnation itself // runs across a universe swap; `reaped` are zombies // already declared interrupted. - let mut frontier = frontier; + let mut boundary = boundary; let mut survivors: BTreeSet = BTreeSet::new(); let mut reaped: BTreeSet = BTreeSet::new(); + let mut adjudicated: Option = None; // Announce our build. if let Some((rumors, _)) = &gossip { @@ -1293,8 +1501,7 @@ impl StateManager { break; } Some((version, message)) => { - // Past the join frontier means the message is live. - let live = frontier.as_ref().is_none_or(|f| version > f); + let live = tx_state.borrow().is_live(&version); apply_message( &log, &tx_state, @@ -1322,6 +1529,15 @@ impl StateManager { &survivors, &mut reaped, ); + adjudicate_boundary( + &log, + &tx_state, + gossip.as_ref().map(|(rumors, _)| rumors), + &own_baseboard, + boundary.as_ref(), + &survivors, + &mut adjudicated, + ); } }, }, @@ -1356,17 +1572,31 @@ impl StateManager { causal_messages = fresh.rumors.causal_messages(); // A reaped zombie may still show as running (its // error event races the swap); it is no survivor. + // A job the executor is still launching is a + // survivor too: its start event may not have + // applied yet, but its events land in the new + // universe like any running job's. survivors = tx_state.borrow().own_running_jobs(); + survivors.extend(executor.in_flight()); survivors.retain(|job_id| !reaped.contains(job_id)); reaped = BTreeSet::new(); - frontier = fresh.frontier.clone(); + boundary = tx_state.borrow().past.boundary.boundary(); // TODO: re-inject local job state (policy pending). + let lost_session = lost( + boundary.as_ref(), + fresh.rumors.network(), + fresh.frontier.as_ref(), + ); tx_state.send_modify(|state| { *state = State::new( own_baseboard.clone(), &roots, state.session_sush_nonce.clone(), - fresh.frontier.clone(), + Past::new( + fresh.frontier.clone(), + state.past.boundary.clone(), + lost_session, + ), ) .expect("roots validated at startup"); state.cubbies = cubbies.borrow().clone(); @@ -1397,6 +1627,15 @@ impl StateManager { &survivors, &mut reaped, ); + adjudicate_boundary( + &log, + &tx_state, + Some(&*rumors), + &own_baseboard, + boundary.as_ref(), + &survivors, + &mut adjudicated, + ); } } }), diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index 594e316..d4f500a 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -7,24 +7,30 @@ mod common; use std::collections::BTreeSet; +use std::fs::{read, read_to_string, write}; use std::net::SocketAddrV6; +use std::slice::from_ref; +use std::time::Duration; use camino::Utf8PathBuf; +use function_name::named; use sled_hardware_types::BaseboardId; use slog::Logger; use tempfile::TempDir; use tokio::sync::watch; +use tokio::time::sleep; use tokio_util::sync::CancellationToken; use chrono::Utc; use sush_api::{JobStartParams, JobWait}; use sush_common::jobs::{ - JobId, JobOutputState, JobStatus, ProcessError, Session, SessionId, SessionSignerNonce, + JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, SessionId, + SessionSignerNonce, SignedJob, }; -use sush_common::keys::pem_cert_chain; -use sush_common::targets::{Cubbies, SledHealth}; +use sush_common::keys::{EphemeralKey, Signer as _, pem_cert_chain}; +use sush_common::targets::{Cubbies, SledHealth, SledId, Target}; use sush_common::version::VersionInfo; -use sush_server::bookmark::{BOOKMARK, BookmarkSource}; +use sush_server::bookmark::BOOKMARK; use sush_server::executor::PathIsolation; use sush_server::gossip::spawn_gossip; use sush_server::locker::Locker; @@ -55,25 +61,18 @@ impl Sled { root_pem: &Utf8PathBuf, shutdown: &CancellationToken, ) -> Sled { - Self::start_with_bookmarks( - log, - dir, - identity, - root_pem, - BookmarkSource::null(), - shutdown, - ) - .await + Self::start_with_locker(log, dir, identity, root_pem, Locker::null(), shutdown).await } - async fn start_with_bookmarks( + async fn start_with_locker( log: &Logger, dir: &Utf8PathBuf, identity: usize, root_pem: &Utf8PathBuf, - bookmarks: BookmarkSource, + locker: Locker, shutdown: &CancellationToken, ) -> Sled { + let seed = seed_gossip(log, &locker).await; let (peers, peers_rx) = watch::channel(BTreeSet::new()); let (addr, universe, linked) = spawn_gossip( log, @@ -82,8 +81,7 @@ impl Sled { corpus(dir), localhost(), peers_rx, - seed_gossip(&bookmarks).await, - bookmarks, + seed, shutdown.clone(), ) .await @@ -101,7 +99,8 @@ impl Sled { cubbies, universe.clone(), linked, - std::slice::from_ref(root_pem), + &locker, + from_ref(root_pem), shutdown.clone(), ) .await @@ -117,18 +116,19 @@ impl Sled { } } +#[named] #[tokio::test] async fn jobs_gossip_between_sleds() { let (_tmp, dir) = pki("sush-distributed-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("jobs_gossip_between_sleds"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let b = Sled::start(&log, &dir, 2, &root_pem, &shutdown).await; @@ -228,18 +228,19 @@ async fn jobs_gossip_between_sleds() { shutdown.cancel(); } +#[named] #[tokio::test] async fn rejoining_replays_without_reexecuting() { let (_tmp, dir) = pki("sush-replay-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("rejoining_replays_without_reexecuting"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); // Sled A runs a whole job before B exists. @@ -343,18 +344,19 @@ async fn rejoining_replays_without_reexecuting() { shutdown.cancel(); } +#[named] #[tokio::test] async fn interrupted_jobs_get_stopped() { let (_tmp, dir) = pki("sush-interrupted-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("interrupted_jobs_get_stopped"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; @@ -422,18 +424,19 @@ async fn interrupted_jobs_get_stopped() { shutdown.cancel(); } +#[named] #[tokio::test] async fn stragglers_do_not_interrupt_live_jobs() { let (_tmp, dir) = pki("sush-straggler-", 3); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("stragglers_do_not_interrupt_live_jobs"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); // A and C converge; C then holds a message A never sees. @@ -531,18 +534,264 @@ async fn stragglers_do_not_interrupt_live_jobs() { shutdown.cancel(); } +/// Sign a job aimed at one sled, so a retry cannot legitimately run +/// anywhere else. +async fn sign_job_for( + root: &mut EphemeralKey, + job_id: JobId, + session_id: SessionId, + command: &str, + sled: &BaseboardId, +) -> SignedJob { + root.sign(JobStartRequest::new( + job_id, + session_id, + command, + JobMode::Batch, + Target::Sleds(vec![SledId::Baseboard(sled.clone())]), + )) + .await + .unwrap() +} + +#[named] +#[tokio::test] +async fn lost_session_is_refused() { + let (_tmp, dir) = pki("sush-lost-", 2); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + // Sled A anchors the session and survives throughout. + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); + + // Sled B keeps its boundary in a locker, learns the session, and + // is then cut off from gossip while its front door still works. + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("the session gossips to B", 120, async || { + b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session_id) + }) + .await; + a.peers.send(BTreeSet::new()).unwrap(); + b.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + + // Two jobs run on B through its front door. No one else hears of + // them. The first leaves a footprint we can count. + let footprint = boundary_dir.path().join("footprint"); + let j1_id = session.next_job_id(); + let j1 = sign_job_for( + &mut root, + j1_id, + session_id, + &format!("echo run >> {}", footprint.display()), + &b.baseboard, + ) + .await; + b.mgr + .job_start( + &authn_b, + j1.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session.job_started(j1.clone()); + let j2_id = session.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j2, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); + + // B dies with both jobs unwitnessed and rejoins. Its boundary + // proves the rack is missing part of the session's history. + b_shutdown.cancel(); + drop(b); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + + // The recorded job finished before the crash, so B adjudicates + // its true ending on its sole authority. + eventually("the boundary job is adjudicated", 120, async || { + a.mgr.job_status(&authn_a, &j2_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // A retry of the first job's preserved artifact is refused, not + // re-executed: the footprint stays single. + b.mgr + .job_start(&authn_b, j1, JobStartParams::default()) + .await + .unwrap(); + eventually("the retry is refused", 120, async || { + b.mgr.job_status(&authn_b, &j1_id).await.is_ok_and(|map| { + map.get(&b.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Error { + error: ProcessError::Io { .. }, + .. + } + ) + }) + }) + }) + .await; + assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); + + shutdown.cancel(); +} + +#[named] +#[tokio::test] +async fn witnessed_session_survives_restart() { + let (_tmp, dir) = pki("sush-witness-", 2); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); + + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() + }) + .await; + + // A job runs on B and its result is witnessed by A, so B's + // boundary frontier is covered when it returns. The job must be + // live traffic on B: a replayed job never executes. + let j1_id = session.next_job_id(); + let j1 = sign_job_for(&mut root, j1_id, session_id, "true", &b.baseboard).await; + a.mgr + .job_start(&authn_a, j1.clone(), JobStartParams::default()) + .await + .unwrap(); + eventually("B's result reaches A", 120, async || { + a.mgr.job_status(&authn_a, &j1_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { .. })) + }) + }) + .await; + + // B restarts. The session's history is intact, so the session + // keeps working on B. + b_shutdown.cancel(); + drop(b); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe reconvergence", 120, async || { + a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() + }) + .await; + + session.job_started(j1); + let j2_id = session.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session_id, "true", &b.baseboard).await; + a.mgr + .job_start(&authn_a, j2, JobStartParams::default()) + .await + .unwrap(); + eventually("the session's next job runs on B", 120, async || { + a.mgr.job_status(&authn_a, &j2_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { .. })) + }) + }) + .await; + + shutdown.cancel(); +} + +#[named] #[tokio::test] async fn bookmarks_survive_restart() { let (_tmp, dir) = pki("sush-bookmark-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("bookmarks_survive_restart"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); // Sled A holds session history, so it wins every dominance contest. @@ -564,12 +813,12 @@ async fn bookmarks_survive_restart() { let slot = Utf8PathBuf::from_path_buf(bookmark_dir.path().to_path_buf()).unwrap(); let record = slot.join(BOOKMARK.file); let b_shutdown = CancellationToken::new(); - let b = Sled::start_with_bookmarks( + let b = Sled::start_with_locker( &log, &dir, 2, &root_pem, - BookmarkSource::new(&log, &Locker::new(&log, vec![slot.clone()])), + Locker::new(&log, vec![slot.clone()]), &b_shutdown, ) .await; @@ -579,7 +828,7 @@ async fn bookmarks_survive_restart() { record.as_std_path().exists() }) .await; - let before = std::fs::read(&record).unwrap(); + let before = read(&record).unwrap(); // The next incarnation reads the record back, rejoins, and // advances it, reclaiming the previous life's identity. @@ -589,13 +838,13 @@ async fn bookmarks_survive_restart() { // Let the dead incarnation's tasks quiesce, as a real reboot // would. Two live sources over one slot are the store's one // forbidden misuse. - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - let b = Sled::start_with_bookmarks( + sleep(Duration::from_millis(500)).await; + let b = Sled::start_with_locker( &log, &dir, 2, &root_pem, - BookmarkSource::new(&log, &Locker::new(&log, vec![slot.clone()])), + Locker::new(&log, vec![slot.clone()]), &shutdown, ) .await; @@ -608,25 +857,26 @@ async fn bookmarks_survive_restart() { eventually( "the record advances past the previous life", 120, - async || std::fs::read(&record).unwrap() != before, + async || read(&record).unwrap() != before, ) .await; shutdown.cancel(); } +#[named] #[tokio::test] async fn gossip_survives_bookmark_failure() { let (_tmp, dir) = pki("sush-nobookmark-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("gossip_survives_bookmark_failure"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; @@ -646,15 +896,12 @@ async fn gossip_survives_bookmark_failure() { // Sled 2's bookmark points into a directory that does not exist. // It sheds the bookmark and gossips anyway, stranding identities // rather than the rack. - let b = Sled::start_with_bookmarks( + let b = Sled::start_with_locker( &log, &dir, 2, &root_pem, - BookmarkSource::new( - &log, - &Locker::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush")]), - ), + Locker::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush")]), &shutdown, ) .await; @@ -665,7 +912,9 @@ async fn gossip_survives_bookmark_failure() { }) .await; - // Live jobs still run on the degraded sled. + // The degraded sled refuses live jobs rather than run one it + // cannot record, and gossips the refusal. The healthy sled still + // runs it. let job_id = session.next_job_id(); let job = sign_job(&mut root, job_id, session_id, "true").await; a.mgr @@ -679,9 +928,23 @@ async fn gossip_survives_bookmark_failure() { ) .await .unwrap(); - eventually("the job runs on the degraded sled", 120, async || { + eventually("the degraded sled refuses the job", 120, async || { a.mgr.job_status(&authn_a, &job_id).await.is_ok_and(|map| { - map.get(&b.baseboard) + map.get(&b.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Error { + error: ProcessError::Io { .. }, + .. + } + ) + }) + }) + }) + .await; + eventually("the healthy sled runs the job", 120, async || { + a.mgr.job_status(&authn_a, &job_id).await.is_ok_and(|map| { + map.get(&a.baseboard) .is_some_and(|s| matches!(s, JobStatus::Stopped { .. })) }) }) diff --git a/server/tests/gossip.rs b/server/tests/gossip.rs index 77784a2..e9ffe2b 100644 --- a/server/tests/gossip.rs +++ b/server/tests/gossip.rs @@ -11,14 +11,16 @@ use std::collections::BTreeSet; use std::net::SocketAddrV6; use camino::Utf8PathBuf; -use rumors::{Network, Peer, Rumors}; +use function_name::named; +use rumors::{Network, Rumors}; use slog::Logger; use tokio::sync::watch; use tokio_util::sync::CancellationToken; use sush_common::jobs::BaseboardId; -use sush_server::bookmark::{BookmarkSource, SushBookmark}; -use sush_server::gossip::{LinkedBaseboards, Universe, spawn_gossip}; +use sush_server::bookmark::SushBookmark; +use sush_server::gossip::{LinkedBaseboards, Seed, Universe, spawn_gossip}; +use sush_server::locker::Locker; use common::{ baseboard, corpus, eventually, gossip_config, localhost, pki, sprockets_config, test_logger, @@ -36,13 +38,8 @@ struct Node { impl Node { async fn start(log: &Logger, dir: &Utf8PathBuf, identity: usize) -> Node { let shutdown = CancellationToken::new(); - let bookmarks = BookmarkSource::null(); - let seed: Rumors = Peer::seed() - .bookmark(bookmarks.next_handle()) - .await - .expect("a pristine seed never touches its bookmark") - .into_rumors(); - let initial = seed.network(); + let seed: Seed = Seed::grow(log, &Locker::null()).await; + let initial = seed.rumors().network(); let (peers, peers_rx) = watch::channel(BTreeSet::new()); let (addr, universe, linked) = spawn_gossip( log, @@ -52,7 +49,6 @@ impl Node { localhost(), peers_rx, seed, - bookmarks, shutdown.clone(), ) .await @@ -109,10 +105,11 @@ fn converged(nodes: &[&Node]) -> Option { nodes.iter().all(|n| n.network() == first).then_some(first) } +#[named] #[tokio::test] async fn cold_start_converges() { let (_tmp, dir) = pki("sush-gossip-", 3); - let log = test_logger("cold_start_converges"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; let c = Node::start(&log, &dir, 3).await; @@ -132,10 +129,11 @@ async fn cold_start_converges() { .await; } +#[named] #[tokio::test] async fn staggered_start_converges() { let (_tmp, dir) = pki("sush-gossip-", 3); - let log = test_logger("staggered_start_converges"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; mesh(&[&a, &b]); @@ -160,10 +158,11 @@ async fn staggered_start_converges() { .await; } +#[named] #[tokio::test] async fn node_replacement_reconverges() { let (_tmp, dir) = pki("sush-gossip-", 4); - let log = test_logger("node_replacement_reconverges"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; let c = Node::start(&log, &dir, 3).await; @@ -189,10 +188,11 @@ async fn node_replacement_reconverges() { .await; } +#[named] #[tokio::test] async fn linked_follows_live_links() { let (_tmp, dir) = pki("sush-gossip-", 2); - let log = test_logger("linked_follows_live_links"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; assert!(a.linked().is_empty()); diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index 7dbfa4e..559e2b5 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -34,17 +34,17 @@ use sush_common::jobs::{ }; use sush_common::keys::{EphemeralKey, KeyError, KeyId, KeyType, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; -use sush_server::bookmark::BookmarkSource; use sush_server::gossip::{Universe, isolated, lonely}; use sush_server::io::BATCH_OUTPUT_BUFFER_SIZE; +use sush_server::locker::Locker; use sush_server::messages::v0::{CertRequest, IdentityRequest, Message, Request, SessionRequest}; use sush_server::output::{JobOutputDir, OutputDirs}; -use sush_server::{JobError, JobManager, seed_gossip}; +use sush_server::{JobError, JobManager}; use crate::test_utils::{ IntoBytes as _, SignJobRequest as _, ephemeral_test_root, ephemeral_test_subject, fake_identity, manager_and_test_root, manager_login, manager_test_root_and_peer, no_cubbies, - test_baseboard_id, test_logger, + null_gossip, test_baseboard_id, test_logger, }; use sush_server::executor::PathIsolation; @@ -579,8 +579,9 @@ async fn cubby_targets() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), cubbies_rx, - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), ) @@ -685,8 +686,9 @@ async fn root_certs_from_files() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), no_cubbies(), - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[path], CancellationToken::new(), ) @@ -736,8 +738,9 @@ async fn bad_root_cert_files() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), no_cubbies(), - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[path], CancellationToken::new(), ) @@ -767,8 +770,9 @@ async fn job_output_dir_moves() { JobOutputDir::new(rx_dirs), test_baseboard_id(), no_cubbies(), - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), ) @@ -856,13 +860,12 @@ async fn job_output_dir_moves() { #[tokio::test] async fn universe_swap() { // A universe migration resets the state machine. Sessions and history - // die with the old universe, and the manager keeps serving. + // die with the old universe, the boundary carries the last committed + // job across, and the manager keeps serving. let log = test_logger(function_name!()); let dir = TempDir::with_prefix("sush-").unwrap(); let mut root = ephemeral_test_root(); - let (universe, universe_rx) = watch::channel(Universe::genesis( - seed_gossip(&BookmarkSource::null()).await, - )); + let (universe, universe_rx) = watch::channel(Universe::genesis(null_gossip().await)); let mgr = JobManager::with_root_certs( log, PathIsolation::InsecureDisable, @@ -871,6 +874,7 @@ async fn universe_swap() { no_cubbies(), universe_rx, lonely(), + &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), ) @@ -909,9 +913,7 @@ async fn universe_swap() { // Migrate. The session and the job's history are gone. universe - .send(Universe::genesis( - seed_gossip(&BookmarkSource::null()).await, - )) + .send(Universe::genesis(null_gossip().await)) .unwrap(); timeout(Duration::from_secs(30), async { while mgr.session(&authn).is_some() { @@ -920,6 +922,30 @@ async fn universe_swap() { }) .await .expect("state reset"); + // The new universe cannot name the job, so the boundary + // adjudicates its recorded ending on this sled's sole authority. + timeout(Duration::from_secs(30), async { + loop { + if let Ok(map) = mgr.job_status(&authn, &job_id).await + && matches!( + map.get(mgr.own_baseboard()), + Some(JobStatus::Stopped { result: Ok(0), .. }) + ) + { + break; + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("boundary adjudication"); + + // A further swap does not re-adjudicate the same boundary: one + // ruling per boundary. + universe + .send(Universe::genesis(null_gossip().await)) + .unwrap(); + sleep(Duration::from_millis(200)).await; assert!(matches!( mgr.job_status(&authn, &job_id).await, Err(JobError::JobNotFound(_)) @@ -1010,7 +1036,7 @@ async fn cert_chain() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let gossip = isolated(seed_gossip(&BookmarkSource::null()).await); + let gossip = isolated(null_gossip().await); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( log, @@ -1020,6 +1046,7 @@ async fn cert_chain() { no_cubbies(), gossip, lonely(), + &Locker::null(), &roots, shutdown, ) @@ -1784,7 +1811,7 @@ async fn hostile_imports_cannot_displace() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let seed = seed_gossip(&BookmarkSource::null()).await; + let seed = null_gossip().await; let peer = seed.clone(); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( @@ -1795,6 +1822,7 @@ async fn hostile_imports_cannot_displace() { no_cubbies(), isolated(seed), lonely(), + &Locker::null(), from_ref(&root_cert), shutdown, ) @@ -1930,7 +1958,7 @@ async fn homonym_issuer_resolves_to_true_parent() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let seed = seed_gossip(&BookmarkSource::null()).await; + let seed = null_gossip().await; let peer = seed.clone(); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( @@ -1941,6 +1969,7 @@ async fn homonym_issuer_resolves_to_true_parent() { no_cubbies(), isolated(seed), lonely(), + &Locker::null(), from_ref(&root_cert), shutdown, ) diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 54c1bb9..6abf424 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -29,9 +29,9 @@ use sush_common::codephrases::Codephrase; use sush_common::jobs::{JobId, JobMode, JobStartRequest, SessionId, VerifiedJob}; use sush_common::keys::{EphemeralKey, KeyType, Signer}; use sush_common::targets::{Cubbies, Target}; -use sush_server::bookmark::BookmarkSource; use sush_server::executor::PathIsolation; use sush_server::gossip::{isolated, lonely}; +use sush_server::locker::Locker; use sush_server::output::{JobOutputDir, JobOutputFileStream}; use sush_server::state::GossipNetwork; use sush_server::{JobError, JobManager, seed_gossip}; @@ -228,7 +228,7 @@ pub async fn manager_test_root_and_peer( CancellationToken, ) { let dir = TempDir::with_prefix("sush-").unwrap(); - let seed = seed_gossip(&BookmarkSource::null()).await; + let seed = null_gossip().await; let peer = seed.clone(); let gossip = isolated(seed); let shutdown = CancellationToken::new(); @@ -241,6 +241,7 @@ pub async fn manager_test_root_and_peer( no_cubbies(), gossip, lonely(), + &Locker::null(), &[root.cert().to_owned()], shutdown.clone(), ) @@ -254,6 +255,12 @@ pub fn no_cubbies() -> watch::Receiver { watch::channel(Cubbies::new()).1 } +/// A seed over storage that persists nothing, silently. +pub async fn null_gossip() -> GossipNetwork { + let log = Logger::root(slog::Discard, slog::o!()); + seed_gossip(&log, &Locker::null()).await.into_rumors() +} + pub async fn authz( client: &Client, response: ResponseValue, From 6d2a6b6d154759f9d8213a2cc619e812841e5b0f Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Fri, 4 Sep 2026 18:48:04 -0600 Subject: [PATCH 03/12] Add a fixed-size Bloom filter for the burned-network set The set may never drop an entry, and an attacker must not be able to grow the file by inserting garbage, which rules out an exact set. The bit layout and the hash algorithm are on-disk format, pinned by tests. The filter errs only by calling a network burned when it is not; that error refuses a session, and never runs a job. Co-Authored-By: Claude Mythos 5 --- server/src/bloom.rs | 161 ++++++++++++++++++++++++++++++++++++++++++++ server/src/lib.rs | 1 + 2 files changed, 162 insertions(+) create mode 100644 server/src/bloom.rs diff --git a/server/src/bloom.rs b/server/src/bloom.rs new file mode 100644 index 0000000..5ede9cd --- /dev/null +++ b/server/src/bloom.rs @@ -0,0 +1,161 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A fixed-size Bloom filter for durable, grow-only sets. +//! +//! Designed for the boundary record's burned networks set. Entries may +//! never be dropped (once burned, a network stays burned), the size must +//! stay bounded under adversarial growth (we must not fill the M.2s), +//! and the filter may err only by claiming a key it never held. The +//! caller must treat that claim as refusal. +//! +//! The layout and hash algorithm determine the on-disk format, and must +//! not be changed without updating the boundary record's magic. Each key +//! probes the table at seven positions, cut as 16-bit words from its +//! SHA3-256 digest. The table is currently sized at 8192 bits, for +//! which seven probes is the optimal count out to about 800 entries +//! (bits * ln 2 / probes). The false positive rate there is about 0.7%, +//! degrading gradually past it. Real occupancy should stay in the tens, +//! where false positives are negligible. + +use std::array::from_fn; +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use sush_common::hash::{OUT_LEN, hash}; +use sush_common::wire::ExactBytes; + +const BYTES: usize = 1024; +const BITS: u64 = (BYTES * 8) as u64; +const PROBES: usize = 7; + +// The probes must fit in the digest, and the table size must divide +// 2^16, or reducing a 16-bit word to a table slot would favor some +// slots over others. +const _: () = { + assert!(2 * PROBES <= OUT_LEN); + assert!((1u64 << 16).is_multiple_of(BITS)); +}; + +/// A grow-only set of byte-string keys. +#[derive(Clone, Eq, PartialEq)] +pub struct Bloom { + bits: Box<[u8; BYTES]>, +} + +/// The probe positions for `key` are seven 16-bit words cut from +/// the leading bytes of the key's SHA3-256 digest, each reduced to +/// a table slot. +fn probes(key: &[u8]) -> [u64; PROBES] { + let digest = *hash(key).as_bytes(); + from_fn(|i| { + let j = 2 * i; + let word = u16::from_le_bytes(digest[j..j + 2].try_into().expect("two bytes")); + u64::from(word) % BITS + }) +} + +/// The byte index and mask selecting `probe`'s bit. +fn bit(probe: u64) -> (usize, u8) { + ((probe / 8) as usize, 1 << (probe % 8)) +} + +impl Bloom { + pub fn new() -> Self { + Self { + bits: Box::new([0; BYTES]), + } + } + + /// Permanently add `key` to the set. + pub fn insert(&mut self, key: &[u8]) { + for (byte, mask) in probes(key).map(bit) { + self.bits[byte] |= mask; + } + } + + /// Whether `key` may have been inserted. Never returns a false negative, + /// but false positives occur at the documented rate. + pub fn contains(&self, key: &[u8]) -> bool { + probes(key) + .map(bit) + .into_iter() + .all(|(byte, mask)| self.bits[byte] & mask != 0) + } +} + +impl Default for Bloom { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Bloom { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let set: u32 = self.bits.iter().map(|byte| byte.count_ones()).sum(); + write!(f, "Bloom({set}/{BITS} bits set)") + } +} + +impl Serialize for Bloom { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.bits[..]) + } +} + +impl<'de> Deserialize<'de> for Bloom { + fn deserialize>(deserializer: D) -> Result { + let bits = deserializer.deserialize_bytes(ExactBytes::)?; + Ok(Self { + bits: Box::new(bits), + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn round_trip() { + let mut bloom = Bloom::new(); + assert!(!bloom.contains(b"lost-network")); + bloom.insert(b"lost-network"); + assert!(bloom.contains(b"lost-network")); + assert!(!bloom.contains(b"other-network")); + } + + #[test] + fn serde_preserves_bits() { + let mut bloom = Bloom::new(); + bloom.insert(b"burned"); + let mut bytes = Vec::new(); + ciborium::ser::into_writer(&bloom, &mut bytes).unwrap(); + let back: Bloom = ciborium::de::from_reader(bytes.as_slice()).unwrap(); + assert_eq!(bloom, back); + assert!(back.contains(b"burned")); + } + + #[test] + fn false_positives_are_rare() { + let mut bloom = Bloom::new(); + for i in 0..128 { + bloom.insert(format!("burned-{i}").as_bytes()); + } + let hits = (0..10_000) + .filter(|i| bloom.contains(format!("probe-{i}").as_bytes())) + .count(); + assert_eq!( + hits, 0, + "false positives at plausible occupancy: {hits}/10000" + ); + } + + /// If this test fails, STOP! The on-disk format may have changed! + #[test] + fn pin_probes() { + assert_eq!(probes(b"sush"), [3392, 1007, 8018, 5416, 1335, 7804, 1908]); + } +} diff --git a/server/src/lib.rs b/server/src/lib.rs index 89312df..e23bca8 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -10,6 +10,7 @@ extern crate function_name; #[cfg(all(feature = "embedded", feature = "test-support"))] compile_error!("`test-support` must not be enabled for an embedded server"); +pub mod bloom; pub mod bookmark; pub mod boundary; pub mod error; From 991ec17b3e0e86a292458b5597f1e31abdc1869e Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Fri, 4 Sep 2026 18:48:07 -0600 Subject: [PATCH 04/12] Add JobStatus::Skipped for jobs a sled will never run A skip is a decision, not a failure. Without it, a job the boundary machinery has ruled out sits Queued forever on that sled; that hangs client waits and records no outcome. The status is terminal and carries a reason: the job's session sits below the sled's execution floor, or a previous life of the sled already handled it. The client renders a skip with its own row and reason. Nothing emits the status yet; the boundary rework that follows will emit it. Co-Authored-By: Claude Mythos 5 --- api/src/lib.rs | 2 +- client/src/cli.rs | 18 ++++++++++++++++ client/src/commands.rs | 7 +++++- common/src/jobs.rs | 40 +++++++++++++++++++++++++++++++--- sush.json | 49 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 5 deletions(-) diff --git a/api/src/lib.rs b/api/src/lib.rs index 54124a0..4f989e4 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -388,7 +388,7 @@ impl JobWait { match self { Self::None => true, Self::Start => !matches!(status, Queued { .. }), - Self::Stop => matches!(status, Cancelled { .. } | Error { .. } | Stopped { .. }), + Self::Stop => status.is_terminal(), } } } diff --git a/client/src/cli.rs b/client/src/cli.rs index 6f86bd4..77ee694 100644 --- a/client/src/cli.rs +++ b/client/src/cli.rs @@ -203,6 +203,11 @@ fn short_status_row(status: &JobStatus) -> String { JobStatus::Error { time_error, error, .. } => format!("Error at {time_error}: {error}"), + JobStatus::Skipped { + time_skipped, + reason, + .. + } => format!("Skipped at {time_skipped}: {reason}"), } } @@ -900,6 +905,19 @@ impl CommandContext for Cli { Error:\t{error}" ) } + JobStatus::Skipped { + job_id, + time_skipped, + reason, + } => { + println!( + "⏩ Job ID:\t{job_id}\n \ + Target:\t{baseboard_id}\n \ + Job status:\tSkipped\n \ + Skipped at:\t{time_skipped}\n \ + Reason:\t{reason}" + ) + } } } } diff --git a/client/src/commands.rs b/client/src/commands.rs index b7df19b..7f719fb 100644 --- a/client/src/commands.rs +++ b/client/src/commands.rs @@ -51,7 +51,7 @@ use sush_common::interactive::{InteractiveJobError, InteractiveJobMessage}; use sush_common::jobs::JobOutputStream::{self, Stderr, Stdout}; use sush_common::jobs::{ Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStatus, JobStatusMap, - Session, SessionId, SessionSignerNonce, SignedJob, job_status_try_from_json_map, + Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, job_status_try_from_json_map, }; #[cfg(feature = "permslip")] use sush_common::jobs::{JobStartRequest, SessionSushNonce}; @@ -2073,6 +2073,9 @@ async fn job_output_from( Some(JobStatus::Started { job_id, .. }) => { return Err(CommandError::JobStillRunning(job_id.to_owned())); } + Some(JobStatus::Skipped { job_id, reason, .. }) => { + return Err(CommandError::JobSkipped(job_id.to_owned(), *reason)); + } Some(JobStatus::Stopped { output, .. }) => output, }; let len = match stream { @@ -2526,6 +2529,8 @@ pub enum CommandError { JobDidNotRun(JobId), #[error("❌ Job `{0}` is not yet running")] JobNotYetRunning(JobId), + #[error("⏩ Job `{0}` was skipped on this sled: {1}")] + JobSkipped(JobId, SkipReason), #[error("❌ Job `{0}` is still running")] JobStillRunning(JobId), #[error("❌ JSON error: {0}")] diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 186def1..231d404 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -375,6 +375,35 @@ pub enum JobStatus { result: Result, output: JobOutputState, }, + /// The reporting sled decided it will never run this job. A skip + /// is a decision, not a failure: the job may have run on other + /// sleds, and the operator decides whether to resubmit. + Skipped { + job_id: JobId, + time_skipped: DateTime, + reason: SkipReason, + }, +} + +/// Why a sled will never run a job. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum SkipReason { + /// The job's session sits at or below the sled's execution floor, + /// where the sled cannot tell replay from re-run. + BelowFloor, + /// The job's chain position precedes the sled's recorded + /// commitment: a previous life already handled it. + AlreadyHandled, +} + +impl fmt::Display for SkipReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::BelowFloor => "the session sits below this sled's execution floor", + Self::AlreadyHandled => "a previous life of this sled already handled it", + }) + } } pub type JobStatusMap = BTreeMap; @@ -476,7 +505,10 @@ impl JobStatus { pub fn is_terminal(&self) -> bool { matches!( self, - Self::Cancelled { .. } | Self::Error { .. } | Self::Stopped { .. } + Self::Cancelled { .. } + | Self::Error { .. } + | Self::Stopped { .. } + | Self::Skipped { .. } ) } @@ -488,12 +520,13 @@ impl JobStatus { Self::Error { time_error, .. } => *time_error, Self::Started { time_started, .. } => *time_started, Self::Stopped { time_stopped, .. } => *time_stopped, + Self::Skipped { time_skipped, .. } => *time_skipped, } } pub fn time_elapsed(&self) -> TimeDelta { match self { - Self::Cancelled { .. } | Self::Error { .. } => TimeDelta::zero(), + Self::Cancelled { .. } | Self::Error { .. } | Self::Skipped { .. } => TimeDelta::zero(), Self::Queued { time_queued, .. } => Utc::now() - time_queued, Self::Started { time_started, .. } => Utc::now() - time_started, Self::Stopped { @@ -510,7 +543,8 @@ impl JobStatus { | Self::Queued { job_id, .. } | Self::Error { job_id, .. } | Self::Started { job_id, .. } - | Self::Stopped { job_id, .. } => job_id, + | Self::Stopped { job_id, .. } + | Self::Skipped { job_id, .. } => job_id, } } diff --git a/sush.json b/sush.json index cb04437..23579d6 100644 --- a/sush.json +++ b/sush.json @@ -1455,6 +1455,36 @@ "Stopped" ], "additionalProperties": false + }, + { + "description": "The reporting sled decided it will never run this job. A skip is a decision, not a failure: the job may have run on other sleds, and the operator decides whether to resubmit.", + "type": "object", + "properties": { + "Skipped": { + "type": "object", + "properties": { + "job_id": { + "$ref": "#/components/schemas/JobId" + }, + "reason": { + "$ref": "#/components/schemas/SkipReason" + }, + "time_skipped": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "job_id", + "reason", + "time_skipped" + ] + } + }, + "required": [ + "Skipped" + ], + "additionalProperties": false } ] }, @@ -1717,6 +1747,25 @@ "signature" ] }, + "SkipReason": { + "description": "Why a sled will never run a job.", + "oneOf": [ + { + "description": "The job's session sits at or below the sled's execution floor, where the sled cannot tell replay from re-run.", + "type": "string", + "enum": [ + "below-floor" + ] + }, + { + "description": "The job's chain position precedes the sled's recorded commitment: a previous life already handled it.", + "type": "string", + "enum": [ + "already-handled" + ] + } + ] + }, "SledHealth": { "description": "One sled's gossip link health, as the answering sled sees it. A silent death can lag `Linked` at TCP's pace.", "oneOf": [ From bacf655cc057f040728aa74b8b910073fac3ce30 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Fri, 4 Sep 2026 18:48:07 -0600 Subject: [PATCH 05/12] Use wide-width glyphs in client output U+26A0 (warning sign) is East Asian Width neutral, so wcwidth counts one column while the variation selector makes most emulators draw two, and aligned output drifts on those rows. U+2757 (exclamation mark) means the same thing and carries the wide property. Co-Authored-By: Claude Mythos 5 --- client/src/cli.rs | 14 +++++++------- client/src/tunnel.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/src/cli.rs b/client/src/cli.rs index 77ee694..86eadb4 100644 --- a/client/src/cli.rs +++ b/client/src/cli.rs @@ -91,7 +91,7 @@ impl Cli { let path = match BaseDirectories::with_prefix(PREFIX).place_state_file(SESSION_FILE_NAME) { Ok(path) => path, Err(error) => { - eprintln!("⚠️ The session will not persist: {error}"); + eprintln!("❗ The session will not persist: {error}"); return; } }; @@ -102,17 +102,17 @@ impl Cli { session, }) => *self.session.lock().unwrap() = Some(session), Ok(SavedSession { version, .. }) => { - eprintln!("⚠️ Ignoring a version {version} saved session") + eprintln!("❗ Ignoring a version {version} saved session") } - Err(error) => eprintln!("⚠️ Ignoring the saved session: {error}"), + Err(error) => eprintln!("❗ Ignoring the saved session: {error}"), }, Err(error) if error.kind() == ErrorKind::NotFound => (), - Err(error) => eprintln!("⚠️ Ignoring the saved session: {error}"), + Err(error) => eprintln!("❗ Ignoring the saved session: {error}"), } self.session_file = Some(path); match BaseDirectories::with_prefix(PREFIX).place_state_file(TOKEN_FILE_NAME) { Ok(path) => self.token_file = Some(path), - Err(error) => eprintln!("⚠️ Signing tokens will not persist: {error}"), + Err(error) => eprintln!("❗ Signing tokens will not persist: {error}"), } } @@ -149,7 +149,7 @@ impl Cli { }, }; if let Err(error) = result { - eprintln!("⚠️ The session was not saved: {error}"); + eprintln!("❗ The session was not saved: {error}"); } } } @@ -309,7 +309,7 @@ impl CommandContext for Cli { .map_err(io::Error::other) .and_then(|json| write_private(path, &json)); if let Err(error) = result { - eprintln!("⚠️ The token was not saved: {error}"); + eprintln!("❗ The token was not saved: {error}"); } } diff --git a/client/src/tunnel.rs b/client/src/tunnel.rs index 73c937a..d387152 100644 --- a/client/src/tunnel.rs +++ b/client/src/tunnel.rs @@ -125,7 +125,7 @@ impl Tunnel { let target = Arc::clone(&target); connections.spawn(async move { if let Err(error) = forward(&target, stream).await { - eprintln!("⚠️ Tunnel connection failed: {error}"); + eprintln!("❗ Tunnel connection failed: {error}"); } }); } From 70519b1b0b03b92a6f0e170b3a6c1f2007bf4387 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Fri, 4 Sep 2026 18:48:07 -0600 Subject: [PATCH 06/12] Rebuild the execution boundary on chain position The old record stored the causal frontier at commitment and compared it against the join frontier to decide whether history was lost. That comparison does not survive a crash: a restarted sled re-issues its dead predecessor's untransmitted versions as soon as it sends, so the recorded frontier can be re-covered by different messages and the lost-history verdict silently flips. This change removes the frontier from the record; the chain itself becomes the watermark. The record now stores the network, a burned set, the join of executed session starts, and the committed job. The commit also computes the successor's chain id from the signed bytes and stores it. The sled screens each session's start version against this record. The committed session resumes at its stored successor when it activates, so jobs at earlier chain positions drain without executing and a lost suffix no longer strands the session. A session whose start lies strictly above the executed join has never run here and is admitted. A start the record cannot order makes the sled hop: the sled reports a SessionHop error to the gossip set and raises its floor at the frontier that includes the report. The record joins every executed start rather than keeping only the last one, because a record that remembers only its last session cannot rank the session before it, and a third session could then replay the first session's jobs. The burned set holds every network whose watermark this record overwrote. It is a Bloom filter because entries may never be dropped, and a flood of forged networks must not grow the file. The launcher writes the burn and the advance together, so no crash window separates them. A sled that re-enters a burned universe reports a UniverseFlipFlop error and raises its floor the same way. Without the burn, returning to an earlier universe would overwrite the watermark that remembers the jobs it ran there, and those jobs could be resubmitted and re-executed. A sled that enters a universe with history while holding no record raises its floor at the frontier that includes its own announce, since it cannot order any session started before it arrived. Floors live in memory only. A floor's version is created by sending a message, and if the sled dies before that message reaches anyone, no other copy of it ever exists and no future session can dominate the floor; a persisted floor would then refuse every session in that universe until a cold boot. Every restart re-detects the burn, the missing record, or the unordered session and raises the floor again, so persisting the floor gains nothing. Jobs below the floor end as JobStatus::Skipped, and a new JobEvent carries the skip to the gossip set, so client waits resolve instead of hanging on a job the sled will never run. Arrival and birth marks replace the join frontier. The zombie check classified this sled's own job starts against the join frontier, which is one peer's view of history: a start from a previous life could sit above a lagging peer's frontier, be classified as this life's, and leave the rack believing the job still runs. The birth mark, the frontier just past this life's first send, separates the lives exactly: nothing a previous life sent dominates it, and everything this life sends does. The arrival mark, the frontier just before that first send, keeps the general replay line for foreign traffic, which can reach this sled before its sender has seen that first send. Both marks are computed locally and never persisted. They replace every use of Universe::frontier, so that field is deleted. This change also deletes the bookmark generation numbers. The record keeps every universe's identities, so a straggling write costs at most a stranded identity, and the locker's sequence guard already refuses stale writers. The gossip manager stops every session before handing a new peer its handle. Co-Authored-By: Claude Mythos 5 --- common/src/jobs.rs | 22 +- server/src/bookmark.rs | 99 +-- server/src/boundary.rs | 271 +++++--- server/src/executor.rs | 32 +- server/src/gossip.rs | 46 +- server/src/messages.rs | 50 +- server/src/state.rs | 623 ++++++++++++++---- server/tests/distributed.rs | 555 ++++++++++++++-- server/tests/output/job-skipped-event.bin | Bin 0 -> 145 bytes server/tests/output/session-hop-error.bin | 1 + .../tests/output/universe-flip-flop-error.bin | 1 + sush.json | 12 + 12 files changed, 1356 insertions(+), 356 deletions(-) create mode 100644 server/tests/output/job-skipped-event.bin create mode 100644 server/tests/output/session-hop-error.bin create mode 100644 server/tests/output/universe-flip-flop-error.bin diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 231d404..e5f5ed7 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -109,6 +109,7 @@ impl SessionId { LastJob::None => hash(&[b"None", self.0.to_be_bytes().as_slice()].concat()), LastJob::Some(job) => hash(&[b"Some", job.to_be_signed().as_slice()].concat()), LastJob::Burned(job_id) => hash(&[b"Burned", job_id.to_be_bytes().as_slice()].concat()), + LastJob::Resumed(next) => return *next, }) } } @@ -153,6 +154,7 @@ pub enum LastJob { None, Some(SignedJob), Burned(JobId), + Resumed(JobId), } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -201,11 +203,13 @@ impl Session { self.last_job = LastJob::Some(job) } - /// Burn `job_id`, either as the session's next job or by - /// rewinding it from the chain head. The rewind unwinds a - /// signed-but-unrun job on a signer. On a server it converges a - /// skip that raced the start it names, keeping the execution in - /// history. Returns whether the chain moved. + /// Burn `job_id` when it is the session's next job or the job at + /// the chain head. Burning the next job skips it before it runs. + /// Burning the head rewrites the chain to continue from the burn: + /// a signer unwinds a job it signed that never ran, and a server + /// converges with that signer when a skip request arrives after + /// the start of the job it names. An execution already in history + /// stays there. Returns whether the chain moved. pub fn skip_job(&mut self, job_id: JobId) -> bool { if job_id == self.next_job_id() || matches!(&self.last_job, LastJob::Some(job) if *job.job_id() == job_id) @@ -220,6 +224,14 @@ impl Session { pub fn next_job_id(&self) -> JobId { self.session_id.next_job_id(&self.last_job) } + + /// Resume the chain at `successor`, the position a boundary + /// record stored at its last commitment. Every position before + /// `successor` was already handled by the sled that stored the + /// record. + pub fn resume_at(&mut self, successor: JobId) { + self.last_job = LastJob::Resumed(successor); + } } /// How a job runs. The streaming modes allow **unrecorded** I/O. diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index bcbc36e..536f933 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -13,13 +13,12 @@ //! The record format and when to load & store are dictated by rumors. //! We use a [`Tenant`] of a [`Locker`] to store it on disk(s); //! if a load fails or the slots disagree, we assume a new identity -//! rather than risk resuming with a stale one. Generation numbers -//! ensure that a straggler from an abandoned universe can't clobber -//! its successor's record. +//! rather than risk resuming with a stale one. The record keeps every +//! universe's identities, so a lost write costs at most a stranded +//! identity, never a stale one. use std::io::{self, Cursor}; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; use rumors::{Bookmark, BookmarkError, Serialized}; use slog::{Discard, Logger, o, warn}; @@ -40,38 +39,23 @@ pub enum BookmarkIoError { Serialize(#[source] io::Error), #[error("storing the bookmark failed: {0}")] Store(#[source] StoreError), - #[error("the bookmark was handed to a newer peer")] - Superseded, } -/// This server's bookmark storage. Hands out one handle per peer, -/// each superseding the last. +/// This server's bookmark storage. Every handle shares the one record. #[derive(Clone, Debug)] pub struct BookmarkSource { - ratchet: Arc, -} - -#[derive(Debug)] -struct Ratchet { log: Logger, - tenant: Tenant, - generation: AtomicU64, + tenant: Arc, } impl BookmarkSource { /// A source persisting to `locker`. /// [`Seed::grow`](crate::gossip::Seed::grow) makes the one source - /// a locker gets per process. A handle is disabled when its source - /// hands out a newer one. No source can disable another source's - /// handles, so a second source would let an old peer overwrite its - /// replacement's record. + /// a locker gets per process. pub fn new(log: &Logger, locker: &Locker) -> Self { Self { - ratchet: Arc::new(Ratchet { - log: log.new(o!("component" => "bookmark")), - tenant: locker.tenant(BOOKMARK), - generation: AtomicU64::new(0), - }), + log: log.new(o!("component" => "bookmark")), + tenant: Arc::new(locker.tenant(BOOKMARK)), } } @@ -80,12 +64,15 @@ impl BookmarkSource { Self::new(&Logger::root(Discard, o!()), &Locker::null()) } - /// A ratcheting handle for the next peer. - /// All earlier handles are superseded. - pub fn next_handle(&self) -> SushBookmark { + /// A persisting handle for a peer. Rumors persists a bookmark only + /// when a gossip session starts, and the gossip manager stops + /// every session before it hands a new peer its handle, so no two + /// peers persist concurrently; see the migration notes in + /// [`gossip`](crate::gossip). + pub fn handle(&self) -> SushBookmark { SushBookmark { - ratchet: self.ratchet.clone(), - generation: self.ratchet.generation.fetch_add(1, Ordering::SeqCst) + 1, + log: self.log.clone(), + tenant: self.tenant.clone(), shed: false, } } @@ -94,8 +81,8 @@ impl BookmarkSource { /// gossiping after its real bookmark failed. pub fn shed_handle(&self) -> SushBookmark { SushBookmark { - ratchet: self.ratchet.clone(), - generation: 0, + log: self.log.clone(), + tenant: self.tenant.clone(), shed: true, } } @@ -104,18 +91,11 @@ impl BookmarkSource { /// One peer's handle on the [`BookmarkSource`]. #[derive(Debug)] pub struct SushBookmark { - ratchet: Arc, - generation: u64, + log: Logger, + tenant: Arc, shed: bool, } -impl SushBookmark { - /// Has this bookmark been overtaken by events? - fn obe(&self) -> bool { - self.generation < self.ratchet.generation.load(Ordering::SeqCst) - } -} - impl BookmarkError for SushBookmark { type Error = BookmarkIoError; } @@ -127,15 +107,12 @@ impl Bookmark for SushBookmark { if self.shed { return Ok(None); } - let mut guard = self.ratchet.tenant.lock().await; - if self.obe() { - return Err(BookmarkIoError::Superseded); - } + let mut guard = self.tenant.lock().await; match guard.load().await { Verdict::Adopt(record) | Verdict::Restore(record) => Ok(Some(Cursor::new(record))), Verdict::Empty => Ok(None), Verdict::Discard(reason) => { - warn!(self.ratchet.log, "assuming a fresh identity"; "reason" => %reason); + warn!(self.log, "assuming a fresh identity"; "reason" => %reason); Ok(None) } } @@ -152,10 +129,7 @@ impl Bookmark for SushBookmark { write(&mut buf).await.map_err(BookmarkIoError::Serialize)?; let record = buf.into_inner(); - let mut guard = self.ratchet.tenant.lock().await; - if self.obe() { - return Err(BookmarkIoError::Superseded); - } + let mut guard = self.tenant.lock().await; guard.store(&record).await.map_err(BookmarkIoError::Store) } } @@ -210,7 +184,7 @@ mod test { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); let source = source(slots(&dir)); - let handle = source.next_handle(); + let handle = source.handle(); assert!(read_back(&handle).await.is_none()); handle.store(record(b"who we are")).await.unwrap(); assert_eq!(read_back(&handle).await.unwrap(), b"who we are"); @@ -223,26 +197,19 @@ mod test { let slots = slots(&dir); for (slot, bytes) in slots.iter().zip([b"one", b"two"]) { let lone = source(vec![slot.clone()]); - lone.next_handle().store(record(bytes)).await.unwrap(); + lone.handle().store(record(bytes)).await.unwrap(); } - assert!(read_back(&source(slots).next_handle()).await.is_none()); + assert!(read_back(&source(slots).handle()).await.is_none()); } - /// A new handle disables the old one's loads and stores. + /// Handles share the record: one stores, another reads it back. #[tokio::test] - async fn stale_generations_are_disabled() { + async fn handles_share_record() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); let source = source(slots(&dir)); - let old = source.next_handle(); - old.store(record(b"before")).await.unwrap(); - let new = source.next_handle(); - assert!(matches!( - old.store(record(b"after")).await, - Err(BookmarkIoError::Superseded) - )); - assert!(matches!(old.load().await, Err(BookmarkIoError::Superseded))); - assert_eq!(read_back(&new).await.unwrap(), b"before"); + source.handle().store(record(b"shared")).await.unwrap(); + assert_eq!(read_back(&source.handle()).await.unwrap(), b"shared"); } /// A null source and a shed handle persist nothing and never fail, @@ -250,16 +217,16 @@ mod test { #[tokio::test] async fn null_and_shed_touch_nothing() { let null = BookmarkSource::null(); - let handle = null.next_handle(); + let handle = null.handle(); handle.store(record(b"lost")).await.unwrap(); assert!(read_back(&handle).await.is_none()); let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); let source = source(slots(&dir)); - source.next_handle().store(record(b"kept")).await.unwrap(); + source.handle().store(record(b"kept")).await.unwrap(); let shed = source.shed_handle(); assert!(read_back(&shed).await.is_none()); shed.store(record(b"dropped")).await.unwrap(); - assert_eq!(read_back(&source.next_handle()).await.unwrap(), b"kept"); + assert_eq!(read_back(&source.handle()).await.unwrap(), b"kept"); } } diff --git a/server/src/boundary.rs b/server/src/boundary.rs index d7d4168..19919bc 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -4,22 +4,50 @@ //! The boundary between jobs we executed and jobs we only heard about. //! -//! The gossip frontier answers "what have I heard"; the boundary -//! answers "where was I when I last committed to running a job." One -//! record, overwritten in chain order before each spawn, carries the -//! job, its session, our causal frontier at the moment of commitment, -//! and the job's ending, once it has one. +//! One record, rewritten before each spawn, carries what this sled +//! last committed to: the job, its session, the chain position after +//! the job, the universe they belong to, and the job's ending once +//! it has one. A universe is one shared gossip history, identified +//! by its network; sleds join universes, leave them, and sometimes +//! return. Alongside the commitment, the record keeps the join +//! of every session start this sled has executed under in that +//! universe, and the set of universes it burned by leaving. The +//! record is this sled's execution watermark: nothing at or below it +//! may run again. //! -//! After a restart, compare the recorded frontier with the join -//! frontier. If the join frontier covers it, every request we had -//! processed was witnessed: replay refuses old jobs, the session -//! chain never releases a resubmitted one, and zombie reaping reports -//! what we left running. If not, a suffix of our history died with -//! us: jobs may have run here that no sled can name, and their signed -//! artifacts could be resubmitted and run again. The state machine -//! then refuses jobs of the recorded session until a new session -//! supersedes it, and adjudicates the recorded job itself: the -//! recorded ending if the job got one, and interrupted if it did not. +//! After a restart, replayed gossip rebuilds the sessions. When the +//! committed session activates, the sled resumes its chain at the +//! stored successor. The previous life had already moved the chain +//! past every earlier position, so the session's queue never +//! releases them here, and the chain continues from the successor +//! whether its request arrives by replay or by resubmission. A +//! session that starts strictly above the executed join has never +//! run here and is served. A session the record cannot order makes +//! the sled hop: the sled reports the hop to the gossip set as an +//! error and sets a floor, in memory, at the frontier that includes +//! the report. A session that does not start above the floor has +//! its jobs skipped; a session started after the hop is served. The +//! join folds in every session this sled has executed under in this +//! universe, so a session it once ran under can never screen as +//! new: the sled hops, and the session's jobs are skipped rather +//! than re-run. If replay gives the recorded job no status, the sled +//! adjudicates it: it announces the recorded ending when the record +//! holds one, and an interrupted ending when it does not. +//! +//! Universes have no order. The record instead keeps a burned set, +//! holding the network of every universe whose watermark it +//! overwrote by moving on. A sled that re-enters a burned universe +//! has flip-flopped, and raises its floor: it reports the flip-flop +//! to the gossip set and sets the floor, in memory, at the frontier +//! that includes the report. No older message can contain a version +//! born at that instant, so a session started before the re-entry +//! never lies above the floor, and its jobs are skipped. A sled that +//! enters a universe with history while holding no record for it +//! raises a floor the same way. The floor is never persisted: the +//! burn, the missing record, or the unordered session is still there +//! after a restart, and raises it again. A floor written to disk +//! could carry a version that died with the life that created it; +//! no later session could ever dominate such a floor. //! //! A boundary that cannot be written means the job must not run. A //! boundary that cannot be trusted means no job may run at all, since @@ -29,7 +57,7 @@ use std::sync::Mutex as SyncMutex; use std::sync::atomic::{AtomicBool, Ordering}; -use ciborium::{de::from_reader, ser::into_writer}; +use ciborium::{de::from_reader as from_cbor, ser::into_writer as into_cbor}; use rumors::{Network, Version}; use serde::{Deserialize, Serialize}; use slog::{Logger, o, warn}; @@ -37,6 +65,7 @@ use thiserror::Error; use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId}; +use crate::bloom::Bloom; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOUNDARY: TenantSpec = TenantSpec { @@ -44,39 +73,80 @@ pub const BOUNDARY: TenantSpec = TenantSpec { magic: b"SUSHBOUNDARY", }; -/// The execution boundary: the last job this sled committed to -/// running, everything it had seen when it committed, and how far the -/// job got. +/// The execution boundary: what this sled last committed to, in which +/// universe, and which universes it has left behind ("burned"). +/// +/// Versions do not compare across universes, so `network` scopes every +/// version in the record. `burned` holds the network of every +/// universe whose watermark this record overwrote by moving on: a +/// sled re-entering one raises its floor in memory, and the record on +/// disk stays the displaced universe's true watermark until a commit +/// overwrites it. +/// +/// `executed` is the join of the start versions of every session +/// this sled has executed under in this universe. A single stored +/// start would forget the sessions before it, and a third session +/// could then replay the first session's jobs; the join never +/// forgets. Session starts are witnessed messages, and only those +/// keep their meaning across a crash. A start that only this sled +/// ever saw belongs to a session whose history is lost, and refusal +/// is the right answer there anyway. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Boundary { pub network: Network, - pub session: SessionId, - pub job: JobId, + pub burned: Bloom, #[serde(with = "version_bytes")] - pub frontier: Version, - pub outcome: JobOutcome, + pub executed: Version, + pub job: Option, } -/// How far the boundary job got. +/// The last job this sled committed to running, and how far it got. +/// We also store the chain position *after* `job`, computed from the +/// request's signed bytes at commit time, to allow session resumption. #[derive(Clone, Debug, Deserialize, Serialize)] -pub enum JobOutcome { - /// Committed to run, with no ending recorded. After a crash, - /// interrupted is the truth. - Committed, - /// The job's terminal status. - Ended(JobStatus), +pub struct Committed { + pub session: SessionId, + pub job: JobId, + pub successor: JobId, + pub outcome: JobOutcome, } impl Boundary { - /// Whether the rack already knows everything we knew at - /// commitment. Covered means no committed job can be lost. - /// Anything else means a suffix of our history died with us. - /// The comparison includes third-party traffic we had seen, so a - /// join through a lagging peer can look uncovered; the cost is a - /// session refused on this sled until a new one supersedes it. - pub fn covered_by(&self, network: Network, join_frontier: Option<&Version>) -> bool { - self.network == network && join_frontier.is_some_and(|frontier| self.frontier <= *frontier) + /// Whether this record burned `network`: left its universe behind + /// and overwrote its watermark. + pub fn is_burned(&self, network: Network) -> bool { + self.burned.contains(&network_key(network)) } + + /// The burned set for the replacement record, committed in + /// `network`. A replacement in a different universe burns this + /// record's own network. + pub fn burned_for(&self, network: Network) -> Bloom { + let mut burned = self.burned.clone(); + if self.network != network { + burned.insert(&network_key(self.network)); + } + burned + } + + /// The executed-session join for the replacement record, + /// committed in `network` and folding in `started`. Joins never + /// cross universes, so a replacement elsewhere starts its join + /// fresh. + pub fn executed_for(&self, network: Network, started: &Version) -> Version { + if self.network == network { + self.executed.clone() | started.clone() + } else { + started.clone() + } + } +} + +/// A network's Bloom key is its CBOR bytes. +fn network_key(network: Network) -> Vec { + let mut bytes = Vec::new(); + into_cbor(&network, &mut bytes).expect("writing to a Vec cannot fail"); + bytes } mod version_bytes { @@ -94,14 +164,23 @@ mod version_bytes { } } +/// How far the boundary job got. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum JobOutcome { + /// Committed to run, with no ending recorded. + Committed, + /// The job's terminal status, error endings included. + Ended(JobStatus), +} + fn encode(boundary: &Boundary) -> Vec { let mut bytes = Vec::new(); - into_writer(boundary, &mut bytes).expect("writing to a Vec cannot fail"); + into_cbor(boundary, &mut bytes).expect("writing to a Vec cannot fail"); bytes } fn decode(record: &[u8]) -> Option { - from_reader(record).ok() + from_cbor(record).ok() } #[derive(Debug, Error)] @@ -119,8 +198,9 @@ pub struct BoundaryStore { tenant: Tenant, /// The record, readable synchronously by the state machine. boundary: SyncMutex>, - /// Untrusted until loaded, and forever if the load discards: - /// writing would launder the disagreement into false agreement. + /// Untrusted until loaded, and forever if the load discards: a + /// write would overwrite the disagreeing slots, and the next load + /// would see agreement that never happened. untrusted: AtomicBool, loaded: AtomicBool, } @@ -167,10 +247,9 @@ impl BoundaryStore { self.boundary.lock().unwrap().clone() } - /// Record how the boundary job ended, so the next life can tell - /// the truth instead of guessing. A stop displaces an adjudicated - /// Interrupted, mirroring the status arms in the state machine; - /// nothing else is overwritten, and a record that has moved on to + /// Record how the boundary job ended. A stop displaces an adjudicated + /// `Interrupted`, mirroring the status arms in the state machine. + /// Nothing else is overwritten, and a record that has moved on to /// a newer job ignores the old job's ending. pub async fn record_outcome(&self, job_id: &JobId, outcome: &JobStatus) { debug_assert!(outcome.is_terminal()); @@ -180,11 +259,14 @@ impl BoundaryStore { let mut guard = self.tenant.lock().await; let updated = { let recorded = self.boundary.lock().unwrap(); - let Some(boundary) = recorded.as_ref().filter(|b| b.job == *job_id) else { + let Some(boundary) = recorded.as_ref() else { + return; + }; + let Some(committed) = boundary.job.as_ref().filter(|c| c.job == *job_id) else { return; }; let displaces = matches!( - (&boundary.outcome, outcome), + (&committed.outcome, outcome), (JobOutcome::Committed, _) | ( JobOutcome::Ended(JobStatus::Error { @@ -198,8 +280,13 @@ impl BoundaryStore { return; } Boundary { - outcome: JobOutcome::Ended(outcome.clone()), - ..boundary.clone() + job: Some(Committed { + outcome: JobOutcome::Ended(outcome.clone()), + ..committed.clone() + }), + network: boundary.network, + burned: boundary.burned.clone(), + executed: boundary.executed.clone(), } }; if let Err(error) = guard.store(&encode(&updated)).await { @@ -212,7 +299,7 @@ impl BoundaryStore { *self.boundary.lock().unwrap() = Some(updated); } - /// Commit to executing the job at `boundary`. On failure the + /// Commit to executing the job in `boundary`. On failure the /// caller must not run the job. pub async fn advance(&self, boundary: &Boundary) -> Result<(), BoundaryError> { if self.untrusted() { @@ -264,16 +351,36 @@ mod test { serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() } - fn boundary(seed: u8) -> Boundary { + fn boundary() -> Boundary { Boundary { - network: network(seed), - session: SessionId::random(), - job: JobId::random(), - frontier: "(1, 1, (0, 0, 2))".parse().unwrap(), - outcome: JobOutcome::Committed, + network: network(1), + burned: Bloom::new(), + executed: "(1, 1, (0, 0, 2))".parse().unwrap(), + job: Some(Committed { + session: SessionId::random(), + job: JobId::random(), + successor: JobId::random(), + outcome: JobOutcome::Committed, + }), } } + fn job_of(boundary: &Boundary) -> JobId { + boundary.job.as_ref().expect("a committed job").job + } + + /// The burned set's keys are on-disk format: a change to the + /// network's serde shape would silently forget every burn, and a + /// forgotten burn admits a flip-flop instead of refusing it. If + /// this fails, STOP, and see the warning on [`crate::bloom`]. + #[test] + fn pin_network_keys() { + assert_eq!( + network_key(network(1)), + [0x50, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + #[tokio::test] async fn commitments_survive_restarts() { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); @@ -281,7 +388,9 @@ mod test { let first = store(slots.clone()).await; assert!(first.boundary().is_none()); - let (a, b) = (boundary(1), boundary(2)); + let (a, mut b) = (boundary(), boundary()); + b.network = network(2); + b.burned = a.burned_for(b.network); first.advance(&a).await.unwrap(); first.advance(&b).await.unwrap(); @@ -289,9 +398,13 @@ mod test { assert!(!next.untrusted()); let recorded = next.boundary().unwrap(); assert_eq!(recorded.network, b.network); - assert_eq!(recorded.session, b.session); - assert_eq!(recorded.job, b.job); - assert_eq!(recorded.frontier, b.frontier); + assert_eq!(job_of(&recorded), job_of(&b)); + assert!(recorded.is_burned(network(1))); + assert!(!recorded.is_burned(network(3))); + assert_eq!(recorded.executed, b.executed); + let (recorded, expected) = (recorded.job.unwrap(), b.job.unwrap()); + assert_eq!(recorded.session, expected.session); + assert_eq!(recorded.successor, expected.successor); assert!(matches!(recorded.outcome, JobOutcome::Committed)); } @@ -303,30 +416,31 @@ mod test { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slots = slots(&dir); let first = store(slots.clone()).await; - let b = boundary(1); + let b = boundary(); + let job = job_of(&b); first.advance(&b).await.unwrap(); let interrupted = JobStatus::Error { - job_id: b.job, + job_id: job, time_error: chrono::Utc::now(), error: ProcessError::Interrupted, }; let killed = JobStatus::Error { - job_id: b.job, + job_id: job, time_error: chrono::Utc::now(), error: ProcessError::Killed(9), }; first.record_outcome(&JobId::random(), &killed).await; assert!(matches!( - first.boundary().unwrap().outcome, + first.boundary().unwrap().job.unwrap().outcome, JobOutcome::Committed )); - first.record_outcome(&b.job, &interrupted).await; - first.record_outcome(&b.job, &killed).await; + first.record_outcome(&job, &interrupted).await; + first.record_outcome(&job, &killed).await; let next = store(slots).await; assert!(matches!( - next.boundary().unwrap().outcome, + next.boundary().unwrap().job.unwrap().outcome, JobOutcome::Ended(JobStatus::Error { error: ProcessError::Interrupted, .. @@ -334,33 +448,20 @@ mod test { )); } - #[tokio::test] - async fn coverage_requires_frontier_and_universe() { - let boundary = boundary(1); - let covered: Version = "(2, 2, (0, 0, 3))".parse().unwrap(); - let behind: Version = "(1, 0, (0, 0, 2))".parse().unwrap(); - - assert!(boundary.covered_by(network(1), Some(&boundary.frontier))); - assert!(boundary.covered_by(network(1), Some(&covered))); - assert!(!boundary.covered_by(network(1), Some(&behind))); - assert!(!boundary.covered_by(network(2), Some(&covered))); - assert!(!boundary.covered_by(network(1), None)); - } - #[tokio::test] async fn disagreement_is_untrusted_and_pins() { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slots = slots(&dir); - for (slot, seed) in slots.iter().zip([1, 2]) { + for slot in &slots { let lone = store(vec![slot.clone()]).await; - lone.advance(&boundary(seed)).await.unwrap(); + lone.advance(&boundary()).await.unwrap(); } let untrusted = store(slots.clone()).await; assert!(untrusted.untrusted()); assert!(untrusted.boundary().is_none()); assert!(matches!( - untrusted.advance(&boundary(3)).await, + untrusted.advance(&boundary()).await, Err(BoundaryError::Untrusted) )); @@ -386,7 +487,7 @@ mod test { let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots(&dir))); assert!(store.untrusted()); assert!(matches!( - store.advance(&boundary(1)).await, + store.advance(&boundary()).await, Err(BoundaryError::Untrusted) )); } diff --git a/server/src/executor.rs b/server/src/executor.rs index 347d8b1..1bf2950 100644 --- a/server/src/executor.rs +++ b/server/src/executor.rs @@ -32,7 +32,8 @@ use tokio_util::sync::CancellationToken; use sush_api::JobStartParams; use sush_common::interactive::WindowSize; use sush_common::jobs::{ - JobId, JobMode, JobOutputStream, JobStartRequest, ProcessError, SignedJob, VerifiedJob, + JobId, JobMode, JobOutputStream, JobStartRequest, ProcessError, SignedJob, SkipReason, + VerifiedJob, }; use crate::boundary::{Boundary, BoundaryStore}; @@ -51,8 +52,10 @@ pub const DEFAULT_TERM: &str = "vt100"; /// experience backpressure; if we do, something is wrong. const EVENTS_CHANNEL_CAPACITY: usize = 16; -/// The launcher performs one small fsync per job, so at human job -/// rates this never fills. A full queue refuses the job. +/// Each queued launch waits on one small fsync, so the queue drains in +/// milliseconds. A burst of concurrent jobs can still fill it, and a +/// full queue refuses the job with an error event rather than block +/// the state machine. const LAUNCH_CHANNEL_CAPACITY: usize = 16; pub struct Executor { @@ -112,7 +115,13 @@ impl Executor { what: "recording the execution boundary".to_string(), error: error.to_string(), }; - send_error(&launch.log, &launch.boundary.job, &launch.events, error).await; + send_error( + &launch.log, + launch.request.payload().job_id(), + &launch.events, + error, + ) + .await; continue; } spawn(job_spawn(launch)); @@ -229,6 +238,21 @@ impl Executor { }); } + /// Report that this sled will never run `job_id`; see + /// [`JobStatus::Skipped`](sush_common::jobs::JobStatus). + pub fn job_skipped(&self, job_id: JobId, reason: SkipReason) { + let Some(events) = self.events.read().unwrap().as_ref().cloned() else { + return; + }; + let log = self.log.clone(); + spawn(async move { + let event = Event::Job(JobEvent::Skipped(job_id, Utc::now(), reason)); + if let Err(error) = events.send(event).await { + warn!(log, "failed to send skip event"; "job_id" => %job_id, "error" => %error); + } + }); + } + pub fn output_dir(&self) -> &JobOutputDir { &self.output_dir } diff --git a/server/src/gossip.rs b/server/src/gossip.rs index 0ffe21b..e08f02b 100644 --- a/server/src/gossip.rs +++ b/server/src/gossip.rs @@ -29,7 +29,7 @@ use std::net::{SocketAddr, SocketAddrV6}; use std::time::Duration; use futures::StreamExt as _; -use rumors::{Error, Joined, Network, Peer, Rumors, Ticks, Version}; +use rumors::{Error, Joined, Network, Peer, Rumors, Ticks}; use serde::Serialize; use serde::de::DeserializeOwned; use sled_hardware_types::BaseboardId; @@ -72,22 +72,16 @@ impl Default for GossipConfig { } } -/// A gossip universe and where we entered it. +/// A gossip universe. #[derive(Clone, Debug)] pub struct Universe { /// The gossiped set. pub rumors: Rumors, - /// The causal frontier of the set received when we joined, - /// or `None` if we seeded the universe ourselves. - pub frontier: Option, } impl Universe { pub fn genesis(rumors: Rumors) -> Self { - Self { - rumors, - frontier: None, - } + Self { rumors } } } @@ -105,18 +99,20 @@ impl Seed { /// Seed a fresh universe with this server as its only peer, over /// `locker`'s storage, making the locker's one [`BookmarkSource`]. /// - /// A pristine seed's bookmark touches no storage, and identities - /// recorded there are reclaimed only after a migration returns us - /// to their universe. Bad storage would abort every session at the - /// persist gate, before the seed could even learn to migrate. The - /// probe runs first, and a failed probe sheds the bookmark. + /// The probe runs first because broken storage would otherwise + /// wedge gossip: rumors stores the bookmark at the start of every + /// session, a failed store aborts the session, and a peer that can + /// never hold a session can never join another universe. When the + /// probe fails we gossip with a shed handle instead, which + /// persists nothing; each restart then strands an identity, which + /// is harmless. pub async fn grow(log: &Logger, locker: &Locker) -> Self where T: DeserializeOwned + Serialize + Send + Sync + 'static, { let bookmarks = BookmarkSource::new(log, locker); let handle = match locker.probe().await { - Ok(()) => bookmarks.next_handle(), + Ok(()) => bookmarks.handle(), Err(_) => bookmarks.shed_handle(), }; let rumors = match Peer::seed().bookmark(handle).await { @@ -452,11 +448,17 @@ where /// the next link retries; either way all links are rebuilt, since the /// old ones belong to the universe we are leaving. /// - /// The new peer gets a fresh bookmark handle, fencing off all the - /// abandoned universe's stores. If the received identity cannot be - /// persisted, we keep gossiping with a shed handle rather than take - /// the sled out of gossip; a stranded identity is harmless, unlike - /// a support shell that cannot reach a degraded rack. + /// The new peer gets its own handle on the same bookmark storage. + /// That is safe because rumors persists a bookmark only when a + /// session starts, and aborting the drivers above ends every + /// session before the handle exists: the abandoned peer can never + /// store again. A store it already had in flight either loses to + /// the locker's sequence guard, or records a session that was + /// aborted before it sent anything, so nothing on the wire + /// outruns the record. If the received identity cannot be + /// persisted, we keep gossiping with a shed handle rather than + /// take the sled out of gossip; a stranded identity is harmless, + /// unlike a support shell that cannot reach a degraded rack. async fn migrate(&mut self, peer: SocketAddr, mut link: SprocketsLink) { self.drivers.abort_all(); self.live.clear(); @@ -464,7 +466,7 @@ where self.log, "joining the universe that beat ours"; "peer" => %peer, "ours" => %self.rumors.network(), ); - let bootstrap = Peer::bootstrap().bookmark(self.bookmarks.next_handle()); + let bootstrap = Peer::bootstrap().bookmark(self.bookmarks.handle()); match timeout(self.config.join_timeout, bootstrap.join(&mut link)).await { Ok(Joined::Joined { peer }) => self.adopt(peer), Ok(Joined::Unbookmarked(unbookmarked)) => { @@ -493,10 +495,8 @@ where /// Follow the joined peer into its universe. fn adopt(&mut self, peer: Peer) { self.rumors = peer.into_rumors(); - let frontier = self.rumors.snapshot().latest().clone(); let _ = self.publish.send(Universe { rumors: self.rumors.clone(), - frontier: Some(frontier), }); self.joins.clear(); info!(self.log, "migrated"; "network" => %self.rumors.network()); diff --git a/server/src/messages.rs b/server/src/messages.rs index e7c732e..7eff26d 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -14,7 +14,7 @@ use x509_cert::Certificate; use sush_api::JobStartParams; use sush_common::authn::SignedLogin; use sush_common::jobs::JobOutputState; -use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob}; +use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob, SkipReason}; use sush_common::keys::{KeyId, SshPublicKey}; use sush_common::version::VersionInfo; @@ -257,6 +257,7 @@ pub mod v0 { JobOutputState, ), Error(JobId, DateTime, ProcessError), + Skipped(JobId, DateTime, SkipReason), } #[derive(Clone, Debug, Deserialize, Eq, Error, PartialEq, Serialize)] @@ -272,6 +273,10 @@ pub mod v0 { incoming_session: SessionId, incoming_version: Version, }, + #[error("Sled re-entered a burned universe")] + UniverseFlipFlop, + #[error("Sled hopped into a session its record cannot order")] + SessionHop, } } @@ -606,6 +611,49 @@ mod wire_format { assert_wire_format("concurrent-sessions-error", msg); } + #[test] + fn job_skipped_event() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Job(JobEvent::Skipped( + JobId::from_str("zoo-zero").unwrap(), + "2026-09-04T20:00:00Z".parse().unwrap(), + SkipReason::BelowFloor, + )), + ) + .into(); + assert_wire_format("job-skipped-event", msg); + } + + #[test] + fn session_hop_error() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Error(Error::SessionHop), + ) + .into(); + assert_wire_format("session-hop-error", msg); + } + + #[test] + fn universe_flip_flop_error() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Error(Error::UniverseFlipFlop), + ) + .into(); + assert_wire_format("universe-flip-flop-error", msg); + } + #[test] fn job_start_interactive_request() { use sush_api::JobWait; diff --git a/server/src/state.rs b/server/src/state.rs index 1291f1e..2743f51 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -27,15 +27,16 @@ use x509_cert::der::Encode as _; use sush_api::JobStartParams; use sush_common::authn::{Identity, Nonce, RequestVerifier, SignedLogin}; use sush_common::jobs::{ - Access, JobId, JobStatus, JobStatusMap, ProcessError, Session, SessionId, SessionSushNonce, - SignedJob, + Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, + SessionSushNonce, SignedJob, SkipReason, }; use sush_common::keys::{KeyError, KeyId, Signature, SshPublicKey}; use sush_common::targets::Cubbies; use sush_common::version::{VersionInfo, VersionMap}; +use crate::bloom::Bloom; use crate::bookmark::SushBookmark; -use crate::boundary::{Boundary, BoundaryStore, JobOutcome}; +use crate::boundary::{Boundary, BoundaryStore, Committed, JobOutcome}; use crate::executor::{Executor, PathIsolation}; use crate::gossip::{Seed, Universe}; use crate::history::JobHistory; @@ -108,7 +109,6 @@ pub enum SessionState { Active { /// Last observed session request. frontier: Version, - /// Start of this session; identity anchor. started: Version, /// The active session. session: Box, @@ -135,10 +135,12 @@ impl SessionState { Active { session, queued_jobs, + started, .. } => Some(SessionGuard { inner: session, queued_jobs, + started, }), } } @@ -190,6 +192,8 @@ impl Default for SessionState { struct SessionGuard<'a> { inner: &'a mut Session, queued_jobs: &'a mut QueuedJobs, + /// The version of the session's start message; identity anchor. + started: &'a Version, } impl<'a> SessionGuard<'a> { @@ -209,10 +213,6 @@ impl<'a> SessionGuard<'a> { self.inner.skip_job(*job_id) } - pub fn next_queued_job(&mut self) -> Option { - self.queued_jobs.remove(&self.inner.next_job_id()) - } - #[allow(clippy::too_many_arguments)] pub fn enqueue_job( &mut self, @@ -228,10 +228,16 @@ impl<'a> SessionGuard<'a> { ) { let job_id = *job.job_id(); let targeted = job.payload().runs_on(own_baseboard, cubbies); - if history.contains(&job_id) { + // Adjudication can give the boundary job a status before its + // request replays. The queue releases jobs in order, so + // dropping that request would wedge the session behind the + // boundary job forever. A replayed request therefore always + // joins the queue; only a live duplicate is dropped. + if !replayed && history.contains(&job_id) { // Note but otherwise ignore the duplicate job. info!(log, "already started job"; "job_id" => %job_id); - } else if self.queued_jobs.len() >= MAX_QUEUED_JOBS + } else if !replayed + && self.queued_jobs.len() >= MAX_QUEUED_JOBS && !self.queued_jobs.contains_key(&job_id) { // We have no choice; drop the job on the floor. @@ -311,21 +317,40 @@ impl<'a> SessionGuard<'a> { history: &mut JobHistory, executor: &mut Executor, attachments: &mut AttachmentPoints, - past: &Past, + past: &mut Past, rumors: Option<&GossipNetwork>, ) { - while let Some(QueuedJob { - job: request, - params, - replayed, - }) = self.next_queued_job() - { + loop { + let next_id = self.inner.next_job_id(); + if !self.queued_jobs.contains_key(&next_id) { + break; + } + // A hop can raise the floor mid-drain, so screen on every pass. + let admission = match rumors { + None => Admission::Admit, + Some(rumors) => past.screen(rumors.network(), self.session_id(), self.started), + }; + // A hop raises the floor and screens again. + if matches!(admission, Admission::Hop) { + let Some(rumors) = rumors else { + break; + }; + warn!( + log, "raising the execution floor above a session the boundary record cannot order"; + "session_id" => %self.session_id(), "started" => ?self.started, + ); + past.hop(rumors, own_baseboard, Error::SessionHop); + continue; + } + let QueuedJob { + job: request, + params, + replayed, + } = self.queued_jobs.remove(&next_id).expect("checked above"); let (tx_attachment, rx_attachment) = watch::channel(None); let job_id = request.payload().job_id().to_owned(); if request.payload().runs_on(own_baseboard, cubbies) { - if replayed { - warn!(log, "not executing replayed job"; "job_id" => %job_id); - } else if past.boundary.untrusted() { + if past.boundary.untrusted() { warn!( log, "refusing job, the execution boundary is untrusted"; "job_id" => %job_id, @@ -334,22 +359,21 @@ impl<'a> SessionGuard<'a> { job_id, ProcessError::Io { what: "consulting the execution boundary".to_string(), - error: "the store is untrusted; this sled needs service".to_string(), - }, - ); - } else if past.lost_session == Some(self.session_id()) { - warn!( - log, "refusing job for session with lost history"; - "job_id" => %job_id, - ); - executor.job_refused( - job_id, - ProcessError::Io { - what: "consulting the execution boundary".to_string(), - error: "a restart lost part of this session's history on this sled; start a new session" + error: "the boundary records on this sled's M.2s disagree \ + or are corrupt; an M.2 may have failed and the sled \ + needs service" .to_string(), }, ); + } else if matches!(admission, Admission::Refuse) { + // Below the floor the queue skips forward in chain + // order. A live job gets the terminal skip status; + // a replayed one is history, and re-reporting it + // on every rejoin would grow the message set. + if !replayed { + warn!(log, "skipping job below the execution floor"; "job_id" => %job_id); + executor.job_skipped(job_id, SkipReason::BelowFloor); + } } else if history .get_job_status(&job_id) .map(|status| { @@ -362,12 +386,19 @@ impl<'a> SessionGuard<'a> { self.job_started(request); continue; }; + let successor = self + .session_id() + .next_job_id(&LastJob::Some(request.clone())); let boundary = Boundary { network: rumors.network(), - session: self.session_id(), - job: job_id, - frontier: rumors.snapshot().latest().clone(), - outcome: JobOutcome::Committed, + burned: past.burned_for(rumors.network()), + executed: past.executed_for(rumors.network(), self.started), + job: Some(Committed { + session: self.session_id(), + job: job_id, + successor, + outcome: JobOutcome::Committed, + }), }; if executor.job_start(certs, request.clone(), params, tx_attachment, boundary) { attachments.insert(job_id, rx_attachment); @@ -379,47 +410,156 @@ impl<'a> SessionGuard<'a> { } } +/// What [`Past::screen`] decides about a live job of the active +/// session. +enum Admission { + /// Execute normally. + Admit, + /// The session does not start above the floor; skip the job. + Refuse, + /// The boundary record cannot order this session, so a previous + /// life of this sled may have run its jobs. The sled reports an + /// error and raises its floor, as if it had just entered the + /// universe at this instant. + Hop, +} + /// This incarnation's relationship to its own past: where it entered /// the universe, what a previous life committed to, and what it left /// running. #[derive(Debug)] pub struct Past { - /// The causal frontier we joined this universe at, if we joined - /// rather than seeded it. - join_frontier: Option, + /// The frontier just before this life's first send into this + /// universe: the general replay line. Messages that do not + /// strictly follow it count as replayed. + arrival: Version, + /// The frontier just past this life's first send. Own-baseboard + /// events at or below it belong to a previous life; everything + /// this life sends causally follows it. Never persisted: each + /// life computes its own, so the split is exact and independent of + /// which peer we joined through. It is not the general replay + /// line, because a neighbor's fresh traffic reaches us before our + /// first send reaches the neighbor. + birth: Version, /// The last job this sled committed to executing, durable across /// restarts. boundary: Arc, - /// The session a previous life of this sled lost history of. Its - /// jobs must not execute here; see [`Boundary::covered_by`]. - lost_session: Option, + /// The boundary record as we found it on entering this universe. + /// The live store advances with our own jobs; [`Past::screen`] + /// compares sessions against the previous life's commitment, so + /// it reads this frozen copy. + committed: Option, + /// The execution floor, raised in memory at entry into a burned + /// universe, at record-less entry into a universe with history, + /// and on a session hop. Only sessions started strictly above it + /// are served; below it this sled cannot tell replay from re-run. + /// Never persisted: whatever raised it is still there after a + /// restart and raises it again. See [`entry_floor`]. + floor: Option, /// Jobs whose replayed start events say a previous life ran them. /// The end of such a job may replay later and remove it from the /// set, so the survivors are known only when replay finishes. zombies: BTreeSet, } +/// Whether `version` strictly dominates `mark`. Equal and concurrent +/// versions do not. +fn dominates(version: &Version, mark: &Version) -> bool { + version > mark +} + impl Past { pub fn new( - join_frontier: Option, + arrival: Version, + birth: Version, boundary: Arc, - lost_session: Option, + committed: Option, + floor: Option, ) -> Self { Self { - join_frontier, + arrival, + birth, boundary, - lost_session, + committed, + floor, zombies: BTreeSet::new(), } } + /// Decide whether the active session, identified by its id and + /// the version of its start message, may run a live job here. + fn screen(&self, network: Network, session: SessionId, started: &Version) -> Admission { + // We must refuse anything below our floor, because we cannot + // tell replay from re-run there. + if self + .floor + .as_ref() + .is_some_and(|floor| !dominates(started, floor)) + { + return Admission::Refuse; + } + + // If we have committed to nothing, anything is safe to admit. + let Some(committed) = &self.committed else { + return Admission::Admit; + }; + + // Jobs cannot cross universes, and anything a previous life + // ran in this universe is below the floor checked above. + if committed.network != network { + return Admission::Admit; + } + + match &committed.job { + Some(job) if job.session == session => Admission::Admit, + _ => { + if dominates(started, &committed.executed) { + Admission::Admit + } else { + Admission::Hop + } + } + } + } + + /// Report the cause of an [`Admission::Hop`] to the gossip set, + /// then set the floor at the frontier that includes the report. + fn hop(&mut self, rumors: &GossipNetwork, own_baseboard: &BaseboardId, report: Error) { + rumors.send(Message::Event(own_baseboard.clone(), Event::Error(report)).into()); + self.floor = Some(rumors.snapshot().latest().clone()); + } + + /// The burned set for a record committed in `network`; see + /// [`Boundary::burned_for`]. + fn burned_for(&self, network: Network) -> Bloom { + self.committed + .as_ref() + .map(|boundary| boundary.burned_for(network)) + .unwrap_or_default() + } + + /// The executed-session join for a record committed in `network` + /// under a session started at `started`; see + /// [`Boundary::executed_for`]. + fn executed_for(&self, network: Network, started: &Version) -> Version { + self.committed + .as_ref() + .map(|boundary| boundary.executed_for(network, started)) + .unwrap_or_else(|| started.clone()) + } + /// Whether a message at `version` is live traffic rather than - /// replayed history. Only strict causal descendants of the join - /// frontier are live. + /// replayed history: only strict causal descendants of the + /// arrival are live. fn is_live(&self, version: &Version) -> bool { - self.join_frontier - .as_ref() - .is_none_or(|frontier| version > frontier) + dominates(version, &self.arrival) + } + + /// Whether this sled's own event at `version` came from a + /// previous life. The split is exact: nothing a previous life + /// sent dominates the birth, and everything this life sends does. + fn is_from_previous_life(&self, version: &Version) -> bool { + !dominates(version, &self.birth) } } @@ -694,10 +834,30 @@ impl State { log, "session started"; "session_id" => %session_id, "actor" => %actor, ); + let mut session = Session::started(*session_id, actor.clone()); + // The committed session resumes at its + // stored successor: every earlier chain + // position was already handled by the life + // that stored it, and the chain needs no + // replay to verify the next job. + if let Some(committed) = self + .past + .committed + .as_ref() + .and_then(|boundary| boundary.job.as_ref()) + .filter(|committed| committed.session == *session_id) + { + info!( + log, "resuming the committed session at its successor"; + "session_id" => %session_id, + "successor" => %committed.successor, + ); + session.resume_at(committed.successor); + } self.session = Active { frontier: self.session.frontier() | incoming_version.clone(), started: incoming_version.clone(), - session: Box::new(Session::started(*session_id, actor.clone())), + session: Box::new(session), queued_jobs: QueuedJobs::new(), attach_grants: BTreeMap::new(), }; @@ -847,7 +1007,7 @@ impl State { &mut self.history, executor, &mut self.attachments, - &self.past, + &mut self.past, rumors, ); } else { @@ -916,7 +1076,7 @@ impl State { &mut self.history, executor, &mut self.attachments, - &self.past, + &mut self.past, rumors, ); } @@ -1019,9 +1179,19 @@ impl State { } info!(log, "job started"; "job_id" => %job_id, "when" => %when); self.running.insert((*job_id, baseboard_id.clone()), *when); - // A replayed start on our own baseboard is a - // previous life's. Only these can be zombies. - if *baseboard_id == self.own_baseboard && !self.is_live(incoming_version) { + // A start on our own baseboard sent by a + // previous life names a job whose process + // died with that life. No executor here will + // ever end it, so the job is a zombie: unless + // replay delivers its ending, we report it + // interrupted once replay drains. The birth + // mark splits the lives exactly; the arrival + // mark would misread a previous life's start + // as this life's own whenever the join peer + // lagged behind that start. + if *baseboard_id == self.own_baseboard + && self.past.is_from_previous_life(incoming_version) + { self.past.zombies.insert(*job_id); } self.history.set_job_status( @@ -1115,6 +1285,33 @@ impl State { self.record_boundary_outcome(job_id); } } + JobEvent::Skipped(job_id, when, reason) => { + info!( + log, "job skipped"; + "job_id" => %job_id, "when" => %when, "reason" => %reason, + ); + self.running.remove(&(*job_id, baseboard_id.clone())); + self.history.transition_job_status( + job_id, + baseboard_id, + Some(incoming_version.rank()), + // A skip is the reporting sled's decision + // about itself, and never displaces a + // terminal status it already reported. + |old_status| match old_status { + None + | Some(JobStatus::Queued { .. }) + | Some(JobStatus::Started { .. }) => Some(JobStatus::Skipped { + job_id: *job_id, + time_skipped: *when, + reason: *reason, + }), + _ => None, + }, + self.session.queued_jobs(), + &self.running, + ); + } }, Event::Error(error) => { error!(log, "session error"; "error" => %error); @@ -1236,9 +1433,10 @@ fn reap_zombies( /// no other sled can ever report it. A replayed start makes it a /// zombie instead, and a replayed end settles it. Adjudicate only /// after replay drains, like [`reap_zombies`], and once per universe. -/// Sleds that witnessed a real terminal event keep it; the ruling -/// convinces only sleds that knew nothing, so verdicts can differ -/// across sleds. +/// The ruling is gossiped to every sled, but a sled that already +/// holds a terminal status for the job keeps it; only sleds with +/// none adopt the ruling, so one sled may show the real ending while +/// another shows interrupted. fn adjudicate_boundary( log: &Logger, tx_state: &watch::Sender, @@ -1248,34 +1446,36 @@ fn adjudicate_boundary( survivors: &BTreeSet, adjudicated: &mut Option, ) { - let Some(boundary) = snapshot else { + // A floor commits to no job, so there is nothing to rule on. + let Some(Committed { job, outcome, .. }) = snapshot.and_then(|boundary| boundary.job.as_ref()) + else { return; }; + let job_id = *job; // One ruling per boundary: a sled that swaps universes again // without running a job must not re-adjudicate the same job. - if *adjudicated == Some(boundary.job) { + if *adjudicated == Some(job_id) { return; } // A survivor's events land in the new universe when it finishes, // so it needs no verdict now and may need one at a later swap. - if survivors.contains(&boundary.job) { + if survivors.contains(&job_id) { return; } - *adjudicated = Some(boundary.job); + *adjudicated = Some(job_id); let witnessed = { let state = tx_state.borrow(); state - .get_job_status(&boundary.job) + .get_job_status(&job_id) .is_some_and(|status| status.get(own_baseboard).is_some()) }; if witnessed { return; } - let job_id = boundary.job; // A stopped ending announces the start and stop pair, so the // ordinary status transitions apply on every sled; a bare stop // with no prior start would be dropped. - let events = match &boundary.outcome { + let events = match outcome { JobOutcome::Ended(JobStatus::Stopped { time_started, time_stopped, @@ -1303,6 +1503,56 @@ fn adjudicate_boundary( warn!(log, "adjudicated an unwitnessed job from a previous life"; "job_id" => %job_id); } +/// The execution floor for entering a universe, or `None` when the +/// record can order everything this sled may meet there. Two kinds +/// of entry have history the record cannot order. Re-entering a +/// burned universe is the flip-flop: the record overwrote this +/// universe's watermark when it left, and the sled reports the +/// return as an error. Entering a universe that has history while +/// holding no record leaves the sled unable to tell a first visit +/// from a return after a clean slate: the universe may hold jobs +/// this baseboard already ran, and they must not run twice. Both +/// set the floor at the frontier that includes this life's first +/// send. +/// +/// The floor must live in memory only. Its version is created by +/// sending a message, and if the sled dies before the message +/// reaches anyone, no other copy of it ever exists. A floor written +/// to disk would carry that dead version into the next life, where +/// no future session start could ever dominate it, and every one +/// would be refused until a cold boot. A floor raised fresh at each +/// entry is built from a message the living sled is actively +/// gossiping, so future sessions come to dominate it. +fn entry_floor( + log: &Logger, + snapshot: Option<&Boundary>, + arrival: &Version, + rumors: &GossipNetwork, + own_baseboard: &BaseboardId, +) -> Option { + let network = rumors.network(); + match snapshot { + // A record committed in this universe is its watermark, even + // when its burned set names this network: a sled that returns + // and commits burns its own network into the replacement + // record, and the Bloom set can never drop the stale entry. + Some(boundary) if boundary.network != network && boundary.is_burned(network) => { + warn!( + log, "re-entered a universe this sled's boundary record burned"; + "committed" => ?boundary.job, + ); + rumors.send( + Message::Event(own_baseboard.clone(), Event::Error(Error::UniverseFlipFlop)).into(), + ); + } + None if *arrival != Version::new() => { + warn!(log, "no boundary record, and this universe has history"); + } + _ => return None, + } + Some(rumors.snapshot().latest().clone()) +} + /// Grow a fresh gossip seed over sush's message type. /// /// A peer that seeds its own network has no one to gossip with, so jobs run @@ -1355,46 +1605,28 @@ impl StateManager { // in the face of arbitrary *causal* reorderings. // `borrow_and_update` marks the value seen, so a migration // that landed before we subscribed does not replay as a swap. - let Universe { - rumors: initial, - frontier, - } = universe.borrow_and_update().clone(); + let Universe { rumors: initial } = universe.borrow_and_update().clone(); let mut causal_messages = initial.causal_messages(); - // The boundary's session counts as lost unless the universe - // already knows everything the boundary knew. See - // [`Boundary::covered_by`]. Decisions read a snapshot of the - // boundary, never the live store: the launcher advances the - // store concurrently, and a job committed after the snapshot - // is this incarnation's, not the past's. - let lost = { - let log = log.clone(); - move |snapshot: Option<&Boundary>, network: Network, frontier: Option<&Version>| { - snapshot.and_then(|boundary| { - let covered = boundary.covered_by(network, frontier); - debug!( - log, "boundary coverage"; - "covered" => covered, - "recorded_network" => ?boundary.network, - "network" => ?network, - "recorded_frontier" => ?boundary.frontier, - "join_frontier" => ?frontier, - ); - (!covered).then_some(boundary.session) - }) - } - }; + // Decisions read a snapshot of the boundary, never the live + // store: the launcher advances the store concurrently, and a + // job committed after the snapshot is this incarnation's, not + // the past's. let boundary = store.boundary(); + let boundary_store = store.clone(); - // We report our current state through a watch channel. + // We report our current state through a watch channel. The + // placeholder birth is replaced before any message applies. let mut initial_state = State::new( own_baseboard.clone(), roots, session_sush_nonce.clone(), Past::new( - frontier.clone(), + Version::new(), + Version::new(), store.clone(), - lost(boundary.as_ref(), initial.network(), frontier.as_ref()), + boundary.clone(), + None, ), )?; initial_state.cubbies = cubbies.borrow_and_update().clone(); @@ -1421,27 +1653,54 @@ impl StateManager { spawn(async move { info!(log, "managing state"); - // Replay bookkeeping. `frontier` classifies incoming - // messages (at or concurrent with it means replayed - // history); `survivors` are jobs this incarnation itself - // runs across a universe swap; `reaped` are zombies - // already declared interrupted. + // Before any message is processed: announce our build + // (the send that starts this life's causal presence), + // raise the floor in case the record burned the + // initial universe, and take the birth mark that + // splits this life's traffic from replayed history. + let boundary = match &gossip { + Some((rumors, _)) => { + let arrival = rumors.snapshot().latest().clone(); + rumors.send( + Message::Event( + own_baseboard.clone(), + Event::Version(VersionInfo::current()), + ) + .into(), + ); + let floor = + entry_floor(&log, boundary.as_ref(), &arrival, rumors, &own_baseboard); + let birth = rumors.snapshot().latest().clone(); + tx_state.send_modify(|state| { + state.past = Past::new( + arrival, + birth, + boundary_store.clone(), + boundary.clone(), + floor.clone(), + ); + }); + boundary + } + None => boundary, + }; + if let Some(Committed { session, job, .. }) = + boundary.as_ref().and_then(|boundary| boundary.job.as_ref()) + { + info!( + log, "inherited an execution boundary; the committed session resumes at its stored successor"; + "session_id" => %session, "job_id" => %job, + ); + } + + // Replay bookkeeping. `survivors` are jobs this + // incarnation itself runs across a universe swap; + // `reaped` are zombies already declared interrupted. let mut boundary = boundary; let mut survivors: BTreeSet = BTreeSet::new(); let mut reaped: BTreeSet = BTreeSet::new(); let mut adjudicated: Option = None; - // Announce our build. - if let Some((rumors, _)) = &gossip { - rumors.send( - Message::Event( - own_baseboard.clone(), - Event::Version(VersionInfo::current()), - ) - .into(), - ); - } - // These flip both to `true` once our two input streams (local // requests and local events from the executor) terminate or // we're shutting down. At this point, we must drop `gossip` @@ -1580,35 +1839,37 @@ impl StateManager { survivors.extend(executor.in_flight()); survivors.retain(|job_id| !reaped.contains(job_id)); reaped = BTreeSet::new(); - boundary = tx_state.borrow().past.boundary.boundary(); - // TODO: re-inject local job state (policy pending). - let lost_session = lost( - boundary.as_ref(), - fresh.rumors.network(), - fresh.frontier.as_ref(), + *rumors = fresh.rumors; + let arrival = rumors.snapshot().latest().clone(); + rumors.send( + Message::Event( + own_baseboard.clone(), + Event::Version(VersionInfo::current()), + ) + .into(), ); + let boundary_store = tx_state.borrow().past.boundary.clone(); + boundary = boundary_store.boundary(); + let floor = + entry_floor(&log, boundary.as_ref(), &arrival, rumors, &own_baseboard); + let birth = rumors.snapshot().latest().clone(); + // TODO: re-inject local job state (policy pending). tx_state.send_modify(|state| { *state = State::new( own_baseboard.clone(), &roots, state.session_sush_nonce.clone(), Past::new( - fresh.frontier.clone(), - state.past.boundary.clone(), - lost_session, + arrival, + birth, + boundary_store.clone(), + boundary.clone(), + floor.clone(), ), ) .expect("roots validated at startup"); state.cubbies = cubbies.borrow().clone(); }); - *rumors = fresh.rumors; - rumors.send( - Message::Event( - own_baseboard.clone(), - Event::Version(VersionInfo::current()), - ) - .into(), - ); // The set received at join is already local: // drain it, then reap. drain_ready( @@ -1817,3 +2078,103 @@ pub fn cert_chain(certs: &Certificates, key_id: &KeyId) -> Result Network { + serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() + } + + fn past(committed: Option, floor: Option) -> Past { + let log = Logger::root(slog::Discard, o!()); + Past::new( + Version::new(), + Version::new(), + Arc::new(BoundaryStore::new(&log, &Locker::null())), + committed, + floor, + ) + } + + /// The admission rules: a sled with no record admits, a record + /// from a foreign universe admits, the committed session admits + /// outright, a session started strictly above the executed join + /// admits, and everything else hops. + #[test] + fn admission_rules() { + let started: Version = "(1, 1, (0, 0, 2))".parse().unwrap(); + let older: Version = "(1, 0, (0, 0, 2))".parse().unwrap(); + let newer: Version = "(2, 1, (0, 0, 3))".parse().unwrap(); + let concurrent: Version = "(1, 2, (0, 0, 1))".parse().unwrap(); + let session = SessionId::random(); + let job = JobId::random(); + let committed = Boundary { + network: network(1), + burned: Bloom::new(), + executed: started.clone(), + job: Some(Committed { + session, + job, + successor: JobId::random(), + outcome: JobOutcome::Committed, + }), + }; + + let recordless = past(None, None); + assert!(matches!( + recordless.screen(network(1), session, &started), + Admission::Admit + )); + + let past = past(Some(committed), None); + assert!(matches!( + past.screen(network(2), session, &started), + Admission::Admit + )); + // The committed session admits outright: it resumes at the + // stored successor when it activates, and chain position + // keeps every earlier job from popping. + assert!(matches!( + past.screen(network(1), session, &started), + Admission::Admit + )); + assert!(matches!( + past.screen(network(1), SessionId::random(), &newer), + Admission::Admit + )); + for unordered in [&older, &started, &concurrent] { + assert!(matches!( + past.screen(network(1), SessionId::random(), unordered), + Admission::Hop + )); + } + } + + /// A floor refuses every session not started strictly above it, + /// in every universe: the floor belongs to this life, not to any + /// record. + #[test] + fn floors_refuse_below() { + let floor: Version = "(1, 1, (0, 0, 2))".parse().unwrap(); + let at: Version = floor.clone(); + let below: Version = "(1, 0, (0, 0, 2))".parse().unwrap(); + let above: Version = "(2, 1, (0, 0, 3))".parse().unwrap(); + let concurrent: Version = "(1, 2, (0, 0, 1))".parse().unwrap(); + let past = past(None, Some(floor.clone())); + + for started in [&at, &below, &concurrent] { + for net in [network(1), network(2)] { + assert!(matches!( + past.screen(net, SessionId::random(), started), + Admission::Refuse + )); + } + } + assert!(matches!( + past.screen(network(1), SessionId::random(), &above), + Admission::Admit + )); + } +} diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index d4f500a..27c0bc1 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -25,7 +25,7 @@ use chrono::Utc; use sush_api::{JobStartParams, JobWait}; use sush_common::jobs::{ JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, SessionId, - SessionSignerNonce, SignedJob, + SessionSignerNonce, SignedJob, SkipReason, }; use sush_common::keys::{EphemeralKey, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, SledHealth, SledId, Target}; @@ -295,6 +295,7 @@ async fn rejoining_replays_without_reexecuting() { // Live traffic still executes everywhere: a fresh session's job runs // on both sleds. + sees(&a, &b).await; let successor_nonce = SessionSignerNonce::random(); let successor = SessionId::compute( a.mgr.own_baseboard(), @@ -480,6 +481,7 @@ async fn stragglers_do_not_interrupt_live_jobs() { a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() }) .await; + sees(&a, &b).await; let authn_a = fake_identity(&mut root).await; let signer_nonce = SessionSignerNonce::random(); let session_id = SessionId::compute( @@ -534,6 +536,20 @@ async fn stragglers_do_not_interrupt_live_jobs() { shutdown.cancel(); } +/// Wait until `anchor` has applied `joiner`'s build announcement. A +/// session started on `anchor` afterward causally follows everything +/// `joiner` held at entry, so it clears the joiner's entry floor. +async fn sees(anchor: &Sled, joiner: &Sled) { + eventually("the anchor sees the joiner", 60, async || { + anchor + .mgr + .versions() + .iter() + .any(|row| row.baseboard == joiner.baseboard) + }) + .await; +} + /// Sign a job aimed at one sled, so a retry cannot legitimately run /// anywhere else. async fn sign_job_for( @@ -556,7 +572,7 @@ async fn sign_job_for( #[named] #[tokio::test] -async fn lost_session_is_refused() { +async fn lost_suffix_never_reruns() { let (_tmp, dir) = pki("sush-lost-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); @@ -572,6 +588,22 @@ async fn lost_session_is_refused() { // Sled A anchors the session and survives throughout. let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; + // Sled B keeps its boundary in a locker. It joins before the + // session starts: a record-less sled raises its floor at entry, + // and serves only sessions started after it arrived. + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() + }) + .await; + sees(&a, &b).await; let signer_nonce = SessionSignerNonce::random(); let session_id = SessionId::compute( a.mgr.own_baseboard(), @@ -583,17 +615,6 @@ async fn lost_session_is_refused() { .session_start(&authn_a, session_id, signer_nonce, true) .await .unwrap(); - - // Sled B keeps its boundary in a locker, learns the session, and - // is then cut off from gossip while its front door still works. - let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); - let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); - let b_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); - let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; - let authn_b = fake_identity(&mut root).await; - a.peers.send(BTreeSet::from([b.addr])).unwrap(); - b.peers.send(BTreeSet::from([a.addr])).unwrap(); eventually("the session gossips to B", 120, async || { b.mgr .session(&authn_b) @@ -664,35 +685,77 @@ async fn lost_session_is_refused() { }) .await; - // A retry of the first job's preserved artifact is refused, not - // re-executed: the footprint stays single. + // A retry of the first job's preserved artifact stays queued + // instead of running: the stored successor resumes the chain past + // it, so its position never pops, and the footprint file still + // shows one run. b.mgr .job_start(&authn_b, j1, JobStartParams::default()) .await .unwrap(); - eventually("the retry is refused", 120, async || { + eventually("the retry queues", 60, async || { b.mgr.job_status(&authn_b, &j1_id).await.is_ok_and(|map| { - map.get(&b.baseboard).is_some_and(|s| { - matches!( - s, - JobStatus::Error { - error: ProcessError::Io { .. }, - .. - } - ) - }) + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Queued { .. })) }) }) .await; + sleep(Duration::from_secs(1)).await; + assert!( + b.mgr.job_status(&authn_b, &j1_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Queued { .. })) + }), + "a job below the stored successor must stay queued" + ); assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); + // A new session serves immediately. + let signer_nonce = SessionSignerNonce::random(); + let session2_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let session2 = Session::new(session2_id); + a.mgr + .session_start(&authn_a, session2_id, signer_nonce, true) + .await + .unwrap(); + eventually("the new session gossips to B", 120, async || { + b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session2_id) + }) + .await; + let j3_id = session2.next_job_id(); + let j3 = sign_job_for(&mut root, j3_id, session2_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j3, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + b.mgr.job_status(&authn_b, &j3_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }), + "a new session must not be held" + ); + shutdown.cancel(); } #[named] #[tokio::test] -async fn witnessed_session_survives_restart() { - let (_tmp, dir) = pki("sush-witness-", 2); +async fn session_resumes_at_stored_successor() { + let (_tmp, dir) = pki("sush-resume-", 3); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); write( @@ -704,8 +767,29 @@ async fn witnessed_session_survives_restart() { let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); + // Three sleds converge, then the session starts, so everyone + // serves it. B keeps its boundary in a locker. let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; + let c = Sled::start(&log, &dir, 3, &root_pem, &shutdown).await; + let authn_c = fake_identity(&mut root).await; + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr, c.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + c.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + let network = a.universe.borrow().rumors.network(); + b.universe.borrow().rumors.network() == network + && c.universe.borrow().rumors.network() == network + }) + .await; + sees(&a, &b).await; + sees(&a, &c).await; let signer_nonce = SessionSignerNonce::random(); let session_id = SessionId::compute( a.mgr.own_baseboard(), @@ -717,6 +801,382 @@ async fn witnessed_session_survives_restart() { .session_start(&authn_a, session_id, signer_nonce, true) .await .unwrap(); + eventually("the session gossips to B and C", 120, async || { + [(&b.mgr, &authn_b), (&c.mgr, &authn_c)] + .iter() + .all(|(mgr, authn)| { + mgr.session(authn) + .is_some_and(|s| s.session_id() == session_id) + }) + }) + .await; + + // C falls behind: it hears nothing of what follows. + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + c.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + + // B runs two jobs submitted through its own API; only A + // witnesses them. + let j1_id = session.next_job_id(); + let j1 = sign_job_for(&mut root, j1_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j1.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session.job_started(j1); + let j2_id = session.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j2.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session.job_started(j2); + eventually("A witnesses the runs", 60, async || { + a.mgr.job_status(&authn_a, &j2_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // B dies and rejoins through lagging C alone. C holds none of the + // jobs, but B's record stores the successor of its last + // commitment, so the session resumes there with no witness at + // all: the next job runs immediately. + b_shutdown.cancel(); + drop(b); + a.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + let authn_b = fake_identity(&mut root).await; + b.peers.send(BTreeSet::from([c.addr])).unwrap(); + c.peers.send(BTreeSet::from([b.addr])).unwrap(); + eventually("the session replays to B", 120, async || { + b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session_id) + }) + .await; + let j3_id = session.next_job_id(); + let j3 = sign_job_for(&mut root, j3_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j3, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + b.mgr.job_status(&authn_b, &j3_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }), + "the resumed session must serve its next job without a witness" + ); + + // A returns, and the resumed chain reconciles rack-wide. + a.peers.send(BTreeSet::from([b.addr, c.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr, c.addr])).unwrap(); + c.peers.send(BTreeSet::from([a.addr, b.addr])).unwrap(); + eventually("the resumed run reaches A", 120, async || { + a.mgr.job_status(&authn_a, &j3_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + shutdown.cancel(); +} + +#[named] +#[tokio::test] +async fn universe_flip_flop_raises_floor() { + let (_tmp, dir) = pki("sush-flipflop-", 3); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + // Two universes that never meet: A anchors one session, D another. + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + let d = Sled::start(&log, &dir, 3, &root_pem, &shutdown).await; + let authn_d = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session2_id = SessionId::compute( + d.mgr.own_baseboard(), + d.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session2 = Session::new(session2_id); + d.mgr + .session_start(&authn_d, session2_id, signer_nonce, true) + .await + .unwrap(); + + // X joins A's universe and runs a job there. + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let x_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; + let authn_x = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + a.universe.borrow().rumors.network() == x.universe.borrow().rumors.network() + }) + .await; + sees(&a, &x).await; + let signer_nonce = SessionSignerNonce::random(); + let session1_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session1 = Session::new(session1_id); + a.mgr + .session_start(&authn_a, session1_id, signer_nonce, true) + .await + .unwrap(); + eventually("session one gossips to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session1_id) + }) + .await; + let j1_id = session1.next_job_id(); + let j1 = sign_job_for(&mut root, j1_id, session1_id, "true", &x.baseboard).await; + x.mgr + .job_start( + &authn_x, + j1.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session1.job_started(j1); + // A must witness the job: X's replayed chain in its third life + // can only be rebuilt from what A holds. + eventually("A witnesses the first job", 60, async || { + a.mgr.job_status(&authn_a, &j1_id).await.is_ok_and(|map| { + map.get(&x.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // X dies and rejoins D's universe instead, running a job there. + // That commit burns A's universe out of X's record. + x_shutdown.cancel(); + drop(x); + a.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let x_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; + let authn_x = fake_identity(&mut root).await; + d.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([d.addr])).unwrap(); + eventually("session two gossips to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session2_id) + }) + .await; + let j2_id = session2.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session2_id, "true", &x.baseboard).await; + x.mgr + .job_start( + &authn_x, + j2.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session2.job_started(j2); + // D must witness the job: X's replayed chain in its fourth life + // can only be rebuilt from what D holds. + eventually("D witnesses the second job", 60, async || { + d.mgr.job_status(&authn_d, &j2_id).await.is_ok_and(|map| { + map.get(&x.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // X dies again and flip-flops back to A's universe. Its record + // burned that universe, so X raises its floor: session one's next + // job is refused, and nothing re-runs. + x_shutdown.cancel(); + drop(x); + d.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let x_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; + let authn_x = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("session one replays to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session1_id) + }) + .await; + let j3_id = session1.next_job_id(); + let j3 = sign_job_for(&mut root, j3_id, session1_id, "true", &x.baseboard).await; + x.mgr + .job_start(&authn_x, j3, JobStartParams::default()) + .await + .unwrap(); + eventually("the floor skips the old session", 120, async || { + a.mgr.job_status(&authn_a, &j3_id).await.is_ok_and(|map| { + map.get(&x.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Skipped { + reason: SkipReason::BelowFloor, + .. + } + ) + }) + }) + }) + .await; + + // A witnessed the refusal, so a session started now begins above + // X's floor, and serves X again. + let signer_nonce = SessionSignerNonce::random(); + let session3_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let session3 = Session::new(session3_id); + a.mgr + .session_start(&authn_a, session3_id, signer_nonce, true) + .await + .unwrap(); + eventually("session three gossips to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session3_id) + }) + .await; + let j4_id = session3.next_job_id(); + let j4 = sign_job_for(&mut root, j4_id, session3_id, "true", &x.baseboard).await; + x.mgr + .job_start( + &authn_x, + j4, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + x.mgr.job_status(&authn_x, &j4_id).await.is_ok_and(|map| { + map.get(&x.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }), + "a session started above the floor must serve" + ); + + // X flops back to D's universe a second time. The job X served in + // A's universe displaced D's watermark, so that write must have + // burned D: session two's next job is skipped there, never re-run. + x_shutdown.cancel(); + drop(x); + a.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + let authn_x = fake_identity(&mut root).await; + d.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([d.addr])).unwrap(); + eventually("session two replays to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session2_id) + }) + .await; + let j5_id = session2.next_job_id(); + let j5 = sign_job_for(&mut root, j5_id, session2_id, "true", &x.baseboard).await; + x.mgr + .job_start(&authn_x, j5, JobStartParams::default()) + .await + .unwrap(); + eventually("the floor skips session two as well", 120, async || { + x.mgr.job_status(&authn_x, &j5_id).await.is_ok_and(|map| { + map.get(&x.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Skipped { + reason: SkipReason::BelowFloor, + .. + } + ) + }) + }) + }) + .await; + + shutdown.cancel(); +} + +#[named] +#[tokio::test] +async fn witnessed_session_survives_restart() { + let (_tmp, dir) = pki("sush-witness-", 2); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); @@ -729,10 +1189,22 @@ async fn witnessed_session_survives_restart() { a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() }) .await; + sees(&a, &b).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); - // A job runs on B and its result is witnessed by A, so B's - // boundary frontier is covered when it returns. The job must be - // live traffic on B: a replayed job never executes. + // A job runs on B and its request is witnessed by A, so replay + // reaches B's boundary when it returns. The job must be live + // traffic on B: a replayed job never executes. let j1_id = session.next_job_id(); let j1 = sign_job_for(&mut root, j1_id, session_id, "true", &b.baseboard).await; a.mgr @@ -881,17 +1353,6 @@ async fn gossip_survives_bookmark_failure() { let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), - signer_nonce, - ); - let session = Session::new(session_id); - a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) - .await - .unwrap(); // Sled 2's bookmark points into a directory that does not exist. // It sheds the bookmark and gossips anyway, stranding identities @@ -911,6 +1372,18 @@ async fn gossip_survives_bookmark_failure() { a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() }) .await; + sees(&a, &b).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); // The degraded sled refuses live jobs rather than run one it // cannot record, and gossips the refusal. The healthy sled still diff --git a/server/tests/output/job-skipped-event.bin b/server/tests/output/job-skipped-event.bin new file mode 100644 index 0000000000000000000000000000000000000000..074aa944cf6d5823062abeb3c059915205ff59f0 GIT binary patch literal 145 zcmZ3O6lSn6)wL`&ucT>Fc0poMNqk;uZc=Jdwxyx5t^o)bTILp~7G);pz+{|)d`*mu zj0_Bn4HhPQ Date: Fri, 4 Sep 2026 21:02:19 -0600 Subject: [PATCH 07/12] Version the locker records and gossip message formats Both tenant records are now stored in a two-element CBOR envelope: the format's version number, then the body. A trait chains each version to the one it superseded, and the chain's Previous bound requires the conversion, so declaring a new version does not compile until the upgrade from the old one exists. Decoding matches the stored version against the chain, parses the body at that version, and converts the result up to the latest. The chain follows the ledger versioning in omicron's config-reconciler (see omicron#11249), with three changes. The version is stored explicitly instead of inferred from the body's shape, because two versions that differ only by an optional field encode identically. A record that fails to parse or convert is reported and left to the caller, so the boundary store stays untrusted and the bookmark assumes a fresh identity; omicron panics instead. Nothing is written back at load time; the first ordinary write persists the latest format. A Versioned ancestor carries the version number, and two policy traits carry the chain and its conversion. Record covers tenant records, and its conversion may fail: the caller can quarantine a bad record. Wire covers gossip messages, and its conversion is infallible: delivery is prefix-closed, so the state machine could neither skip a message that failed to convert nor stop at it. The VersionedMessage enum tag already serves as the wire envelope, so the message chain converts up when the state machine unwraps it; v0::Message is wire version 0. Version 0 is the shipped baseline for both tenants: the boundary record, and the bookmark record wrapping the bytes rumors writes. The latest format of each tenant is the live type itself, with no frozen copy. A pinned snapshot freezes its bytes instead, so a change to the live type fails the pin rather than silently changing version 0. The format module documents the steps a format change requires. A version newer than the software recognizes gets its own error, so a downgraded sled reports what happened instead of calling the record corrupt. An adversarial review of this machinery drove five hardening changes. Chain version numbers are checked at compile time: walk carries a const assertion that each version exceeds its predecessor, so a duplicate number cannot silently decode old records as the new format, and a decreasing one cannot refuse a known version as Future. The envelope refuses trailing elements instead of silently ignoring them. Schema snapshots freeze the shapes of the two signed types every sled rebuilds from the wire (the job request and the challenge response), because a field added under the house serde extension idiom would otherwise change what verifies, silently and only in a mixed rack. And the certificate and login-key wire fields now carry raw DER and OpenSSH bytes, parsed where used, so an artifact a future dependency refuses to parse costs one import or one login, never the gossip sessions that replay it. The Unknown message variant carries the whole received message re-encoded, so its Serialize cannot fail; rumors treats a message that fails to serialize as a panic. The wire bytes of all of these are unchanged; every pinned snapshot passes as before. Co-Authored-By: Claude Mythos 5 --- common/src/authn.rs | 22 +- common/src/jobs.rs | 17 + common/src/keys.rs | 8 + .../output/challenge-response-schema.json | 32 ++ .../output/job-start-request-schema.json | 49 +++ server/src/bookmark.rs | 39 +- server/src/boundary.rs | 101 ++++- server/src/format.rs | 393 ++++++++++++++++++ server/src/lib.rs | 1 + server/src/manager.rs | 7 +- server/src/messages.rs | 134 +++--- server/src/state.rs | 15 +- server/tests/output/boundary-record-v0.bin | Bin 0 -> 1225 bytes tests/src/manager_tests.rs | 13 +- 14 files changed, 733 insertions(+), 98 deletions(-) create mode 100644 common/tests/output/challenge-response-schema.json create mode 100644 common/tests/output/job-start-request-schema.json create mode 100644 server/src/format.rs create mode 100644 server/tests/output/boundary-record-v0.bin diff --git a/common/src/authn.rs b/common/src/authn.rs index 38111cc..42966d2 100644 --- a/common/src/authn.rs +++ b/common/src/authn.rs @@ -153,7 +153,7 @@ impl RequestKey { codephrase_newtype! { /// The server half of an ephemeral request-signing key. - #[derive(Clone, Deserialize, Eq, PartialEq, Serialize)] + #[derive(Clone, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct RequestVerifier = Full; } @@ -347,7 +347,7 @@ impl SeqWindow { /// Response to an authentication challenge, containing the server-chosen /// nonce and a fresh client-chosen nonce. This is the structure that is /// signed and verified as authentication credentials. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct ChallengeResponse { nonce: Nonce, cnonce: Nonce, @@ -613,6 +613,24 @@ mod test { use super::*; + /// A login is signed, gossiped, and re-verified by every sled, + /// which rebuilds it from the wire to check the signature. Two + /// shapes in one rack disagree about which logins verify. This + /// pin freezes the shape: any field change fails here. Do not + /// re-pin; add a new wire version. + #[test] + fn pin_challenge_response_schema() { + let schema = + serde_json::to_string_pretty(&schemars::schema_for!(ChallengeResponse)).unwrap(); + let path = "tests/output/challenge-response-schema.json"; + if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { + std::fs::write(path, &schema).unwrap(); + } else { + let expected = std::fs::read_to_string(path).expect("missing snapshot"); + assert_eq!(schema, expected, "the signed login's shape changed"); + } + } + /// Values to be signed must match even across versions. #[test] fn pin_to_be_signed() { diff --git a/common/src/jobs.rs b/common/src/jobs.rs index e5f5ed7..fa76133 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -731,6 +731,23 @@ mod test { use crate::keys::{EccR, EccS, EncodedSignature}; use crate::targets::SledId; + /// A job request is signed, and every sled rebuilds it from the + /// wire to check the signature, so two sleds with different + /// request shapes disagree about what verifies. This pin freezes + /// the shape: any field change fails here. Do not re-pin; add a + /// new wire version. + #[test] + fn pin_job_start_request_schema() { + let schema = serde_json::to_string_pretty(&schemars::schema_for!(JobStartRequest)).unwrap(); + let path = "tests/output/job-start-request-schema.json"; + if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { + std::fs::write(path, &schema).unwrap(); + } else { + let expected = std::fs::read_to_string(path).expect("missing snapshot"); + assert_eq!(schema, expected, "the signed request's shape changed"); + } + } + /// A request's defaulted fields stay out of the signed material, /// so a signature made before a field existed still verifies /// after it is added. The literal hash pins the scheme for diff --git a/common/src/keys.rs b/common/src/keys.rs index 6d62799..bb89ac6 100644 --- a/common/src/keys.rs +++ b/common/src/keys.rs @@ -113,6 +113,14 @@ impl<'de> Deserialize<'de> for SshPublicKey { } impl SshPublicKey { + pub fn to_openssh(&self) -> Result { + Ok(self.0.to_openssh()?) + } + + pub fn from_openssh(openssh: &str) -> Result { + Ok(Self(ssh_key::PublicKey::from_openssh(openssh)?)) + } + pub fn key_id(&self) -> Result { KeyId::try_from(&self.0) } diff --git a/common/tests/output/challenge-response-schema.json b/common/tests/output/challenge-response-schema.json new file mode 100644 index 0000000..f57ce26 --- /dev/null +++ b/common/tests/output/challenge-response-schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ChallengeResponse", + "description": "Response to an authentication challenge, containing the server-chosen nonce and a fresh client-chosen nonce. This is the structure that is signed and verified as authentication credentials.", + "type": "object", + "required": [ + "cnonce", + "epk", + "nonce" + ], + "properties": { + "cnonce": { + "$ref": "#/definitions/Nonce" + }, + "epk": { + "$ref": "#/definitions/RequestVerifier" + }, + "nonce": { + "$ref": "#/definitions/Nonce" + } + }, + "definitions": { + "Nonce": { + "description": "A unique random string. Authentication credentials have two of these: one generated by the server, and one by the client. This structure is agnostic to the syntax of the string.", + "type": "string" + }, + "RequestVerifier": { + "description": "The server half of an ephemeral request-signing key.", + "type": "string" + } + } +} \ No newline at end of file diff --git a/common/tests/output/job-start-request-schema.json b/common/tests/output/job-start-request-schema.json new file mode 100644 index 0000000..c5cd765 --- /dev/null +++ b/common/tests/output/job-start-request-schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "JobStartRequest", + "description": "A request to run the given `command` as `job_id`.", + "type": "object", + "required": [ + "command", + "job_id", + "session_id" + ], + "properties": { + "command": { + "type": "string" + }, + "job_id": { + "$ref": "#/definitions/JobId" + }, + "mode": { + "$ref": "#/definitions/JobMode" + }, + "session_id": { + "$ref": "#/definitions/SessionId" + }, + "target": { + "description": "The sleds this job runs on.", + "type": "string" + } + }, + "definitions": { + "JobId": { + "description": "A globally unique identifier for a job within a session.", + "type": "string" + }, + "JobMode": { + "description": "How a job runs. The streaming modes allow **unrecorded** I/O.", + "type": "string", + "enum": [ + "batch", + "interactive", + "stream-input", + "stream-output" + ] + }, + "SessionId": { + "description": "A globally unique identifier for a session.", + "type": "string" + } + } +} \ No newline at end of file diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index 536f933..351ce4d 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -25,6 +25,7 @@ use slog::{Discard, Logger, o, warn}; use thiserror::Error; use tokio::io::AsyncWrite; +use crate::format::{self, NoFormat, Record, Versioned}; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOOKMARK: TenantSpec = TenantSpec { @@ -32,6 +33,25 @@ pub const BOOKMARK: TenantSpec = TenantSpec { magic: b"SUSHBOOKMARK", }; +/// Wrap the opaque bytes rumors writes. +#[derive(serde::Deserialize, serde::Serialize)] +struct BookmarkRecord(#[serde(with = "format::cbor_bytes")] Vec); + +impl Versioned for BookmarkRecord { + const VERSION: u16 = 0; +} + +impl Record for BookmarkRecord { + type Previous = NoFormat; +} + +impl TryFrom for BookmarkRecord { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } +} + /// What a bookmark load or store failed at. #[derive(Debug, Error)] pub enum BookmarkIoError { @@ -109,7 +129,17 @@ impl Bookmark for SushBookmark { } let mut guard = self.tenant.lock().await; match guard.load().await { - Verdict::Adopt(record) | Verdict::Restore(record) => Ok(Some(Cursor::new(record))), + Verdict::Adopt(record) | Verdict::Restore(record) => { + match format::decode::(&record) { + Ok(BookmarkRecord(bytes)) => Ok(Some(Cursor::new(bytes))), + // Stranding the old identity is harmless; + // resuming from a misread record is not. + Err(error) => { + warn!(self.log, "assuming a fresh identity"; "reason" => %error); + Ok(None) + } + } + } Verdict::Empty => Ok(None), Verdict::Discard(reason) => { warn!(self.log, "assuming a fresh identity"; "reason" => %reason); @@ -127,10 +157,13 @@ impl Bookmark for SushBookmark { } let mut buf = Cursor::new(Vec::new()); write(&mut buf).await.map_err(BookmarkIoError::Serialize)?; - let record = buf.into_inner(); + let record = BookmarkRecord(buf.into_inner()); let mut guard = self.tenant.lock().await; - guard.store(&record).await.map_err(BookmarkIoError::Store) + guard + .store(&format::encode(&record)) + .await + .map_err(BookmarkIoError::Store) } } diff --git a/server/src/boundary.rs b/server/src/boundary.rs index 19919bc..7ca2c61 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -57,7 +57,7 @@ use std::sync::Mutex as SyncMutex; use std::sync::atomic::{AtomicBool, Ordering}; -use ciborium::{de::from_reader as from_cbor, ser::into_writer as into_cbor}; +use ciborium::ser::into_writer as into_cbor; use rumors::{Network, Version}; use serde::{Deserialize, Serialize}; use slog::{Logger, o, warn}; @@ -66,6 +66,7 @@ use thiserror::Error; use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId}; use crate::bloom::Bloom; +use crate::format::{self, NoFormat, Record, Versioned}; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOUNDARY: TenantSpec = TenantSpec { @@ -173,14 +174,22 @@ pub enum JobOutcome { Ended(JobStatus), } -fn encode(boundary: &Boundary) -> Vec { - let mut bytes = Vec::new(); - into_cbor(boundary, &mut bytes).expect("writing to a Vec cannot fail"); - bytes +/// Version 0 is the shipped baseline. The pinned record snapshot in +/// this module's tests freezes its bytes; see [`crate::format`] for +/// the steps a format change requires. +impl Versioned for Boundary { + const VERSION: u16 = 0; +} + +impl Record for Boundary { + type Previous = NoFormat; } -fn decode(record: &[u8]) -> Option { - from_cbor(record).ok() +impl TryFrom for Boundary { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } } #[derive(Debug, Error)] @@ -225,13 +234,19 @@ impl BoundaryStore { "the boundary store loads once, at startup", ); let boundary = match self.tenant.load().await { - Verdict::Adopt(record) | Verdict::Restore(record) => match decode(&record) { - Some(boundary) => Some(boundary), - None => { - warn!(self.log, "undecodable boundary record"); - return; + Verdict::Adopt(record) | Verdict::Restore(record) => { + match format::decode::(&record) { + Ok(boundary) => Some(boundary), + // An unreadable record is not an absent one: + // absent would mean a clean slate, forgetting the + // previous life's commitments. The store stays + // untrusted instead, and no job runs. + Err(error) => { + warn!(self.log, "unusable boundary record"; "error" => %error); + return; + } } - }, + } Verdict::Empty => None, Verdict::Discard(_) => return, }; @@ -289,7 +304,7 @@ impl BoundaryStore { executed: boundary.executed.clone(), } }; - if let Err(error) = guard.store(&encode(&updated)).await { + if let Err(error) = guard.store(&format::encode(&updated)).await { warn!( self.log, "failed to record the boundary job's outcome"; "job_id" => %job_id, "error" => %error, @@ -309,7 +324,7 @@ impl BoundaryStore { return Err(BoundaryError::Untrusted); } let mut guard = self.tenant.lock().await; - guard.store(&encode(boundary)).await?; + guard.store(&format::encode(boundary)).await?; *self.boundary.lock().unwrap() = Some(boundary.clone()); Ok(()) } @@ -369,6 +384,62 @@ mod test { boundary.job.as_ref().expect("a committed job").job } + /// The record's bytes are on-disk format, frozen at version 0. If + /// this fails, STOP: do not re-pin. Copy the old shape into a + /// frozen module and add a new version instead; see + /// [`crate::format`]. + #[test] + fn pin_record_format_v0() { + let record = Boundary { + network: network(1), + burned: { + let mut burned = Bloom::new(); + burned.insert(&network_key(network(2))); + burned + }, + executed: "(1, 1, (0, 0, 2))".parse().unwrap(), + job: Some(Committed { + session: "abandon-ability".parse().unwrap(), + job: "zoo-zero".parse().unwrap(), + successor: "able-about".parse().unwrap(), + outcome: JobOutcome::Committed, + }), + }; + let bytes = format::encode(&record); + let path = "tests/output/boundary-record-v0.bin"; + if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { + std::fs::write(path, &bytes).unwrap(); + } else { + let expected = std::fs::read(path).expect("missing snapshot"); + assert_eq!(bytes, expected, "record format changed: {bytes:02x?}"); + } + let decoded: Boundary = format::decode(&bytes).unwrap(); + assert_eq!(decoded.network, record.network); + assert!(decoded.is_burned(network(2))); + assert_eq!(decoded.executed, record.executed); + assert_eq!( + decoded.job.unwrap().successor, + record.job.unwrap().successor + ); + } + + /// A record from a newer software version loads as untrusted, and + /// the sled reports it instead of guessing at the format. + #[tokio::test] + async fn future_record_is_untrusted() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + #[derive(Serialize)] + struct Envelope(u16, #[serde(with = "crate::format::cbor_bytes")] Vec); + let mut bytes = Vec::new(); + into_cbor(&Envelope(1, b"from the future".to_vec()), &mut bytes).unwrap(); + let scratch = Locker::new(&test_log(), slots.clone()); + scratch.tenant(BOUNDARY).store(&bytes).await.unwrap(); + + let store = store(slots).await; + assert!(store.untrusted()); + } + /// The burned set's keys are on-disk format: a change to the /// network's serde shape would silently forget every burn, and a /// forgotten burn admits a flip-flop instead of refusing it. If diff --git a/server/src/format.rs b/server/src/format.rs new file mode 100644 index 0000000..a6b0e94 --- /dev/null +++ b/server/src/format.rs @@ -0,0 +1,393 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Versioned durable formats. +//! +//! Every version names its predecessor as its `Previous`. [`Record`] and +//! [`Wire`] each require a conversion from it; fallible for [`Record`], +//! infallible for [`Wire`]. A new version therefore does not compile +//! until the conversion from the old one exists, so by induction every +//! version ever shipped can be upgraded to the latest. +//! +//! A [`Locker`](crate::locker::Locker) tenant record is stored as a +//! two-element CBOR array, `[version, bytes]`, the byte string holding +//! the record's own CBOR encoding. The version lets a newer release +//! read every record an older release ever wrote: [`decode`] matches +//! the stored version against the chain of known formats, parses the +//! body as the format that matches, and converts the result up the +//! chain to the latest format. A gossip message instead carries its +//! version as the [`VersionedMessage`](crate::messages::VersionedMessage) +//! variant tag, and converts up when the state machine unwraps it. +//! +//! Tests must pin each version's serialized bytes. To change a format, +//! copy the live type into a frozen module under its version's name, +//! point the pin test at the copy, give the live type the next version, +//! name the frozen type as its `Previous`, and write the conversion the +//! compiler requires. + +use std::fmt::Display; + +use ciborium::{de::from_reader as from_cbor, ser::into_writer as into_cbor}; +use serde::{Serialize, de::DeserializeOwned}; +use thiserror::Error; + +/// A type that is one version of a durable format. Version numbers +/// must be unique within a chain and increase along it, so that any +/// version above the latest must belong to newer software. +pub trait Versioned: Sized { + const VERSION: u16; +} + +/// A [`Locker`](crate::locker::Locker) tenant's record format. +/// New versions will not compile without a conversion from their +/// predecessors. The conversion may fail, because the caller can +/// quarantine a record that will not convert; the boundary store, +/// for example, stays untrusted. +pub trait Record: Versioned + Serialize + DeserializeOwned { + /// The format this one supersedes. + type Previous: Record + TryInto; + + /// Parse `body` as the chain member whose version is `version`, + /// converting the result up to `Self`. This is not a public + /// interface; use [`decode`]. + fn walk(version: u16, body: &[u8]) -> Result { + const { + assert!( + Self::Previous::VERSION == NoFormat::VERSION + || Self::Previous::VERSION < Self::VERSION, + "chain versions must increase", + ) + }; + + if version == Self::VERSION { + return from_cbor(body).map_err(|error: ciborium::de::Error<_>| FormatError::Body { + version, + message: error.to_string(), + }); + } + + let previous = Self::Previous::walk(version, body)?; + previous.try_into().map_err(|error| FormatError::Convert { + from: Self::Previous::VERSION, + message: error.to_string(), + }) + } +} + +/// A wire format for messages, with the same upgrade guarantee +/// as [`Record`]. But here the conversion is infallible, because +/// message processing cannot in general skip a replayed message +/// or stop at it. +pub trait Wire: Versioned { + /// The format this one supersedes. + type Previous: Wire + Into; +} + +/// The end of every version chain. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub enum NoFormat {} + +impl Versioned for NoFormat { + // Reserved for the terminus. + const VERSION: u16 = u16::MAX; +} + +impl Record for NoFormat { + type Previous = NoFormat; + + fn walk(version: u16, _body: &[u8]) -> Result { + Err(FormatError::Unknown { version }) + } +} + +impl Wire for NoFormat { + type Previous = NoFormat; +} + +/// Why a record could not be read. +#[derive(Debug, Error)] +pub enum FormatError { + #[error("the record's version envelope did not parse")] + Envelope, + #[error("the record's format version {version} is newer than this software")] + Future { version: u16 }, + #[error("no format in this record's chain has version {version}")] + Unknown { version: u16 }, + #[error("the record did not parse as format version {version}: {message}")] + Body { version: u16, message: String }, + #[error("converting the record from format version {from} failed: {message}")] + Convert { from: u16, message: String }, +} + +/// The version envelope: `[version, body]`, with the body nested +/// as a CBOR byte string holding the encoded record. Nesting +/// keeps the body out of `ciborium::Value`, whose deserializer is +/// stricter than the byte-stream one and refuses serde adapters that +/// read CBOR byte strings; [`decode`] reads the version and hands +/// the untouched body bytes to the stream deserializer. +#[derive(serde::Serialize)] +struct Envelope(u16, #[serde(with = "cbor_bytes")] Vec); + +// Manual impl to refuse trailing elements. +impl<'de> serde::Deserialize<'de> for Envelope { + fn deserialize>(deserializer: D) -> Result { + use serde::de::{Error as _, IgnoredAny, SeqAccess, Visitor}; + + struct Body(Vec); + impl<'de> serde::Deserialize<'de> for Body { + fn deserialize>(deserializer: D) -> Result { + cbor_bytes::deserialize(deserializer).map(Body) + } + } + + struct EnvelopeVisitor; + impl<'de> Visitor<'de> for EnvelopeVisitor { + type Value = Envelope; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a two-element version envelope") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let version = seq + .next_element::()? + .ok_or_else(|| A::Error::custom("an envelope without a version"))?; + let Body(body) = seq + .next_element::()? + .ok_or_else(|| A::Error::custom("an envelope without a body"))?; + if seq.next_element::()?.is_some() { + return Err(A::Error::custom("trailing elements in the envelope")); + } + Ok(Envelope(version, body)) + } + } + + deserializer.deserialize_seq(EnvelopeVisitor) + } +} + +/// Serialize a byte buffer as a CBOR byte string rather than an +/// array of integers. +pub(crate) mod cbor_bytes { + use serde::de::Visitor; + use serde::{Deserializer, Serializer}; + use std::fmt; + + pub fn serialize(bytes: &[u8], serializer: S) -> Result { + serializer.serialize_bytes(bytes) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + struct Bytes; + impl<'de> Visitor<'de> for Bytes { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("bytes") + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(v.to_vec()) + } + } + deserializer.deserialize_bytes(Bytes) + } +} + +/// Encode `value` at the latest version, inside the version envelope. +pub fn encode(value: &T) -> Vec { + let mut body = Vec::new(); + into_cbor(value, &mut body).expect("writing to a Vec cannot fail"); + let mut bytes = Vec::new(); + into_cbor(&Envelope(T::VERSION, body), &mut bytes).expect("writing to a Vec cannot fail"); + bytes +} + +/// Decode a record at whatever version it was stored, converting up +/// the chain to `T`. +pub fn decode(bytes: &[u8]) -> Result { + let Envelope(version, body) = from_cbor(bytes).map_err(|_| FormatError::Envelope)?; + if version > T::VERSION { + return Err(FormatError::Future { version }); + } + T::walk(version, &body) +} + +#[cfg(test)] +mod test { + use super::*; + + use serde::Deserialize; + + #[derive(Debug, Deserialize, Serialize)] + struct TestV0 { + count: u8, + } + + #[derive(Debug, Deserialize, Serialize, PartialEq)] + struct TestV1 { + count: u32, + label: String, + } + + impl Versioned for TestV0 { + const VERSION: u16 = 0; + } + impl Record for TestV0 { + type Previous = NoFormat; + } + + // The gap at version 1 is deliberate: unknown_version_is_refused + // walks the whole chain without matching it. + impl Versioned for TestV1 { + const VERSION: u16 = 2; + } + impl Record for TestV1 { + type Previous = TestV0; + } + + impl TryFrom for TestV0 { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } + } + + impl TryFrom for TestV1 { + type Error = &'static str; + fn try_from(old: TestV0) -> Result { + if old.count == u8::MAX { + return Err("saturated count"); + } + Ok(TestV1 { + count: old.count.into(), + label: String::new(), + }) + } + } + + #[test] + fn trailing_envelope_elements_are_refused() { + #[derive(Serialize)] + struct Extended(u16, #[serde(with = "cbor_bytes")] Vec, u16); + let mut body = Vec::new(); + into_cbor(&TestV0 { count: 7 }, &mut body).unwrap(); + let mut bytes = Vec::new(); + into_cbor(&Extended(0, body, 7), &mut bytes).unwrap(); + assert!(matches!( + decode::(&bytes), + Err(FormatError::Envelope) + )); + } + + #[test] + fn round_trip_at_latest() { + let value = TestV1 { + count: 7, + label: "seven".to_string(), + }; + let decoded: TestV1 = decode(&encode(&value)).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn old_version_converts_up() { + let old = encode(&TestV0 { count: 7 }); + let new: TestV1 = decode(&old).unwrap(); + assert_eq!( + new, + TestV1 { + count: 7, + label: String::new(), + } + ); + } + + #[test] + fn future_version_is_refused() { + let futuristic = encode(&TestV1 { + count: 1, + label: String::new(), + }); + assert!(matches!( + decode::(&futuristic), + Err(FormatError::Future { version: 2 }) + )); + } + + #[test] + fn unknown_version_is_refused() { + let mut bytes = Vec::new(); + into_cbor(&Envelope(1, Vec::new()), &mut bytes).unwrap(); + assert!(matches!( + decode::(&bytes), + Err(FormatError::Unknown { version: 1 }) + )); + } + + #[test] + fn failed_conversion_is_reported() { + let saturated = encode(&TestV0 { count: u8::MAX }); + assert!(matches!( + decode::(&saturated), + Err(FormatError::Convert { from: 0, .. }) + )); + } + + #[test] + fn garbage_is_refused() { + assert!(matches!( + decode::(b"scribble"), + Err(FormatError::Envelope) + )); + } + + #[derive(Debug, PartialEq)] + struct WireV0(u8); + #[derive(Debug, PartialEq)] + struct WireV1(u32); + + impl Versioned for WireV0 { + const VERSION: u16 = 0; + } + impl Wire for WireV0 { + type Previous = NoFormat; + } + + impl Versioned for WireV1 { + const VERSION: u16 = 1; + } + impl Wire for WireV1 { + type Previous = WireV0; + } + + impl From for WireV0 { + fn from(none: NoFormat) -> Self { + match none {} + } + } + + impl From for WireV1 { + fn from(old: WireV0) -> Self { + WireV1(old.0.into()) + } + } + + /// A wire chain converts infallibly; the bound will not accept a + /// fallible conversion. + #[test] + fn wire_chain_converts() { + assert_eq!(WireV1::from(WireV0(7)), WireV1(7)); + } + + #[test] + fn wrong_body_is_reported() { + let mut bytes = Vec::new(); + let mut body = Vec::new(); + into_cbor(&"not a struct", &mut body).unwrap(); + into_cbor(&Envelope(0, body), &mut bytes).unwrap(); + assert!(matches!( + decode::(&bytes), + Err(FormatError::Body { version: 0, .. }) + )); + } +} diff --git a/server/src/lib.rs b/server/src/lib.rs index e23bca8..8f59f09 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -15,6 +15,7 @@ pub mod bookmark; pub mod boundary; pub mod error; pub mod executor; +pub mod format; pub mod gossip; pub mod history; pub mod io; diff --git a/server/src/manager.rs b/server/src/manager.rs index fb83347..989f6f0 100644 --- a/server/src/manager.rs +++ b/server/src/manager.rs @@ -23,7 +23,7 @@ use tokio::time::timeout; use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; use x509_cert::Certificate; -use x509_cert::der::DecodePem as _; +use x509_cert::der::{DecodePem as _, Encode as _}; use sush_api::{JobStartParams, JobStopParams, JobWait}; use sush_common::authn::{ @@ -258,7 +258,8 @@ impl JobManager { return Err(KeyError::SelfSigned.into()); } let key_id = KeyId::try_from(&cert)?; - self.cert_request(authn, CertRequest::Import(cert)).await?; + let der = cert.to_der().map_err(KeyError::from)?; + self.cert_request(authn, CertRequest::Import(der)).await?; if wait { self.wait_for(self.wait_for_cert(key_id)).await?; } @@ -421,7 +422,7 @@ impl JobManager { last_used: Instant::now(), }, ); - let login = IdentityRequest::Login(public_key, response); + let login = IdentityRequest::Login(public_key.to_openssh()?, response); self.identity_request(key_id, login) .await .map(|()| identity) diff --git a/server/src/messages.rs b/server/src/messages.rs index 7eff26d..ce68e81 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -5,26 +5,33 @@ //! Messages gossiped via rumors. use chrono::{DateTime, Utc}; +use ciborium::{Value, de::from_reader as from_cbor, ser::into_writer as into_cbor}; use rumors::Version; use serde::{Deserialize, Serialize}; use sled_hardware_types::BaseboardId; use thiserror::Error; -use x509_cert::Certificate; use sush_api::JobStartParams; use sush_common::authn::SignedLogin; use sush_common::jobs::JobOutputState; use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob, SkipReason}; -use sush_common::keys::{KeyId, SshPublicKey}; +use sush_common::keys::KeyId; use sush_common::version::VersionInfo; +use crate::format::{NoFormat, Versioned, Wire}; + /// Once a message schema has shipped, it is frozen, since any changes /// could break decoding of *existing* messages. Each version gets its /// own module; everything defined there and all of their dependencies /// (e.g., types shared with the HTTP API, etc.) become part of the frozen -/// version, whose serde shape on the gossip wire must not change. New -/// versions must implement `TryInto` to convert old messages into -/// compatible new ones. +/// version, whose serde shape on the gossip wire must not change. A +/// new version must convert [`From`] the old one; the +/// [`Wire`](crate::format::Wire) chain demands it. New variants must +/// keep this enum's encoding: a single-entry, text-keyed map. Older +/// sleds read an unrecognized text key as [`Unknown`](Self::Unknown) +/// and carry on, but an integer key or an array does not decode at +/// all, and a message that fails to decode aborts gossip sessions +/// instead of being ignored. /// /// Updates go sled by sled, so mixed gossip networks exist for /// the whole rollout. A message from a newer peer decodes as @@ -36,27 +43,31 @@ pub enum VersionedMessage { /// The initial message format. V0(v0::Message), - /// A message from a newer version. - Unknown(String), + Unknown { + version: String, + message: Vec, + }, } impl Serialize for VersionedMessage { fn serialize(&self, serializer: S) -> Result { - use serde::ser::Error as _; match self { Self::V0(message) => { serializer.serialize_newtype_variant("VersionedMessage", 0, "V0", message) } - Self::Unknown(version) => Err(S::Error::custom(format!( - "refusing to send a message from a newer version ({version})" - ))), + // Emit what was received. + Self::Unknown { message, .. } => { + let value: Value = from_cbor(message.as_slice()) + .expect("the bytes were encoded from a value this module decoded"); + value.serialize(serializer) + } } } } impl<'de> Deserialize<'de> for VersionedMessage { fn deserialize>(deserializer: D) -> Result { - use serde::de::{Error as _, IgnoredAny, MapAccess, Visitor}; + use serde::de::{Error as _, MapAccess, Visitor}; struct VersionVisitor; @@ -74,14 +85,22 @@ impl<'de> Deserialize<'de> for VersionedMessage { Ok(match version.as_str() { "V0" => VersionedMessage::V0(map.next_value()?), _ => { - map.next_value::()?; - VersionedMessage::Unknown(version) + let value: Value = map.next_value()?; + let whole = Value::Map(vec![(Value::Text(version.clone()), value)]); + let mut message = Vec::new(); + into_cbor(&whole, &mut message).expect("writing to a Vec cannot fail"); + VersionedMessage::Unknown { version, message } } }) } fn visit_str(self, version: &str) -> Result { - Ok(VersionedMessage::Unknown(version.to_string())) + let mut message = Vec::new(); + into_cbor(&version, &mut message).expect("writing to a Vec cannot fail"); + Ok(VersionedMessage::Unknown { + version: version.to_string(), + message, + }) } } @@ -89,39 +108,6 @@ impl<'de> Deserialize<'de> for VersionedMessage { } } -/// DER bytes for certificates, which have no serde support. -mod cert_der { - use serde::de::Visitor; - use serde::ser::Error as _; - use serde::{Deserializer, Serializer}; - use x509_cert::Certificate; - use x509_cert::der::{Decode as _, Encode as _}; - - pub fn serialize(cert: &Certificate, serializer: S) -> Result { - serializer.serialize_bytes(&cert.to_der().map_err(S::Error::custom)?) - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result { - struct DerVisitor; - - impl<'de> Visitor<'de> for DerVisitor { - type Value = Certificate; - - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str("a DER-encoded certificate") - } - - fn visit_bytes(self, der: &[u8]) -> Result { - Certificate::from_der(der).map_err(E::custom) - } - } - - deserializer.deserialize_bytes(DerVisitor) - } -} - pub mod v0 { use super::*; @@ -138,6 +124,23 @@ pub mod v0 { } } + /// Version 0 is the initial wire format; see [`crate::format`]. + /// A later format converts up from this one infallibly, and the + /// state machine handles only the latest. + impl Versioned for Message { + const VERSION: u16 = 0; + } + + impl Wire for Message { + type Previous = NoFormat; + } + + impl From for Message { + fn from(none: NoFormat) -> Self { + match none {} + } + } + /// A request and the key that made it. /// /// The actor is attribution, not authority: it names the key whose @@ -210,7 +213,8 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[allow(clippy::large_enum_variant)] pub enum CertRequest { - Import(#[serde(with = "cert_der")] Certificate), + /// The certificate's DER bytes, decoded at use. + Import(#[serde(with = "crate::format::cbor_bytes")] Vec), Revoke(KeyId, DateTime), } @@ -236,7 +240,8 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[allow(clippy::large_enum_variant)] pub enum IdentityRequest { - Login(SshPublicKey, SignedLogin), + /// OpenSSH public keys are string encoded and decoded at use. + Login(String, SignedLogin), Revoke(KeyId, DateTime), } @@ -302,7 +307,7 @@ mod wire_format { #[track_caller] fn assert_wire_format(name: &str, message: VersionedMessage) { let mut bytes = Vec::new(); - ciborium::ser::into_writer(&message, &mut bytes).unwrap(); + into_cbor(&message, &mut bytes).unwrap(); let path = format!("tests/output/{name}.bin"); if env::var("EXPECTORATE").as_deref() == Ok("overwrite") { write(&path, &bytes).unwrap(); @@ -310,7 +315,7 @@ mod wire_format { let expected = read(&path).expect("missing snapshot"); assert_eq!(bytes, expected, "gossip wire format changed: {bytes:02x?}"); } - let decoded: VersionedMessage = ciborium::de::from_reader(bytes.as_slice()).unwrap(); + let decoded: VersionedMessage = from_cbor(bytes.as_slice()).unwrap(); assert_eq!(decoded, message, "wire format should round-trip"); } @@ -402,7 +407,7 @@ mod wire_format { #[test] fn identity_login_request() { use sush_common::authn::{ChallengeResponse, RequestVerifier}; - use sush_common::keys::{EncodedSignature, Signed, SshPublicKey}; + use sush_common::keys::{EncodedSignature, Signed}; // Craft deterministic evidence: nonces, then the ed25519 // basepoint as the verifier. @@ -416,7 +421,6 @@ mod wire_format { .unwrap(); let openssh = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILM+rvN+ot98qgEN796jTiQfZfG1KaT0PtFDJ13gEiGB test@sush"; - let public_key: SshPublicKey = serde_json::from_value(serde_json::json!(openssh)).unwrap(); let signed = Signed::new( response, @@ -430,7 +434,7 @@ mod wire_format { ); let msg: VersionedMessage = Message::Request(Request::identity( KeyId::from_str("zoo-zero").unwrap(), - IdentityRequest::Login(public_key, signed), + IdentityRequest::Login(openssh.to_string(), signed), )) .into(); assert_wire_format("identity-login-request", msg); @@ -457,8 +461,6 @@ mod wire_format { /// still fails. #[test] fn unknown_version_tolerated() { - use ciborium::value::Value; - let v1 = Value::Map(vec![( Value::Text("V1".to_string()), Value::Map(vec![( @@ -467,14 +469,17 @@ mod wire_format { )]), )]); let mut bytes = Vec::new(); - ciborium::ser::into_writer(&v1, &mut bytes).unwrap(); - let decoded: VersionedMessage = ciborium::de::from_reader(bytes.as_slice()).unwrap(); - assert_eq!(decoded, VersionedMessage::Unknown("V1".to_string())); + into_cbor(&v1, &mut bytes).unwrap(); + let decoded: VersionedMessage = from_cbor(bytes.as_slice()).unwrap(); + assert!(matches!(&decoded, VersionedMessage::Unknown { version, .. } if version == "V1")); + // Rumors panics when a message fails to serialize, so this + // must not fail, and it must emit what was received. let mut resent = Vec::new(); - assert!(ciborium::ser::into_writer(&decoded, &mut resent).is_err()); + into_cbor(&decoded, &mut resent).unwrap(); + assert_eq!(resent, bytes); - let corrupt: Result = ciborium::de::from_reader([0x01].as_slice()); + let corrupt: Result = from_cbor([0x01].as_slice()); assert!(corrupt.is_err()); } @@ -509,12 +514,13 @@ mod wire_format { #[test] fn cert_requests() { - use x509_cert::der::DecodePem as _; + use x509_cert::Certificate; + use x509_cert::der::{DecodePem as _, Encode as _}; let cert = Certificate::from_pem(include_str!("../../client/certs/staging.pem")).unwrap(); let msg: VersionedMessage = Message::Request(Request::cert( KeyId::from_str("zoo-zero").unwrap(), - CertRequest::Import(cert), + CertRequest::Import(cert.to_der().unwrap()), )) .into(); assert_wire_format("cert-import-request", msg); diff --git a/server/src/state.rs b/server/src/state.rs index 2743f51..2629da4 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -22,7 +22,7 @@ use tokio::task::JoinHandle; use tokio::{select, spawn}; use tokio_util::sync::CancellationToken; use x509_cert::Certificate; -use x509_cert::der::Encode as _; +use x509_cert::der::{Decode as _, Encode as _}; use sush_api::JobStartParams; use sush_common::authn::{Identity, Nonce, RequestVerifier, SignedLogin}; @@ -791,7 +791,10 @@ impl State { match message.as_ref() { V0(Message::Request(request)) => match request { Request::Cert(attributed) => match attributed.as_parts() { - (actor, CertRequest::Import(cert)) => match self.cert_import(cert) { + (actor, CertRequest::Import(der)) => match Certificate::from_der(der) + .map_err(KeyError::from) + .and_then(|cert| self.cert_import(&cert)) + { Ok(key_id) => { info!(log, "imported certificate"; "key_id" => %key_id, "actor" => %actor); self.validate_certs(&self.roots.clone()); @@ -1118,8 +1121,10 @@ impl State { } }, Request::Identity(attributed) => match attributed.as_parts() { - (actor, IdentityRequest::Login(public_key, signed)) => { - match verify_login(public_key, signed) { + (actor, IdentityRequest::Login(openssh, signed)) => { + match SshPublicKey::from_openssh(openssh) + .and_then(|public_key| verify_login(&public_key, signed)) + { Ok(registered) => { let identity = ®istered.identity; if self.revoked_keys.peek(&identity.key_id).is_some() { @@ -1325,7 +1330,7 @@ impl State { } } }, - Unknown(version) => { + Unknown { version, .. } => { if self.unknown_versions.insert(version.clone()) { warn!(log, "ignoring messages from a newer peer"; "version" => version); } diff --git a/server/tests/output/boundary-record-v0.bin b/server/tests/output/boundary-record-v0.bin new file mode 100644 index 0000000000000000000000000000000000000000..8915286d3eab11ddad7a95756bfa191c4c26f415 GIT binary patch literal 1225 zcmZo-h-5jkBt0*+q&&YUJAjb^6{IDV7UiX;M6#d=H=yunz(KW+Di+Y(%~WwE*@g&^ zt#ec(okGBYfIS=p`bwKWRyNacXgKW`16T0uC)O tVMdU00;=r)|II8eO-=?HnO{UeErS4adVXn1a(-@VrgMI7Ze|J4SpXv^CgK19 literal 0 HcmV?d00001 diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index 559e2b5..e43e19d 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -23,6 +23,7 @@ use tokio::fs::{metadata, read, write}; use tokio::sync::watch; use tokio::time::{sleep, timeout}; use tokio_util::sync::CancellationToken; +use x509_cert::der::Encode as _; use x509_cert::time::Validity; use sush_api::{JobStartParams, JobStopParams, JobWait}; @@ -1247,7 +1248,7 @@ async fn revocation_tombstones() { peer.send( Message::Request(Request::cert( authn.key_id.clone(), - CertRequest::Import(doomed.cert().clone()), + CertRequest::Import(doomed.cert().to_der().unwrap()), )) .into(), ); @@ -1445,7 +1446,7 @@ async fn gossiped_identities() { peer.send( Message::Request(Request::identity( root_key_id.clone(), - IdentityRequest::Login(root_pk.clone(), signed_by_liar), + IdentityRequest::Login(root_pk.to_openssh().unwrap(), signed_by_liar), )) .into(), ); @@ -1460,7 +1461,7 @@ async fn gossiped_identities() { peer.send( Message::Request(Request::identity( root_key_id.clone(), - IdentityRequest::Login(root_pk.clone(), signed), + IdentityRequest::Login(root_pk.to_openssh().unwrap(), signed), )) .into(), ); @@ -1844,7 +1845,7 @@ async fn hostile_imports_cannot_displace() { let import = |key: &EphemeralKey| { Message::Request(Request::cert( key.key_id().clone(), - CertRequest::Import(key.cert().clone()), + CertRequest::Import(key.cert().to_der().unwrap()), )) .into() }; @@ -1873,7 +1874,7 @@ async fn hostile_imports_cannot_displace() { peer.send( Message::Request(Request::cert( child.key_id().clone(), - CertRequest::Import(conflict), + CertRequest::Import(conflict.to_der().unwrap()), )) .into(), ); @@ -1983,7 +1984,7 @@ async fn homonym_issuer_resolves_to_true_parent() { peer.send( Message::Request(Request::cert( homonym.key_id().clone(), - CertRequest::Import(homonym.cert().clone()), + CertRequest::Import(homonym.cert().to_der().unwrap()), )) .into(), ); From 6d1ddc1e5606df15b147a767596eeb99999ffa96 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sat, 5 Sep 2026 19:19:18 -0600 Subject: [PATCH 08/12] Skip a session's queued jobs when the session ends Fixes #76. A session can end while jobs still wait in its queue: stopped by its starter, superseded by a causally later start, or annihilated by a concurrent one. Until now the sled dropped the queue silently, and each waiting job stayed Queued forever. Now every queued job this sled would have run gets the terminal skip status with a new reason, SessionEnded, and jobs already started finish and report as usual. A job request arriving after its session ended gets the same skip in place of the old error status, so a job's fate does not depend on whether its request arrived before or after the end. Replayed requests report nothing, as with floor skips: re-reporting history on every rejoin would grow the message set. The new reason joins the pinned wire snapshots. Co-Authored-By: Claude Mythos 5 --- common/src/jobs.rs | 2 + server/src/messages.rs | 17 ++ server/src/state.rs | 35 ++-- .../tests/output/session-ended-skip-event.bin | Bin 0 -> 147 bytes sush.json | 6 + tests/src/manager_tests.rs | 155 +++++++++++++++--- 6 files changed, 180 insertions(+), 35 deletions(-) create mode 100644 server/tests/output/session-ended-skip-event.bin diff --git a/common/src/jobs.rs b/common/src/jobs.rs index fa76133..26eeb0e 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -407,6 +407,7 @@ pub enum SkipReason { /// The job's chain position precedes the sled's recorded /// commitment: a previous life already handled it. AlreadyHandled, + SessionEnded, } impl fmt::Display for SkipReason { @@ -414,6 +415,7 @@ impl fmt::Display for SkipReason { f.write_str(match self { Self::BelowFloor => "the session sits below this sled's execution floor", Self::AlreadyHandled => "a previous life of this sled already handled it", + Self::SessionEnded => "the session ended before the job could start", }) } } diff --git a/server/src/messages.rs b/server/src/messages.rs index ce68e81..b6885ee 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -634,6 +634,23 @@ mod wire_format { assert_wire_format("job-skipped-event", msg); } + #[test] + fn session_ended_skip_event() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Job(JobEvent::Skipped( + JobId::from_str("zoo-zero").unwrap(), + "2026-09-04T20:00:00Z".parse().unwrap(), + SkipReason::SessionEnded, + )), + ) + .into(); + assert_wire_format("session-ended-skip-event", msg); + } + #[test] fn session_hop_error() { let msg: VersionedMessage = Message::Event( diff --git a/server/src/state.rs b/server/src/state.rs index 2629da4..f9fe3d9 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -778,6 +778,19 @@ impl State { self.revoked_keys.peek(key_id).is_some() } + fn skip_queued_jobs(&self, executor: &Executor) { + for (job_id, queued) in self.session.queued_jobs().into_iter().flatten() { + if !queued.replayed + && queued + .job + .payload() + .runs_on(&self.own_baseboard, &self.cubbies) + { + executor.job_skipped(*job_id, SkipReason::SessionEnded); + } + } + } + #[allow(clippy::result_large_err)] fn update( &mut self, @@ -837,6 +850,7 @@ impl State { log, "session started"; "session_id" => %session_id, "actor" => %actor, ); + self.skip_queued_jobs(executor); let mut session = Session::started(*session_id, actor.clone()); // The committed session resumes at its // stored successor: every earlier chain @@ -884,9 +898,9 @@ impl State { incoming_session: *session_id, incoming_version: incoming_version.clone(), }; - self.session = Inactive { - frontier: &*frontier | incoming_version.clone(), - }; + let frontier = &*frontier | incoming_version.clone(); + self.skip_queued_jobs(executor); + self.session = Inactive { frontier }; return Err(error); } @@ -929,6 +943,7 @@ impl State { log, "session stopped"; "session_id" => %session_id, "actor" => %actor, ); + self.skip_queued_jobs(executor); self.session = Inactive { frontier: frontier.clone(), } @@ -1054,8 +1069,9 @@ impl State { // session's own start (since each session is // linearized by its accepting server). // - // Refused jobs targeting this sled record an error - // status so the submitter learns their fate. + // Refused jobs targeting this sled record the + // terminal skip status so the submitter learns + // their fate. let session_id = signed.payload().session_id(); let live = self.is_live(incoming_version); match self.session.active_session() { @@ -1086,7 +1102,7 @@ impl State { _ => { let job_id = *signed.job_id(); warn!( - log, "refusing job for inactive session"; + log, "skipping job for inactive session"; "job_id" => %job_id, "session_id" => %session_id, "actor" => %actor, @@ -1095,12 +1111,7 @@ impl State { && signed.payload().runs_on(&self.own_baseboard, &self.cubbies) && !self.history.contains(&job_id) { - executor.job_refused( - job_id, - ProcessError::InvalidJob(format!( - "session `{session_id}` is not active" - )), - ); + executor.job_skipped(job_id, SkipReason::SessionEnded); } } } diff --git a/server/tests/output/session-ended-skip-event.bin b/server/tests/output/session-ended-skip-event.bin new file mode 100644 index 0000000000000000000000000000000000000000..505c78992bd71b49e20692042c938ba94174bda9 GIT binary patch literal 147 zcmZ3O6lSn6)wL`&ucT>Fc0poMNqk;uZc=Jdwxyx5t^o)bTILp~7G);pz+{|)d`*mu zj0_Bn4HhPQ JobId { + let session_id = session.session_id(); + let hole_id = session.next_job_id(); + let hole = root + .sign_job_request(hole_id, session_id, "true", false) + .await; + session.job_started(hole.into_signed()); + let job_id = session.next_job_id(); + let job = root + .sign_job_request(job_id, session_id, "false", false) + .await; + mgr.job_start(authn, job.clone().into_signed(), JobStartParams::default()) + .await + .expect("should be able to queue the job"); + session.job_started(job.into_signed()); + mgr.wait_for_job_status(&job_id).await.unwrap(); + job_id +} + #[named] #[tokio::test] async fn cancel_queued_job() { @@ -457,28 +484,7 @@ async fn cancel_queued_job() { .expect("should be able to start job A"); session.job_started(job_a.into_signed()); - // Queue job B behind a hole in the job chain, so it cannot start - // before we cancel it: the executor only runs the job whose id the - // chain expects next, and it never sees this one. - let hole_id = session.next_job_id(); - let hole = root - .sign_job_request(hole_id, session_id, "true", false) - .await; - session.job_started(hole.into_signed()); - let command_b = "false"; - let job_id_b = session.next_job_id(); - let job_b = root - .sign_job_request(job_id_b, session_id, command_b, false) - .await; - mgr.job_start( - &authn, - job_b.clone().into_signed(), - JobStartParams::default(), - ) - .await - .expect("should be able to queue job B"); - session.job_started(job_b.into_signed()); - mgr.wait_for_job_status(&job_id_b).await.unwrap(); + let job_id_b = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; assert!(matches!( &mgr.job_status(&authn, &job_id_b).await.unwrap()[mgr.own_baseboard()], JobStatus::Queued { job_id: jid, time_queued, .. } if *jid == job_id_b && *time_queued <= Utc::now() @@ -513,6 +519,109 @@ async fn cancel_queued_job() { .expect("should be able to stop job A"); } +async fn wait_for_session_ended_skip(mgr: &JobManager, authn: &Identity, job_id: &JobId) { + timeout(Duration::from_secs(30), async { + loop { + if let Ok(map) = mgr.job_status(authn, job_id).await + && matches!( + map.get(mgr.own_baseboard()), + Some(JobStatus::Skipped { + reason: SkipReason::SessionEnded, + .. + }) + ) + { + break; + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("job skipped for ended session"); +} + +#[named] +#[tokio::test] +async fn session_stop_skips_queued_jobs() { + let log = test_logger(function_name!()); + let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; + let authn = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = + SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let mut session = Session::new(session_id); + mgr.session_start(&authn, session_id, signer_nonce, true) + .await + .unwrap(); + + let job_id_a = session.next_job_id(); + let job_a = root + .sign_job_request(job_id_a, session_id, "sleep 10", false) + .await; + mgr.job_start( + &authn, + job_a.clone().into_signed(), + JobStartParams { + wait: JobWait::Start, + ..Default::default() + }, + ) + .await + .expect("should be able to start job A"); + session.job_started(job_a.into_signed()); + + let job_id_b = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; + + mgr.session_stop(&authn, session_id) + .await + .expect("should be able to stop the session"); + wait_for_session_ended_skip(&mgr, &authn, &job_id_b).await; + + assert!(matches!( + &mgr.job_status(&authn, &job_id_a).await.unwrap()[mgr.own_baseboard()], + JobStatus::Started { .. } + )); + + mgr.job_stop( + &authn, + &job_id_a, + JobStopParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .expect("should be able to stop job A"); +} + +#[named] +#[tokio::test] +async fn superseding_session_skips_queued_jobs() { + let log = test_logger(function_name!()); + let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; + let authn = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = + SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let mut session = Session::new(session_id); + mgr.session_start(&authn, session_id, signer_nonce, true) + .await + .unwrap(); + + let job_id = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; + + let new_signer_nonce = SessionSignerNonce::random(); + let new_session_id = SessionId::compute( + mgr.own_baseboard(), + mgr.session_sush_nonce(), + new_signer_nonce, + ); + mgr.session_start(&authn, new_session_id, new_signer_nonce, true) + .await + .expect("should be able to start a superseding session"); + wait_for_session_ended_skip(&mgr, &authn, &job_id).await; +} + #[named] #[tokio::test] async fn job_output_perms() { From 94762dc96e1ecad8a75f9280461adee701a066d6 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 6 Sep 2026 13:23:51 -0600 Subject: [PATCH 09/12] Strip `sush-` prefix from locker file names We'll use a `sush` directory on the Omicron side. Co-Authored-By: Claude Mythos 5 --- server/src/bookmark.rs | 2 +- server/src/boundary.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index 351ce4d..cdb22c1 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -29,7 +29,7 @@ use crate::format::{self, NoFormat, Record, Versioned}; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOOKMARK: TenantSpec = TenantSpec { - file: "sush-bookmark", + file: "bookmark", magic: b"SUSHBOOKMARK", }; diff --git a/server/src/boundary.rs b/server/src/boundary.rs index 7ca2c61..8514e3c 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -70,7 +70,7 @@ use crate::format::{self, NoFormat, Record, Versioned}; use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; pub const BOUNDARY: TenantSpec = TenantSpec { - file: "sush-boundary", + file: "boundary", magic: b"SUSHBOUNDARY", }; From 5c9b721ec0b7ac7e9f037c7553aee49b8b617cc6 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 6 Sep 2026 17:52:43 -0600 Subject: [PATCH 10/12] Bump toolchain Match current omicron/main. Co-Authored-By: Claude Mythos 5 --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 59277fe..ef662bf 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.97.1" +channel = "1.98.1" components = ["clippy", "rustfmt"] From 2ceb9d92ba2d607018d044ce709cc49b94bf040b Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 21:35:18 -0600 Subject: [PATCH 11/12] Make `isolated` & `lonely` associated functions rather than free ones Co-Authored-By: Claude Mythos 5 --- server/src/gossip.rs | 43 ++++++++++++++++++++++---------------- server/src/main.rs | 6 +++--- tests/src/manager_tests.rs | 32 ++++++++++++++-------------- tests/src/test_utils.rs | 6 +++--- 4 files changed, 47 insertions(+), 40 deletions(-) diff --git a/server/src/gossip.rs b/server/src/gossip.rs index e08f02b..418848f 100644 --- a/server/src/gossip.rs +++ b/server/src/gossip.rs @@ -48,7 +48,20 @@ use crate::link::{AttestedBaseboards, CorpusSource, SprocketsDial, SprocketsLink use crate::locker::Locker; /// The attested baseboards of our live gossip peers. -pub type LinkedBaseboards = watch::Receiver>; +#[derive(Clone, Debug)] +pub struct LinkedBaseboards(watch::Receiver>); + +impl LinkedBaseboards { + /// A set that is forever empty, to accompany [`Universe::isolated`]. + pub fn lonely() -> Self { + let (_tx, rx) = watch::channel(BTreeSet::new()); + Self(rx) + } + + pub fn borrow(&self) -> watch::Ref<'_, BTreeSet> { + self.0.borrow() + } +} /// Manager timing. The defaults suit a rack, tests shrink them. #[derive(Clone, Debug)] @@ -83,6 +96,14 @@ impl Universe { pub fn genesis(rumors: Rumors) -> Self { Self { rumors } } + + /// A single-peer universe that never changes. The standalone server + /// uses this, as does a sled that cannot gossip. The receiver + /// outlives its sender. + pub fn isolated(seed: Rumors) -> watch::Receiver { + let (_tx, rx) = watch::channel(Universe::genesis(seed)); + rx + } } /// A seeded network paired with the source persisting its identity. @@ -130,26 +151,12 @@ impl Seed { } /// The network alone, for a seed that will never gossip - /// (see [`isolated`]). + /// (see [`Universe::isolated`]). pub fn into_rumors(self) -> Rumors { self.rumors } } -/// A single-peer universe that never changes. The standalone server uses -/// this, as does a sled that cannot gossip. The receiver outlives its -/// sender. -pub fn isolated(seed: Rumors) -> watch::Receiver> { - let (_tx, rx) = watch::channel(Universe::genesis(seed)); - rx -} - -/// A linked set that is forever empty, to accompany [`isolated`]. -pub fn lonely() -> LinkedBaseboards { - let (_tx, rx) = watch::channel(BTreeSet::new()); - rx -} - /// Whether the peer's universe dominates ours, by rumors' documented rule. fn remote_dominates( local_events: &Ticks, @@ -169,7 +176,7 @@ fn remote_dominates( /// over it until `shutdown`. Returns the address the listener bound, the /// channel following the current universe, and the channel following the /// baseboards we hold live links to. A caller that cannot bind may fall -/// back to [`isolated`]. +/// back to [`Universe::isolated`]. #[allow(clippy::too_many_arguments)] pub async fn spawn_gossip( log: &Logger, @@ -239,7 +246,7 @@ where shutdown, }; spawn(manager.run()); - (subscribe, subscribe_linked) + (subscribe, LinkedBaseboards(subscribe_linked)) } /// A link establishment that finished, and the peer it was aimed at. diff --git a/server/src/main.rs b/server/src/main.rs index 4528dfa..bc4222f 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -21,7 +21,7 @@ use x509_cert::der::DecodePem as _; use sush_api::sush_api_mod::api_description; use sush_common::targets::Cubbies; use sush_server::executor::PathIsolation; -use sush_server::gossip::{isolated, lonely}; +use sush_server::gossip::{LinkedBaseboards, Universe}; use sush_server::locker::Locker; use sush_server::manager::JobManager; use sush_server::output::JobOutputDir; @@ -95,7 +95,7 @@ async fn main() -> Result<(), String> { }; // TODO: get/seed Rumors network - let gossip = isolated(seed_gossip(&log, &Locker::null()).await.into_rumors()); + let gossip = Universe::isolated(seed_gossip(&log, &Locker::null()).await.into_rumors()); #[cfg(feature = "test-support")] let roots = overridable_root_certs(&override_root_certs).await?; @@ -111,7 +111,7 @@ async fn main() -> Result<(), String> { baseboard, cubbies, gossip, - lonely(), + LinkedBaseboards::lonely(), &Locker::null(), &roots, shutdown.clone(), diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index 4961c23..085ed9a 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -35,7 +35,7 @@ use sush_common::jobs::{ }; use sush_common::keys::{EphemeralKey, KeyError, KeyId, KeyType, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; -use sush_server::gossip::{Universe, isolated, lonely}; +use sush_server::gossip::{LinkedBaseboards, Universe}; use sush_server::io::BATCH_OUTPUT_BUFFER_SIZE; use sush_server::locker::Locker; use sush_server::messages::v0::{CertRequest, IdentityRequest, Message, Request, SessionRequest}; @@ -689,8 +689,8 @@ async fn cubby_targets() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), cubbies_rx, - isolated(null_gossip().await), - lonely(), + Universe::isolated(null_gossip().await), + LinkedBaseboards::lonely(), &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), @@ -796,8 +796,8 @@ async fn root_certs_from_files() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), no_cubbies(), - isolated(null_gossip().await), - lonely(), + Universe::isolated(null_gossip().await), + LinkedBaseboards::lonely(), &Locker::null(), &[path], CancellationToken::new(), @@ -848,8 +848,8 @@ async fn bad_root_cert_files() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), no_cubbies(), - isolated(null_gossip().await), - lonely(), + Universe::isolated(null_gossip().await), + LinkedBaseboards::lonely(), &Locker::null(), &[path], CancellationToken::new(), @@ -880,8 +880,8 @@ async fn job_output_dir_moves() { JobOutputDir::new(rx_dirs), test_baseboard_id(), no_cubbies(), - isolated(null_gossip().await), - lonely(), + Universe::isolated(null_gossip().await), + LinkedBaseboards::lonely(), &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), @@ -983,7 +983,7 @@ async fn universe_swap() { test_baseboard_id(), no_cubbies(), universe_rx, - lonely(), + LinkedBaseboards::lonely(), &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), @@ -1146,7 +1146,7 @@ async fn cert_chain() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let gossip = isolated(null_gossip().await); + let gossip = Universe::isolated(null_gossip().await); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( log, @@ -1155,7 +1155,7 @@ async fn cert_chain() { baseboard, no_cubbies(), gossip, - lonely(), + LinkedBaseboards::lonely(), &Locker::null(), &roots, shutdown, @@ -1930,8 +1930,8 @@ async fn hostile_imports_cannot_displace() { JobOutputDir::fixed(dir.path()), baseboard, no_cubbies(), - isolated(seed), - lonely(), + Universe::isolated(seed), + LinkedBaseboards::lonely(), &Locker::null(), from_ref(&root_cert), shutdown, @@ -2077,8 +2077,8 @@ async fn homonym_issuer_resolves_to_true_parent() { JobOutputDir::fixed(dir.path()), baseboard, no_cubbies(), - isolated(seed), - lonely(), + Universe::isolated(seed), + LinkedBaseboards::lonely(), &Locker::null(), from_ref(&root_cert), shutdown, diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 6abf424..0f4a539 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -30,7 +30,7 @@ use sush_common::jobs::{JobId, JobMode, JobStartRequest, SessionId, VerifiedJob} use sush_common::keys::{EphemeralKey, KeyType, Signer}; use sush_common::targets::{Cubbies, Target}; use sush_server::executor::PathIsolation; -use sush_server::gossip::{isolated, lonely}; +use sush_server::gossip::{LinkedBaseboards, Universe}; use sush_server::locker::Locker; use sush_server::output::{JobOutputDir, JobOutputFileStream}; use sush_server::state::GossipNetwork; @@ -230,7 +230,7 @@ pub async fn manager_test_root_and_peer( let dir = TempDir::with_prefix("sush-").unwrap(); let seed = null_gossip().await; let peer = seed.clone(); - let gossip = isolated(seed); + let gossip = Universe::isolated(seed); let shutdown = CancellationToken::new(); let root = ephemeral_test_root(); let mgr = JobManager::with_root_certs( @@ -240,7 +240,7 @@ pub async fn manager_test_root_and_peer( test_baseboard_id(), no_cubbies(), gossip, - lonely(), + LinkedBaseboards::lonely(), &Locker::null(), &[root.cert().to_owned()], shutdown.clone(), From fa405c21143fc1cda26eb74aa4a470adc10d99be Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 9 Sep 2026 12:52:03 -0600 Subject: [PATCH 12/12] Hold an advisory lock on every locker slot A second locker over any of the same slots, in this process or another, now fails to construct instead of clobbering records. Co-Authored-By: Claude Mythos 5 --- server/src/bookmark.rs | 2 +- server/src/boundary.rs | 16 +++++--- server/src/locker.rs | 76 +++++++++++++++++++++++++++++++++---- server/tests/distributed.rs | 26 ++++++------- 4 files changed, 93 insertions(+), 27 deletions(-) diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index cdb22c1..eeecd7e 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -201,7 +201,7 @@ mod test { } fn source(slots: Vec) -> BookmarkSource { - BookmarkSource::new(&test_log(), &Locker::new(&test_log(), slots)) + BookmarkSource::new(&test_log(), &Locker::new(&test_log(), slots).unwrap()) } async fn read_back(handle: &SushBookmark) -> Option> { diff --git a/server/src/boundary.rs b/server/src/boundary.rs index 8514e3c..61ccfed 100644 --- a/server/src/boundary.rs +++ b/server/src/boundary.rs @@ -357,7 +357,7 @@ mod test { } async fn store(slots: Vec) -> BoundaryStore { - let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots)); + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots).unwrap()); store.load().await; store } @@ -433,8 +433,9 @@ mod test { struct Envelope(u16, #[serde(with = "crate::format::cbor_bytes")] Vec); let mut bytes = Vec::new(); into_cbor(&Envelope(1, b"from the future".to_vec()), &mut bytes).unwrap(); - let scratch = Locker::new(&test_log(), slots.clone()); + let scratch = Locker::new(&test_log(), slots.clone()).unwrap(); scratch.tenant(BOUNDARY).store(&bytes).await.unwrap(); + drop(scratch); let store = store(slots).await; assert!(store.untrusted()); @@ -464,6 +465,7 @@ mod test { b.burned = a.burned_for(b.network); first.advance(&a).await.unwrap(); first.advance(&b).await.unwrap(); + drop(first); let next = store(slots).await; assert!(!next.untrusted()); @@ -509,6 +511,7 @@ mod test { first.record_outcome(&job, &interrupted).await; first.record_outcome(&job, &killed).await; + drop(first); let next = store(slots).await; assert!(matches!( next.boundary().unwrap().job.unwrap().outcome, @@ -535,6 +538,7 @@ mod test { untrusted.advance(&boundary()).await, Err(BoundaryError::Untrusted) )); + drop(untrusted); let reload = store(slots).await; assert!(reload.untrusted()); @@ -544,10 +548,11 @@ mod test { async fn undecodable_record_is_untrusted() { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slots = slots(&dir); - let scratch = Locker::new(&test_log(), slots.clone()); + let scratch = Locker::new(&test_log(), slots.clone()).unwrap(); scratch.tenant(BOUNDARY).store(b"scribble").await.unwrap(); + drop(scratch); - let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots)); + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots).unwrap()); store.load().await; assert!(store.untrusted()); } @@ -555,7 +560,8 @@ mod test { #[tokio::test] async fn unloaded_is_untrusted() { let dir = TempDir::with_prefix("sush-boundary-").unwrap(); - let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots(&dir))); + let store = + BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots(&dir)).unwrap()); assert!(store.untrusted()); assert!(matches!( store.advance(&boundary()).await, diff --git a/server/src/locker.rs b/server/src/locker.rs index c63dab7..a9171f3 100644 --- a/server/src/locker.rs +++ b/server/src/locker.rs @@ -28,11 +28,16 @@ //! Discarding is a verdict, not an error, since each tenant decides //! what starting over means. //! +//! A locker holds an advisory lock on every slot until it and all +//! of its tenants drop. Trying to construct a second locker over +//! any of the same slots, in this process or another, will fail +//! instead of clobbering records. +//! //! All of this is necessary to ensure the basic constraint that //! **we must never adopt stale data**. use std::collections::BTreeSet; -use std::fs::Permissions; +use std::fs::{File, Permissions}; use std::io::{self, Write as _}; use std::os::unix::fs::PermissionsExt as _; use std::sync::{Arc, Mutex as SyncMutex}; @@ -40,6 +45,7 @@ use std::sync::{Arc, Mutex as SyncMutex}; use atomicwrites::{AtomicFile, OverwriteBehavior}; use camino::Utf8PathBuf; use futures::TryFutureExt as _; +use rustix::fs::{FlockOperation, flock}; use slog::{Logger, o, warn}; use thiserror::Error; use tokio::fs::{read, remove_file, write}; @@ -49,6 +55,7 @@ use tokio::task::spawn_blocking; use sush_common::authn::Nonce; use sush_common::hash::Hasher; +const LOCK_FILE: &str = "lock"; const MAGIC_LEN: usize = 12; const NONCE_LEN: usize = 32; const SEQ_LEN: usize = 8; @@ -96,6 +103,15 @@ pub enum Discard { }, } +/// Some other locker holds a requested slot. +#[derive(Debug, Error)] +#[error("another locker holds `{path}`: {error}")] +pub struct Locked { + path: Utf8PathBuf, + #[source] + error: io::Error, +} + #[derive(Debug, Error)] pub enum StoreError { #[error("storing `{path}` failed: {error}")] @@ -122,23 +138,48 @@ pub struct Locker { /// file would have its own sequence state, silently defeating /// the straggler guard. claimed: Arc>>, + /// Advisory locks held by this locker for as long as it lives. + /// Every non-broken slot gets a lock. + locks: Arc>, } impl Locker { /// A locker spans `slots`. Empty means nothing persists - /// (the standalone server, tests). - pub fn new(log: &Logger, slots: Vec) -> Self { - Self { - log: log.new(o!("component" => "locker")), + /// (the standalone server, tests). Fails when another locker + /// holds any of the slots. + pub fn new(log: &Logger, slots: Vec) -> Result { + let log = log.new(o!("component" => "locker")); + let mut locks = Vec::with_capacity(slots.len()); + for slot in &slots { + let path = slot.join(LOCK_FILE); + match File::create(&path) { + Ok(file) => match flock(&file, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => locks.push(file), + Err(errno) => { + return Err(Locked { + path, + error: errno.into(), + }); + } + }, + Err(error) => { + warn!(log, "cannot create slot lock file"; "path" => %path, "error" => %error); + } + } + } + Ok(Self { + log, slots: Arc::new(slots), nonce: Nonce::random(), - claimed: Arc::new(SyncMutex::new(BTreeSet::new())), - } + claimed: Arc::new(SyncMutex::new(BTreeSet::from([LOCK_FILE]))), + locks: Arc::new(locks), + }) } /// A locker that loads and persists nothing. pub fn null() -> Self { Self::new(&Logger::root(slog::Discard, o!()), Vec::new()) + .expect("a locker without slots takes no locks") } /// A tenant of this locker, described by a (constant) specification. @@ -156,6 +197,7 @@ impl Locker { nonce: self.nonce.clone(), reserved: Mutex::new(0), committed: Arc::new(SyncMutex::new(0)), + _locks: Arc::clone(&self.locks), } } @@ -184,6 +226,9 @@ pub struct Tenant { reserved: Mutex, /// The newest sequence number written to all slots. committed: Arc>, + /// Holds the slot locks for this tenant's lifetime, so that the + /// locker may drop before its tenants without freeing the slots. + _locks: Arc>, } impl Tenant { @@ -391,7 +436,7 @@ mod test { } fn locker(slots: Vec) -> Locker { - Locker::new(&test_log(), slots) + Locker::new(&test_log(), slots).unwrap() } fn files(slots: &[Utf8PathBuf]) -> Vec { @@ -456,6 +501,7 @@ mod test { let b = locker(vec![slots[1].clone()]).tenant(SPEC); b.store(b"junk").await.unwrap(); b.store(b"another").await.unwrap(); + drop((a, b)); let tenant = locker(slots).tenant(SPEC); assert!(matches!( @@ -475,6 +521,7 @@ mod test { let b = locker(vec![slots[1].clone()]).tenant(SPEC); b.store(b"junk").await.unwrap(); b.store(b"same").await.unwrap(); + drop((a, b)); let tenant = locker(slots.clone()).tenant(SPEC); assert!(matches!( @@ -537,6 +584,7 @@ mod test { tenant.load().await, Verdict::Adopt(record) if record == b"new" )); + drop(tenant); let next = locker(slots.clone()).tenant(SPEC); assert!(matches!( @@ -617,6 +665,18 @@ mod test { )); } + /// A slot admits one locker at a time, and a tenant keeps the + /// locks alive after its locker drops. + #[tokio::test] + async fn second_locker_is_locked_out() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + assert!(Locker::new(&test_log(), slots.clone()).is_err()); + drop(tenant); + assert!(Locker::new(&test_log(), slots).is_ok()); + } + /// A null locker persists nothing and never fails. #[tokio::test] async fn null_touches_nothing() { diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index 27c0bc1..a3a8398 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -594,7 +594,7 @@ async fn lost_suffix_never_reruns() { let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); let b_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); + let locker = Locker::new(&log, vec![slot.clone()]).unwrap(); let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; let authn_b = fake_identity(&mut root).await; a.peers.send(BTreeSet::from([b.addr])).unwrap(); @@ -669,7 +669,7 @@ async fn lost_suffix_never_reruns() { b_shutdown.cancel(); drop(b); sleep(Duration::from_millis(500)).await; - let locker = Locker::new(&log, vec![slot]); + let locker = Locker::new(&log, vec![slot]).unwrap(); let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; let authn_b = fake_identity(&mut root).await; a.peers.send(BTreeSet::from([b.addr])).unwrap(); @@ -776,7 +776,7 @@ async fn session_resumes_at_stored_successor() { let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); let b_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); + let locker = Locker::new(&log, vec![slot.clone()]).unwrap(); let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; let authn_b = fake_identity(&mut root).await; a.peers.send(BTreeSet::from([b.addr, c.addr])).unwrap(); @@ -862,7 +862,7 @@ async fn session_resumes_at_stored_successor() { drop(b); a.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; - let locker = Locker::new(&log, vec![slot]); + let locker = Locker::new(&log, vec![slot]).unwrap(); let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; let authn_b = fake_identity(&mut root).await; b.peers.send(BTreeSet::from([c.addr])).unwrap(); @@ -945,7 +945,7 @@ async fn universe_flip_flop_raises_floor() { let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); let x_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); + let locker = Locker::new(&log, vec![slot.clone()]).unwrap(); let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; let authn_x = fake_identity(&mut root).await; a.peers.send(BTreeSet::from([x.addr])).unwrap(); @@ -1003,7 +1003,7 @@ async fn universe_flip_flop_raises_floor() { a.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; let x_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); + let locker = Locker::new(&log, vec![slot.clone()]).unwrap(); let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; let authn_x = fake_identity(&mut root).await; d.peers.send(BTreeSet::from([x.addr])).unwrap(); @@ -1046,7 +1046,7 @@ async fn universe_flip_flop_raises_floor() { d.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; let x_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); + let locker = Locker::new(&log, vec![slot.clone()]).unwrap(); let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; let authn_x = fake_identity(&mut root).await; a.peers.send(BTreeSet::from([x.addr])).unwrap(); @@ -1125,7 +1125,7 @@ async fn universe_flip_flop_raises_floor() { drop(x); a.peers.send(BTreeSet::new()).unwrap(); sleep(Duration::from_millis(500)).await; - let locker = Locker::new(&log, vec![slot]); + let locker = Locker::new(&log, vec![slot]).unwrap(); let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; let authn_x = fake_identity(&mut root).await; d.peers.send(BTreeSet::from([x.addr])).unwrap(); @@ -1181,7 +1181,7 @@ async fn witnessed_session_survives_restart() { let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); let b_shutdown = CancellationToken::new(); - let locker = Locker::new(&log, vec![slot.clone()]); + let locker = Locker::new(&log, vec![slot.clone()]).unwrap(); let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; a.peers.send(BTreeSet::from([b.addr])).unwrap(); b.peers.send(BTreeSet::from([a.addr])).unwrap(); @@ -1224,7 +1224,7 @@ async fn witnessed_session_survives_restart() { b_shutdown.cancel(); drop(b); sleep(Duration::from_millis(500)).await; - let locker = Locker::new(&log, vec![slot]); + let locker = Locker::new(&log, vec![slot]).unwrap(); let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; a.peers.send(BTreeSet::from([b.addr])).unwrap(); b.peers.send(BTreeSet::from([a.addr])).unwrap(); @@ -1290,7 +1290,7 @@ async fn bookmarks_survive_restart() { &dir, 2, &root_pem, - Locker::new(&log, vec![slot.clone()]), + Locker::new(&log, vec![slot.clone()]).unwrap(), &b_shutdown, ) .await; @@ -1316,7 +1316,7 @@ async fn bookmarks_survive_restart() { &dir, 2, &root_pem, - Locker::new(&log, vec![slot.clone()]), + Locker::new(&log, vec![slot.clone()]).unwrap(), &shutdown, ) .await; @@ -1362,7 +1362,7 @@ async fn gossip_survives_bookmark_failure() { &dir, 2, &root_pem, - Locker::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush")]), + Locker::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush")]).unwrap(), &shutdown, ) .await;