From c2bb9d22cd2558443953f3225463c56e73f1eab8 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 10:08:27 -0800 Subject: [PATCH 01/35] disk-thin: dm-clone tables, skip discards, hydration control, cutover `SessionDisk` gains three methods with a "not on this platform" default: `hydrate` makes a device over a destination volume that a guest can open before the volume holds its data, `hydration` reports how far the kernel has copied it in, and `settle` maps the device straight onto the destination once every region is in. `ThinPoolDisk` implements them with a `clone` target: destination = the thin volume, source = any read-only block device, metadata = an 8 MiB thin named `-md`, one region per chunk. The device is created with hydration off; a `BLKDISCARD` of every chunk the destination already holds (`no_discard_passdown`) marks those regions copied without a write, so the blocks the volume shares with its origin stay shared and only the delta is ever copied. Then hydration is turned on with a 1 MiB batch and 8 MiB in flight. `settle` suspends, loads a `linear` table over the destination and resumes in one owned task, so the guest's held I/O is released whatever happens to the caller; the metadata thin goes after. `delete` dispatches on the table kind so a hydrating or settled device takes its metadata with it. The review pass folded `delete`'s device removal into `remove_device`, shared by the thin and clone paths, and `thin_of` onto `target_of`. Root tests in `tests/clone.rs` run over the CI pool with a loop device as the source: the delta is copied, a guest write through the device lands, the settled destination reads as source plus write, `dirty_blocks` against the origin lists only the copied and written chunks (the skipped chunk is still shared), a settle before the copy is done is refused, and a delete mid-copy leaves nothing. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .github/workflows/ci.yml | 1 + crates/engram-disk-thin/src/disk.rs | 242 ++++++++++++++++++++++--- crates/engram-disk-thin/src/dm.rs | 39 ++++ crates/engram-disk-thin/src/parse.rs | 49 +++++ crates/engram-disk-thin/tests/clone.rs | 200 ++++++++++++++++++++ crates/engram-host-core/src/disk.rs | 51 ++++++ 6 files changed, 561 insertions(+), 21 deletions(-) create mode 100644 crates/engram-disk-thin/tests/clone.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5062b5b71..21f3c48dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1193,6 +1193,7 @@ jobs: sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-disk-thin \ --test thin_pool \ + --test clone \ --run-ignored ignored-only \ --no-fail-fast \ --test-threads=1 diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 906241e9d..f6a4ae403 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -1,5 +1,6 @@ //! The disk over the ioctl layer; Linux only. +use std::fs::File; use std::io; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; @@ -9,7 +10,9 @@ use std::time::Duration; use crate::dm; use crate::parse::{self, DevNo, ThinParams, METADATA_BLOCK_BYTES}; use async_trait::async_trait; -use engram_host_core::disk::{Capacity, SessionDisk, Volume, VolumeName}; +use engram_host_core::disk::{ + punch_hole, Capacity, ChunkIndex, Hydration, SessionDisk, Volume, VolumeName, +}; const SECTOR_BYTES: u64 = 512; /// dm-thin device ids are 24 bits wide. @@ -18,6 +21,14 @@ const ID_MASK: u32 = 0x00FF_FFFF; /// to close it. const REMOVE_ATTEMPTS: u32 = 10; const REMOVE_RETRY: Duration = Duration::from_millis(100); +/// A dm-clone metadata volume: a superblock and one bit per region in +/// 4 KiB blocks. 8 MiB holds the bitset of a 2 million region volume +/// with room for its transaction shadows. +const CLONE_METADATA_BYTES: u64 = 8 << 20; +/// Regions per background copy and copies in flight: 1 MiB bios, 8 MiB +/// in flight. The kernel's defaults of 1 and 1 copy at a few MiB/s. +const HYDRATION_BATCH_REGIONS: u64 = 16; +const HYDRATION_THRESHOLD_REGIONS: u64 = 128; pub struct ThinPoolDisk { dm: dm::Dm, @@ -130,11 +141,28 @@ impl ThinPoolDisk { Ok(thins) } + /// The one target of the device behind `name`. + async fn target_of(&self, name: &VolumeName) -> io::Result { + match self.dm.table(&self.device_name(name)).await?.as_slice() { + [target] => Ok(target.clone()), + targets => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{name} has {} targets, not one", targets.len()), + )), + } + } + /// The thin behind `name` and its size, refused when it belongs to /// another pool. async fn thin_of(&self, name: &VolumeName) -> io::Result<(ThinParams, u64)> { - let targets = self.dm.table(&self.device_name(name)).await?; - let params = parse::thin_params(&single_target(&targets, "thin")?)?; + let target = self.target_of(name).await?; + if target.kind != "thin" { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {} device, not a thin", target.kind), + )); + } + let params = parse::thin_params(&target.params)?; if params.pool != self.pool_dev { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -144,7 +172,43 @@ impl ThinPoolDisk { ), )); } - Ok((params, targets[0].length_sectors)) + Ok((params, target.length_sectors)) + } + + /// Remove the device, waiting a bounded time for the last holder to + /// close it, then its node. + async fn remove_device(&self, name: &VolumeName) -> io::Result<()> { + let mut attempt = 1; + loop { + match self.dm.remove(&self.device_name(name)).await { + Err(e) if e.kind() == io::ErrorKind::ResourceBusy && attempt < REMOVE_ATTEMPTS => { + attempt += 1; + tokio::time::sleep(REMOVE_RETRY).await; + } + removed => break removed?, + } + } + match tokio::fs::remove_file(self.node_of(name)).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } + } + + async fn delete_thin(&self, name: &VolumeName) -> io::Result<()> { + let (thin, _) = self.thin_of(name).await?; + self.remove_device(name).await?; + self.dm + .message(&self.pool, &format!("delete {}", thin.id)) + .await + } + + /// Delete the metadata thin of the hydrating device `name`, if a + /// settle or an earlier delete has not. + async fn delete_metadata(&self, name: &VolumeName) -> io::Result<()> { + match self.delete_thin(&metadata_name(name)?).await { + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + deleted => deleted, + } } /// A device over thin `id`. On failure the thin is deleted from the @@ -230,6 +294,46 @@ impl Drop for MetadataSnap<'_> { } } +/// The metadata volume of a hydrating device. +fn metadata_name(name: &VolumeName) -> io::Result { + VolumeName::new(format!("{name}-md")) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) +} + +/// Mark the `skip` chunks of a hydrating device as copied: under +/// `no_discard_passdown`, a discard of regions the kernel has not copied +/// marks them copied and leaves the destination's blocks as they are. +/// Consecutive chunks go in one discard; a skip list is most of a +/// volume, in long runs. +async fn skip_regions(device: &Volume, skip: &[ChunkIndex]) -> io::Result<()> { + let file = File::options().read(true).write(true).open(&device.path)?; + let (chunk, size) = (device.chunk_size, device.size_bytes); + let mut chunks: Vec = skip.iter().map(|c| c.0).collect(); + chunks.sort_unstable(); + chunks.dedup(); + tokio::task::spawn_blocking(move || { + let mut chunks = chunks.into_iter().peekable(); + while let Some(first) = chunks.next() { + let mut last = first; + while chunks.next_if_eq(&(last + 1)).is_some() { + last += 1; + } + let offset = first * chunk; + let end = (last + 1) * chunk; + if end > size { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("skip chunk {last} lies beyond the {size}-byte volume"), + )); + } + punch_hole(&file, offset, end - offset)?; + } + Ok(()) + }) + .await + .map_err(io::Error::other)? +} + fn single_target(targets: &[dm::Target], kind: &str) -> io::Result { match targets { [target] if target.kind == kind => Ok(target.params.clone()), @@ -302,26 +406,21 @@ impl SessionDisk for ThinPoolDisk { self.volume(name, sectors).await } - /// Waits a bounded time for the last holder to close the device. + /// Waits a bounded time for the last holder to close the device. A + /// hydrating or settled device takes its metadata thin with it. async fn delete(&self, name: &VolumeName) -> io::Result<()> { - let (thin, _) = self.thin_of(name).await?; - let mut attempt = 1; - loop { - match self.dm.remove(&self.device_name(name)).await { - Err(e) if e.kind() == io::ErrorKind::ResourceBusy && attempt < REMOVE_ATTEMPTS => { - attempt += 1; - tokio::time::sleep(REMOVE_RETRY).await; - } - removed => break removed?, + let target = self.target_of(name).await?; + match target.kind.as_str() { + "thin" => self.delete_thin(name).await, + "clone" | "linear" => { + self.remove_device(name).await?; + self.delete_metadata(name).await } + kind => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {kind} device, which this disk did not make"), + )), } - match tokio::fs::remove_file(self.node_of(name)).await { - Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), - _ => {} - } - self.dm - .message(&self.pool, &format!("delete {}", thin.id)) - .await } async fn list(&self) -> io::Result> { @@ -359,6 +458,107 @@ impl SessionDisk for ThinPoolDisk { }) } + /// A `clone` target over `dest` with `source` as its source and a + /// region per pool block (the unit the pool shares and `dirty_blocks` + /// sees; a guest write to a cold region waits for one region's copy), + /// created with hydration off so the skip discards land before any + /// copy; then hydration on. A failure leaves nothing. + async fn hydrate( + &self, + dest: &Volume, + source: &Path, + name: &VolumeName, + skip: &[ChunkIndex], + ) -> io::Result> { + let (_, sectors) = self.thin_of(&dest.name).await?; + let region_sectors = self.block_bytes / SECTOR_BYTES; + if !sectors.is_multiple_of(region_sectors) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{}: {sectors} sectors is not whole pool blocks", dest.name), + )); + } + let metadata = self + .create(&metadata_name(name)?, CLONE_METADATA_BYTES) + .await?; + let params = format!( + "{} {} {} {region_sectors} 2 no_hydration no_discard_passdown \ + 4 hydration_threshold {HYDRATION_THRESHOLD_REGIONS} \ + hydration_batch_size {HYDRATION_BATCH_REGIONS}", + dm::devno_of(&metadata.path)?, + dm::devno_of(&dest.path)?, + dm::devno_of(source)?, + ); + let device = self.device_name(name); + let hydrating = async { + self.dm.create(&device, sectors, "clone", ¶ms).await?; + let volume = self.volume(name, sectors).await?; + skip_regions(&volume, skip).await?; + self.dm.message(&device, "enable_hydration").await?; + Ok(volume) + } + .await; + if hydrating.is_err() { + let removed = match self.remove_device(name).await { + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + removed => removed, + }; + if let Err(error) = removed.and(self.delete_metadata(name).await) { + tracing::warn!(%name, %error, "hydrating device left behind by a failed hydrate"); + } + } + hydrating.map(Some) + } + + async fn hydration(&self, name: &VolumeName) -> io::Result { + let status = self.dm.status(&self.device_name(name)).await?; + match status.as_slice() { + [target] if target.kind == "clone" => { + let status = parse::clone_status(&target.params)?; + Ok(Hydration::Hydrating { + hydrated: status.hydrated_regions, + total: status.total_regions, + }) + } + [target] if target.kind == "linear" => Ok(Hydration::Settled), + _ => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is not a hydrating device: {status:?}"), + )), + } + } + + /// Reloads the device as `linear` over `dest`. The suspend holds the + /// guest's I/O for the reload, so the three calls are one owned task. + /// A failed reload resumes the clone table. The metadata thin goes + /// last: the kernel closes it when the clone table is destroyed. + async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { + match self.hydration(name).await? { + Hydration::Hydrating { hydrated, total } if hydrated < total => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name}: {hydrated} of {total} regions are hydrated"), + )); + } + Hydration::Hydrating { .. } => { + let (_, sectors) = self.thin_of(&dest.name).await?; + let params = format!("{} 0", dm::devno_of(&dest.path)?); + let dm = self.dm.clone(); + let device = self.device_name(name); + tokio::spawn(async move { + dm.suspend(&device).await?; + let loaded = dm.load(&device, sectors, "linear", ¶ms).await; + let resumed = dm.resume(&device).await; + loaded.and(resumed) + }) + .await + .map_err(io::Error::other)??; + } + Hydration::Settled => {} + } + self.delete_metadata(name).await + } + /// Reads the pool's mapping tree instead of the volumes. A block is /// dirty when any pool block inside it differs; `block_bytes` is a /// multiple of the pool's block size. diff --git a/crates/engram-disk-thin/src/dm.rs b/crates/engram-disk-thin/src/dm.rs index 073caaa5b..e36a900ea 100644 --- a/crates/engram-disk-thin/src/dm.rs +++ b/crates/engram-disk-thin/src/dm.rs @@ -352,6 +352,28 @@ impl Dm { ready } + /// Load a new single-target table as the inactive table of `name`; + /// `resume` makes it live. A failed load leaves the active table. + pub async fn load( + &self, + name: &str, + length_sectors: u64, + kind: &str, + params: &str, + ) -> io::Result<()> { + let mut load = Request::new(name, 0)?; + load.push_target(length_sectors, kind, params); + self.call( + "load", + name, + load, + DM_TABLE_LOAD_CMD, + io::ErrorKind::ResourceBusy, + ) + .await + .map(drop) + } + pub async fn remove(&self, name: &str) -> io::Result<()> { self.call( "remove", @@ -486,6 +508,23 @@ pub async fn device_path(devno: DevNo) -> io::Result { Ok(PathBuf::from("/dev").join(parse::devname(&uevent)?)) } +/// The device number of the block device node at `path`. +pub fn devno_of(path: &Path) -> io::Result { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + let meta = std::fs::metadata(path)?; + if !meta.file_type().is_block_device() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a block device", path.display()), + )); + } + let rdev = meta.rdev(); + Ok(DevNo { + major: libc::major(rdev), + minor: libc::minor(rdev), + }) +} + /// Create a block device node, replacing one that points elsewhere. #[allow(unsafe_code)] pub fn make_node(path: &Path, devno: DevNo) -> io::Result<()> { diff --git a/crates/engram-disk-thin/src/parse.rs b/crates/engram-disk-thin/src/parse.rs index 1fa11c43c..8ff0f2b7d 100644 --- a/crates/engram-disk-thin/src/parse.rs +++ b/crates/engram-disk-thin/src/parse.rs @@ -110,6 +110,35 @@ pub fn pool_status(status: &str) -> io::Result { } } +/// `clone` status: ` / +/// / +/// ... ... `. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CloneStatus { + pub hydrated_regions: u64, + pub total_regions: u64, + pub hydrating_regions: u64, +} + +/// A clone whose metadata is not read-write (`ro`, or the single word +/// `Fail`) is an error: it copies nothing more and must not be settled. +pub fn clone_status(status: &str) -> io::Result { + let fields: Vec<&str> = status.split_whitespace().collect(); + match fields.as_slice() { + ["Fail" | "Error"] => Err(invalid(format!("clone is {}", fields[0]))), + [.., mode] if *mode != "rw" => Err(invalid(format!("clone is {mode}"))), + [_, _, _, regions, hydrating, ..] => { + let (hydrated_regions, total_regions) = fraction(regions)?; + Ok(CloneStatus { + hydrated_regions, + total_regions, + hydrating_regions: number(hydrating)?, + }) + } + _ => Err(invalid(format!("not clone status: {status:?}"))), + } +} + /// The `DEVNAME=` line of a sysfs `uevent` file. pub fn devname(uevent: &str) -> io::Result { uevent @@ -242,6 +271,26 @@ mod tests { } } + #[test] + fn clone_status_reads_the_region_counts() { + let status = clone_status( + "8 25/2048 128 1500/4096 16 2 no_hydration no_discard_passdown 4 hydration_threshold 128 hydration_batch_size 16 rw", + ) + .unwrap(); + assert_eq!( + status, + CloneStatus { + hydrated_regions: 1500, + total_regions: 4096, + hydrating_regions: 16, + } + ); + for bad in ["8 25/2048 128 4096/4096 0 0 0 ro", "Fail"] { + let err = clone_status(bad).unwrap_err().to_string(); + assert!(err.starts_with("clone is "), "{bad}: {err}"); + } + } + #[test] fn devname_finds_the_node_name() { assert_eq!( diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs new file mode 100644 index 000000000..b9cd88131 --- /dev/null +++ b/crates/engram-disk-thin/tests/clone.rs @@ -0,0 +1,200 @@ +#![cfg(target_os = "linux")] +//! A hydrating device over the pool, as root, with a loop device as the +//! source. `ENGRAM_TEST_THIN_POOL` names the pool as in `thin_pool.rs`; +//! unset, the test skips. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use engram_disk_thin::ThinPoolDisk; +use engram_host_core::disk::{ChunkIndex, Hydration, SessionDisk, VolumeName}; + +/// Two pool blocks per chunk: a skip discard spans regions, and the +/// kernel copies a chunk as more than one region. +const CHUNK: u64 = 2 * 65536; +const CHUNKS: u64 = 8; +const REGIONS: u64 = CHUNKS * 2; + +macro_rules! pool_or_skip { + () => { + match std::env::var("ENGRAM_TEST_THIN_POOL") { + Ok(pool) => ThinPoolDisk::open(&pool, CHUNK, Path::new("/dev/engram-test")) + .await + .expect("open pool"), + Err(_) => { + eprintln!("SKIP: ENGRAM_TEST_THIN_POOL is not set"); + return; + } + } + }; +} + +fn name(role: &str) -> VolumeName { + VolumeName::new(format!("t-clone-{role}-{}", std::process::id())).unwrap() +} + +fn fill(byte: u8) -> Vec { + vec![byte; CHUNK as usize] +} + +/// A loop device over a file, detached on drop. +struct Loop(PathBuf); + +impl Loop { + async fn over(file: &Path) -> Self { + let output = tokio::process::Command::new("losetup") + .args(["--find", "--show"]) + .arg(file) + .output() + .await + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + Self(PathBuf::from( + String::from_utf8(output.stdout).unwrap().trim(), + )) + } +} + +impl Drop for Loop { + fn drop(&mut self) { + let _ = std::process::Command::new("losetup") + .arg("-d") + .arg(&self.0) + .status(); + } +} + +/// The source holds chunks 0, 1 and 3; the destination's origin holds +/// chunks 0 and 1 with chunk 0 equal to the source's. Skipped: chunk 0 +/// (held) and the zero chunks. The kernel copies 1 and 3; the guest +/// writes 5 through the device. After the settle the destination reads +/// as the source plus the write, and chunk 0 still shares its blocks +/// with the origin. +#[tokio::test] +#[ignore] +async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination() { + let pool = pool_or_skip!(); + let base = pool.create(&name("base"), CHUNKS * CHUNK).await.unwrap(); + base.write_chunk(ChunkIndex(0), fill(1)).await.unwrap(); + base.write_chunk(ChunkIndex(1), fill(2)).await.unwrap(); + let dest = pool.snapshot(&base, &name("l")).await.unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let source_file = dir.path().join("source"); + let mut source = vec![0u8; (CHUNKS * CHUNK) as usize]; + for (chunk, byte) in [(0, 1u8), (1, 3), (3, 4)] { + let at = (chunk * CHUNK) as usize; + source[at..at + CHUNK as usize].fill(byte); + } + std::fs::write(&source_file, &source).unwrap(); + let source = Loop::over(&source_file).await; + + let skip: Vec = [0, 2, 4, 5, 6, 7].map(ChunkIndex).to_vec(); + let device = pool + .hydrate(&dest, &source.0, &name("io"), &skip) + .await + .unwrap() + .expect("dm-clone on linux"); + assert!(pool.list().await.unwrap().contains(&name("io-md"))); + + assert_eq!(device.read_chunk(ChunkIndex(0)).await.unwrap(), fill(1)); + assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); + assert_eq!(device.read_chunk(ChunkIndex(2)).await.unwrap(), fill(0)); + assert_eq!(device.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); + device.write_chunk(ChunkIndex(5), fill(5)).await.unwrap(); + + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match pool.hydration(&device.name).await.unwrap() { + Hydration::Hydrating { hydrated, total } if hydrated == total => { + assert_eq!(total, REGIONS); + break; + } + Hydration::Hydrating { .. } if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + state => panic!("hydration did not finish: {state:?}"), + } + } + pool.settle(&device.name, &dest).await.unwrap(); + assert_eq!( + pool.hydration(&device.name).await.unwrap(), + Hydration::Settled + ); + assert!(!pool.list().await.unwrap().contains(&name("io-md"))); + + assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); + assert_eq!(device.read_chunk(ChunkIndex(5)).await.unwrap(), fill(5)); + for (chunk, byte) in [(0, 1u8), (1, 3), (2, 0), (3, 4), (5, 5)] { + assert_eq!( + dest.read_chunk(ChunkIndex(chunk)).await.unwrap(), + fill(byte), + "chunk {chunk}" + ); + } + assert_eq!( + pool.dirty_blocks(&base, &dest, CHUNK).await.unwrap(), + vec![1, 3, 5], + "the skipped chunk 0 still shares its blocks with the origin" + ); + + let (device, dest, base) = (device.name, dest.name, base.name); + pool.delete(&device).await.unwrap(); + pool.delete(&dest).await.unwrap(); + pool.delete(&base).await.unwrap(); + assert!(!pool + .list() + .await + .unwrap() + .iter() + .any(|v| v.as_str().starts_with("t-clone-"))); +} + +/// A settle before the copy is done is refused and changes nothing. +#[tokio::test] +#[ignore] +async fn settle_refuses_a_device_that_is_still_hydrating() { + let pool = pool_or_skip!(); + let dest = pool.create(&name("early-l"), CHUNKS * CHUNK).await.unwrap(); + let dir = tempfile::tempdir().unwrap(); + let source_file = dir.path().join("source"); + std::fs::write(&source_file, vec![7u8; (CHUNKS * CHUNK) as usize]).unwrap(); + let source = Loop::over(&source_file).await; + // A suspended destination takes no write, so the copy cannot finish. + dmsetup(&["suspend", &pool.device_name(&dest.name)]).await; + let device = pool + .hydrate(&dest, &source.0, &name("early-io"), &[]) + .await + .unwrap() + .unwrap(); + let err = pool.settle(&device.name, &dest).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput, "{err}"); + assert!(matches!( + pool.hydration(&device.name).await.unwrap(), + Hydration::Hydrating { .. } + )); + dmsetup(&["resume", &pool.device_name(&dest.name)]).await; + // A delete while the kernel is still copying leaks nothing. + let (device, dest) = (device.name, dest.name); + pool.delete(&device).await.unwrap(); + pool.delete(&dest).await.unwrap(); + assert!(!pool + .list() + .await + .unwrap() + .iter() + .any(|v| v.as_str().starts_with("t-clone-early"))); +} + +async fn dmsetup(args: &[&str]) { + let status = tokio::process::Command::new("dmsetup") + .args(args) + .status() + .await + .unwrap(); + assert!(status.success(), "dmsetup {args:?}"); +} diff --git a/crates/engram-host-core/src/disk.rs b/crates/engram-host-core/src/disk.rs index ea8c8a51d..2f0870467 100644 --- a/crates/engram-host-core/src/disk.rs +++ b/crates/engram-host-core/src/disk.rs @@ -72,6 +72,17 @@ impl Capacity { } } +/// How far the kernel has copied a hydrating volume's regions in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Hydration { + /// `hydrated` of `total` regions are in the destination; the rest + /// still read from the source. + Hydrating { hydrated: u64, total: u64 }, + /// Every region is in the destination and the device maps onto it + /// directly ([`SessionDisk::settle`] ran). + Settled, +} + /// An open volume. Clones share the file handle. #[derive(Clone, Debug)] pub struct Volume { @@ -416,6 +427,46 @@ pub trait SessionDisk: Send + Sync { async fn capacity(&self) -> io::Result; + /// A device named `name` over `dest` that a guest can open before + /// `dest` holds its data: a read of a region not yet in `dest` is + /// served from the read-only block device at `source`, a write goes + /// to `dest` (after the kernel copies the rest of its region in), + /// and the kernel copies every other region from `source` into + /// `dest` in the background. `skip` lists the chunks `dest` already + /// holds in their final form; they are never copied, so blocks + /// `dest` shares with its origin stay shared. `Ok(None)` on a + /// platform without such a device; the caller then writes `dest` in + /// full before the guest starts. + async fn hydrate( + &self, + dest: &Volume, + source: &Path, + name: &VolumeName, + skip: &[ChunkIndex], + ) -> io::Result> { + let _ = (dest, source, name, skip); + Ok(None) + } + + /// Progress of a device [`Self::hydrate`] made. + async fn hydration(&self, name: &VolumeName) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("{name} is not a hydrating device on this platform"), + )) + } + + /// Map a fully hydrated device straight onto `dest`. Every I/O in + /// flight completes first; open handles keep working; `source` is + /// not read again. + async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { + let _ = dest; + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("{name} is not a hydrating device on this platform"), + )) + } + /// Ascending indices of the `block_bytes`-sized blocks that differ /// between two snapshots of one origin; `block_bytes` divides the /// chunk size. The default reads both in full and is the oracle a From 1bf0f9d59a75a24dfc76f48661cd4af04ab21c94 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 10:11:42 -0800 Subject: [PATCH 02/35] disk-thin: suspend, operation and resume as one task; resume on open `snapshot` and `settle` each spawned an owned suspend-op-resume task. `while_suspended` is that task once, and hands both outcomes back so a caller decides what a failed resume costs it (`snapshot` deletes the unactivated thin). `Dm::suspended` reads the kernel's `DM_SUSPEND_FLAG`, and `open` resumes every device of its pool that a host-agent dying inside that task left suspended: the device holds every I/O until then, and its inactive table, if any, is the one the task meant to make live. Root test: a device suspended behind the disk's back reads again after the next `open`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-disk-thin/src/disk.rs | 105 +++++++++++++-------- crates/engram-disk-thin/src/dm.rs | 15 +++ crates/engram-disk-thin/tests/thin_pool.rs | 31 ++++++ 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index f6a4ae403..6c5397362 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -75,6 +75,16 @@ impl ThinPoolDisk { next_id: Arc::new(AtomicU32::new(1)), metadata_snap: tokio::sync::Mutex::new(()), }; + // A host-agent that died inside `while_suspended` left a device + // holding every I/O; its inactive table, if any, is the one it + // meant to make live. + let prefix = format!("{pool}."); + for (device, _) in disk.dm.list().await? { + if device.starts_with(&prefix) && disk.dm.suspended(&device).await? { + tracing::warn!(device, "resuming a device left suspended"); + disk.dm.resume(&device).await?; + } + } let in_pool = disk.thin_ids().await?; let active = disk.active_ids().await?; for id in in_pool.iter().filter(|id| !active.contains(id)) { @@ -175,6 +185,30 @@ impl ThinPoolDisk { Ok((params, target.length_sectors)) } + /// Run `op` while `device` is suspended, then resume it, as one owned + /// task: a suspended device holds every I/O, so the resume runs + /// whatever happens to the caller. Both outcomes come back; what a + /// failed resume costs is the caller's to decide. + async fn while_suspended( + &self, + device: String, + op: impl FnOnce(dm::Dm) -> Fut + Send + 'static, + ) -> io::Result<(io::Result, io::Result<()>)> + where + T: Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + let dm = self.dm.clone(); + tokio::spawn(async move { + dm.suspend(&device).await?; + let done = op(dm.clone()).await; + let resumed = dm.resume(&device).await; + Ok((done, resumed)) + }) + .await + .map_err(io::Error::other)? + } + /// Remove the device, waiting a bounded time for the last holder to /// close it, then its node. async fn remove_device(&self, name: &VolumeName) -> io::Result<()> { @@ -364,40 +398,33 @@ impl SessionDisk for ThinPoolDisk { async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { let (thin, sectors) = self.thin_of(&origin.name).await?; - let dm = self.dm.clone(); let pool = self.pool.clone(); let next_id = self.next_id.clone(); - let origin_name = self.device_name(&origin.name); - // The origin holds all I/O while it is suspended, so the resume - // must run whatever happens to the caller: the calls are one owned - // task. - let id = tokio::spawn(async move { - dm.suspend(&origin_name).await?; - let created = loop { - let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; - match dm - .message(&pool, &format!("create_snap {id} {}", thin.id)) - .await - { - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, - created => break created.map(|()| id), - } - }; - let resumed = dm.resume(&origin_name).await; - match (created, resumed) { - (Ok(id), Ok(())) => Ok(id), - (Ok(id), Err(e)) => { - // The snapshot exists but nothing will activate it. - if let Err(error) = dm.message(&pool, &format!("delete {id}")).await { - tracing::warn!(pool = %pool, id, %error, "thin left behind by a failed resume"); + let (created, resumed) = self + .while_suspended(self.device_name(&origin.name), move |dm| async move { + loop { + let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; + match dm + .message(&pool, &format!("create_snap {id} {}", thin.id)) + .await + { + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + created => return created.map(|()| id), } - Err(e) } - (Err(e), _) => Err(e), + }) + .await?; + let id = match (created, resumed) { + (Ok(id), Ok(())) => id, + (Ok(id), Err(e)) => { + // The snapshot exists but nothing will activate it. + if let Err(error) = self.dm.message(&self.pool, &format!("delete {id}")).await { + tracing::warn!(pool = %self.pool, id, %error, "thin left behind by a failed resume"); + } + return Err(e); } - }) - .await - .map_err(io::Error::other)??; + (Err(e), _) => return Err(e), + }; self.activate(name, id, sectors).await } @@ -528,9 +555,8 @@ impl SessionDisk for ThinPoolDisk { } } - /// Reloads the device as `linear` over `dest`. The suspend holds the - /// guest's I/O for the reload, so the three calls are one owned task. - /// A failed reload resumes the clone table. The metadata thin goes + /// Reloads the device as `linear` over `dest` under a suspend. A + /// failed reload resumes the clone table. The metadata thin goes /// last: the kernel closes it when the clone table is destroyed. async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { match self.hydration(name).await? { @@ -543,16 +569,13 @@ impl SessionDisk for ThinPoolDisk { Hydration::Hydrating { .. } => { let (_, sectors) = self.thin_of(&dest.name).await?; let params = format!("{} 0", dm::devno_of(&dest.path)?); - let dm = self.dm.clone(); let device = self.device_name(name); - tokio::spawn(async move { - dm.suspend(&device).await?; - let loaded = dm.load(&device, sectors, "linear", ¶ms).await; - let resumed = dm.resume(&device).await; - loaded.and(resumed) - }) - .await - .map_err(io::Error::other)??; + let (loaded, resumed) = self + .while_suspended(device.clone(), move |dm| async move { + dm.load(&device, sectors, "linear", ¶ms).await + }) + .await?; + loaded.and(resumed)?; } Hydration::Settled => {} } diff --git a/crates/engram-disk-thin/src/dm.rs b/crates/engram-disk-thin/src/dm.rs index e36a900ea..ece66ae86 100644 --- a/crates/engram-disk-thin/src/dm.rs +++ b/crates/engram-disk-thin/src/dm.rs @@ -420,6 +420,21 @@ impl Dm { .map(drop) } + /// Whether `name` is suspended: the kernel sets `DM_SUSPEND_FLAG` in + /// every reply about a suspended device. + pub async fn suspended(&self, name: &str) -> io::Result { + let reply = self + .call( + "status", + name, + Request::new(name, 0)?, + DM_DEV_STATUS_CMD, + io::ErrorKind::ResourceBusy, + ) + .await?; + Ok(reply.flags() & DM_SUSPEND_FLAG != 0) + } + pub async fn resume(&self, name: &str) -> io::Result<()> { self.call( "resume", diff --git a/crates/engram-disk-thin/tests/thin_pool.rs b/crates/engram-disk-thin/tests/thin_pool.rs index 4a5ce09e1..67df8bdf4 100644 --- a/crates/engram-disk-thin/tests/thin_pool.rs +++ b/crates/engram-disk-thin/tests/thin_pool.rs @@ -318,6 +318,37 @@ async fn open_deletes_a_thin_whose_device_is_gone_and_create_skips_a_taken_id() assert!(deleted.success()); } +/// A host-agent that died between a suspend and its resume left the +/// device holding every I/O; the next `open` resumes it. +#[tokio::test] +#[ignore] +async fn open_resumes_a_device_left_suspended() { + let disk = pool_or_skip!(); + let held = name("suspended", "v"); + let v = disk.create(&held, CHUNK).await.unwrap(); + let suspended = tokio::process::Command::new("dmsetup") + .args(["suspend", &disk.device_name(&held)]) + .status() + .await + .unwrap(); + assert!(suspended.success()); + + let disk = ThinPoolDisk::open(disk.pool(), CHUNK, Path::new("/dev/engram-test")) + .await + .unwrap(); + let read = tokio::time::timeout( + std::time::Duration::from_secs(5), + v.read_chunk(ChunkIndex(0)), + ) + .await + .expect("the device was resumed") + .unwrap(); + assert_eq!(read, fill(0)); + + drop(v); + disk.delete(&held).await.unwrap(); +} + /// A reserve left behind by a holder that died: the next read releases /// it, reserves its own and releases that one too. #[tokio::test] From c7bcefad82464e05bc9c102531d66308d9dbc56e Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 11:11:29 -0800 Subject: [PATCH 03/35] host-agent: BlkSource, a ublk device served by the fill engine A hydrating session volume copies its cold regions from a read-only block device. BlkSource is that device: a ublk device the host-agent serves in process, one per role, with the bytes coming from the fill engine directly. There is no helper process, no second fetch path and no second cache copy; the engine's classes, hedging, peer racing and admission are the whole policy. The source cannot be told which reads are the kernel's background copy and which are the guest waiting, so it tells them apart by position. The copy walks the delta chunks in ascending order; the source keeps a window of whole chunks around that stream, fetched at `Hydrate` and four chunks ahead, and hands them out region by region, so the store sees one GET per chunk. A read anywhere else is a guest read: a ranged GET at `Demand`, and its chunk is recorded as touched (the first-touch prefix of a later restore). A host-agent that dies leaves the device to the kernel: with `UBLK_F_USER_RECOVERY` the reads it left queued wait, and `recover` takes the device over from the next host-agent and answers them. A stop is refused while a device still maps onto the source: a ublk device with reads queued and no server holds the kernel until reboot. The queue thread sleeps in `io_uring_enter`; finished reads cross back from tokio through an eventfd read queued on the same ring. libublk sends control commands over a thread-local ring that only the thread which built the `UblkCtrl` has, so the server thread and every blocking task that issues one make their own first. The root lane loads `ublk_drv` and runs the two tests: a served manifest answers a demand read and a region-by-region stream with only the demand read recorded, and a server killed with a read queued is recovered by a new one that completes the read and deletes the device. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .github/workflows/ci.yml | 6 +- Cargo.lock | 348 +++++++++- crates/engram-host-agent/Cargo.toml | 4 + crates/engram-host-agent/src/blk_source.rs | 696 +++++++++++++++++++ crates/engram-host-agent/src/lib.rs | 2 + crates/engram-host-agent/tests/blk_source.rs | 215 ++++++ workspace-hack/Cargo.toml | 22 +- 7 files changed, 1281 insertions(+), 12 deletions(-) create mode 100644 crates/engram-host-agent/src/blk_source.rs create mode 100644 crates/engram-host-agent/tests/blk_source.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21f3c48dc..86619454b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1178,6 +1178,7 @@ jobs: sudo apt-get install -y -qq --no-install-recommends thin-provisioning-tools sudo env POOL_DIR=/var/lib/engram/thin POOL_DATA_SIZE=4G POOL_META_SIZE=128M \ sh deploy/helm/engram-host-fleet/files/thin-pool.sh + sudo modprobe ublk_drv # Every state node prep can find: a used disk, a disk without room, # a crash before the first activation, a restart. @@ -1185,10 +1186,12 @@ jobs: run: sudo sh scripts/thin-pool-states-test.sh # The disk over the pool, then the session disk under a real guest - # over the same pool (`session_disk` reads ENGRAM_TEST_THIN_POOL). + # over the same pool (`session_disk` reads ENGRAM_TEST_THIN_POOL) + # and the ublk source a hydrating volume reads from. - name: dm-thin session-disk tests (root) env: ENGRAM_TEST_THIN_POOL: engram-pool + ENGRAM_TEST_UBLK: "1" run: | sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-disk-thin \ @@ -1200,6 +1203,7 @@ jobs: sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-host-agent \ --test session_disk \ + --test blk_source \ --run-ignored ignored-only \ --no-fail-fast \ --test-threads=1 diff --git a/Cargo.lock b/Cargo.lock index 59e4b459c..e91ce87b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,6 +247,43 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + [[package]] name = "async-graphql-parser" version = "7.2.1" @@ -271,6 +308,82 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.6", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.6", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.6", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -293,6 +406,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.92" @@ -989,12 +1108,15 @@ dependencies = [ "itertools 0.12.1", "lazy_static", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex 1.3.0", "syn 2.0.117", + "which", ] [[package]] @@ -1055,6 +1177,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitmaps" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" + [[package]] name = "blake3" version = "1.8.7" @@ -1095,6 +1223,19 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "borsh" version = "1.8.1" @@ -1549,7 +1690,7 @@ dependencies = [ "futures-core", "mio", "parking_lot 0.12.5", - "rustix", + "rustix 1.1.6", "signal-hook", "signal-hook-mio", "winapi", @@ -1662,6 +1803,16 @@ dependencies = [ "darling_macro 0.20.11", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + [[package]] name = "darling" version = "0.24.1" @@ -1686,6 +1837,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + [[package]] name = "darling_core" version = "0.24.1" @@ -1710,6 +1875,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.24.1" @@ -1883,6 +2059,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_setters" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e6f6fa1f03c14ae082120b84b3c7fbd7b8588d924cf2d7c3daf9afd49df8b9" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -2692,8 +2880,10 @@ dependencies = [ "futures", "futures-util", "hex", + "io-uring", "jemalloc_pprof", "libc", + "libublk", "metrics", "metrics-exporter-prometheus", "nix 0.31.3", @@ -3259,7 +3449,7 @@ dependencies = [ "proptest", "pulldown-cmark", "ratatui", - "rustix", + "rustix 1.1.6", "serde", "serde_json", "syntect", @@ -3599,6 +3789,19 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.34" @@ -3622,6 +3825,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.34" @@ -4356,6 +4565,18 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "io-uring" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878c26bc5e45fe089ce2ca25689260c152a87ce1d7de2ca86101efeea6973932" +dependencies = [ + "bindgen", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.12.1" @@ -4829,6 +5050,41 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libublk" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212246283957aba80cbd6f04586d19eb57cf94aa49232c2a8a6b25b72c356239" +dependencies = [ + "async-lock", + "bitflags 2.13.2", + "bitmaps", + "derive_setters", + "futures-timer", + "io-uring", + "libc", + "libublk-rs-sys", + "log", + "serde", + "serde_json", + "slab", + "smol", + "thiserror 1.0.69", +] + +[[package]] +name = "libublk-rs-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd981fd7cf597c782453c2520f0d8f511dd6f66fc184b68577d7c7cf7108e44" +dependencies = [ + "anyhow", + "bindgen", + "libc", + "regex", + "serde", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -4838,6 +5094,12 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -5948,6 +6210,17 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -5981,6 +6254,20 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.6", + "windows-sys 0.61.2", +] + [[package]] name = "polyval" version = "0.7.3" @@ -6708,6 +6995,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.6" @@ -6717,7 +7017,7 @@ dependencies = [ "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -7240,6 +7540,23 @@ dependencies = [ "serde", ] +[[package]] +name = "smol" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33bd3e260892199c3ccfc487c88b2da2265080acb316cd920da72fdfd7c599f" +dependencies = [ + "async-channel", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-net", + "async-process", + "blocking", + "futures-lite", +] + [[package]] name = "socket2" version = "0.6.5" @@ -7638,7 +7955,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.6", "windows-sys 0.61.2", ] @@ -7650,7 +7967,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.2", "parking_lot 0.12.5", - "rustix", + "rustix 1.1.6", "signal-hook", "windows-sys 0.61.2", ] @@ -8879,6 +9196,18 @@ dependencies = [ "wezterm-dynamic", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "1.6.1" @@ -9268,10 +9597,12 @@ dependencies = [ "aws-smithy-types", "axum", "axum-core", + "bindgen", "bitflags 2.13.2", "bytes", "cc", "chrono", + "concurrent-queue", "crc32fast", "crossbeam-utils", "crypto-common 0.1.7", @@ -9284,6 +9615,7 @@ dependencies = [ "displaydoc", "either", "errno", + "event-listener-strategy", "flate2", "form_urlencoded", "futures-channel", @@ -9307,6 +9639,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "libc", + "linux-raw-sys 0.12.1", "log", "memchr", "mio", @@ -9321,6 +9654,7 @@ dependencies = [ "percent-encoding", "phf_shared", "ppv-lite86", + "prettyplease", "proc-macro2", "prost", "prost-build", @@ -9335,7 +9669,7 @@ dependencies = [ "regex-syntax", "reqwest", "rsa", - "rustix", + "rustix 1.1.6", "rustls", "serde", "serde_core", @@ -9404,7 +9738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08" dependencies = [ "libc", - "rustix", + "rustix 1.1.6", ] [[package]] diff --git a/crates/engram-host-agent/Cargo.toml b/crates/engram-host-agent/Cargo.toml index 2c3accc95..b63f126aa 100644 --- a/crates/engram-host-agent/Cargo.toml +++ b/crates/engram-host-agent/Cargo.toml @@ -106,6 +106,10 @@ workspace-hack = { version = "0.1", path = "../../workspace-hack" } # linux5_13: `RegisterMode::MINOR` — the exact ioctl mode the probe # exercises. userfaultfd = { version = "0.9", features = ["linux5_13"] } +# The ublk device `blk_source` serves; `io-uring` for the eventfd read +# that wakes its queue thread. +libublk = "0.4.8" +io-uring = "0.7" # #1003: jemalloc as the global allocator (prod is musl-static; musl # mallocng retained 23 GB of ~48 MiB groups under chunk-buffer churn) # + the sampled heap profiler (SIGUSR2 → pprof dump) + allocator diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs new file mode 100644 index 000000000..3d6c2071b --- /dev/null +++ b/crates/engram-host-agent/src/blk_source.rs @@ -0,0 +1,696 @@ +//! A read-only block device served from the fill engine: the source a +//! hydrating session volume copies its cold regions from +//! (`SessionDisk::hydrate`). The device is a ublk device the host-agent +//! serves in process; the kernel reads it for the regions the guest +//! touches before they are copied and for its background copy, and +//! nothing else opens it. +//! +//! Two kinds of read arrive and the source cannot be told which is +//! which, so it tells them apart by position. The background copy walks +//! the volume's cold regions in ascending order, one region or a batch +//! at a time; the source keeps a window of whole chunks around that +//! stream, fetched at `Hydrate` and a few chunks ahead, and hands them +//! out region by region, so the store sees one GET per chunk and the +//! copy runs at the engine's rate. A read anywhere else is the guest +//! waiting (or a write the kernel first copies the region for): it gets +//! a ranged GET at `Demand`, and its chunk is recorded as touched. + +use std::collections::{HashMap, VecDeque}; +use std::fs::File; +use std::io::{self, Write}; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use engram_chunk_store::{ChunkHash, Manifest}; +use engram_fill::{Class, FillEngine}; +use io_uring::{opcode, types}; +use libublk::ctrl::{UblkCtrl, UblkCtrlBuilder}; +use libublk::io::{UblkDev, UblkIOCtx, UblkQueue}; +use libublk::{sys, BufDesc, BufDescList, UblkError, UblkFlags, UblkIORes}; +use tokio::sync::{mpsc, oneshot, OnceCell}; + +/// Chunks fetched ahead of the stream chunk. With 16 MiB chunks this is +/// 64 MiB in flight per device, which keeps the copy near the engine's +/// rate; the window holds at most this many plus two chunks. +const WINDOW_AHEAD: usize = 4; +/// Region-shaped reads at ascending delta chunks, outside the window, +/// that move the stream there (a host-agent restart loses the cursor). +const RESYNC_READS: u32 = 4; +/// The tag the eventfd read carries; real tags are below the depth. +const EVENT_TAG: u16 = u16::MAX; +const QUEUE_DEPTH: u16 = 64; +/// The largest read the kernel sends; the block layer splits above it. +const IO_BUF_BYTES: u32 = 512 << 10; +const DEVICE_NODE_WAIT: Duration = Duration::from_secs(5); + +/// A served device. [`Self::stop`] ends it; a dropped one keeps serving +/// Entries in a thread's ublk control ring; commands go one at a time. +const CTRL_RING_DEPTH: u32 = 16; +/// until the process exits, after which [`Self::recover`] takes it over. +pub struct BlkSource { + ctrl: Arc, + path: PathBuf, + shared: Arc, + server: Option>, +} + +impl BlkSource { + /// Serve `manifest` through a new device. `delta` lists, ascending, + /// the chunks the kernel will copy (the ones not skipped), which is + /// where the stream runs; `region_bytes` is the kernel's copy unit. + pub async fn serve( + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result { + Self::start(engine, manifest, delta, region_bytes, None).await + } + + /// Take over device `dev_id` from a host-agent that died: the reads it + /// left queued are reissued to this server. + pub async fn recover( + dev_id: u32, + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result { + Self::start(engine, manifest, delta, region_bytes, Some(dev_id)).await + } + + async fn start( + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + recover: Option, + ) -> io::Result { + let shared = Arc::new(Shared::new( + engine, + manifest, + delta, + region_bytes, + recover.is_none(), + )?); + let builder = UblkCtrlBuilder::default() + .name("engram-source") + .nr_queues(1) + .depth(QUEUE_DEPTH) + .io_buf_bytes(IO_BUF_BYTES) + .ctrl_flags((sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64); + let builder = match recover { + None => builder.dev_flags(UblkFlags::UBLK_DEV_F_ADD_DEV), + Some(id) => builder + .id(id as i32) + .dev_flags(UblkFlags::UBLK_DEV_F_RECOVER_DEV), + }; + let ctrl = Arc::new(builder.build().map_err(ublk_err)?); + if recover.is_some() { + let ctrl = ctrl.clone(); + tokio::task::spawn_blocking(move || with_ctrl_ring(|| ctrl.start_user_recover())) + .await + .map_err(io::Error::other)??; + } + let (requests, queue) = mpsc::unbounded_channel(); + let completions = Arc::new(Completions::new()?); + tokio::spawn(dispatch(shared.clone(), queue, completions.clone())); + + let (ready, started) = oneshot::channel(); + let server = std::thread::Builder::new() + .name("engram-source".into()) + .spawn({ + let ctrl = ctrl.clone(); + let shared = shared.clone(); + move || { + let size_bytes = shared.size_bytes; + let ready = Arc::new(Mutex::new(Some(ready))); + let queue_fn = { + let shared = shared.clone(); + move |qid: u16, dev: &UblkDev| { + queue_loop(qid, dev, &shared, &requests, &completions) + } + }; + let run = with_ctrl_ring(|| { + ctrl.run_target( + |dev| { + dev.set_default_params(size_bytes); + dev.tgt.params.basic.attrs |= sys::UBLK_ATTR_READ_ONLY; + Ok(()) + }, + queue_fn, + { + let ready = ready.clone(); + move |_| { + if let Some(ready) = ready.lock().unwrap().take() { + let _ = ready.send(Ok(())); + } + } + }, + ) + }); + if let Err(e) = run { + match ready.lock().unwrap().take() { + Some(ready) => { + let _ = ready.send(Err(e)); + } + None => { + tracing::warn!(error = %e, "ublk source server ended with an error") + } + } + } + } + })?; + started.await.map_err(|_| { + io::Error::other("the ublk server thread ended before the device started") + })??; + let path = PathBuf::from(ctrl.get_bdev_path()); + wait_for_node(&path).await?; + Ok(Self { + ctrl, + path, + shared, + server: Some(server), + }) + } + + /// The block device: `/dev/ublkb`. + pub fn path(&self) -> &Path { + &self.path + } + + pub fn dev_id(&self) -> u32 { + self.ctrl.dev_info().dev_id + } + + /// Chunks the guest read before the copy reached them, in first-read + /// order. + pub fn touched(&self) -> Vec { + self.shared.touched.lock().unwrap().order.clone() + } + + /// Stop serving and delete the device. Refused while a device still + /// maps onto it: a device with reads queued and no server to answer + /// them holds the kernel until reboot, so the mapping goes first. + pub async fn stop(mut self) -> io::Result<()> { + let holders = self + .path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| format!("/sys/block/{n}/holders")); + if let Some(holders) = holders { + let held = std::fs::read_dir(&holders)?.count(); + if held > 0 { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{} is still the source of {held} device(s)", + self.path.display() + ), + )); + } + } + self.shared.stopping.store(true, Ordering::Relaxed); + let ctrl = self.ctrl.clone(); + let server = self.server.take(); + tokio::task::spawn_blocking(move || -> io::Result<()> { + with_ctrl_ring(|| ctrl.kill_dev())?; + if let Some(server) = server { + server + .join() + .map_err(|_| io::Error::other("the ublk server thread panicked"))?; + } + with_ctrl_ring(|| ctrl.del_dev())?; + Ok(()) + }) + .await + .map_err(io::Error::other)? + } +} + +impl Drop for BlkSource { + fn drop(&mut self) { + if self.server.is_some() { + tracing::warn!(path = %self.path.display(), "ublk source dropped while serving; it serves until the process exits"); + } + } +} + +/// What the engine side and the queue thread share. +struct Shared { + engine: FillEngine, + /// The manifest's hash per chunk; `None` reads as zeros. + hashes: Vec>, + chunk_size: u64, + size_bytes: u64, + region_bytes: u64, + /// Ascending chunks the kernel copies. + delta: Vec, + window: Mutex, + touched: Mutex, + stopping: AtomicBool, +} + +struct Window { + /// Position in `delta` of the chunk the kernel's copy is in; `None` + /// until the stream shows itself. + cursor: Option, + chunks: HashMap>>, + /// The last region-shaped read outside the window and how many came + /// in ascending order. + resync: Option<(u64, u32)>, +} + +struct Touched { + order: Vec, + seen: Vec, +} + +enum Serve { + Stream(Arc>), + Demand, +} + +impl Shared { + fn new( + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + fresh: bool, + ) -> io::Result { + let chunk_size = manifest.chunk_size.0; + if region_bytes == 0 + || chunk_size == 0 + || !chunk_size.is_multiple_of(region_bytes) + || !manifest.total_bytes.is_multiple_of(512) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "manifest of {} bytes in {chunk_size}-byte chunks does not fit {region_bytes}-byte regions", + manifest.total_bytes + ), + )); + } + let count = manifest.expected_chunk_count() as usize; + let mut hashes = vec![None; count]; + for chunk in &manifest.chunks { + let index = (chunk.offset / chunk_size) as usize; + match hashes.get_mut(index) { + Some(slot) if chunk.offset.is_multiple_of(chunk_size) => *slot = Some(chunk.hash), + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "manifest chunk at offset {} lies outside the volume", + chunk.offset + ), + )) + } + } + } + if !delta.is_sorted() || delta.last().is_some_and(|last| *last as usize >= count) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "delta chunks are not ascending indices of the volume", + )); + } + let shared = Self { + engine, + hashes, + chunk_size, + size_bytes: manifest.total_bytes, + region_bytes, + delta, + window: Mutex::new(Window { + cursor: None, + chunks: HashMap::new(), + resync: None, + }), + touched: Mutex::new(Touched { + order: Vec::new(), + seen: vec![false; count], + }), + stopping: AtomicBool::new(false), + }; + // A new device's copy starts at the first delta chunk; fetch it + // and the chunks after it before the kernel asks. + if fresh && !shared.delta.is_empty() { + let mut window = shared.window.lock().unwrap(); + shared.advance(&mut window, 0); + } + Ok(shared) + } + + async fn read(&self, offset: u64, len: u64) -> io::Result { + let end = offset + .checked_add(len) + .filter(|end| *end <= self.size_bytes) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "read of {len} at {offset} lies outside the {}-byte device", + self.size_bytes + ), + ) + })?; + let mut out = BytesMut::with_capacity(len as usize); + let mut at = offset; + while at < end { + let chunk = at / self.chunk_size; + let in_chunk = at % self.chunk_size; + let piece = (self.chunk_size - in_chunk).min(end - at); + match self.hashes[chunk as usize] { + None => out.resize(out.len() + piece as usize, 0), + Some(hash) => { + let bytes = match self.classify(chunk, at, piece) { + Serve::Stream(cell) => { + let whole = cell + .get_or_try_init(|| self.engine.read(hash, Class::Hydrate)) + .await + .map_err(|e| io::Error::other(e.to_string()))?; + whole.slice(in_chunk as usize..(in_chunk + piece) as usize) + } + Serve::Demand => { + self.touch(chunk); + self.engine + .read_range(hash, in_chunk, piece as usize, Class::Demand) + .await + .map_err(|e| io::Error::other(e.to_string()))? + } + }; + if bytes.len() as u64 != piece { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "chunk {chunk} gave {} bytes for a {piece}-byte read", + bytes.len() + ), + )); + } + out.extend_from_slice(&bytes); + } + } + at += piece; + } + Ok(out.freeze()) + } + + /// Stream or demand, by position against the window. A stream read + /// at the chunk after the cursor moves the cursor; the chunk before + /// it still serves, since the copy's reads of one chunk and the next + /// overlap in flight. + fn classify(&self, chunk: u64, at: u64, len: u64) -> Serve { + let mut window = self.window.lock().unwrap(); + let near = window.cursor.and_then(|pos| { + let before = pos.checked_sub(1).map(|p| self.delta[p]); + if chunk == self.delta[pos] || Some(chunk) == before { + Some(pos) + } else if self.delta.get(pos + 1) == Some(&chunk) { + Some(pos + 1) + } else { + None + } + }); + let pos = match near { + Some(pos) => { + window.resync = None; + pos + } + None => { + let shaped = + at.is_multiple_of(self.region_bytes) && len.is_multiple_of(self.region_bytes); + match self.resync(&mut window, chunk, shaped) { + Some(pos) => pos, + None => return Serve::Demand, + } + } + }; + self.advance(&mut window, pos); + Serve::Stream(window.chunks.entry(chunk).or_default().clone()) + } + + /// A region-shaped read of a delta chunk outside the window: the + /// first one places a stream the device has not seen yet; otherwise + /// `RESYNC_READS` of them in ascending order move the stream there. + fn resync(&self, window: &mut Window, chunk: u64, shaped: bool) -> Option { + if !shaped { + return None; + } + let pos = self.delta.binary_search(&chunk).ok()?; + let run = match window.resync { + Some((last, run)) if chunk >= last => run + 1, + _ => 1, + }; + window.resync = Some((chunk, run)); + if window.cursor.is_none() || run >= RESYNC_READS { + window.resync = None; + window.chunks.clear(); + return Some(pos); + } + None + } + + /// Put the cursor at `pos`, drop the chunks behind the one before it, + /// and fetch the chunks up to `WINDOW_AHEAD` past it that are not in + /// the window yet. + fn advance(&self, window: &mut Window, pos: usize) { + if window.cursor != Some(pos) { + window.cursor = Some(pos); + let keep_from = pos.checked_sub(1).map_or(0, |p| self.delta[p]); + window.chunks.retain(|chunk, _| *chunk >= keep_from); + } + let last = (pos + WINDOW_AHEAD).min(self.delta.len() - 1); + for chunk in self.delta[pos..=last].iter().copied() { + let Some(hash) = self.hashes[chunk as usize] else { + continue; + }; + if let std::collections::hash_map::Entry::Vacant(slot) = window.chunks.entry(chunk) { + let cell = slot.insert(Arc::default()).clone(); + let engine = self.engine.clone(); + tokio::spawn(async move { + if let Err(e) = cell + .get_or_try_init(|| engine.read(hash, Class::Hydrate)) + .await + { + tracing::debug!(chunk, error = %e, "stream prefetch failed; the read retries it"); + } + }); + } + } + } + + fn touch(&self, chunk: u64) { + let mut touched = self.touched.lock().unwrap(); + if !std::mem::replace(&mut touched.seen[chunk as usize], true) { + touched.order.push(chunk as u32); + } + } +} + +struct Request { + tag: u16, + offset: u64, + len: u32, +} + +/// Finished reads on their way back to the queue thread, which sleeps +/// in `io_uring_enter`; the eventfd wakes it. +struct Completions { + done: Mutex)>>, + event: File, +} + +impl Completions { + fn new() -> io::Result { + // SAFETY: eventfd takes two integers and returns a new descriptor + // or -1; nothing is borrowed. + let fd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: `fd` is a fresh descriptor this process owns and nothing + // else closes. + let event = File::from(unsafe { OwnedFd::from_raw_fd(fd) }); + Ok(Self { + done: Mutex::new(VecDeque::new()), + event, + }) + } + + fn push(&self, tag: u16, result: io::Result) { + self.done.lock().unwrap().push_back((tag, result)); + if let Err(e) = (&self.event).write_all(&1u64.to_ne_bytes()) { + tracing::error!(error = %e, "ublk source: eventfd write failed; a read stays unanswered"); + } + } + + fn drain(&self) -> Vec<(u16, io::Result)> { + self.done.lock().unwrap().drain(..).collect() + } +} + +async fn dispatch( + shared: Arc, + mut requests: mpsc::UnboundedReceiver, + completions: Arc, +) { + while let Some(Request { tag, offset, len }) = requests.recv().await { + let shared = shared.clone(); + let completions = completions.clone(); + tokio::spawn(async move { + let result = shared.read(offset, len as u64).await; + completions.push(tag, result); + }); + } +} + +/// The queue thread: hands each read to the engine side and completes +/// it when the eventfd says its bytes are back. Runs until the device +/// stops. +fn queue_loop( + qid: u16, + dev: &UblkDev, + shared: &Shared, + requests: &mpsc::UnboundedSender, + completions: &Completions, +) { + let bufs = dev.alloc_queue_io_bufs(); + let queue = match UblkQueue::new(qid, dev) + .and_then(|q| q.submit_fetch_commands_unified(BufDescList::Slices(Some(&bufs)))) + { + Ok(queue) => queue, + Err(e) => { + tracing::error!(qid, error = %e, "ublk source queue did not start"); + return; + } + }; + let mut counter = 0u64; + arm_event(&queue, &completions.event, &mut counter); + queue.wait_and_handle_io(|queue, tag, io| { + if io.is_tgt_io() { + for (tag, result) in completions.drain() { + let buf = &bufs[tag as usize]; + let res = match result { + Ok(bytes) if bytes.len() <= buf.len() => { + // SAFETY: the buffer is `buf.len()` bytes; between + // this request's arrival and its commit below, this + // thread is its only writer and the kernel does not + // read it. + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + buf.as_mut_ptr(), + bytes.len(), + ); + } + bytes.len() as i32 + } + Ok(bytes) => { + tracing::error!( + tag, + len = bytes.len(), + "ublk source: read answered with more than its buffer" + ); + -libc::EIO + } + Err(e) => { + tracing::warn!(tag, error = %e, "ublk source: read failed"); + -libc::EIO + } + }; + complete(queue, tag, buf.as_slice(), res); + } + arm_event(queue, &completions.event, &mut counter); + return; + } + let iod = queue.get_iod(tag); + let buf = &bufs[tag as usize]; + let op = iod.op_flags & 0xff; + if op != sys::UBLK_IO_OP_READ { + complete(queue, tag, buf.as_slice(), -libc::EOPNOTSUPP); + return; + } + if shared.stopping.load(Ordering::Relaxed) { + complete(queue, tag, buf.as_slice(), -libc::EIO); + return; + } + let request = Request { + tag, + offset: iod.start_sector << 9, + len: iod.nr_sectors << 9, + }; + if requests.send(request).is_err() { + complete(queue, tag, buf.as_slice(), -libc::EIO); + } + }); +} + +fn complete(queue: &UblkQueue, tag: u16, buf: &[u8], res: i32) { + if let Err(e) = + queue.complete_io_cmd_unified(tag, BufDesc::Slice(buf), Ok(UblkIORes::Result(res))) + { + tracing::error!(tag, error = %e, "ublk source: completion not submitted"); + } +} + +/// Queue a read of the eventfd so a completion wakes the queue thread. +fn arm_event(queue: &UblkQueue, event: &File, counter: &mut u64) { + let sqe = opcode::Read::new( + types::Fd(event.as_raw_fd()), + std::ptr::from_mut(counter).cast::(), + std::mem::size_of::() as u32, + ) + .build() + .user_data(UblkIOCtx::build_user_data(EVENT_TAG, 0, 0, true)); + if let Err(e) = queue.ublk_submit_sqe_sync(sqe) { + tracing::error!(error = %e, "ublk source: eventfd read not submitted; completions stall"); + } +} + +async fn wait_for_node(path: &Path) -> io::Result<()> { + let appears = async { + while tokio::fs::metadata(path).await.is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + tokio::time::timeout(DEVICE_NODE_WAIT, appears) + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::NotFound, + format!("{} did not appear", path.display()), + ) + }) +} + +fn ublk_err(e: UblkError) -> io::Error { + match e { + UblkError::IOError(e) => e, + other => io::Error::other(other.to_string()), + } +} + +/// Run `f`, a libublk control command, on this thread. libublk sends +/// control commands over a thread-local ring that only the thread which +/// built the `UblkCtrl` has; every other thread makes its own first. +fn with_ctrl_ring(f: impl FnOnce() -> Result) -> io::Result { + libublk::ublk_init_ctrl_task_ring(|ring| { + if ring.is_none() { + *ring = Some( + io_uring::IoUring::::builder() + .build(CTRL_RING_DEPTH) + .map_err(UblkError::IOError)?, + ); + } + Ok(()) + }) + .map_err(ublk_err)?; + f().map_err(ublk_err) +} diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index b0406445f..e10e1a5ab 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -87,6 +87,8 @@ pub(crate) mod snapshot_post; pub mod substrate_server; pub use host_client::{IdleEvictionHarness, LocalHostClient}; #[cfg(target_os = "linux")] +pub mod blk_source; +#[cfg(target_os = "linux")] pub mod heap_profile; pub mod heartbeat; pub mod idle_evictor; diff --git a/crates/engram-host-agent/tests/blk_source.rs b/crates/engram-host-agent/tests/blk_source.rs new file mode 100644 index 000000000..eb33d3c30 --- /dev/null +++ b/crates/engram-host-agent/tests/blk_source.rs @@ -0,0 +1,215 @@ +//! The ublk source under a real kernel, as root, with `ublk_drv` loaded. +//! `ENGRAM_TEST_UBLK` set runs the tests (the CI root lane); unset, they +//! skip. +// tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) +#![allow(clippy::disallowed_methods)] +#![cfg(target_os = "linux")] + +use std::fs::File; +use std::os::unix::fs::{FileExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind}; +use engram_fill::FillEngine; +use engram_host_agent::blk_source::BlkSource; +use engram_storage_local::LocalBlobStorage; +use libublk::helpers::IoBuf; + +const CHUNK: u64 = 256 << 10; +const REGION: u64 = 64 << 10; +const CHUNKS: u64 = 4; +/// Chunks 0, 1 and 3 hold bytes; chunk 2 is a hole. +const HELD: [(u64, u8); 3] = [(0, 0x11), (1, 0x22), (3, 0x33)]; +const DELTA: [u64; 3] = [0, 1, 3]; + +fn gate() -> bool { + if std::env::var_os("ENGRAM_TEST_UBLK").is_none() { + eprintln!("SKIP: ENGRAM_TEST_UBLK is not set"); + return false; + } + true +} + +fn image() -> Vec { + let mut image = vec![0u8; (CHUNKS * CHUNK) as usize]; + for (chunk, byte) in HELD { + let at = (chunk * CHUNK) as usize; + image[at..at + CHUNK as usize].fill(byte); + } + image +} + +/// A store holding the image's chunks under `dir`, and its manifest. +async fn engine_and_manifest(dir: &Path) -> (FillEngine, Manifest) { + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.join("blobs")))); + let mut chunks = Vec::new(); + for (chunk, byte) in HELD { + let hash = store.put_chunk(&vec![byte; CHUNK as usize]).await.unwrap(); + chunks.push(ChunkRef { + offset: chunk * CHUNK, + hash, + }); + } + let manifest = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks, + ..Manifest::empty(ManifestKind::Disk, CHUNKS * CHUNK) + }; + (FillEngine::new(store, Default::default()), manifest) +} + +/// The device opened for direct I/O, so every read reaches the source +/// with the offset and length the test gives it. +fn open_direct(path: &Path) -> File { + File::options() + .read(true) + .custom_flags(libc::O_DIRECT) + .open(path) + .unwrap() +} + +fn read_at(device: &File, offset: u64, len: usize) -> Vec { + let mut buf = IoBuf::::new(len); + device.read_exact_at(buf.as_mut_slice(), offset).unwrap(); + buf.as_slice().to_vec() +} + +#[tokio::test] +#[ignore = "requires root and the ublk_drv module"] +async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { + if !gate() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let (engine, manifest) = engine_and_manifest(dir.path()).await; + let source = BlkSource::serve(engine, &manifest, DELTA.to_vec(), REGION) + .await + .unwrap(); + let path = source.path().to_path_buf(); + let device = open_direct(&path); + let image = image(); + + // A guest read far from where the copy starts: served, and recorded. + let at = 3 * CHUNK + 8192; + assert_eq!( + read_at(&device, at, 4096), + image[at as usize..at as usize + 4096] + ); + assert_eq!(source.touched(), vec![3]); + + // The copy's stream: the whole device, region by region, ascending. + for at in (0..CHUNKS * CHUNK).step_by(REGION as usize) { + assert_eq!( + read_at(&device, at, REGION as usize), + image[at as usize..(at + REGION) as usize], + "region at {at}" + ); + } + assert_eq!(source.touched(), vec![3], "the stream is not a guest read"); + + drop(device); + source.stop().await.unwrap(); + assert!(!path.exists(), "{} was deleted", path.display()); +} + +/// The server half of the recovery test: serves a source from the +/// directory `ENGRAM_UBLK_CHILD_DIR` names until it is killed. +#[tokio::test] +#[ignore = "a helper the recovery test runs as a child process"] +async fn child_serves_until_killed() { + let Some(dir) = std::env::var_os("ENGRAM_UBLK_CHILD_DIR") else { + return; + }; + let dir = PathBuf::from(dir); + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.join("blobs")))); + let manifest: Manifest = + serde_json::from_slice(&std::fs::read(dir.join("manifest.json")).unwrap()).unwrap(); + let source = BlkSource::serve( + FillEngine::new(store, Default::default()), + &manifest, + DELTA.to_vec(), + REGION, + ) + .await + .unwrap(); + std::fs::write(dir.join("dev_id"), source.dev_id().to_string()).unwrap(); + std::future::pending::<()>().await; +} + +/// A host-agent killed with a read queued at its source: the device +/// stays, the read waits, and the next host-agent's server answers it. +#[tokio::test] +#[ignore = "requires root and the ublk_drv module"] +async fn a_source_killed_with_a_queued_read_is_recovered_by_a_new_server() { + if !gate() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let (engine, manifest) = engine_and_manifest(dir.path()).await; + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "child_serves_until_killed", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("ENGRAM_UBLK_CHILD_DIR", dir.path()) + .spawn() + .unwrap(); + let dev_id_file = dir.path().join("dev_id"); + let deadline = Instant::now() + Duration::from_secs(20); + while !dev_id_file.exists() { + assert!( + Instant::now() < deadline, + "the child did not serve a device" + ); + assert!(child.try_wait().unwrap().is_none(), "the child exited"); + std::thread::sleep(Duration::from_millis(50)); + } + let dev_id: u32 = std::fs::read_to_string(&dev_id_file) + .unwrap() + .trim() + .parse() + .unwrap(); + let path = PathBuf::from(format!("/dev/ublkb{dev_id}")); + let image = image(); + assert_eq!( + read_at(&open_direct(&path), 0, REGION as usize), + image[..REGION as usize] + ); + + // Freeze the server, queue a read it cannot answer, then kill it. + let pid = nix::unistd::Pid::from_raw(child.id() as i32); + nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGSTOP).unwrap(); + let at = 3 * CHUNK + REGION; + let queued = std::thread::spawn({ + let path = path.clone(); + move || read_at(&open_direct(&path), at, REGION as usize) + }); + std::thread::sleep(Duration::from_millis(300)); + assert!(!queued.is_finished(), "the read waits on the frozen server"); + nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGKILL).unwrap(); + child.wait().unwrap(); + + let recovered = BlkSource::recover(dev_id, engine, &manifest, DELTA.to_vec(), REGION) + .await + .unwrap(); + let bytes = tokio::time::timeout( + Duration::from_secs(10), + tokio::task::spawn_blocking(move || queued.join().unwrap()), + ) + .await + .expect("the queued read completes after recovery") + .unwrap(); + assert_eq!(bytes, image[at as usize..(at + REGION) as usize]); + + recovered.stop().await.unwrap(); + assert!(!path.exists(), "{} was deleted", path.display()); +} diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 0e4efc808..7350f29b4 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -183,47 +183,61 @@ winnow = { version = "1" } zeroize = { version = "1", features = ["derive"] } [target.x86_64-unknown-linux-gnu.dependencies] +concurrent-queue = { version = "2" } +event-listener-strategy = { version = "0.5" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } libc = { version = "0.2", default-features = false, features = ["use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nix = { version = "0.31", features = ["hostname", "ioctl", "mount", "socket"] } nom = { version = "7" } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } tower = { version = "0.5", default-features = false, features = ["timeout"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-unknown-linux-gnu.build-dependencies] +bindgen = { version = "0.69" } +concurrent-queue = { version = "2" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nom = { version = "7" } +prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-unknown-linux-musl.dependencies] +concurrent-queue = { version = "2" } +event-listener-strategy = { version = "0.5" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } libc = { version = "0.2", default-features = false, features = ["use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nix = { version = "0.31", features = ["hostname", "ioctl", "mount", "socket"] } nom = { version = "7" } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } tower = { version = "0.5", default-features = false, features = ["timeout"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-unknown-linux-musl.build-dependencies] +bindgen = { version = "0.69" } +concurrent-queue = { version = "2" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nom = { version = "7" } +prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-apple-darwin.dependencies] From ea0b6e8ca285b247e4e361aa5b2b3b8e33aa2122 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 12:07:10 -0800 Subject: [PATCH 04/35] host-core: one lazy stage path A role whose manifest is not on the host used to take one of three eager paths (`OntoKept`, `FromImage`, `Whole`): the delta was written in full before the guest started, 27-29 s of a 39 s cold resume. They are now one path, `bring_up`: the nearest base this host holds (a kept checkpoint of the lineage, the staged image, or nothing) is snapshotted and the difference comes in behind the running guest, through `SessionDisk::hydrate` and a source the fill engine serves (`SourceServer`, the trait `BlkSource` implements). On a platform without a hydrating device the same path writes the difference first (`Staged`). The names stay the only record. A hydrating role has no checkpoint volume until it settles; two names carry what a checkpoint volume carried: sb---dest- the live device's destination, which becomes that manifest sb---base- what it was snapshotted from From them and the kernel's own state (the clone table names the source device) a restarted host-agent rebuilds everything: `reattach` finds the hydrating roles, recomputes the plan from the two manifests, and serves the source again. No caller tracks a manifest ref or a device id between calls. `hydration_plan` is the pure core: chunks the base holds in final form are skipped (the kernel marks them hydrated through a discard, so the destination keeps sharing their blocks), chunks the manifest leaves out are zeroed before the copy, and the rest is the delta the source serves. `SessionVolumes::settle` is one step of the way to a settled volume, for a driver loop; it cuts the device over and stops the source. A release or purge deletes the live device before the volumes under it and stops its source after, which is the order the ublk rule needs. A hydrating volume whose source this process does not serve is never deleted: the kernel removes a device only after its reads complete, and nothing would answer them. Its destination and base stay with it, so a later `reattach` can rebuild the plan. `SessionDisk::hydration` reports `Settled` for a volume that never hydrated, so one question serves every volume. The thin-pool disk lists and opens hydrating and settled devices, and reports the source device of a hydrating one. Tests drive the lazy path through a fake hydrating disk (hard link for the device, one chunk copied per poll) and a file-backed source server that refuses to stop while a device still copies from it, so the order rules are asserted, not assumed. The eager fallback keeps every existing test. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-disk-thin/src/disk.rs | 92 +- crates/engram-disk-thin/src/parse.rs | 21 + crates/engram-disk-thin/tests/clone.rs | 28 +- crates/engram-host-core/src/disk.rs | 80 +- crates/engram-host-core/src/disk_flows.rs | 85 +- crates/engram-host-core/src/lib.rs | 5 +- .../engram-host-core/src/session_volumes.rs | 984 ++++++++++++++---- crates/engram-host-core/src/test_support.rs | 242 +++++ 8 files changed, 1295 insertions(+), 242 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 6c5397362..eb97ada18 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -117,20 +117,11 @@ impl ThinPoolDisk { self.nodes_dir.join(self.device_name(name)) } + /// The id of every thin on this pool that has a device, from one + /// device list and one table read per device. A device that goes + /// away between the two is not on the pool. async fn active_ids(&self) -> io::Result> { - Ok(self - .thins() - .await? - .into_iter() - .map(|t| t.params.id) - .collect()) - } - - /// Every thin device on this pool, from one device list and one - /// table read per device. A device that goes away between the two - /// is not on the pool. - async fn thins(&self) -> io::Result> { - let mut thins = Vec::new(); + let mut ids = Vec::new(); for (name, _) in self.dm.list().await? { let targets = match self.dm.table(&name).await { Ok(targets) => targets, @@ -145,10 +136,10 @@ impl ThinPoolDisk { } let params = parse::thin_params(&target.params)?; if params.pool == self.pool_dev { - thins.push(Thin { name, params }); + ids.push(params.id); } } - Ok(thins) + Ok(ids) } /// The one target of the device behind `name`. @@ -309,11 +300,6 @@ impl ThinPoolDisk { } } -struct Thin { - name: String, - params: ThinParams, -} - /// A reserved metadata snapshot; released on drop. struct MetadataSnap<'a> { dm: &'a dm::Dm, @@ -429,7 +415,17 @@ impl SessionDisk for ThinPoolDisk { } async fn open(&self, name: &VolumeName) -> io::Result { - let (_, sectors) = self.thin_of(name).await?; + let target = self.target_of(name).await?; + let sectors = match target.kind.as_str() { + "thin" => self.thin_of(name).await?.1, + "clone" | "linear" => target.length_sectors, + kind => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {kind} device, which this disk did not make"), + )) + } + }; self.volume(name, sectors).await } @@ -450,16 +446,31 @@ impl SessionDisk for ThinPoolDisk { } } + /// The pool's thins and the hydrating and settled devices over them, + /// all under this pool's name. async fn list(&self) -> io::Result> { - Ok(self - .thins() - .await? - .into_iter() - .filter_map(|t| { - let volume = t.name.strip_prefix(&self.pool)?.strip_prefix('.')?; - VolumeName::new(volume.to_string()).ok() - }) - .collect()) + let prefix = format!("{}.", self.pool); + let mut names = Vec::new(); + for (device, _) in self.dm.list().await? { + let Some(volume) = device.strip_prefix(&prefix) else { + continue; + }; + let Ok(name) = VolumeName::new(volume.to_string()) else { + continue; + }; + match self.dm.table(&device).await { + Ok(targets) => { + if let [target] = targets.as_slice() { + if ["thin", "clone", "linear"].contains(&target.kind.as_str()) { + names.push(name); + } + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + } + } + Ok(names) } async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { @@ -485,6 +496,10 @@ impl SessionDisk for ThinPoolDisk { }) } + fn region_bytes(&self) -> Option { + Some(self.block_bytes) + } + /// A `clone` target over `dest` with `source` as its source and a /// region per pool block (the unit the pool shares and `dirty_blocks` /// sees; a guest write to a cold region waits for one region's copy), @@ -496,7 +511,7 @@ impl SessionDisk for ThinPoolDisk { source: &Path, name: &VolumeName, skip: &[ChunkIndex], - ) -> io::Result> { + ) -> io::Result { let (_, sectors) = self.thin_of(&dest.name).await?; let region_sectors = self.block_bytes / SECTOR_BYTES; if !sectors.is_multiple_of(region_sectors) { @@ -534,23 +549,26 @@ impl SessionDisk for ThinPoolDisk { tracing::warn!(%name, %error, "hydrating device left behind by a failed hydrate"); } } - hydrating.map(Some) + hydrating } async fn hydration(&self, name: &VolumeName) -> io::Result { - let status = self.dm.status(&self.device_name(name)).await?; + let device = self.device_name(name); + let status = self.dm.status(&device).await?; match status.as_slice() { [target] if target.kind == "clone" => { let status = parse::clone_status(&target.params)?; + let table = single_target(&self.dm.table(&device).await?, "clone")?; Ok(Hydration::Hydrating { hydrated: status.hydrated_regions, total: status.total_regions, + source: dm::device_path(parse::clone_source(&table)?).await?, }) } - [target] if target.kind == "linear" => Ok(Hydration::Settled), + [target] if target.kind == "linear" || target.kind == "thin" => Ok(Hydration::Settled), _ => Err(io::Error::new( io::ErrorKind::InvalidInput, - format!("{name} is not a hydrating device: {status:?}"), + format!("{name} is not a session volume: {status:?}"), )), } } @@ -560,7 +578,9 @@ impl SessionDisk for ThinPoolDisk { /// last: the kernel closes it when the clone table is destroyed. async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { match self.hydration(name).await? { - Hydration::Hydrating { hydrated, total } if hydrated < total => { + Hydration::Hydrating { + hydrated, total, .. + } if hydrated < total => { return Err(io::Error::new( io::ErrorKind::InvalidInput, format!("{name}: {hydrated} of {total} regions are hydrated"), diff --git a/crates/engram-disk-thin/src/parse.rs b/crates/engram-disk-thin/src/parse.rs index 8ff0f2b7d..ddde1d712 100644 --- a/crates/engram-disk-thin/src/parse.rs +++ b/crates/engram-disk-thin/src/parse.rs @@ -139,6 +139,15 @@ pub fn clone_status(status: &str) -> io::Result { } } +/// The source device of a `clone` table: +/// ` ...`. +pub fn clone_source(params: &str) -> io::Result { + match params.split_whitespace().collect::>().as_slice() { + [_, _, source, ..] => DevNo::parse(source), + _ => Err(invalid(format!("not clone params: {params:?}"))), + } +} + /// The `DEVNAME=` line of a sysfs `uevent` file. pub fn devname(uevent: &str) -> io::Result { uevent @@ -271,6 +280,18 @@ mod tests { } } + #[test] + fn clone_source_is_the_third_device() { + assert_eq!( + clone_source("252:3 252:4 259:0 128 2 no_hydration no_discard_passdown 0").unwrap(), + DevNo { + major: 259, + minor: 0 + } + ); + assert!(clone_source("252:3 252:4").is_err()); + } + #[test] fn clone_status_reads_the_region_counts() { let status = clone_status( diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs index b9cd88131..48ea1f151 100644 --- a/crates/engram-disk-thin/tests/clone.rs +++ b/crates/engram-disk-thin/tests/clone.rs @@ -97,9 +97,13 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati let device = pool .hydrate(&dest, &source.0, &name("io"), &skip) .await - .unwrap() - .expect("dm-clone on linux"); - assert!(pool.list().await.unwrap().contains(&name("io-md"))); + .unwrap(); + let listed = pool.list().await.unwrap(); + assert!(listed.contains(&name("io")) && listed.contains(&name("io-md"))); + assert_eq!( + pool.open(&name("io")).await.unwrap().size_bytes, + device.size_bytes + ); assert_eq!(device.read_chunk(ChunkIndex(0)).await.unwrap(), fill(1)); assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); @@ -110,8 +114,12 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati let deadline = Instant::now() + Duration::from_secs(10); loop { match pool.hydration(&device.name).await.unwrap() { - Hydration::Hydrating { hydrated, total } if hydrated == total => { - assert_eq!(total, REGIONS); + Hydration::Hydrating { + hydrated, + total, + source: from, + } if hydrated == total => { + assert_eq!((total, from), (REGIONS, source.0.clone())); break; } Hydration::Hydrating { .. } if Instant::now() < deadline => { @@ -142,7 +150,10 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati "the skipped chunk 0 still shares its blocks with the origin" ); - let (device, dest, base) = (device.name, dest.name, base.name); + // A volume holds its device open; a delete needs it closed. + let names = (device.name.clone(), dest.name.clone(), base.name.clone()); + drop((device, dest, base)); + let (device, dest, base) = names; pool.delete(&device).await.unwrap(); pool.delete(&dest).await.unwrap(); pool.delete(&base).await.unwrap(); @@ -169,7 +180,6 @@ async fn settle_refuses_a_device_that_is_still_hydrating() { let device = pool .hydrate(&dest, &source.0, &name("early-io"), &[]) .await - .unwrap() .unwrap(); let err = pool.settle(&device.name, &dest).await.unwrap_err(); assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput, "{err}"); @@ -179,7 +189,9 @@ async fn settle_refuses_a_device_that_is_still_hydrating() { )); dmsetup(&["resume", &pool.device_name(&dest.name)]).await; // A delete while the kernel is still copying leaks nothing. - let (device, dest) = (device.name, dest.name); + let names = (device.name.clone(), dest.name.clone()); + drop((device, dest)); + let (device, dest) = names; pool.delete(&device).await.unwrap(); pool.delete(&dest).await.unwrap(); assert!(!pool diff --git a/crates/engram-host-core/src/disk.rs b/crates/engram-host-core/src/disk.rs index 2f0870467..d0c34629b 100644 --- a/crates/engram-host-core/src/disk.rs +++ b/crates/engram-host-core/src/disk.rs @@ -15,6 +15,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use async_trait::async_trait; +use engram_chunk_store::Manifest; +use engram_fill::FillEngine; /// The name a volume is created and reopened by. #[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -73,11 +75,15 @@ impl Capacity { } /// How far the kernel has copied a hydrating volume's regions in. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum Hydration { /// `hydrated` of `total` regions are in the destination; the rest - /// still read from the source. - Hydrating { hydrated: u64, total: u64 }, + /// still read from the block device at `source`. + Hydrating { + hydrated: u64, + total: u64, + source: PathBuf, + }, /// Every region is in the destination and the device maps onto it /// directly ([`SessionDisk::settle`] ran). Settled, @@ -427,6 +433,13 @@ pub trait SessionDisk: Send + Sync { async fn capacity(&self) -> io::Result; + /// The unit [`Self::hydrate`] copies in, which divides the chunk + /// size; `None` on a platform without a hydrating device, where a + /// volume is written in full before the guest starts. + fn region_bytes(&self) -> Option { + None + } + /// A device named `name` over `dest` that a guest can open before /// `dest` holds its data: a read of a region not yet in `dest` is /// served from the read-only block device at `source`, a write goes @@ -434,28 +447,28 @@ pub trait SessionDisk: Send + Sync { /// and the kernel copies every other region from `source` into /// `dest` in the background. `skip` lists the chunks `dest` already /// holds in their final form; they are never copied, so blocks - /// `dest` shares with its origin stay shared. `Ok(None)` on a - /// platform without such a device; the caller then writes `dest` in - /// full before the guest starts. + /// `dest` shares with its origin stay shared. async fn hydrate( &self, dest: &Volume, source: &Path, name: &VolumeName, skip: &[ChunkIndex], - ) -> io::Result> { - let _ = (dest, source, name, skip); - Ok(None) - } - - /// Progress of a device [`Self::hydrate`] made. - async fn hydration(&self, name: &VolumeName) -> io::Result { + ) -> io::Result { + let _ = (dest, source, skip); Err(io::Error::new( io::ErrorKind::Unsupported, - format!("{name} is not a hydrating device on this platform"), + format!("{name}: this platform has no hydrating device"), )) } + /// Progress of a device [`Self::hydrate`] made; `Settled` for a volume + /// that never hydrated or has settled. + async fn hydration(&self, name: &VolumeName) -> io::Result { + let _ = name; + Ok(Hydration::Settled) + } + /// Map a fully hydrated device straight onto `dest`. Every I/O in /// flight completes first; open handles keep working; `source` is /// not read again. @@ -511,6 +524,45 @@ pub trait SessionDisk: Send + Sync { } } +/// A read-only block device that serves a manifest from the fill +/// engine: what a hydrating volume copies its cold regions from. +#[async_trait] +pub trait SourceDevice: Send + Sync { + fn path(&self) -> &Path; + + /// Chunks a guest read before the copy reached them, in first-read + /// order. + fn touched(&self) -> Vec; + + /// Stop serving and delete the device. Refused while a device still + /// maps onto it. + async fn stop(self: Box) -> io::Result<()>; +} + +/// Makes [`SourceDevice`]s. `delta` lists, ascending, the chunks the kernel +/// copies from the device; `region_bytes` is its copy unit. +#[async_trait] +pub trait SourceServer: Send + Sync { + async fn serve( + &self, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result>; + + /// Take over the device at `path` from a process that died serving + /// it; the reads it left queued are answered. + async fn recover( + &self, + path: &Path, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result>; +} + /// The data extents of `file` as `(offset, len)` ranges, from /// `SEEK_DATA`/`SEEK_HOLE`. Holes read as zeros and are not listed. A /// filesystem may coalesce extents, which only widens a range. diff --git a/crates/engram-host-core/src/disk_flows.rs b/crates/engram-host-core/src/disk_flows.rs index 0a7baeb62..e587e29a8 100644 --- a/crates/engram-host-core/src/disk_flows.rs +++ b/crates/engram-host-core/src/disk_flows.rs @@ -1,5 +1,6 @@ //! Chunks between a [`Volume`] and the chunk store: [`stage`] turns a -//! manifest into a volume, [`checkpoint`] turns a volume into a manifest. +//! manifest into a volume, [`hydration_plan`] says what a kernel copy +//! does instead, [`checkpoint`] turns a volume into a manifest. use std::io; @@ -23,7 +24,7 @@ fn store_err(e: engram_chunk_store::ChunkStoreError) -> io::Error { io::Error::other(e) } -fn shape_matches(volume: &Volume, manifest: &Manifest) -> io::Result<()> { +pub(crate) fn shape_matches(volume: &Volume, manifest: &Manifest) -> io::Result<()> { if volume.chunk_size != manifest.chunk_size.as_u64() || volume.size_bytes != manifest.total_bytes { @@ -185,6 +186,47 @@ pub async fn stage_at( Ok(volume) } +/// What brings a snapshot of `base`'s volume (an empty volume when +/// `None`) to `wanted` when the kernel copies the difference in behind +/// a running guest ([`SessionDisk::hydrate`]). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HydrationPlan { + /// The chunks the kernel copies from the source, ascending: listed + /// in `wanted` with a hash the base does not hold at that offset. + pub delta: Vec, + /// Chunks to zero on the destination before the copy starts: the + /// base holds them, `wanted` does not. + pub zero: Vec, + /// Every chunk not in `delta`: the destination holds it already. + pub skip: Vec, +} + +pub fn hydration_plan(wanted: &Manifest, base: Option<&Manifest>) -> io::Result { + wanted.validate().map_err(store_err)?; + let chunk_size = wanted.chunk_size.as_u64(); + let (write, zero) = match base { + Some(base) => { + let delta = wanted.delta_from(base).map_err(store_err)?; + (delta.write, delta.zero) + } + None => (wanted.chunks.clone(), Vec::new()), + }; + let mut delta: Vec = write.iter().map(|c| c.offset / chunk_size).collect(); + delta.sort_unstable(); + let skip = (0..wanted.expected_chunk_count()) + .filter(|chunk| delta.binary_search(chunk).is_err()) + .map(ChunkIndex) + .collect(); + Ok(HydrationPlan { + delta, + zero: zero + .iter() + .map(|offset| ChunkIndex(offset / chunk_size)) + .collect(), + skip, + }) +} + /// A snapshot of `live` and the manifest describing it. With `previous` /// (the last snapshot and its manifest) only the chunks that differ are /// uploaded. The guest must be paused across [`checkpoint_split::take`]; @@ -414,6 +456,45 @@ mod tests { assert_eq!(bytes_of(&over).await, bytes_of(&c2).await); } + /// Base [1, 2, 3, 0, 5]; wanted [1, 9, 0, 4, 5]: chunk 1 changed, + /// chunk 2 dropped, chunk 3 new, the rest held. + #[test] + fn a_hydration_plan_copies_the_changed_chunks_zeroes_the_dropped_and_skips_the_rest() { + let manifest = |pattern: &[u8]| Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: pattern + .iter() + .enumerate() + .filter(|(_, byte)| **byte != 0) + .map(|(i, byte)| ChunkRef { + offset: i as u64 * CHUNK, + hash: engram_chunk_store::ChunkHash::of(&fill(*byte)), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, CHUNK * pattern.len() as u64) + }; + let base = manifest(&[1, 2, 3, 0, 5]); + let wanted = manifest(&[1, 9, 0, 4, 5]); + assert_eq!( + hydration_plan(&wanted, Some(&base)).unwrap(), + HydrationPlan { + delta: vec![1, 3], + zero: vec![ChunkIndex(2)], + skip: [0, 2, 4].map(ChunkIndex).to_vec(), + } + ); + assert_eq!( + hydration_plan(&wanted, None).unwrap(), + HydrationPlan { + delta: vec![0, 1, 3, 4], + zero: vec![], + skip: vec![ChunkIndex(2)], + } + ); + let other_shape = manifest(&[1, 2]); + assert!(hydration_plan(&wanted, Some(&other_shape)).is_err()); + } + #[tokio::test] async fn stage_refuses_a_manifest_whose_shape_differs_from_its_base() { let r = rig(); diff --git a/crates/engram-host-core/src/lib.rs b/crates/engram-host-core/src/lib.rs index c3cf5b005..989f148c1 100644 --- a/crates/engram-host-core/src/lib.rs +++ b/crates/engram-host-core/src/lib.rs @@ -41,9 +41,10 @@ mod test_support; pub use checkpoint::checkpoint_tail_admits_publish; pub use coord::{CoordControlPlane, CoordError}; pub use disk::{ - data_extents, is_zero_chunk, Capacity, ChunkIndex, FileDisk, SessionDisk, Volume, VolumeName, + data_extents, is_zero_chunk, Capacity, ChunkIndex, FileDisk, Hydration, SessionDisk, + SourceDevice, SourceServer, Volume, VolumeName, }; -pub use disk_flows::{checkpoint, checkpoint_split, stage}; +pub use disk_flows::{checkpoint, checkpoint_split, hydration_plan, stage, HydrationPlan}; pub use finalize::{plan_finalize_retry, FinalizeRetry, FinalizeStage}; pub use fs::{HostFs, TokioFs}; pub use session_volumes::SessionVolumes; diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 9d5f32dc6..f75bee325 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -6,6 +6,8 @@ //! sb-- live //! sb---ckpt--v the last checkpoint; holds that manifest //! tmp-sb-- a stage or a checkpoint in flight +//! sb---dest--v a hydrating live volume's destination; becomes that manifest +//! sb---base--v what the destination was snapshotted from; holds that manifest //! ``` //! //! A checkpoint volume is named after a manifest the chunk store holds, @@ -14,12 +16,19 @@ //! and the next checkpoint publishes under it, so no caller tracks a //! manifest ref between calls. //! +//! A role whose manifest is not on the host starts on a hydrating +//! device: the live volume reads the base's blocks and a source the +//! fill engine serves, and the kernel copies the difference in behind +//! the guest ([`SessionDisk::hydrate`]). Until that device settles there +//! is no checkpoint volume; the `dest` and `base` names record what +//! the live data converges to and what it is measured against. +//! //! A released sandbox keeps its newest checkpoint volume per role, so a //! resume on this host is a snapshot of it instead of a stage from blob //! storage. [`SessionVolumes::reap`] deletes kept checkpoints, oldest //! release first, when the pool needs the space. -use std::collections::VecDeque; +use std::collections::{HashMap, HashSet, VecDeque}; use std::io; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -32,13 +41,19 @@ use engram_fill::{Class, FillEngine, PeerSource}; use tracing::Instrument as _; use uuid::Uuid; -use crate::disk::{Capacity, SessionDisk, Volume, VolumeName}; -use crate::disk_flows::{checkpoint_split, stage}; +use crate::disk::{ + Capacity, Hydration, SessionDisk, SourceDevice, SourceServer, Volume, VolumeName, +}; +use crate::disk_flows::{checkpoint_split, hydration_plan, shape_matches, stage}; use crate::image_volumes::{self, ImageVolumes}; const LIVE_PREFIX: &str = "sb-"; const TRANSIENT_PREFIX: &str = "tmp-sb-"; const CHECKPOINT_INFIX: &str = "-ckpt-"; +const DEST_INFIX: &str = "-dest-"; +const BASE_INFIX: &str = "-base-"; +/// The base of a hydration that started from zeros. +const BASE_NONE: &str = "none"; const PRECOPY_SUFFIX: &str = "-pre"; /// What a volume name says. @@ -47,6 +62,11 @@ pub enum Kind { Live, Checkpoint(ManifestRef), Transient, + /// The destination of a hydrating live volume; holds the manifest + /// once the hydration settles. + Dest(ManifestRef), + /// What a hydration's destination was snapshotted from. + Base(Option), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -77,27 +97,49 @@ pub fn live_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { } pub fn checkpoint_name(sandbox: SandboxId, role: DiskRole, manifest: ManifestRef) -> VolumeName { + named_after(sandbox, role, CHECKPOINT_INFIX, &manifest_suffix(manifest)) +} + +fn dest_name(sandbox: SandboxId, role: DiskRole, manifest: ManifestRef) -> VolumeName { + named_after(sandbox, role, DEST_INFIX, &manifest_suffix(manifest)) +} + +fn base_name(sandbox: SandboxId, role: DiskRole, manifest: Option) -> VolumeName { + let suffix = manifest.map_or(BASE_NONE.to_string(), manifest_suffix); + named_after(sandbox, role, BASE_INFIX, &suffix) +} + +fn named_after(sandbox: SandboxId, role: DiskRole, infix: &str, suffix: &str) -> VolumeName { VolumeName::new(format!( - "{LIVE_PREFIX}{}-{}{CHECKPOINT_INFIX}{}-v{}", + "{LIVE_PREFIX}{}-{role}{infix}{suffix}", sandbox.as_uuid().simple(), - role, - manifest.manifest_id.simple(), - manifest.version )) - .expect("a checkpoint volume name is at most 100 bytes of [a-z0-9-]") + .expect("a named volume is at most 100 bytes of [a-z0-9-]") +} + +fn manifest_suffix(manifest: ManifestRef) -> String { + format!("{}-v{}", manifest.manifest_id.simple(), manifest.version) +} + +fn parse_manifest_suffix(suffix: &str) -> Option { + let (id, version) = suffix.split_once("-v")?; + Some(ManifestRef { + manifest_id: Uuid::try_parse(id).ok()?, + version: version.parse().ok()?, + }) } /// How [`SessionVolumes`] brought up a role's disk. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StagePath { /// The role's checkpoint was already on this host. Kept, - /// Only the chunks that changed, written onto a kept checkpoint. - OntoKept, - /// The delta from the image, written onto its staged volume. - FromImage, - /// Every chunk, written into an empty volume: the image's volume is - /// not on this host. - Whole, + /// A hydrating device over a snapshot of the nearest base this host + /// holds: the guest starts at once and the kernel copies the + /// difference in behind it. + Lazy, + /// The difference from the nearest base written in full before the + /// guest starts: this platform has no hydrating device. + Staged, /// A snapshot of a manifest this host had staged. StagedSnapshot, /// A fork of the image's volume. @@ -110,9 +152,8 @@ impl StagePath { pub fn as_str(self) -> &'static str { match self { StagePath::Kept => "kept", - StagePath::OntoKept => "onto_kept", - StagePath::FromImage => "from_image", - StagePath::Whole => "whole", + StagePath::Lazy => "lazy", + StagePath::Staged => "staged", StagePath::StagedSnapshot => "staged_snapshot", StagePath::ImageFork => "image_fork", StagePath::Empty => "empty", @@ -148,38 +189,48 @@ pub fn parse(name: &VolumeName) -> Option { }; let (sandbox, rest) = rest.split_once('-')?; let sandbox = SandboxId::from(Uuid::try_parse(sandbox).ok()?); - let (role, rest) = match rest.split_once(CHECKPOINT_INFIX) { - Some((role, manifest)) => (role, Some(manifest)), - None => (rest, None), - }; - let role = match (transient, role.strip_suffix(PRECOPY_SUFFIX)) { - (true, Some(role)) => role, - _ => role, - }; - let role: DiskRole = role.parse().ok()?; - let kind = match (transient, rest) { - (true, None) => Kind::Transient, - (false, None) => Kind::Live, - (false, Some(manifest)) => { - let (id, version) = manifest.split_once("-v")?; - Kind::Checkpoint(ManifestRef { - manifest_id: Uuid::try_parse(id).ok()?, - version: version.parse().ok()?, - }) + let infixed = [CHECKPOINT_INFIX, DEST_INFIX, BASE_INFIX] + .into_iter() + .find_map(|infix| { + rest.split_once(infix) + .map(|(role, suffix)| (role, infix, suffix)) + }); + let (role, kind) = match (transient, infixed) { + (true, None) => ( + rest.strip_suffix(PRECOPY_SUFFIX).unwrap_or(rest), + Kind::Transient, + ), + (false, None) => (rest, Kind::Live), + (false, Some((role, CHECKPOINT_INFIX, suffix))) => { + (role, Kind::Checkpoint(parse_manifest_suffix(suffix)?)) + } + (false, Some((role, DEST_INFIX, suffix))) => { + (role, Kind::Dest(parse_manifest_suffix(suffix)?)) } - (true, Some(_)) => return None, + (false, Some((role, BASE_INFIX, BASE_NONE))) => (role, Kind::Base(None)), + (false, Some((role, BASE_INFIX, suffix))) => { + (role, Kind::Base(Some(parse_manifest_suffix(suffix)?))) + } + _ => return None, }; Some(Parsed { sandbox, - role, + role: role.parse().ok()?, kind, }) } -/// Where a live volume's contents come from. A manifest is written in -/// full before the sandbox starts; when `base` names an image staged on -/// this host, that volume is the origin and only the difference is -/// written. A manifest that is itself a staged image is snapshotted. +/// Live volumes first: a hydrating one holds its destination open, so +/// the destination can go only after it. +fn live_first(names: &mut [(VolumeName, Parsed)]) { + names.sort_by_key(|(_, p)| p.kind != Kind::Live); +} + +/// Where a live volume's contents come from. A manifest that is itself a +/// staged image is snapshotted. Otherwise, when `base` names an image +/// staged on this host, the volume starts as a snapshot of it and only +/// the difference comes in: behind the running guest where the platform +/// has a hydrating device, in full before it starts elsewhere. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Source { /// An image's disk: the volume starts a new lineage as a snapshot of @@ -264,6 +315,17 @@ pub struct Report { pub kept: Vec, } +/// A volume on this host that holds a manifest near the one wanted: +/// what a role's volume starts from. +struct Base { + volume: Volume, + manifest: Manifest, + manifest_ref: ManifestRef, + /// The sandbox whose kept checkpoint volume this is; a hydration + /// takes the volume over as its base. + kept_by: Option, +} + /// The owner of every `sb-` volume on a host. pub struct SessionVolumes { disk: Arc, @@ -273,6 +335,11 @@ pub struct SessionVolumes { /// first. A sandbox `reconcile` finds with no live volume joins at /// the front: its age is unknown, so it goes first. kept: Mutex>, + /// Serves the sources hydrating volumes copy from; without it every + /// volume is written in full before its guest starts. + sources: Option>, + /// The source of each hydrating live volume this process serves. + serving: Mutex>>, } impl SessionVolumes { @@ -282,9 +349,23 @@ impl SessionVolumes { disk, fill, kept: Mutex::new(VecDeque::new()), + sources: None, + serving: Mutex::new(HashMap::new()), } } + /// Start guests on hydrating volumes, with `sources` serving what + /// the kernel copies from. + pub fn with_sources(mut self, sources: Arc) -> Self { + self.sources = Some(sources); + self + } + + /// The source server and the copy unit, when this host hydrates. + fn hydration_support(&self) -> Option<(&dyn SourceServer, u64)> { + Some((self.sources.as_deref()?, self.disk.region_bytes()?)) + } + pub fn disk(&self) -> &dyn SessionDisk { self.disk.as_ref() } @@ -358,11 +439,10 @@ impl SessionVolumes { } } - /// The checkpoint volume first, under the transient name until its - /// manifest is in the store; then the live volume as its snapshot. - /// A `Continue` of a checkpoint this host kept for the lineage skips - /// the stage; one of a newer manifest stages only the delta onto it. - /// A stage fetches through `fill`. + /// The live volume of a role. A manifest this host holds in full (a + /// staged image, a kept checkpoint of it) is snapshotted; a + /// `Continue` or `Fork` of one it does not hold starts from the + /// nearest base it has ([`Self::bring_up`]). #[tracing::instrument( name = "disk.provision", skip_all, @@ -377,90 +457,61 @@ impl SessionVolumes { ) -> io::Result<(PathBuf, ManifestRef, StagePath)> { let tmp = transient_name(sandbox, role); self.delete_if_present(&tmp).await?; - let ((checkpoint, lineage), path) = match source { + let (live, lineage, path) = match source { Source::Image { manifest } => { - let wanted = self - .fill - .store() - .get_manifest(manifest) - .instrument(tracing::info_span!("disk.manifest_get")) - .await - .map_err(store_err)?; + let wanted = self.manifest(manifest).await?; self.images .fork(manifest, &tmp) .instrument(tracing::info_span!("disk.image_fork")) .await?; - ( - self.new_lineage(&tmp, sandbox, role, &wanted).await?, - StagePath::ImageFork, - ) + let lineage = self.new_lineage(&wanted).await?; + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + (live, lineage, StagePath::ImageFork) } Source::Fork { manifest, base } => { - let wanted = self - .fill - .store() - .get_manifest(manifest) - .instrument(tracing::info_span!("disk.manifest_get")) - .await - .map_err(store_err)?; - let path = match self.staged(manifest).await { + let wanted = self.manifest(manifest).await?; + let lineage = self.new_lineage(&wanted).await?; + match self.staged(manifest).await { Some((origin, _)) => { self.disk.snapshot(&origin, &tmp).await?; - StagePath::StagedSnapshot + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + (live, lineage, StagePath::StagedSnapshot) } None => { - self.stage_into(fill, &tmp, &wanted, base) - .instrument(tracing::info_span!("disk.stage")) - .await? - .1 + let base = self.image_base(base).await; + let (live, path) = self + .bring_up(fill, sandbox, role, &wanted, lineage, base) + .await?; + (live, lineage, path) } - }; - (self.new_lineage(&tmp, sandbox, role, &wanted).await?, path) + } } Source::Continue { manifest, base } => { match self.kept(sandbox, role, manifest).await? { - Some((owner, volume, _, held)) if held == manifest => ( - ( - self.adopt(owner, volume, sandbox, role, held).await?, - manifest, - ), - StagePath::Kept, - ), + Some((owner, volume, _, held)) if held == manifest => { + let checkpoint = self.adopt(owner, volume, sandbox, role, held).await?; + let live = self.live_snapshot(&checkpoint, sandbox, role).await?; + (live, manifest, StagePath::Kept) + } kept => { - let wanted = self - .fill - .store() - .get_manifest(manifest) - .instrument(tracing::info_span!("disk.manifest_get")) - .await - .map_err(store_err)?; - let path = match kept { - Some((_, volume, held_manifest, _)) + let wanted = self.manifest(manifest).await?; + let base = match kept { + Some((owner, volume, held_manifest, held)) if volume.size_bytes == wanted.total_bytes => { - stage( - self.disk.as_ref(), - fill, - Class::Demand, - &tmp, - &wanted, - Some((&volume, &held_manifest)), - ) - .instrument(tracing::info_span!("disk.stage", onto_kept = true)) - .await?; - StagePath::OntoKept - } - _ => { - self.stage_into(fill, &tmp, &wanted, base) - .instrument(tracing::info_span!("disk.stage")) - .await? - .1 + Some(Base { + volume, + manifest: held_manifest, + manifest_ref: held, + kept_by: Some(owner), + }) } + _ => self.image_base(base).await, }; - ( - (self.commit(&tmp, sandbox, role, manifest).await?, manifest), - path, - ) + let (live, path) = self + .bring_up(fill, sandbox, role, &wanted, manifest, base) + .await?; + (live, manifest, path) } } } @@ -470,29 +521,25 @@ impl SessionVolumes { chunk_size: ChunkSize::bytes(volume.chunk_size), ..Manifest::empty(ManifestKind::Disk, size_bytes) }; - ( - self.new_lineage(&tmp, sandbox, role, &manifest).await?, - StagePath::Empty, - ) + let lineage = self.new_lineage(&manifest).await?; + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + (live, lineage, StagePath::Empty) } }; - let live = self - .disk - .snapshot(&checkpoint, &live_name(sandbox, role)) - .instrument(tracing::info_span!("disk.live_snapshot")) - .await?; Ok((live.path, lineage, path)) } - /// Start a lineage that holds `manifest`, whose bytes `tmp` holds, - /// and commit `tmp` as its first checkpoint. - async fn new_lineage( - &self, - tmp: &VolumeName, - sandbox: SandboxId, - role: DiskRole, - manifest: &Manifest, - ) -> io::Result<(Volume, ManifestRef)> { + async fn manifest(&self, manifest: ManifestRef) -> io::Result { + self.fill + .store() + .get_manifest(manifest) + .instrument(tracing::info_span!("disk.manifest_get")) + .await + .map_err(store_err) + } + + /// Start a lineage that holds `manifest`. + async fn new_lineage(&self, manifest: &Manifest) -> io::Result { let lineage = ManifestRef::new(); self.fill .store() @@ -500,7 +547,32 @@ impl SessionVolumes { .instrument(tracing::info_span!("disk.manifest_put")) .await .map_err(store_err)?; - Ok((self.commit(tmp, sandbox, role, lineage).await?, lineage)) + Ok(lineage) + } + + /// Commit `tmp`, which holds `lineage`'s manifest, as the role's + /// checkpoint volume, and make the live volume its snapshot. + async fn commit_live( + &self, + tmp: &VolumeName, + sandbox: SandboxId, + role: DiskRole, + lineage: ManifestRef, + ) -> io::Result { + let checkpoint = self.commit(tmp, sandbox, role, lineage).await?; + self.live_snapshot(&checkpoint, sandbox, role).await + } + + async fn live_snapshot( + &self, + checkpoint: &Volume, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result { + self.disk + .snapshot(checkpoint, &live_name(sandbox, role)) + .instrument(tracing::info_span!("disk.live_snapshot")) + .await } /// Give the staged transient volume its checkpoint name and delete @@ -536,6 +608,145 @@ impl SessionVolumes { self.disk.open(&name).await } + /// The live volume of a role whose manifest `wanted` is not on this + /// host, under `lineage`: a hydrating device when this host has one + /// and serves sources, else the difference from `base` written in + /// full first. + async fn bring_up( + &self, + fill: &FillEngine, + sandbox: SandboxId, + role: DiskRole, + wanted: &Manifest, + lineage: ManifestRef, + base: Option, + ) -> io::Result<(Volume, StagePath)> { + if self.hydration_support().is_some() { + let live = self + .hydrate(fill, sandbox, role, wanted, lineage, base) + .instrument(tracing::info_span!( + "disk.hydrate", + held = tracing::field::Empty, + delta = tracing::field::Empty, + zero = tracing::field::Empty, + )) + .await?; + return Ok((live, StagePath::Lazy)); + } + let tmp = transient_name(sandbox, role); + stage( + self.disk.as_ref(), + fill, + Class::Demand, + &tmp, + wanted, + base.as_ref().map(|b| (&b.volume, &b.manifest)), + ) + .instrument(tracing::info_span!("disk.stage")) + .await?; + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + Ok((live, StagePath::Staged)) + } + + /// A hydrating live volume: its destination is a snapshot of the + /// base (a kept checkpoint volume is taken over; an empty volume + /// stands in for no base) under the `dest` name, and its source + /// serves `wanted` through `fill`. Chunks the base holds in final + /// form are skipped; chunks `wanted` leaves out are zeroed first. + /// Records on the current span how many chunks the base `held`, the + /// kernel copies (`delta`) and were zeroed (`zero`). + async fn hydrate( + &self, + fill: &FillEngine, + sandbox: SandboxId, + role: DiskRole, + wanted: &Manifest, + lineage: ManifestRef, + base: Option, + ) -> io::Result { + let (sources, region_bytes) = self.hydration_support().ok_or_else(|| { + io::Error::new(io::ErrorKind::Unsupported, "this host does not hydrate") + })?; + if let Some(base) = &base { + shape_matches(&base.volume, &base.manifest)?; + } + let plan = hydration_plan(wanted, base.as_ref().map(|b| &b.manifest))?; + let base_volume = match base { + Some(Base { + volume, + manifest_ref, + kept_by: Some(_), + .. + }) => { + let name = base_name(sandbox, role, Some(manifest_ref)); + self.disk.rename(&volume.name, &name).await?; + drop(volume); + self.disk.open(&name).await? + } + Some(Base { + volume, + manifest_ref, + kept_by: None, + .. + }) => { + self.disk + .snapshot(&volume, &base_name(sandbox, role, Some(manifest_ref))) + .await? + } + None => { + self.disk + .create(&base_name(sandbox, role, None), wanted.total_bytes) + .await? + } + }; + let dest = self + .disk + .snapshot(&base_volume, &dest_name(sandbox, role, lineage)) + .await?; + drop(base_volume); + shape_matches(&dest, wanted)?; + for chunk in &plan.zero { + dest.zero_chunk(*chunk).await?; + } + let source = sources + .serve(fill.clone(), wanted, plan.delta.clone(), region_bytes) + .await?; + let live = match self + .disk + .hydrate(&dest, source.path(), &live_name(sandbox, role), &plan.skip) + .await + { + Ok(live) => live, + Err(e) => { + if let Err(error) = source.stop().await { + tracing::warn!(sandbox_id = %sandbox, %role, %error, "source of a failed hydration not stopped"); + } + return Err(e); + } + }; + let span = tracing::Span::current(); + span.record("held", (plan.skip.len() - plan.zero.len()) as u64); + span.record("delta", plan.delta.len() as u64); + span.record("zero", plan.zero.len() as u64); + self.serving + .lock() + .unwrap() + .insert(live.name.clone(), source); + Ok(live) + } + + /// The staged volume of `image` as a base, when this host has it. + async fn image_base(&self, image: Option) -> Option { + let image = image?; + let (volume, manifest) = self.staged(image).await?; + Some(Base { + volume, + manifest, + manifest_ref: image, + kept_by: None, + }) + } + /// The newest checkpoint volume of `role` that `sandbox` or a released /// sandbox kept for the lineage of `wanted`, no newer than it: the /// sandbox that kept it, the volume, its manifest and ref. A resume @@ -597,36 +808,6 @@ impl SessionVolumes { self.disk.open(&name).await } - /// Write `wanted` into a new volume `name` through `fill`, from the - /// staged volume of `base` when this host has it. Without it, every - /// chunk of `wanted` is written. - async fn stage_into( - &self, - fill: &FillEngine, - name: &VolumeName, - wanted: &Manifest, - base: Option, - ) -> io::Result<(Volume, StagePath)> { - let base = match base { - Some(image) => self.staged(image).await, - None => None, - }; - let path = match base { - Some(_) => StagePath::FromImage, - None => StagePath::Whole, - }; - let volume = stage( - self.disk.as_ref(), - fill, - Class::Demand, - name, - wanted, - base.as_ref().map(|(v, m)| (v, m)), - ) - .await?; - Ok((volume, path)) - } - /// The staged volume of an image and its manifest, when this host /// has both. async fn staged(&self, image: ManifestRef) -> Option<(Volume, Manifest)> { @@ -639,6 +820,130 @@ impl SessionVolumes { } } + /// One step of a hydrating role's way to a settled volume: its + /// progress, and once every region is in, the cutover onto the + /// destination and the end of its source. `Settled` when the live + /// volume maps straight onto its destination. + pub async fn settle(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + let live = live_name(sandbox, role); + match self.disk.hydration(&live).await? { + state @ Hydration::Hydrating { + hydrated, total, .. + } if hydrated < total => return Ok(state), + Hydration::Hydrating { .. } => { + let (dest, _) = self.lazy_layout(sandbox, role).await?; + let dest = self.disk.open(&dest).await?; + self.disk.settle(&live, &dest).await?; + } + Hydration::Settled => {} + } + self.stop_source(&live).await?; + Ok(Hydration::Settled) + } + + /// The roles of `sandbox` whose live volume is still hydrating. + pub async fn hydrating(&self, sandbox: SandboxId) -> io::Result> { + let mut roles = Vec::new(); + for role in self.lazy_roles(sandbox).await? { + if let Hydration::Hydrating { .. } = + self.disk.hydration(&live_name(sandbox, role)).await? + { + roles.push(role); + } + } + Ok(roles) + } + + /// After a restart: serve again the sources of `sandbox`'s hydrating + /// roles, which the process that died left to the kernel. Returns + /// those roles. + pub async fn reattach(&self, sandbox: SandboxId) -> io::Result> { + let Some((sources, region_bytes)) = self.hydration_support() else { + return Ok(Vec::new()); + }; + let mut roles = Vec::new(); + for role in self.lazy_roles(sandbox).await? { + let live = live_name(sandbox, role); + let Hydration::Hydrating { source, .. } = self.disk.hydration(&live).await? else { + continue; + }; + let (dest, base) = self.lazy_layout(sandbox, role).await?; + let (Some(Kind::Dest(wanted)), Some(Kind::Base(base))) = + (parse(&dest).map(|p| p.kind), parse(&base).map(|p| p.kind)) + else { + unreachable!("lazy_layout returns a dest and a base name"); + }; + let wanted = self.manifest(wanted).await?; + let base = match base { + Some(base) => Some(self.manifest(base).await?), + None => None, + }; + let plan = hydration_plan(&wanted, base.as_ref())?; + let recovered = sources + .recover( + &source, + self.fill.clone(), + &wanted, + plan.delta, + region_bytes, + ) + .await?; + self.serving.lock().unwrap().insert(live, recovered); + roles.push(role); + } + Ok(roles) + } + + /// The roles of `sandbox` brought up on a hydrating device whose + /// first checkpoint has not run: they have a `dest` volume. + async fn lazy_roles(&self, sandbox: SandboxId) -> io::Result> { + Ok(self + .disk + .list() + .await? + .iter() + .filter_map(parse) + .filter(|p| p.sandbox == sandbox && matches!(p.kind, Kind::Dest(_))) + .map(|p| p.role) + .collect()) + } + + /// The `dest` and `base` volume names of a lazy role. + async fn lazy_layout( + &self, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result<(VolumeName, VolumeName)> { + let (mut dest, mut base) = (None, None); + for name in self.disk.list().await? { + match parse(&name) { + Some(p) if p.sandbox == sandbox && p.role == role => match p.kind { + Kind::Dest(_) => dest = Some(name), + Kind::Base(_) => base = Some(name), + _ => {} + }, + _ => {} + } + } + match (dest, base) { + (Some(dest), Some(base)) => Ok((dest, base)), + _ => Err(io::Error::new( + io::ErrorKind::NotFound, + format!("sandbox {sandbox} has no hydrating {role} volume"), + )), + } + } + + /// End the source of `live`, when this process serves one. The + /// device that read it must be gone or settled first. + async fn stop_source(&self, live: &VolumeName) -> io::Result<()> { + let source = self.serving.lock().unwrap().remove(live); + match source { + Some(source) => source.stop().await, + None => Ok(()), + } + } + /// The roles the sandbox has a live volume for. pub async fn roles(&self, sandbox: SandboxId) -> io::Result> { let names = self.disk.list().await?; @@ -836,14 +1141,15 @@ impl SessionVolumes { /// open stays and is reported as failed. pub async fn purge(&self, sandbox: SandboxId) -> io::Result { let mut report = Report::default(); - for name in self.disk.list().await? { - if parse(&name).is_some_and(|p| p.sandbox == sandbox) { - match self.disk.delete(&name).await { - Ok(()) => report.deleted.push(name), - Err(e) => report.failed.push((name, e)), - } - } - } + let names: Vec<(VolumeName, Parsed)> = self + .disk + .list() + .await? + .into_iter() + .filter_map(|name| parse(&name).map(|p| (name, p))) + .filter(|(_, p)| p.sandbox == sandbox) + .collect(); + self.delete_all(names, &mut report).await; if report.failed.is_empty() { self.kept.lock().unwrap().retain(|s| *s != sandbox); } @@ -895,7 +1201,7 @@ impl SessionVolumes { live: &[SandboxId], ) -> io::Result { let mut report = Report::default(); - let mut newest: std::collections::HashMap<(SandboxId, DiskRole), u64> = Default::default(); + let mut newest: HashMap<(SandboxId, DiskRole), u64> = Default::default(); let parsed: Vec<(VolumeName, Parsed)> = self .disk .list() @@ -910,23 +1216,89 @@ impl SessionVolumes { *v = (*v).max(m.version); } } + let mut unwanted = Vec::new(); for (name, p) in parsed { - let unwanted = match p.kind { - Kind::Live | Kind::Transient => !live.contains(&p.sandbox), - Kind::Checkpoint(m) => m.version < newest[&(p.sandbox, p.role)], + let wanted = match p.kind { + Kind::Live | Kind::Transient | Kind::Dest(_) | Kind::Base(_) => { + live.contains(&p.sandbox) + } + Kind::Checkpoint(m) => m.version == newest[&(p.sandbox, p.role)], }; - if !unwanted { - if matches!(p.kind, Kind::Checkpoint(_)) && !report.kept.contains(&p.sandbox) { - report.kept.push(p.sandbox); + match wanted { + true if matches!(p.kind, Kind::Checkpoint(_)) => { + if !report.kept.contains(&p.sandbox) { + report.kept.push(p.sandbox); + } } + true => {} + false => unwanted.push((name, p)), + } + } + self.delete_all(unwanted, &mut report).await; + Ok(report) + } + + /// Delete `names`, each sandbox's live volume first. A live volume + /// that stays keeps the destination and base under it: the kernel + /// copies from them and `reattach` rebuilds its plan from them. + async fn delete_all(&self, mut names: Vec<(VolumeName, Parsed)>, report: &mut Report) { + live_first(&mut names); + let mut live_stays = HashSet::new(); + for (name, p) in names { + if matches!(p.kind, Kind::Dest(_) | Kind::Base(_)) && live_stays.contains(&p.sandbox) { continue; } - match self.disk.delete(&name).await { - Ok(()) => report.deleted.push(name), - Err(e) => report.failed.push((name, e)), + let deleted = self.delete_into(&name, p.kind, report).await; + if p.kind == Kind::Live && !deleted { + live_stays.insert(p.sandbox); + } + } + } + + /// Delete one volume into `report`; whether it is gone. A deleted + /// live volume's source stops with it. A hydrating volume whose + /// source this process does not serve is not deleted: the kernel + /// removes a device only after its reads complete, and nothing + /// answers them, so the delete would hang (`reattach` first). + async fn delete_into(&self, name: &VolumeName, kind: Kind, report: &mut Report) -> bool { + if kind == Kind::Live && !self.serving.lock().unwrap().contains_key(name) { + match self.disk.hydration(name).await { + Ok(Hydration::Hydrating { source, .. }) => { + report.failed.push(( + name.clone(), + io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{name} hydrates from {}, which nothing serves", + source.display() + ), + ), + )); + return false; + } + Ok(Hydration::Settled) => {} + Err(e) => { + report.failed.push((name.clone(), e)); + return false; + } + } + } + match self.disk.delete(name).await { + Ok(()) => { + if kind == Kind::Live { + if let Err(e) = self.stop_source(name).await { + report.failed.push((name.clone(), e)); + return true; + } + } + report.deleted.push(name.clone()); + true + } + Err(e) => { + report.failed.push((name.clone(), e)); + false } } - Ok(report) } /// The released sandboxes whose kept checkpoints a resume may take. @@ -989,7 +1361,8 @@ mod tests { use super::*; use crate::disk::{ChunkIndex, FileDisk}; use crate::disk_flows::checkpoint; - use crate::test_support::CountingBlob; + use crate::test_support::{CountingBlob, FileSources, HydratingDisk}; + use std::sync::atomic::Ordering; const CHUNK: u64 = 4096; @@ -999,6 +1372,8 @@ mod tests { store: ChunkStore, volumes: SessionVolumes, blob: Arc, + /// Set on a rig whose disk hydrates. + sources: Option>, } fn rig() -> Rig { @@ -1007,6 +1382,31 @@ mod tests { rig_on(dir, disk) } + /// A rig whose disk has a hydrating device and whose host serves + /// sources: resumes take the lazy path. + fn lazy_rig() -> Rig { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(HydratingDisk::new(&dir.path().join("volumes"), CHUNK).unwrap()); + let sources = Arc::new(disk.sources(&dir.path().join("sources"))); + let mut r = rig_on(dir, disk); + r.volumes = r.volumes.with_sources(sources.clone()); + r.sources = Some(sources); + r + } + + /// Drive `settle` until the role's volume is settled; the steps it + /// took. + async fn settled(r: &Rig, sandbox: SandboxId, role: DiskRole) -> u64 { + let mut steps = 0; + loop { + steps += 1; + match r.volumes.settle(sandbox, role).await.unwrap() { + Hydration::Settled => return steps, + Hydration::Hydrating { .. } => assert!(steps < 100, "the hydration never ends"), + } + } + } + /// A rig whose pool reports `POOL_TOTAL` bytes with `VOLUME_COST` /// used per volume, so reaping frees space. fn pressured_rig() -> Rig { @@ -1030,6 +1430,7 @@ mod tests { disk, store, blob, + sources: None, _dir: dir, } } @@ -1198,6 +1599,24 @@ mod tests { }) ); } + assert_eq!( + parse(&dest_name(sandbox, DiskRole::Root, manifest)), + Some(Parsed { + sandbox, + role: DiskRole::Root, + kind: Kind::Dest(manifest) + }) + ); + for base in [Some(manifest), None] { + assert_eq!( + parse(&base_name(sandbox, DiskRole::Swap, base)), + Some(Parsed { + sandbox, + role: DiskRole::Swap, + kind: Kind::Base(base) + }) + ); + } assert_eq!(parse(&image_volumes::volume_name(manifest)), None); assert_eq!(parse(&VolumeName::new("sb-nothex-root").unwrap()), None); assert_eq!( @@ -2144,4 +2563,209 @@ mod tests { assert_eq!(reaped.deleted.len(), 1); assert!(checkpoints_of(&r, dead, DiskRole::Root).await.is_empty()); } + + /// Image [1, 2, 3]; the session changed chunk 1 and zeroed chunk 2. + /// Its resume on a host that hydrates starts on a device over a + /// snapshot of the image volume: chunk 1 is the whole delta, chunk 2 + /// is zeroed before the copy, chunk 0 is never touched. + #[tokio::test] + async fn a_lazy_resume_starts_on_a_hydrating_volume_and_settles_onto_it() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + assert_eq!(p.root_stage, StagePath::StagedSnapshot); + write_byte(&p.root, 1, 9).await; + write_byte(&p.root, 2, 0).await; + let published = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + let gets = r.blob.gets(); + let p = r + .volumes + .provision( + resumed, + Source::Continue { + manifest: published, + base: Some(image), + }, + None, + ) + .await + .unwrap(); + assert_eq!( + (p.root_stage, p.root_manifest), + (StagePath::Lazy, published) + ); + assert_eq!( + r.blob.gets(), + gets + 1, + "the source serves only the changed chunk" + ); + assert_eq!( + names(&r).await, + vec![ + image_volumes::volume_name(image).to_string(), + live_name(resumed, DiskRole::Root).to_string(), + base_name(resumed, DiskRole::Root, Some(image)).to_string(), + dest_name(resumed, DiskRole::Root, published).to_string(), + ] + .into_iter() + .map(|n| n.to_string()) + .collect::>() + .into_iter() + .collect::>() + ); + // The fake device has no lazy read: until the copy lands, the + // destination shows the base with the dropped chunk zeroed. + assert_eq!(bytes_at(&p.root).await, vec![1, 2, 0]); + assert_eq!( + r.volumes.hydrating(resumed).await.unwrap(), + vec![DiskRole::Root] + ); + + let steps = settled(&r, resumed, DiskRole::Root).await; + assert_eq!( + steps, 1, + "the one chunk copies on the first step, then the cutover" + ); + assert_eq!(bytes_at(&p.root).await, vec![1, 9, 0]); + assert!(r.volumes.hydrating(resumed).await.unwrap().is_empty()); + let sources = r.sources.as_ref().unwrap(); + assert_eq!(sources.stopped.load(Ordering::Relaxed), 1); + assert_eq!( + r.volumes.settle(resumed, DiskRole::Root).await.unwrap(), + Hydration::Settled, + "settle is idempotent" + ); + + r.volumes.release(resumed).await.unwrap(); + assert_eq!( + names(&r).await, + vec![image_volumes::volume_name(image).to_string()], + "a hydrating role keeps no checkpoint volume" + ); + } + + /// A release during the window deletes the device before its + /// source; the source refuses the other order. + #[tokio::test] + async fn a_release_during_hydration_ends_the_device_then_its_source() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2]).await; + let sandbox = SandboxId::new(); + let mut newer = r.store.get_manifest(image).await.unwrap(); + newer.chunks[0].hash = r.store.put_chunk(&vec![7u8; CHUNK as usize]).await.unwrap(); + let next = ManifestRef::new(); + r.store.put_manifest(next, &newer).await.unwrap(); + let p = r + .volumes + .provision( + sandbox, + Source::Continue { + manifest: next, + base: Some(image), + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert_eq!( + r.volumes.hydrating(sandbox).await.unwrap(), + vec![DiskRole::Root] + ); + + r.volumes.release(sandbox).await.unwrap(); + let sources = r.sources.as_ref().unwrap(); + assert_eq!(sources.stopped.load(Ordering::Relaxed), 1); + assert_eq!( + names(&r).await, + vec![image_volumes::volume_name(image).to_string()] + ); + } + + /// A host-agent restart mid-window: the device and its source + /// outlive the process; the next process cannot delete the device + /// before it serves the source again, then settles the volume. + #[tokio::test] + async fn a_restart_serves_a_hydrating_volume_again() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2]).await; + let sandbox = SandboxId::new(); + let mut newer = r.store.get_manifest(image).await.unwrap(); + newer.chunks[0].hash = r.store.put_chunk(&vec![7u8; CHUNK as usize]).await.unwrap(); + let next = ManifestRef::new(); + r.store.put_manifest(next, &newer).await.unwrap(); + let p = r + .volumes + .provision( + sandbox, + Source::Continue { + manifest: next, + base: Some(image), + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + + let sources = r.sources.clone().unwrap(); + let restarted = SessionVolumes::new( + r.disk.clone(), + engram_fill::FillEngine::new(r.store.clone(), engram_fill::Config::default()), + ) + .with_sources(sources.clone()); + // Until the source is served again, nothing deletes the device: + // its removal would wait on reads nothing answers. + let refused = restarted.release(sandbox).await.unwrap_err(); + assert_eq!(refused.kind(), std::io::ErrorKind::ResourceBusy); + assert_eq!( + restarted.reattach(sandbox).await.unwrap(), + vec![DiskRole::Root] + ); + loop { + if let Hydration::Settled = restarted.settle(sandbox, DiskRole::Root).await.unwrap() { + break; + } + } + assert_eq!(bytes_at(&p.root).await, vec![7, 2]); + assert_eq!(sources.stopped.load(Ordering::Relaxed), 1); + assert!(restarted.reattach(sandbox).await.unwrap().is_empty()); + } + + /// Without a base on the host, the destination starts from zeros and + /// every listed chunk is the delta. + #[tokio::test] + async fn a_lazy_resume_without_a_base_starts_from_zeros() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 0, 3]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision( + sandbox, + Source::Continue { + manifest: image, + base: None, + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert_eq!(bytes_at(&p.root).await, vec![0, 0, 0]); + assert_eq!( + settled(&r, sandbox, DiskRole::Root).await, + 2, + "two chunks to copy" + ); + assert_eq!(bytes_at(&p.root).await, vec![1, 0, 3]); + assert!(names(&r) + .await + .contains(&base_name(sandbox, DiskRole::Root, None).to_string())); + } } diff --git a/crates/engram-host-core/src/test_support.rs b/crates/engram-host-core/src/test_support.rs index 42c6fa17d..42e49b516 100644 --- a/crates/engram-host-core/src/test_support.rs +++ b/crates/engram-host-core/src/test_support.rs @@ -1,12 +1,25 @@ //! Test doubles shared by this crate's unit tests. +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use engram_chunk_store::Manifest; use engram_core::error::BlobError; use engram_core::traits::storage::{BlobObjectMeta, ByteStream}; use engram_core::traits::BlobStorage; +use engram_fill::{Class, FillEngine}; use engram_storage_local::LocalBlobStorage; +use crate::disk::{ + Capacity, ChunkIndex, FileDisk, Hydration, SessionDisk, SourceDevice, SourceServer, Volume, + VolumeName, +}; + /// Counts the chunk GETs a stage makes; manifest reads do not count. pub(crate) struct CountingBlob { inner: LocalBlobStorage, @@ -47,3 +60,232 @@ impl BlobStorage for CountingBlob { self.inner.list_prefix(prefix).await } } + +/// A [`FileDisk`] with a hydrating device, for the lazy path without a +/// kernel: `hydrate` links `name` to the destination file, so a write +/// to either lands in both, and each `hydration` call copies one chunk +/// the plan did not skip from the source file. `settle` needs every +/// chunk copied and changes nothing: the link is direct already. +pub(crate) struct HydratingDisk { + inner: FileDisk, + chunk_size: u64, + clones: Mutex>, + /// Sources a device still copies from; a stop of one is refused. + holders: Arc>>, +} + +struct Clone { + source: PathBuf, + dest: PathBuf, + pending: Vec, + total: u64, + settled: bool, +} + +impl HydratingDisk { + pub(crate) fn new(root: &Path, chunk_size: u64) -> io::Result { + Ok(Self { + inner: FileDisk::new(root, chunk_size)?, + chunk_size, + clones: Mutex::new(HashMap::new()), + holders: Arc::new(Mutex::new(HashSet::new())), + }) + } + + /// A source server whose stops this disk's devices refuse while + /// they copy. + pub(crate) fn sources(&self, dir: &Path) -> FileSources { + FileSources { + dir: dir.to_path_buf(), + holders: self.holders.clone(), + next: AtomicU64::new(0), + stopped: Arc::new(AtomicU64::new(0)), + } + } +} + +#[async_trait::async_trait] +impl SessionDisk for HydratingDisk { + async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { + self.inner.create(name, size_bytes).await + } + async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { + self.inner.snapshot(origin, name).await + } + async fn open(&self, name: &VolumeName) -> io::Result { + self.inner.open(name).await + } + async fn delete(&self, name: &VolumeName) -> io::Result<()> { + if let Some(clone) = self.clones.lock().unwrap().remove(name) { + self.holders.lock().unwrap().remove(&clone.source); + } + self.inner.delete(name).await + } + async fn list(&self) -> io::Result> { + self.inner.list().await + } + async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { + self.inner.rename(from, to).await + } + async fn capacity(&self) -> io::Result { + self.inner.capacity().await + } + fn region_bytes(&self) -> Option { + Some(self.chunk_size) + } + async fn hydrate( + &self, + dest: &Volume, + source: &Path, + name: &VolumeName, + skip: &[ChunkIndex], + ) -> io::Result { + let path = self.inner.root().join(name.as_str()); + std::fs::hard_link(&dest.path, &path)?; + let pending = (0..dest.chunk_count()) + .map(ChunkIndex) + .filter(|c| !skip.contains(c)) + .collect(); + self.clones.lock().unwrap().insert( + name.clone(), + Clone { + source: source.to_path_buf(), + dest: dest.path.clone(), + pending, + total: dest.chunk_count(), + settled: false, + }, + ); + self.holders.lock().unwrap().insert(source.to_path_buf()); + Volume::open(name.clone(), &path, dest.size_bytes, dest.chunk_size) + } + async fn hydration(&self, name: &VolumeName) -> io::Result { + let mut clones = self.clones.lock().unwrap(); + let Some(clone) = clones.get_mut(name) else { + return Ok(Hydration::Settled); + }; + if clone.settled { + return Ok(Hydration::Settled); + } + if let Some(chunk) = clone.pending.first().copied() { + let mut bytes = vec![0u8; self.chunk_size as usize]; + File::open(&clone.source)?.read_exact_at(&mut bytes, chunk.0 * self.chunk_size)?; + File::options() + .write(true) + .open(&clone.dest)? + .write_all_at(&bytes, chunk.0 * self.chunk_size)?; + clone.pending.remove(0); + } + Ok(Hydration::Hydrating { + hydrated: clone.total - clone.pending.len() as u64, + total: clone.total, + source: clone.source.clone(), + }) + } + async fn settle(&self, name: &VolumeName, _dest: &Volume) -> io::Result<()> { + let mut clones = self.clones.lock().unwrap(); + let clone = clones.get_mut(name).ok_or_else(|| { + io::Error::new( + io::ErrorKind::Unsupported, + format!("{name} is not hydrating"), + ) + })?; + if !clone.pending.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name}: {} chunks are not copied", clone.pending.len()), + )); + } + clone.settled = true; + self.holders.lock().unwrap().remove(&clone.source); + Ok(()) + } +} + +/// Serves a manifest as a file holding its delta chunks' bytes, written +/// from the fill engine up front; the kernel reads nothing else. +pub(crate) struct FileSources { + dir: PathBuf, + holders: Arc>>, + next: AtomicU64, + /// Sources stopped so far. + pub(crate) stopped: Arc, +} + +struct FileSource { + path: PathBuf, + holders: Arc>>, + stopped: Arc, +} + +#[async_trait::async_trait] +impl SourceDevice for FileSource { + fn path(&self) -> &Path { + &self.path + } + fn touched(&self) -> Vec { + Vec::new() + } + async fn stop(self: Box) -> io::Result<()> { + if self.holders.lock().unwrap().contains(&self.path) { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!("{} is still a device's source", self.path.display()), + )); + } + std::fs::remove_file(&self.path)?; + self.stopped.fetch_add(1, Ordering::Relaxed); + Ok(()) + } +} + +#[async_trait::async_trait] +impl SourceServer for FileSources { + async fn serve( + &self, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + _region_bytes: u64, + ) -> io::Result> { + std::fs::create_dir_all(&self.dir)?; + let path = self + .dir + .join(format!("src-{}", self.next.fetch_add(1, Ordering::Relaxed))); + let file = File::create_new(&path)?; + file.set_len(manifest.total_bytes)?; + let chunk_size = manifest.chunk_size.as_u64(); + for chunk in manifest + .chunks + .iter() + .filter(|c| delta.contains(&(c.offset / chunk_size))) + { + let bytes = fill + .read(chunk.hash, Class::Hydrate) + .await + .map_err(io::Error::other)?; + file.write_all_at(&bytes, chunk.offset)?; + } + Ok(Box::new(FileSource { + path, + holders: self.holders.clone(), + stopped: self.stopped.clone(), + })) + } + + async fn recover( + &self, + path: &Path, + _fill: FillEngine, + _manifest: &Manifest, + _delta: Vec, + _region_bytes: u64, + ) -> io::Result> { + std::fs::metadata(path)?; + Ok(Box::new(FileSource { + path: path.to_path_buf(), + holders: self.holders.clone(), + stopped: self.stopped.clone(), + })) + } +} From e16160ac562da2d5865cbf0f33691ff73a6f235e Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 12:13:18 -0800 Subject: [PATCH 05/35] host-agent: the restore boots before the disk hydrates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pooled backend takes the lazy path the previous commit opened. The host's `SourceServer` is `UblkSources`: a `BlkSource` per hydrating role, served in process. A restore that comes up on a hydrating volume gets a settle driver, a thin loop over `SessionVolumes::settle` that polls the kernel's progress, cuts the device over when every region is in, stops the source and logs `disk settled role=… regions=… elapsed_ms=…`. Destroy drops the driver before the VM goes, so no settle runs against a volume being released; the release itself ends the device before its source, the order the ublk rule needs. At startup, before the volume reconcile, `reattach_all` serves again every source the previous process left to the kernel: a hydrating volume whose source nothing serves cannot be deleted (the kernel waits for reads nothing answers), so the reconcile would otherwise refuse the dead sandboxes' volumes. Live sandboxes' roles get a settle driver; dead sandboxes' volumes are deleted as before; a ublk device nothing maps onto and nothing serves is reaped. `REATTACH_GENERATION` goes to 3: a build before this one cannot serve a source again after a restart, so the upgrade drain evicts every session in both directions. The CHANGELOG carries the upgrade note; node-prep loads `ublk_drv` on every start. The root test `lazy_restore` runs the whole path over the pool: a cold read through the source returns the manifest's bytes while the kernel copies, a write to a hole survives the settle, the sources stop, and a process killed mid-copy leaves a volume the next process must serve again before it can delete it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 4 +- crates/engram-host-agent/src/blk_source.rs | 102 ++++++- crates/engram-host-agent/src/lib.rs | 7 +- .../engram-host-agent/src/pooled_backend.rs | 109 ++++++- .../engram-host-agent/tests/lazy_restore.rs | 265 ++++++++++++++++++ .../engram-host-core/src/session_volumes.rs | 29 ++ crates/engram-protocol/src/wire.rs | 6 +- .../helm/engram-host-fleet/files/thin-pool.sh | 4 + 9 files changed, 510 insertions(+), 17 deletions(-) create mode 100644 crates/engram-host-agent/tests/lazy_restore.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86619454b..202247375 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1204,6 +1204,7 @@ jobs: -p engram-host-agent \ --test session_disk \ --test blk_source \ + --test lazy_restore \ --run-ignored ignored-only \ --no-fail-fast \ --test-threads=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index c7cc6ce8b..a572b3fc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,7 +68,9 @@ starts with **Upgrade notes**: read them before you upgrade. The upgrade procedu - **The kernel owns the session disk.** A VM's disk is a thin volume in a device-mapper pool; no host process is in the data path, and a host agent restart does not touch a running VM's disk. A base image is staged into the pool once per host and every session's volume - is a snapshot of it, so a create writes nothing. A checkpoint pauses the VM for tens of + is a snapshot of it, so a create writes nothing. A resume starts the VM on a snapshot of + the nearest base the host holds and copies the difference in behind it, so the time to + the first prompt does not grow with the size of the session's disk. A checkpoint pauses the VM for tens of milliseconds however much it wrote and uploads only the blocks that changed. The disk daemon, the copy-on-write view, the flush pipeline, the durability-rollback path, and the quarantined-survivor eviction are gone with it. diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 3d6c2071b..a3de554f7 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -24,9 +24,11 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; +use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use engram_chunk_store::{ChunkHash, Manifest}; use engram_fill::{Class, FillEngine}; +use engram_host_core::{SourceDevice, SourceServer}; use io_uring::{opcode, types}; use libublk::ctrl::{UblkCtrl, UblkCtrlBuilder}; use libublk::io::{UblkDev, UblkIOCtx, UblkQueue}; @@ -46,10 +48,10 @@ const QUEUE_DEPTH: u16 = 64; /// The largest read the kernel sends; the block layer splits above it. const IO_BUF_BYTES: u32 = 512 << 10; const DEVICE_NODE_WAIT: Duration = Duration::from_secs(5); - -/// A served device. [`Self::stop`] ends it; a dropped one keeps serving /// Entries in a thread's ublk control ring; commands go one at a time. const CTRL_RING_DEPTH: u32 = 16; + +/// A served device. [`Self::stop`] ends it; a dropped one keeps serving /// until the process exits, after which [`Self::recover`] takes it over. pub struct BlkSource { ctrl: Arc, @@ -694,3 +696,99 @@ fn with_ctrl_ring(f: impl FnOnce() -> Result) -> io::Result .map_err(ublk_err)?; f().map_err(ublk_err) } + +/// The host's [`SourceServer`]: every source is a [`BlkSource`]. +pub struct UblkSources; + +#[async_trait] +impl SourceServer for UblkSources { + async fn serve( + &self, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result> { + Ok(Box::new( + BlkSource::serve(fill, manifest, delta, region_bytes).await?, + )) + } + + async fn recover( + &self, + path: &Path, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result> { + Ok(Box::new( + BlkSource::recover(dev_id_of(path)?, fill, manifest, delta, region_bytes).await?, + )) + } +} + +#[async_trait] +impl SourceDevice for BlkSource { + fn path(&self) -> &Path { + BlkSource::path(self) + } + + fn touched(&self) -> Vec { + BlkSource::touched(self) + } + + async fn stop(self: Box) -> io::Result<()> { + BlkSource::stop(*self).await + } +} + +/// The id in `/dev/ublkb`. +fn dev_id_of(path: &Path) -> io::Result { + path.file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_prefix("ublkb")) + .and_then(|id| id.parse().ok()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a ublk block device", path.display()), + ) + }) +} + +/// Delete every ublk device nothing maps onto and no server in this +/// process serves (`served` lists their paths): what a process left +/// when it died between making a device and mapping a volume onto it. +/// A device a volume still maps onto stays, with a warning: deleting it +/// would wait on reads nothing answers. +pub async fn reap_orphans(served: &[PathBuf]) -> io::Result> { + let mut reaped = Vec::new(); + let mut entries = match tokio::fs::read_dir("/dev").await { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(reaped), + Err(e) => return Err(e), + }; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let Ok(id) = dev_id_of(&path) else { continue }; + if served.contains(&path) { + continue; + } + let held = std::fs::read_dir(format!("/sys/block/ublkb{id}/holders")) + .map(|holders| holders.count()) + .unwrap_or(0); + if held > 0 { + tracing::warn!(device = %path.display(), holders = held, "an unserved ublk device is still mapped; it is not deleted"); + continue; + } + tokio::task::spawn_blocking(move || { + with_ctrl_ring(|| UblkCtrl::new_simple(id as i32)?.del_dev()) + }) + .await + .map_err(io::Error::other)??; + tracing::info!(device = %path.display(), "deleted an orphaned ublk device"); + reaped.push(id); + } + Ok(reaped) +} diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index e10e1a5ab..ea9d89a66 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -304,9 +304,10 @@ impl HostAgent { } if let Some(fill) = self.fill.clone() { if let Some(disk) = self.session_disk.clone() { - p = p.with_session_volumes(Arc::new( - engram_host_core::SessionVolumes::new(disk, fill.clone()), - )); + let volumes = engram_host_core::SessionVolumes::new(disk, fill.clone()); + #[cfg(target_os = "linux")] + let volumes = volumes.with_sources(Arc::new(blk_source::UblkSources)); + p = p.with_session_volumes(Arc::new(volumes)); } p = p.with_fill(fill); // ADR 0080 §C: the MaterializeImage scratch root — diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index b9a43d489..f023bb61d 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -828,6 +828,10 @@ pub struct PooledBackend { /// A lazy restore's background memory prefetch; dropping the entry /// cancels it. memory_prefetches: DashMap, + /// The driver that settles a sandbox's hydrating disks; dropping the + /// entry ends it. Destroy drops it before the VM goes, so no settle + /// runs against a volume being released. + disk_settles: DashMap, /// The adaptive checkpoint controller's input: what the last /// capture observed about the memory dirty rate (ADR 0101 B). checkpoint_pacing: Arc>, @@ -1014,6 +1018,44 @@ fn record_warm_hook_failure_metric(kind: engram_core::types::CaptureFailureKind) .increment(1); } +/// How often a settle driver reads a hydrating disk's progress, and how +/// long it waits after a step fails. +const SETTLE_POLL: std::time::Duration = std::time::Duration::from_millis(250); +const SETTLE_RETRY: std::time::Duration = std::time::Duration::from_secs(5); + +/// Settle each of the sandbox's hydrating disks: poll until the kernel +/// has every region, then cut the device over and stop its source +/// (`SessionVolumes::settle` is the step). A failed step is retried; a +/// destroy ends the driver by dropping its task. +async fn settle_disks(volumes: Arc, id: SandboxId, roles: Vec) { + for role in roles { + let started = crate::time_source::metrics_now(); + let mut regions = 0; + loop { + match volumes.settle(id, role).await { + Ok(engram_host_core::Hydration::Settled) => { + tracing::info!( + sandbox_id = %id, + role = %role, + regions, + elapsed_ms = started.elapsed().as_millis() as u64, + "disk settled", + ); + break; + } + Ok(engram_host_core::Hydration::Hydrating { total, .. }) => { + regions = total; + tokio::time::sleep(SETTLE_POLL).await; + } + Err(error) => { + tracing::warn!(sandbox_id = %id, role = %role, %error, "disk settle step failed; retrying"); + tokio::time::sleep(SETTLE_RETRY).await; + } + } + } + } +} + /// Abort the wrapped task on drop, including hint polling after DrainDone. /// A leg wrapped by /// [`spawn_leg_keepalive`] stops resending stale progress the moment @@ -1911,17 +1953,43 @@ impl PooledBackend { } .await; remove_staged_manifests(staged_manifests, &src, id, &staged_refs).await; - if restore.is_err() { - self.drop_failed_restore(id); - if let Some(volumes) = self.volumes.as_ref() { - if let Err(error) = volumes.release(id).await { - tracing::warn!(sandbox_id = %id, %error, "release after a failed restore failed"); + let volumes = self.volumes.as_ref(); + match (&restore, volumes) { + (Err(_), _) => { + self.drop_failed_restore(id); + if let Some(volumes) = volumes { + if let Err(error) = volumes.release(id).await { + tracing::warn!(sandbox_id = %id, %error, "release after a failed restore failed"); + } } } + (Ok(()), Some(volumes)) => match volumes.hydrating(id).await { + Ok(roles) => self.settle_disks(id, roles), + Err(error) => { + tracing::warn!(sandbox_id = %id, %error, "the hydrating disks of a restore are unknown; none settles") + } + }, + (Ok(()), None) => {} } restore } + /// Drive `roles`, the sandbox's hydrating disks, to settled volumes + /// in the background. + fn settle_disks(&self, id: SandboxId, roles: Vec) { + let Some(volumes) = self.volumes.clone() else { + return; + }; + if roles.is_empty() { + return; + } + let task = tokio::spawn( + settle_disks(volumes, id, roles) + .instrument(tracing::info_span!("disk.settle", sandbox_id = %id)), + ); + self.disk_settles.insert(id, AbortTaskOnDrop(task)); + } + /// End a restore's memory fetches: cancel its background prefetch and /// drop its peer registration, which ends the peer's batching task. fn drop_restore_fetches(&self, id: SandboxId) { @@ -2039,6 +2107,7 @@ impl PooledBackend { peer_registrations: Arc::new(DashMap::new()), sandbox_registrations: Arc::new(DashMap::new()), memory_prefetches: DashMap::new(), + disk_settles: DashMap::new(), checkpoint_pacing: Arc::new(DashMap::new()), checkpoint_dir: None, checkpoint_recorded: Arc::new(tokio::sync::Notify::new()), @@ -2424,10 +2493,11 @@ impl PooledBackend { )) } - /// Delete the volumes of every sandbox that is neither alive nor - /// awaiting an eviction finalize. Runs once at startup, after the - /// reattach pass fixes the live set and before anything can - /// provision a sandbox. + /// Serve again the sources of every hydrating volume the previous + /// process left, settle the live sandboxes' ones, and delete the + /// volumes of every sandbox that is neither alive nor awaiting an + /// eviction finalize. Runs once at startup, after the reattach pass + /// fixes the live set and before anything can provision a sandbox. pub async fn reconcile_volumes(&self) { let Some(volumes) = &self.volumes else { return; @@ -2439,6 +2509,22 @@ impl PooledBackend { return; } }; + match volumes.reattach_all().await { + Ok(reattached) => { + for id in live.iter().copied() { + let roles: Vec = reattached + .iter() + .filter(|(sandbox, _)| *sandbox == id) + .map(|(_, role)| *role) + .collect(); + self.settle_disks(id, roles); + } + if !reattached.is_empty() { + tracing::info!(volumes = reattached.len(), "hydrating disks served again"); + } + } + Err(error) => tracing::warn!(%error, "startup reattach of hydrating disks failed"), + } let finalizer = self.eviction_finalizer(); match crate::eviction_finalize::reconcile_volumes(volumes, finalizer.as_ref(), live).await { Ok(report) => { @@ -2455,6 +2541,10 @@ impl PooledBackend { } Err(error) => tracing::warn!(%error, "startup volume reconcile failed"), } + #[cfg(target_os = "linux")] + if let Err(error) = crate::blk_source::reap_orphans(&volumes.sources()).await { + tracing::warn!(%error, "orphaned ublk devices were not all deleted"); + } } /// Issue #529: re-drive every un-acked eviction finalize record at @@ -5861,6 +5951,7 @@ impl SandboxBackend for PooledBackend { self.held_snapshots.remove(&id); self.hint_pollers.remove(&id); self.memory_prefetches.remove(&id); + self.disk_settles.remove(&id); self.teardowns .run_or_join(id, || { let session_bindings = self.session_bindings.clone(); diff --git a/crates/engram-host-agent/tests/lazy_restore.rs b/crates/engram-host-agent/tests/lazy_restore.rs new file mode 100644 index 000000000..2bc9d83ca --- /dev/null +++ b/crates/engram-host-agent/tests/lazy_restore.rs @@ -0,0 +1,265 @@ +//! A lazy resume over the dm-thin pool with a ublk source, as root: the +//! device the guest would open answers a cold read with the manifest's +//! bytes while the kernel copies, the volume settles onto its +//! destination, and a process that dies mid-copy leaves a volume the +//! next process must serve again before it can delete it. Needs +//! `ENGRAM_TEST_THIN_POOL` and `ENGRAM_TEST_UBLK` (the CI root lane); +//! unset, the tests skip. +// tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) +#![allow(clippy::disallowed_methods)] +#![cfg(target_os = "linux")] + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind}; +use engram_core::types::manifest::ManifestRef; +use engram_core::types::snapshot::DiskRole; +use engram_core::SandboxId; +use engram_disk_thin::ThinPoolDisk; +use engram_fill::FillEngine; +use engram_host_agent::blk_source::UblkSources; +use engram_host_core::disk::ChunkIndex; +use engram_host_core::session_volumes::{live_name, Source, StagePath}; +use engram_host_core::{Hydration, SessionVolumes}; +use engram_storage_local::LocalBlobStorage; + +/// Two pool blocks per chunk, as in the disk-thin clone test. +const CHUNK: u64 = 2 * 65536; +/// Chunk `i` holds byte `PATTERN[i]`; zero is a hole. +const PATTERN: [u8; 8] = [1, 2, 0, 4, 0, 0, 7, 8]; +const SETTLE_DEADLINE: Duration = Duration::from_secs(30); + +fn gate() -> Option { + let pool = std::env::var("ENGRAM_TEST_THIN_POOL").ok(); + if pool.is_none() || std::env::var_os("ENGRAM_TEST_UBLK").is_none() { + eprintln!("SKIP: ENGRAM_TEST_THIN_POOL or ENGRAM_TEST_UBLK is not set"); + return None; + } + pool +} + +fn fill(byte: u8) -> Vec { + vec![byte; CHUNK as usize] +} + +fn store(dir: &Path) -> ChunkStore { + ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.join("blobs")))) +} + +/// Session volumes over the pool, with the host's ublk sources. +async fn volumes(pool: &str, store: ChunkStore) -> SessionVolumes { + let disk = ThinPoolDisk::open(pool, CHUNK, Path::new("/dev/engram-test")) + .await + .expect("open pool"); + SessionVolumes::new(Arc::new(disk), FillEngine::new(store, Default::default())) + .with_sources(Arc::new(UblkSources)) +} + +/// `PATTERN` as a manifest in `store`. +async fn manifest_in(store: &ChunkStore) -> ManifestRef { + let mut chunks = Vec::new(); + for (i, byte) in PATTERN.iter().enumerate() { + if *byte != 0 { + chunks.push(ChunkRef { + offset: i as u64 * CHUNK, + hash: store.put_chunk(&fill(*byte)).await.unwrap(), + }); + } + } + let manifest = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks, + ..Manifest::empty(ManifestKind::Disk, PATTERN.len() as u64 * CHUNK) + }; + let manifest_ref = ManifestRef::new(); + store.put_manifest(manifest_ref, &manifest).await.unwrap(); + manifest_ref +} + +async fn settled(volumes: &SessionVolumes, sandbox: SandboxId) { + let deadline = Instant::now() + SETTLE_DEADLINE; + loop { + match volumes.settle(sandbox, DiskRole::Root).await.unwrap() { + Hydration::Settled => return, + Hydration::Hydrating { + hydrated, total, .. + } => { + assert!( + Instant::now() < deadline, + "the volume did not settle: {hydrated} of {total} regions" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + } +} + +async fn assert_disk(volumes: &SessionVolumes, sandbox: SandboxId, expected: &[u8]) { + let live = volumes.live(sandbox, DiskRole::Root).await.unwrap(); + for (i, byte) in expected.iter().enumerate() { + assert_eq!( + live.read_chunk(ChunkIndex(i as u64)).await.unwrap(), + fill(*byte), + "chunk {i}" + ); + } +} + +async fn dmsetup(args: &[&str]) { + let status = tokio::process::Command::new("dmsetup") + .args(args) + .status() + .await + .unwrap(); + assert!(status.success(), "dmsetup {args:?}"); +} + +/// The device name of the sandbox's live root volume in the pool. +fn live_device(pool: &str, sandbox: SandboxId) -> String { + format!("{pool}.{}", live_name(sandbox, DiskRole::Root)) +} + +#[tokio::test] +#[ignore = "requires root, a dm-thin pool and the ublk_drv module"] +async fn a_lazy_resume_reads_cold_chunks_through_the_source_and_settles() { + let Some(pool) = gate() else { return }; + let dir = tempfile::tempdir().unwrap(); + let store = store(dir.path()); + let manifest = manifest_in(&store).await; + let volumes = volumes(&pool, store).await; + let sandbox = SandboxId::new(); + + let provisioned = volumes + .provision( + sandbox, + Source::Continue { + manifest, + base: None, + }, + None, + ) + .await + .unwrap(); + assert_eq!(provisioned.root_stage, StagePath::Lazy); + assert_eq!(volumes.sources().len(), 1, "one source serves the root"); + + // The last chunk is the last the ascending copy reaches: read now, it + // comes through the source. A write to a hole lands on the device. + let live = volumes.live(sandbox, DiskRole::Root).await.unwrap(); + assert_eq!(live.read_chunk(ChunkIndex(7)).await.unwrap(), fill(8)); + live.write_chunk(ChunkIndex(2), fill(9)).await.unwrap(); + drop(live); + + settled(&volumes, sandbox).await; + assert!(volumes.hydrating(sandbox).await.unwrap().is_empty()); + assert!(volumes.sources().is_empty(), "the source stopped"); + assert_disk(&volumes, sandbox, &[1, 2, 9, 4, 0, 0, 7, 8]).await; + + volumes.release(sandbox).await.unwrap(); + let prefix = format!("sb-{sandbox}-"); + assert!(!volumes + .disk() + .list() + .await + .unwrap() + .iter() + .any(|n| n.as_str().starts_with(&prefix))); +} + +/// The first half of the recovery test, in a child process: provisions +/// the sandbox lazily, suspends its live device so the copy halts, and +/// waits to be killed. +#[tokio::test] +#[ignore = "a helper the recovery test runs as a child process"] +async fn child_provisions_and_waits() { + let Some(dir) = std::env::var_os("ENGRAM_LAZY_CHILD_DIR") else { + return; + }; + let dir = PathBuf::from(dir); + let pool = std::env::var("ENGRAM_TEST_THIN_POOL").unwrap(); + let store = store(&dir); + let manifest: ManifestRef = + serde_json::from_slice(&std::fs::read(dir.join("manifest.json")).unwrap()).unwrap(); + let volumes = volumes(&pool, store).await; + let sandbox = SandboxId::new(); + let provisioned = volumes + .provision( + sandbox, + Source::Continue { + manifest, + base: None, + }, + None, + ) + .await + .unwrap(); + assert_eq!(provisioned.root_stage, StagePath::Lazy); + dmsetup(&["suspend", &live_device(&pool, sandbox)]).await; + std::fs::write(dir.join("sandbox"), sandbox.to_string()).unwrap(); + std::future::pending::<()>().await; +} + +/// A host-agent killed mid-copy: the next host-agent cannot delete the +/// hydrating volume until it serves the source again, then settles it. +#[tokio::test] +#[ignore = "requires root, a dm-thin pool and the ublk_drv module"] +async fn a_killed_process_leaves_a_volume_the_next_one_serves_again() { + let Some(pool) = gate() else { return }; + let dir = tempfile::tempdir().unwrap(); + let store = store(dir.path()); + let manifest = manifest_in(&store).await; + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "child_provisions_and_waits", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("ENGRAM_LAZY_CHILD_DIR", dir.path()) + .spawn() + .unwrap(); + let sandbox_file = dir.path().join("sandbox"); + let deadline = Instant::now() + Duration::from_secs(30); + while !sandbox_file.exists() { + assert!(Instant::now() < deadline, "the child did not provision"); + assert!(child.try_wait().unwrap().is_none(), "the child exited"); + std::thread::sleep(Duration::from_millis(50)); + } + let sandbox: SandboxId = std::fs::read_to_string(&sandbox_file) + .unwrap() + .trim() + .parse() + .unwrap(); + nix::sys::signal::kill( + nix::unistd::Pid::from_raw(child.id() as i32), + nix::sys::signal::Signal::SIGKILL, + ) + .unwrap(); + child.wait().unwrap(); + + let volumes = volumes(&pool, store).await; + let refused = volumes.release(sandbox).await.unwrap_err(); + assert_eq!( + refused.kind(), + std::io::ErrorKind::ResourceBusy, + "{refused}" + ); + assert_eq!( + volumes.reattach_all().await.unwrap(), + vec![(sandbox, DiskRole::Root)] + ); + assert!(volumes.reattach_all().await.unwrap().is_empty()); + dmsetup(&["resume", &live_device(&pool, sandbox)]).await; + + settled(&volumes, sandbox).await; + assert_disk(&volumes, sandbox, &PATTERN).await; + assert!(volumes.sources().is_empty()); + volumes.release(sandbox).await.unwrap(); +} diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index f75bee325..a4f3493b3 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -894,6 +894,35 @@ impl SessionVolumes { Ok(roles) } + /// [`Self::reattach`] for every sandbox with a hydrating role, live + /// or not: a volume whose source nothing serves cannot be deleted, + /// so this runs before the startup reconcile. + pub async fn reattach_all(&self) -> io::Result> { + let mut sandboxes: Vec = Vec::new(); + for p in self.disk.list().await?.iter().filter_map(parse) { + if matches!(p.kind, Kind::Dest(_)) && !sandboxes.contains(&p.sandbox) { + sandboxes.push(p.sandbox); + } + } + let mut reattached = Vec::new(); + for sandbox in sandboxes { + for role in self.reattach(sandbox).await? { + reattached.push((sandbox, role)); + } + } + Ok(reattached) + } + + /// The devices of the sources this process serves. + pub fn sources(&self) -> Vec { + self.serving + .lock() + .unwrap() + .values() + .map(|s| s.path().to_path_buf()) + .collect() + } + /// The roles of `sandbox` brought up on a hydrating device whose /// first checkpoint has not run: they have a `dest` volume. async fn lazy_roles(&self, sandbox: SandboxId) -> io::Result> { diff --git a/crates/engram-protocol/src/wire.rs b/crates/engram-protocol/src/wire.rs index c76d104ba..e13e5a11b 100644 --- a/crates/engram-protocol/src/wire.rs +++ b/crates/engram-protocol/src/wire.rs @@ -185,8 +185,10 @@ pub const WIRE_VERSION: u32 = 38; /// build that reports nothing counts as 0. 2: the uffd handler speaks only /// substrate protocol version 2 and later, a post-copy `DrainDone` carries /// the hot-set fields, a snapshot create always sends `defer_sync`, and a -/// migration item always carries its hash. -pub const REATTACH_GENERATION: u32 = 2; +/// migration item always carries its hash. 3: a session disk hydrates +/// behind the running guest from a ublk device the host-agent serves, +/// which an earlier build cannot serve again after a restart. +pub const REATTACH_GENERATION: u32 = 3; /// gRPC metadata (header) key carrying the caller's [`WIRE_VERSION`] on /// every coord→host request (issue #229). ASCII, lowercase — tonic diff --git a/deploy/helm/engram-host-fleet/files/thin-pool.sh b/deploy/helm/engram-host-fleet/files/thin-pool.sh index 18ad84993..529e10b6d 100755 --- a/deploy/helm/engram-host-fleet/files/thin-pool.sh +++ b/deploy/helm/engram-host-fleet/files/thin-pool.sh @@ -31,6 +31,10 @@ DATA="$POOL_DIR/data.img" log() { echo "[thin-pool] $*"; } +# The host agent serves the source a resuming session's disk fills from +# as a ublk device; the module is loaded on every start, pool or not. +modprobe ublk_drv 2>/dev/null || true + if dmsetup info "$POOL_NAME" >/dev/null 2>&1; then log "$POOL_NAME is active" dmsetup status "$POOL_NAME" From 470e90694110fc43663bf06b22037d30a3052bcf Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 12:27:40 -0800 Subject: [PATCH 06/35] host-core, disk-thin: checkpoints of a hydrating volume A checkpoint never reads through the source and never loses a write made during the window. The snapshot is taken as before, under the guest pause; the thin-pool disk now snapshots a hydrating or settled device by the thin under it (the clone or linear table's destination), with the device itself suspended so the guest's writes and the kernel's copy both rest. The upload follows one rule, `upload_hydrating`: the volume logically holds the destination's manifest, and a block whose mapping still equals the base's is one the copy has not reached, holding the base's bytes where the manifest's belong. So a block whose mapping changed (hydrated or written) is read from the snapshot and any other is the manifest's: a chunk with no changed block keeps its entry unread, one changed throughout is read whole, and a mixed one is the manifest's chunk with the changed blocks laid over it. Mapping, not content, is the test, so a guest write of bytes equal to the base's is a change. `upload` and `upload_hydrating` share the fan-out that puts chunks. The names stay the record. While the device hydrates, the destination takes the published manifest's name and the snapshot goes, so a restarted host-agent rebuilds the plan from the manifest the disk now holds and no checkpoint volume exists before the settle. Once settled, the snapshot becomes the checkpoint volume and the destination and base names go: the normal layout. `checkpoint_of` answers with the destination's manifest while there is no checkpoint volume, so `take`, `pending` and the publish redrive need no new case. Tests: the composition rule on a mixed chunk over a half-chunk block (the file disk's content diff stands in for the mapping diff), a checkpoint during the window and one after the settle through the fake hydrating disk, and in the root lane a snapshot of the hydrating device that diffs against the base. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-disk-thin/src/disk.rs | 35 ++- crates/engram-disk-thin/src/parse.rs | 36 +++ crates/engram-disk-thin/tests/clone.rs | 15 ++ crates/engram-host-core/src/disk_flows.rs | 166 ++++++++++++- .../engram-host-core/src/session_volumes.rs | 226 +++++++++++++++--- crates/engram-host-core/src/test_support.rs | 9 +- 6 files changed, 441 insertions(+), 46 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index eb97ada18..6ff15412a 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -153,6 +153,39 @@ impl ThinPoolDisk { } } + /// The thin a volume's data lives on, and its size: the volume + /// itself for a thin, the destination for a hydrating or settled + /// device. A snapshot of the volume is a snapshot of that thin. + async fn thin_under(&self, name: &VolumeName) -> io::Result<(ThinParams, u64)> { + let target = self.target_of(name).await?; + let devno = match target.kind.as_str() { + "thin" => return self.thin_of(name).await, + "clone" => parse::clone_dest(&target.params)?, + "linear" => parse::linear_device(&target.params)?, + kind => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {kind} device, which this disk did not make"), + )) + } + }; + let prefix = format!("{}.", self.pool); + let dest = self + .dm + .list() + .await? + .into_iter() + .find(|(_, d)| *d == devno) + .and_then(|(device, _)| VolumeName::new(device.strip_prefix(&prefix)?.to_string()).ok()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("{name} maps onto {devno}, which is not a volume of this pool"), + ) + })?; + self.thin_of(&dest).await + } + /// The thin behind `name` and its size, refused when it belongs to /// another pool. async fn thin_of(&self, name: &VolumeName) -> io::Result<(ThinParams, u64)> { @@ -383,7 +416,7 @@ impl SessionDisk for ThinPoolDisk { } async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { - let (thin, sectors) = self.thin_of(&origin.name).await?; + let (thin, sectors) = self.thin_under(&origin.name).await?; let pool = self.pool.clone(); let next_id = self.next_id.clone(); let (created, resumed) = self diff --git a/crates/engram-disk-thin/src/parse.rs b/crates/engram-disk-thin/src/parse.rs index ddde1d712..ae44a60e0 100644 --- a/crates/engram-disk-thin/src/parse.rs +++ b/crates/engram-disk-thin/src/parse.rs @@ -148,6 +148,22 @@ pub fn clone_source(params: &str) -> io::Result { } } +/// The destination device of a `clone` table. +pub fn clone_dest(params: &str) -> io::Result { + match params.split_whitespace().collect::>().as_slice() { + [_, dest, ..] => DevNo::parse(dest), + _ => Err(invalid(format!("not clone params: {params:?}"))), + } +} + +/// The device of a `linear` table: ` `. +pub fn linear_device(params: &str) -> io::Result { + match params.split_whitespace().collect::>().as_slice() { + [device, _] => DevNo::parse(device), + _ => Err(invalid(format!("not linear params: {params:?}"))), + } +} + /// The `DEVNAME=` line of a sysfs `uevent` file. pub fn devname(uevent: &str) -> io::Result { uevent @@ -280,6 +296,26 @@ mod tests { } } + #[test] + fn clone_dest_is_the_second_device_and_linear_the_only_one() { + let clone = "252:3 252:4 259:0 128 2 no_hydration no_discard_passdown 0"; + assert_eq!( + clone_dest(clone).unwrap(), + DevNo { + major: 252, + minor: 4 + } + ); + assert_eq!( + linear_device("252:4 0").unwrap(), + DevNo { + major: 252, + minor: 4 + } + ); + assert!(linear_device("252:4").is_err()); + } + #[test] fn clone_source_is_the_third_device() { assert_eq!( diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs index 48ea1f151..87f620ae9 100644 --- a/crates/engram-disk-thin/tests/clone.rs +++ b/crates/engram-disk-thin/tests/clone.rs @@ -128,11 +128,26 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati state => panic!("hydration did not finish: {state:?}"), } } + // A snapshot of the hydrating device is a snapshot of its + // destination, under the device's suspend; it diffs against the base. + let mid = pool.snapshot(&device, &name("mid")).await.unwrap(); + assert_eq!( + pool.dirty_blocks(&base, &mid, CHUNK).await.unwrap(), + vec![1, 3, 5] + ); + assert_eq!(mid.read_chunk(ChunkIndex(5)).await.unwrap(), fill(5)); + let mid = mid.name.clone(); + pool.delete(&mid).await.unwrap(); + pool.settle(&device.name, &dest).await.unwrap(); assert_eq!( pool.hydration(&device.name).await.unwrap(), Hydration::Settled ); + let after = pool.snapshot(&device, &name("after")).await.unwrap(); + assert_eq!(after.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); + let after = after.name.clone(); + pool.delete(&after).await.unwrap(); assert!(!pool.list().await.unwrap().contains(&name("io-md"))); assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); diff --git a/crates/engram-host-core/src/disk_flows.rs b/crates/engram-host-core/src/disk_flows.rs index e587e29a8..57cfafb67 100644 --- a/crates/engram-host-core/src/disk_flows.rs +++ b/crates/engram-host-core/src/disk_flows.rs @@ -2,15 +2,18 @@ //! manifest into a volume, [`hydration_plan`] says what a kernel copy //! does instead, [`checkpoint`] turns a volume into a manifest. +use std::collections::{BTreeMap, HashMap}; use std::io; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestDelta, ManifestKind}; +use engram_chunk_store::{ + ChunkHash, ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestDelta, ManifestKind, +}; use engram_fill::{Class, FillEngine}; -use crate::disk::{is_zero_chunk, ChunkIndex, SessionDisk, Volume, VolumeName}; +use crate::disk::{blocks_per_chunk, is_zero_chunk, ChunkIndex, SessionDisk, Volume, VolumeName}; use tracing::Instrument as _; /// Store calls in flight; peak buffered memory is this many chunks. @@ -282,20 +285,96 @@ pub mod checkpoint_split { } }; + let chunks = indices.into_iter().map(|index| { + let snapshot = snapshot.clone(); + (index, async move { snapshot.read_chunk(index).await }) + }); + let delta = put_chunks(store, snapshot.chunk_size, chunks).await?; + Ok(base_manifest.apply(&delta)) + } + + /// Outside the pause, for a snapshot of a hydrating volume. The + /// volume logically holds `wanted`; a block whose mapping still + /// equals `base`'s is one the copy has not reached, and holds the + /// base's bytes where `wanted`'s belong. So a block whose mapping + /// changed since `base` (hydrated or written) is read from the + /// snapshot and any other block is `wanted`'s: a chunk with no + /// changed block keeps `wanted`'s entry unread, one changed + /// throughout is read whole, and a mixed one is `wanted`'s chunk + /// with the changed blocks laid over it. `block_bytes` is the + /// mapping unit; it divides the chunk size. + pub async fn upload_hydrating( + disk: &dyn SessionDisk, + fill: &FillEngine, + snapshot: &Volume, + base: &Volume, + wanted: &Manifest, + block_bytes: u64, + ) -> io::Result { + shape_matches(snapshot, wanted)?; + let per_chunk = blocks_per_chunk(snapshot, block_bytes)?; let chunk_size = snapshot.chunk_size; - // One task per chunk, so the 16 MiB read, the zero check, the - // SHA-256 and the PUT's TLS spread over every core. + let mut changed: BTreeMap> = BTreeMap::new(); + for block in disk.dirty_blocks(base, snapshot, block_bytes).await? { + changed + .entry(block / per_chunk) + .or_default() + .push((block % per_chunk) * block_bytes); + } + let wanted_hashes: HashMap = + wanted.chunks.iter().map(|c| (c.offset, c.hash)).collect(); + let chunks = changed.into_iter().map(|(chunk, blocks)| { + let index = ChunkIndex(chunk); + let offset = chunk * chunk_size; + let (snapshot, fill) = (snapshot.clone(), fill.clone()); + let hash = wanted_hashes.get(&offset).copied(); + (index, async move { + if blocks.len() as u64 == per_chunk { + return snapshot.read_chunk(index).await; + } + let mut bytes = match hash { + Some(hash) => fill + .read(hash, Class::Hydrate) + .await + .map_err(store_err)? + .to_vec(), + None => vec![0; snapshot.read_chunk(index).await?.len()], + }; + for at in blocks { + let end = (at + block_bytes).min(bytes.len() as u64); + let part = snapshot.read_range(offset + at, end - at).await?; + bytes[at as usize..end as usize].copy_from_slice(&part); + } + Ok(bytes) + }) + }); + let delta = put_chunks(fill.store(), chunk_size, chunks).await?; + Ok(wanted.apply(&delta)) + } + + /// Put the chunks `chunks` yields, each a chunk index and the future + /// of its bytes, and return the delta they make: a chunk of zeros is + /// a hole. One task per chunk, so the 16 MiB read, the zero check, + /// the SHA-256 and the PUT's TLS spread over every core. + async fn put_chunks( + store: &ChunkStore, + chunk_size: u64, + chunks: impl Iterator, + ) -> io::Result + where + F: std::future::Future>> + Send + 'static, + { let mut tasks = tokio::task::JoinSet::new(); - let mut results: Vec<(u64, Option)> = Vec::with_capacity(indices.len()); - for index in indices { + let mut results: Vec<(u64, Option)> = Vec::new(); + for (index, bytes) in chunks { while tasks.len() >= CONCURRENCY { let done = tasks.join_next().await.expect("tasks is not empty"); results.push(done.map_err(io::Error::other)??); } - let (snapshot, store) = (snapshot.clone(), store.clone()); + let store = store.clone(); tasks.spawn(async move { let offset = index.0 * chunk_size; - let bytes = snapshot.read_chunk(index).await?; + let bytes = bytes.await?; if is_zero_chunk(&bytes) { return Ok::<_, io::Error>((offset, None)); } @@ -309,7 +388,6 @@ pub mod checkpoint_split { while let Some(done) = tasks.join_next().await { results.push(done.map_err(io::Error::other)??); } - let mut delta = ManifestDelta::default(); for (offset, written) in results { match written { @@ -317,7 +395,7 @@ pub mod checkpoint_split { None => delta.zero.push(offset), } } - Ok(base_manifest.apply(&delta)) + Ok(delta) } } @@ -495,6 +573,74 @@ mod tests { assert!(hydration_plan(&wanted, Some(&other_shape)).is_err()); } + /// Base [1, 2, 3, 0]; wanted [7, 8, 3, 9]. The snapshot of the + /// hydrating volume holds chunk 0 hydrated (7), the first half of + /// chunk 1 written (5) with the rest not yet copied (still 2), chunk + /// 2 held, and chunk 3 not yet copied (still 0). The published disk + /// is the wanted one with the write laid over it. + #[tokio::test] + async fn a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk() { + let r = rig(); + let fill_engine = engram_fill::FillEngine::new(r.store.clone(), Default::default()); + let base = r.disk.create(&name("base"), CHUNK * 4).await.unwrap(); + for (i, byte) in [1u8, 2, 3].into_iter().enumerate() { + base.write_chunk(ChunkIndex(i as u64), fill(byte)) + .await + .unwrap(); + } + let mut wanted_chunks = Vec::new(); + for (i, byte) in [7u8, 8, 3, 9].into_iter().enumerate() { + wanted_chunks.push(ChunkRef { + offset: i as u64 * CHUNK, + hash: r.store.put_chunk(&fill(byte)).await.unwrap(), + }); + } + let wanted = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: wanted_chunks, + ..Manifest::empty(ManifestKind::Disk, CHUNK * 4) + }; + let snapshot = r.disk.snapshot(&base, &name("snap")).await.unwrap(); + snapshot.write_chunk(ChunkIndex(0), fill(7)).await.unwrap(); + snapshot + .write_range(CHUNK, vec![5u8; (CHUNK / 2) as usize]) + .await + .unwrap(); + + let published = checkpoint_split::upload_hydrating( + &r.disk, + &fill_engine, + &snapshot, + &base, + &wanted, + CHUNK / 2, + ) + .await + .unwrap(); + let mut chunk_1 = vec![5u8; (CHUNK / 2) as usize]; + chunk_1.extend(vec![8u8; (CHUNK / 2) as usize]); + let expected = [fill(7), chunk_1, fill(3), fill(9)]; + assert_eq!(published.chunks.len(), 4); + for (i, bytes) in expected.iter().enumerate() { + assert_eq!( + published.chunks[i], + ChunkRef { + offset: i as u64 * CHUNK, + hash: ChunkHash::of(bytes) + }, + "chunk {i}" + ); + } + assert_eq!( + fill_engine + .read(published.chunks[1].hash, Class::Hydrate) + .await + .unwrap() + .to_vec(), + expected[1] + ); + } + #[tokio::test] async fn stage_refuses_a_manifest_whose_shape_differs_from_its_base() { let r = rig(); diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index a4f3493b3..8e35ab74c 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -1085,6 +1085,16 @@ impl SessionVolumes { /// Outside the pause: upload what changed since the last checkpoint, /// publish the manifest at the lineage's next free version, and make /// the snapshot the sandbox's checkpoint volume. + /// + /// A role with no checkpoint volume is on a hydrating device, and + /// `previous` is its destination's manifest: the upload lays the + /// blocks that changed since the base over it + /// ([`checkpoint_split::upload_hydrating`]). While the device still + /// hydrates, the destination takes the published manifest's name + /// and the snapshot goes, so the names stay the record and no + /// checkpoint volume exists before the settle; once settled, the + /// snapshot becomes the checkpoint volume and the destination and + /// base names go with the layout they described. pub async fn publish(&self, pending: PendingCheckpoint) -> io::Result { let PendingCheckpoint { sandbox, @@ -1092,22 +1102,47 @@ impl SessionVolumes { snapshot, previous, } = pending; + let previous_manifest = self.manifest(previous).await?; let previous_name = checkpoint_name(sandbox, role, previous); - let previous_volume = self.disk.open(&previous_name).await?; - let previous_manifest = self - .fill - .store() - .get_manifest(previous) - .await - .map_err(store_err)?; - let manifest = checkpoint_split::upload( - self.disk.as_ref(), - self.fill.store(), - &snapshot, - Some((&previous_volume, &previous_manifest)), - ) - .await?; - drop(previous_volume); + let (manifest, lazy) = match self.disk.open(&previous_name).await { + Ok(previous_volume) => { + let manifest = checkpoint_split::upload( + self.disk.as_ref(), + self.fill.store(), + &snapshot, + Some((&previous_volume, &previous_manifest)), + ) + .await?; + (manifest, None) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let (dest, base) = self.lazy_layout(sandbox, role).await?; + if parse(&dest).map(|p| p.kind) != Some(Kind::Dest(previous)) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{dest} is not the destination for {previous}"), + )); + } + let block_bytes = self.disk.region_bytes().ok_or_else(|| { + io::Error::new( + io::ErrorKind::Unsupported, + format!("{dest} exists on a disk without a hydrating device"), + ) + })?; + let base_volume = self.disk.open(&base).await?; + let manifest = checkpoint_split::upload_hydrating( + self.disk.as_ref(), + &self.fill, + &snapshot, + &base_volume, + &previous_manifest, + block_bytes, + ) + .await?; + (manifest, Some((dest, base))) + } + Err(e) => return Err(e), + }; // The store, not the volume name, says which versions exist: a // crash after `put_manifest` and before the rename leaves a // version the name never reached. @@ -1129,10 +1164,26 @@ impl SessionVolumes { .map_err(store_err)?; let tmp = snapshot.name.clone(); drop(snapshot); - self.disk - .rename(&tmp, &checkpoint_name(sandbox, role, next)) - .await?; - self.disk.delete(&previous_name).await?; + let published = checkpoint_name(sandbox, role, next); + match lazy { + None => { + self.disk.rename(&tmp, &published).await?; + self.disk.delete(&previous_name).await?; + } + Some((dest, base)) => match self.disk.hydration(&live_name(sandbox, role)).await? { + Hydration::Hydrating { .. } => { + self.disk + .rename(&dest, &dest_name(sandbox, role, next)) + .await?; + self.disk.delete(&tmp).await?; + } + Hydration::Settled => { + self.disk.rename(&tmp, &published).await?; + self.disk.delete(&dest).await?; + self.disk.delete(&base).await?; + } + }, + } Ok(next) } @@ -1346,27 +1397,28 @@ impl SessionVolumes { } } - /// The manifest of the sandbox's checkpoint volume: what the last - /// publish made durable. + /// The manifest the role's live volume holds: its checkpoint + /// volume's, or, while it hydrates with no checkpoint volume yet, + /// its destination's. pub async fn checkpoint_of( &self, sandbox: SandboxId, role: DiskRole, ) -> io::Result { - let mut found: Option = None; - for name in self.disk.list().await? { - if let Some(Parsed { - sandbox: s, - role: r, - kind: Kind::Checkpoint(m), - }) = parse(&name) - { - if s == sandbox && r == role && found.is_none_or(|f| m.version > f.version) { - found = Some(m); + let (mut checkpoint, mut dest): (Option, Option) = (None, None); + for p in self.disk.list().await?.iter().filter_map(parse) { + if (p.sandbox, p.role) != (sandbox, role) { + continue; + } + match p.kind { + Kind::Checkpoint(m) if checkpoint.is_none_or(|c| m.version > c.version) => { + checkpoint = Some(m) } + Kind::Dest(m) => dest = Some(m), + _ => {} } } - found.ok_or_else(|| { + checkpoint.or(dest).ok_or_else(|| { io::Error::new( io::ErrorKind::NotFound, format!("sandbox {sandbox} has no {} checkpoint volume", role), @@ -2678,6 +2730,116 @@ mod tests { ); } + /// Base [1, 2, 3, 0, 5], wanted [1, 9, 0, 4, 5]: chunks 1 and 3 + /// copy in, chunk 2 is zeroed. Before the copy the guest writes 7 + /// over the zeroed chunk and 8 over held chunk 4. A checkpoint then + /// publishes the wanted disk with the writes laid over it, under + /// the destination's name: no checkpoint volume exists until the + /// settle. The next checkpoint, after the settle, makes the normal + /// layout. + #[tokio::test] + async fn a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3, 0, 5]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + write_byte(&p.root, 1, 9).await; + write_byte(&p.root, 2, 0).await; + write_byte(&p.root, 3, 4).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + let p = r + .volumes + .provision( + resumed, + Source::Continue { + manifest: wanted, + base: Some(image), + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert_eq!( + r.volumes + .checkpoint_of(resumed, DiskRole::Root) + .await + .unwrap(), + wanted, + "the live volume holds the destination's manifest" + ); + write_byte(&p.root, 2, 7).await; + write_byte(&p.root, 4, 8).await; + + let mid = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + assert_eq!( + (mid.manifest_id, mid.version), + (wanted.manifest_id, wanted.version + 1) + ); + assert_eq!(manifest_bytes(&r, mid).await, vec![1, 9, 7, 4, 8]); + let expected: std::collections::BTreeSet = [ + image_volumes::volume_name(image).to_string(), + live_name(resumed, DiskRole::Root).to_string(), + base_name(resumed, DiskRole::Root, Some(image)).to_string(), + dest_name(resumed, DiskRole::Root, mid).to_string(), + ] + .into_iter() + .collect(); + assert_eq!( + names(&r).await, + expected.into_iter().collect::>(), + "the destination carries the published manifest; no checkpoint volume yet" + ); + assert_eq!( + r.volumes + .checkpoint_of(resumed, DiskRole::Root) + .await + .unwrap(), + mid + ); + + settled(&r, resumed, DiskRole::Root).await; + assert_eq!(bytes_at(&p.root).await, vec![1, 9, 7, 4, 8]); + let after = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + assert_eq!(after.version, mid.version + 1); + assert_eq!(manifest_bytes(&r, after).await, vec![1, 9, 7, 4, 8]); + let expected: std::collections::BTreeSet = [ + image_volumes::volume_name(image).to_string(), + live_name(resumed, DiskRole::Root).to_string(), + checkpoint_name(resumed, DiskRole::Root, after).to_string(), + ] + .into_iter() + .collect(); + assert_eq!( + names(&r).await, + expected.into_iter().collect::>(), + "a settled role has the normal layout" + ); + r.volumes.release(resumed).await.unwrap(); + } + + /// The first byte of each chunk of a published manifest; 0 for a + /// hole. + async fn manifest_bytes(r: &Rig, manifest: ManifestRef) -> Vec { + let fill = engram_fill::FillEngine::new(r.store.clone(), engram_fill::Config::default()); + let manifest = r.store.get_manifest(manifest).await.unwrap(); + let mut out = Vec::new(); + for i in 0..manifest.expected_chunk_count() { + out.push(match manifest.chunk_at(i * CHUNK) { + Some(c) => fill + .read(c.hash, engram_fill::Class::Hydrate) + .await + .unwrap()[0], + None => 0, + }); + } + out + } + /// A release during the window deletes the device before its /// source; the source refuses the other order. #[tokio::test] diff --git a/crates/engram-host-core/src/test_support.rs b/crates/engram-host-core/src/test_support.rs index 42e49b516..29c704b9b 100644 --- a/crates/engram-host-core/src/test_support.rs +++ b/crates/engram-host-core/src/test_support.rs @@ -76,7 +76,10 @@ pub(crate) struct HydratingDisk { struct Clone { source: PathBuf, - dest: PathBuf, + /// The device's own path, a hard link of the destination: the copy + /// writes through it, as the kernel writes through the destination's + /// device number, so a rename of the destination changes nothing. + device: PathBuf, pending: Vec, total: u64, settled: bool, @@ -150,7 +153,7 @@ impl SessionDisk for HydratingDisk { name.clone(), Clone { source: source.to_path_buf(), - dest: dest.path.clone(), + device: path.clone(), pending, total: dest.chunk_count(), settled: false, @@ -172,7 +175,7 @@ impl SessionDisk for HydratingDisk { File::open(&clone.source)?.read_exact_at(&mut bytes, chunk.0 * self.chunk_size)?; File::options() .write(true) - .open(&clone.dest)? + .open(&clone.device)? .write_all_at(&bytes, chunk.0 * self.chunk_size)?; clone.pending.remove(0); } From 6f77201041d3ef000857eb315c5840d22193a467 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 13:10:08 -0800 Subject: [PATCH 07/35] host-core, host-agent: disk first-touch prefix The kernel copies a hydrating volume in ascending order, so the chunks the guest reads first in its window are cold until the copy reaches them, and each read waits on a demand fetch. The source already records the chunk indices of those reads in order. A capture now publishes them as the session's disk first-touch record (`disk-first-touch.v1`, the memory record's encoding, the same cap), beside the memory record. The next restore of the session reads the record with its trace lineage and hands the indices to the root volume's source. The ones the copy must bring in (`prefix_hashes`: the record restricted to the delta, as the wanted manifest's hashes; chunks the base holds and chunks the manifest leaves out need no fetch) are filled at `Prefix` without waiting, so they are resident when the guest asks for them and when the copy reaches them. `Source::Continue` carries the list; a create and a swap pass none. `SessionVolumes::first_touch` answers from the serving source, and after the source stops from what it reported, until the live volume goes: a capture after the settle still publishes what the window read. `Source` is no longer `Copy`. Tests: the prefix restricted to the delta in order; the record through the window, after the settle and after the release on the fake disk; the disk record round trip beside the memory record's publish. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-chunk-store/src/first_touch.rs | 7 + .../engram-host-agent/src/pooled_backend.rs | 81 +++++-- .../engram-host-agent/tests/lazy_restore.rs | 2 + crates/engram-host-core/src/disk_flows.rs | 44 ++++ .../engram-host-core/src/session_volumes.rs | 203 ++++++++++++++---- crates/engram-host-core/src/test_support.rs | 8 +- 6 files changed, 280 insertions(+), 65 deletions(-) diff --git a/crates/engram-chunk-store/src/first_touch.rs b/crates/engram-chunk-store/src/first_touch.rs index 09d4c4fb2..008a761c8 100644 --- a/crates/engram-chunk-store/src/first_touch.rs +++ b/crates/engram-chunk-store/src/first_touch.rs @@ -34,6 +34,13 @@ impl FirstTouch { format!("traces/{session}/first-touch-rest.v1") } + /// The store key of the root disk chunks a session read before the + /// copy of a hydrating restore reached them, in order, encoded as for + /// [`Self::key`]. The next restore fetches the ones it must copy first. + pub fn disk_key(session: Uuid) -> String { + format!("traces/{session}/disk-first-touch.v1") + } + /// The indices touched up to and including `until_ms`, in order. pub fn until(&self, until_ms: i64) -> Vec { self.entries diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index f023bb61d..e0c47a94c 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -266,7 +266,16 @@ mod first_touch_tests { entries: vec![(2, 10), (3, 20), (1, 30)], }; std::fs::write(&file, record.encode()).unwrap(); - publish_first_touch(&plain, session, &file, Some(10)).await; + publish_first_touch(&plain, session, &file, Some(10), vec![5, 2]).await; + assert_eq!( + first_touch_indices( + &plain, + session, + &engram_chunk_store::FirstTouch::disk_key(session.as_uuid()) + ) + .await, + vec![5, 2] + ); let cache = engram_chunk_store::cache::ChunkCache::new( engram_chunk_store::cache::ChunkCacheConfig { @@ -350,6 +359,19 @@ async fn read_trace_lineage(src: &Path) -> Option { serde_json::from_value(value.get("trace_lineage_id")?.clone()).ok() } +/// The chunk indices published under `key` for `session`, in touch +/// order; empty when none were, or the record is unreadable. +async fn first_touch_indices(store: &ChunkStore, session: SessionId, key: &str) -> Vec { + match store.blob_storage().get(key).await { + Ok(bytes) => engram_chunk_store::FirstTouch::decode_indices(&bytes).unwrap_or_default(), + Err(engram_core::BlobError::NotFound) => Vec::new(), + Err(e) => { + tracing::warn!(%session, key, error = %e, "first-touch prefix unreadable"); + Vec::new() + } + } +} + /// The chunk indices under `key` as this manifest's chunk hashes, in /// order. Positions the manifest leaves out are zero pages and need no /// fetch. @@ -359,14 +381,7 @@ async fn first_touch_hashes( key: &str, manifest: &engram_chunk_store::Manifest, ) -> Vec { - let indices = match store.blob_storage().get(key).await { - Ok(bytes) => engram_chunk_store::FirstTouch::decode_indices(&bytes).unwrap_or_default(), - Err(engram_core::BlobError::NotFound) => return Vec::new(), - Err(e) => { - tracing::warn!(%session, error = %e, "first-touch prefix unreadable"); - return Vec::new(); - } - }; + let indices = first_touch_indices(store, session, key).await; let chunk_size = manifest.chunk_size.as_u64(); let by_offset: std::collections::HashMap = manifest.chunks.iter().map(|c| (c.offset, c.hash)).collect(); @@ -426,15 +441,32 @@ fn split_first_touch(record: &engram_chunk_store::FirstTouch, cut_ms: i64) -> (V (prefix, rest) } -/// Publish the chunks the sandbox touched up to its first completed run -/// (all of them before one completes), for the session's next restore to -/// fetch first. Best-effort. +/// Publish the memory chunks the sandbox touched up to its first +/// completed run (all of them before one completes) and the root disk +/// chunks the guest read before a hydrating restore's copy reached them +/// (`disk`, at most [`FIRST_TOUCH_PREFIX_CAP`]), for the session's next +/// restore to fetch first. Best-effort. async fn publish_first_touch( store: &ChunkStore, session: SessionId, file: &Path, first_run_completed: Option, + mut disk: Vec, ) { + let blob = store.blob_storage(); + let encode = engram_chunk_store::FirstTouch::encode_indices; + disk.truncate(FIRST_TOUCH_PREFIX_CAP); + if !disk.is_empty() { + let key = engram_chunk_store::FirstTouch::disk_key(session.as_uuid()); + match blob.put(&key, encode(&disk).into()).await { + Ok(_) => { + tracing::info!(%session, chunks = disk.len(), "published disk first-touch prefix") + } + Err(e) => { + tracing::warn!(%session, error = %e, "publish disk first-touch prefix failed") + } + } + } let Ok(bytes) = tokio::fs::read(file).await else { return; }; @@ -446,8 +478,6 @@ async fn publish_first_touch( return; } let (prefix, rest) = split_first_touch(&record, first_run_completed.unwrap_or(i64::MAX)); - let encode = engram_chunk_store::FirstTouch::encode_indices; - let blob = store.blob_storage(); let key = engram_chunk_store::FirstTouch::key(session.as_uuid()); let rest_key = engram_chunk_store::FirstTouch::rest_key(session.as_uuid()); match tokio::try_join!( @@ -2020,7 +2050,18 @@ impl PooledBackend { let root = if fork_disk { Source::Fork { manifest, base } } else { - Source::Continue { manifest, base } + let first_touch = match (read_trace_lineage(src).await, self.chunk_store.as_ref()) { + (Some(session), Some(store)) => { + let key = engram_chunk_store::FirstTouch::disk_key(session.as_uuid()); + first_touch_indices(store, session, &key).await + } + _ => Vec::new(), + }; + Source::Continue { + manifest, + base, + first_touch, + } }; let sidecar = read_sidecar_swap(src).await?; let swap = match (metadata.swap_manifest, sidecar.swap_mib) { @@ -2038,6 +2079,7 @@ impl PooledBackend { (Some(manifest), _) => Some(Source::Continue { manifest, base: None, + first_touch: Vec::new(), }), (None, _) if !fresh && !sidecar.base_capture => { return Err(SandboxError::MemoryImageUnusable( @@ -2616,8 +2658,13 @@ impl PooledBackend { Some(FirstRun::Completed { at_ms }) => Some(*at_ms), _ => None, }; + let disk = self + .volumes + .as_ref() + .map(|v| v.first_touch(id, DiskRole::Root)) + .unwrap_or_default(); tokio::spawn(async move { - publish_first_touch(&chunk_store, session_id, &file, first_run_completed).await; + publish_first_touch(&chunk_store, session_id, &file, first_run_completed, disk).await; }); } @@ -4683,6 +4730,7 @@ impl SandboxBackend for PooledBackend { Some(manifest) => Source::Continue { manifest, base: None, + first_touch: Vec::new(), }, None => { let Some((uri, cached)) = cached.as_ref() else { @@ -4710,6 +4758,7 @@ impl SandboxBackend for PooledBackend { (Some(manifest), _) => Some(Source::Continue { manifest, base: None, + first_touch: Vec::new(), }), (None, Some(mib)) => Some(Source::Empty { size_bytes: u64::from(mib) << 20, diff --git a/crates/engram-host-agent/tests/lazy_restore.rs b/crates/engram-host-agent/tests/lazy_restore.rs index 2bc9d83ca..a04f10eff 100644 --- a/crates/engram-host-agent/tests/lazy_restore.rs +++ b/crates/engram-host-agent/tests/lazy_restore.rs @@ -137,6 +137,7 @@ async fn a_lazy_resume_reads_cold_chunks_through_the_source_and_settles() { Source::Continue { manifest, base: None, + first_touch: vec![], }, None, ) @@ -190,6 +191,7 @@ async fn child_provisions_and_waits() { Source::Continue { manifest, base: None, + first_touch: vec![], }, None, ) diff --git a/crates/engram-host-core/src/disk_flows.rs b/crates/engram-host-core/src/disk_flows.rs index 57cfafb67..a5df6b246 100644 --- a/crates/engram-host-core/src/disk_flows.rs +++ b/crates/engram-host-core/src/disk_flows.rs @@ -230,6 +230,26 @@ pub fn hydration_plan(wanted: &Manifest, base: Option<&Manifest>) -> io::Result< }) } +/// The chunks of `first_touch` (guest reads of a hydrating volume, in +/// order) that `delta` copies, as `wanted`'s hashes: what a restore +/// fetches at `Prefix` before the copy reaches them. Chunks the base +/// already holds and chunks `wanted` leaves out need no fetch. +pub(crate) fn prefix_hashes( + first_touch: &[u32], + delta: &[u64], + wanted: &Manifest, +) -> Vec { + let chunk_size = wanted.chunk_size.as_u64(); + let by_offset: HashMap = + wanted.chunks.iter().map(|c| (c.offset, c.hash)).collect(); + first_touch + .iter() + .map(|i| u64::from(*i)) + .filter(|i| delta.binary_search(i).is_ok()) + .filter_map(|i| by_offset.get(&(i * chunk_size)).copied()) + .collect() +} + /// A snapshot of `live` and the manifest describing it. With `previous` /// (the last snapshot and its manifest) only the chunks that differ are /// uploaded. The guest must be paused across [`checkpoint_split::take`]; @@ -641,6 +661,30 @@ mod tests { ); } + /// Reads of chunks 3, 0 and 4 came first; the copy brings 1 and 3, + /// so only chunk 3 is fetched ahead. + #[test] + fn the_prefix_is_the_first_touch_in_order_restricted_to_the_delta() { + let wanted = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: [1u8, 9, 0, 4, 5] + .iter() + .enumerate() + .filter(|(_, b)| **b != 0) + .map(|(i, b)| ChunkRef { + offset: i as u64 * CHUNK, + hash: ChunkHash::of(&fill(*b)), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, CHUNK * 5) + }; + assert_eq!( + prefix_hashes(&[3, 0, 4], &[1, 3], &wanted), + vec![ChunkHash::of(&fill(4))] + ); + assert!(prefix_hashes(&[], &[1, 3], &wanted).is_empty()); + } + #[tokio::test] async fn stage_refuses_a_manifest_whose_shape_differs_from_its_base() { let r = rig(); diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 8e35ab74c..d41593101 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -44,7 +44,7 @@ use uuid::Uuid; use crate::disk::{ Capacity, Hydration, SessionDisk, SourceDevice, SourceServer, Volume, VolumeName, }; -use crate::disk_flows::{checkpoint_split, hydration_plan, shape_matches, stage}; +use crate::disk_flows::{checkpoint_split, hydration_plan, prefix_hashes, shape_matches, stage}; use crate::image_volumes::{self, ImageVolumes}; const LIVE_PREFIX: &str = "sb-"; @@ -231,7 +231,7 @@ fn live_first(names: &mut [(VolumeName, Parsed)]) { /// staged on this host, the volume starts as a snapshot of it and only /// the difference comes in: behind the running guest where the platform /// has a hydrating device, in full before it starts elsewhere. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum Source { /// An image's disk: the volume starts a new lineage as a snapshot of /// the image's volume, which is staged first when the host has none @@ -244,9 +244,13 @@ pub enum Source { base: Option, }, /// The sandbox's own manifest: the volume continues its lineage. + /// `first_touch` lists the chunks the guest read before the copy + /// reached them on its last hydrating restore, in order; the ones + /// this restore must copy are fetched first. Continue { manifest: ManifestRef, base: Option, + first_touch: Vec, }, /// Zeros, in a new lineage. Empty { size_bytes: u64 }, @@ -326,6 +330,16 @@ struct Base { kept_by: Option, } +/// What a live volume this host does not hold is brought up to: the +/// manifest, the lineage it continues, the base to start from, and the +/// chunks the guest read first on the last hydrating restore. +struct Wanted<'a> { + manifest: &'a Manifest, + lineage: ManifestRef, + base: Option, + first_touch: &'a [u32], +} + /// The owner of every `sb-` volume on a host. pub struct SessionVolumes { disk: Arc, @@ -340,6 +354,9 @@ pub struct SessionVolumes { sources: Option>, /// The source of each hydrating live volume this process serves. serving: Mutex>>, + /// What each stopped source recorded as read first, kept until the + /// live volume goes. + touched: Mutex>>, } impl SessionVolumes { @@ -351,6 +368,7 @@ impl SessionVolumes { kept: Mutex::new(VecDeque::new()), sources: None, serving: Mutex::new(HashMap::new()), + touched: Mutex::new(HashMap::new()), } } @@ -480,41 +498,63 @@ impl SessionVolumes { None => { let base = self.image_base(base).await; let (live, path) = self - .bring_up(fill, sandbox, role, &wanted, lineage, base) + .bring_up( + fill, + sandbox, + role, + Wanted { + manifest: &wanted, + lineage, + base, + first_touch: &[], + }, + ) .await?; (live, lineage, path) } } } - Source::Continue { manifest, base } => { - match self.kept(sandbox, role, manifest).await? { - Some((owner, volume, _, held)) if held == manifest => { - let checkpoint = self.adopt(owner, volume, sandbox, role, held).await?; - let live = self.live_snapshot(&checkpoint, sandbox, role).await?; - (live, manifest, StagePath::Kept) - } - kept => { - let wanted = self.manifest(manifest).await?; - let base = match kept { - Some((owner, volume, held_manifest, held)) - if volume.size_bytes == wanted.total_bytes => - { - Some(Base { - volume, - manifest: held_manifest, - manifest_ref: held, - kept_by: Some(owner), - }) - } - _ => self.image_base(base).await, - }; - let (live, path) = self - .bring_up(fill, sandbox, role, &wanted, manifest, base) - .await?; - (live, manifest, path) - } + Source::Continue { + manifest, + base, + first_touch, + } => match self.kept(sandbox, role, manifest).await? { + Some((owner, volume, _, held)) if held == manifest => { + let checkpoint = self.adopt(owner, volume, sandbox, role, held).await?; + let live = self.live_snapshot(&checkpoint, sandbox, role).await?; + (live, manifest, StagePath::Kept) } - } + kept => { + let wanted = self.manifest(manifest).await?; + let base = match kept { + Some((owner, volume, held_manifest, held)) + if volume.size_bytes == wanted.total_bytes => + { + Some(Base { + volume, + manifest: held_manifest, + manifest_ref: held, + kept_by: Some(owner), + }) + } + _ => self.image_base(base).await, + }; + let (live, path) = self + .bring_up( + fill, + sandbox, + role, + Wanted { + manifest: &wanted, + lineage: manifest, + base, + first_touch: &first_touch, + }, + ) + .await?; + (live, manifest, path) + } + }, Source::Empty { size_bytes } => { let volume = self.disk.create(&tmp, size_bytes).await?; let manifest = Manifest { @@ -608,27 +648,25 @@ impl SessionVolumes { self.disk.open(&name).await } - /// The live volume of a role whose manifest `wanted` is not on this - /// host, under `lineage`: a hydrating device when this host has one - /// and serves sources, else the difference from `base` written in - /// full first. + /// The live volume of a role whose manifest is not on this host: a + /// hydrating device when this host has one and serves sources, else + /// the difference from the base written in full first. async fn bring_up( &self, fill: &FillEngine, sandbox: SandboxId, role: DiskRole, - wanted: &Manifest, - lineage: ManifestRef, - base: Option, + wanted: Wanted<'_>, ) -> io::Result<(Volume, StagePath)> { if self.hydration_support().is_some() { let live = self - .hydrate(fill, sandbox, role, wanted, lineage, base) + .hydrate(fill, sandbox, role, wanted) .instrument(tracing::info_span!( "disk.hydrate", held = tracing::field::Empty, delta = tracing::field::Empty, zero = tracing::field::Empty, + prefix = tracing::field::Empty, )) .await?; return Ok((live, StagePath::Lazy)); @@ -639,12 +677,14 @@ impl SessionVolumes { fill, Class::Demand, &tmp, - wanted, - base.as_ref().map(|b| (&b.volume, &b.manifest)), + wanted.manifest, + wanted.base.as_ref().map(|b| (&b.volume, &b.manifest)), ) .instrument(tracing::info_span!("disk.stage")) .await?; - let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + let live = self + .commit_live(&tmp, sandbox, role, wanted.lineage) + .await?; Ok((live, StagePath::Staged)) } @@ -653,17 +693,24 @@ impl SessionVolumes { /// stands in for no base) under the `dest` name, and its source /// serves `wanted` through `fill`. Chunks the base holds in final /// form are skipped; chunks `wanted` leaves out are zeroed first. + /// The `first_touch` chunks the copy must bring in are fetched at + /// `Prefix` without waiting, so the guest's first reads find them + /// resident before the copy reaches them. /// Records on the current span how many chunks the base `held`, the - /// kernel copies (`delta`) and were zeroed (`zero`). + /// kernel copies (`delta`), were zeroed (`zero`) and make the `prefix`. async fn hydrate( &self, fill: &FillEngine, sandbox: SandboxId, role: DiskRole, - wanted: &Manifest, - lineage: ManifestRef, - base: Option, + wanted: Wanted<'_>, ) -> io::Result { + let Wanted { + manifest: wanted, + lineage, + base, + first_touch, + } = wanted; let (sources, region_bytes) = self.hydration_support().ok_or_else(|| { io::Error::new(io::ErrorKind::Unsupported, "this host does not hydrate") })?; @@ -724,10 +771,24 @@ impl SessionVolumes { return Err(e); } }; + let prefix = prefix_hashes(first_touch, &plan.delta, wanted); let span = tracing::Span::current(); span.record("held", (plan.skip.len() - plan.zero.len()) as u64); span.record("delta", plan.delta.len() as u64); span.record("zero", plan.zero.len() as u64); + span.record("prefix", prefix.len() as u64); + if !prefix.is_empty() { + let fill = fill.clone(); + tokio::spawn( + async move { + match fill.fill(prefix, Class::Prefix).await { + Ok(stats) => tracing::info!(?stats, "disk first-touch prefix resident"), + Err(error) => tracing::warn!(%error, "disk first-touch prefix failed"), + } + } + .instrument(tracing::info_span!("disk.prefix", sandbox_id = %sandbox, %role)), + ); + } self.serving .lock() .unwrap() @@ -913,6 +974,22 @@ impl SessionVolumes { Ok(reattached) } + /// The chunks the guest read before the copy reached them, in + /// order: from the role's source while it serves, and after its stop + /// until the volume goes. Empty for a role that never hydrated. + pub fn first_touch(&self, sandbox: SandboxId, role: DiskRole) -> Vec { + let live = live_name(sandbox, role); + if let Some(source) = self.serving.lock().unwrap().get(&live) { + return source.touched(); + } + self.touched + .lock() + .unwrap() + .get(&live) + .cloned() + .unwrap_or_default() + } + /// The devices of the sources this process serves. pub fn sources(&self) -> Vec { self.serving @@ -968,7 +1045,13 @@ impl SessionVolumes { async fn stop_source(&self, live: &VolumeName) -> io::Result<()> { let source = self.serving.lock().unwrap().remove(live); match source { - Some(source) => source.stop().await, + Some(source) => { + self.touched + .lock() + .unwrap() + .insert(live.clone(), source.touched()); + source.stop().await + } None => Ok(()), } } @@ -1370,6 +1453,7 @@ impl SessionVolumes { report.failed.push((name.clone(), e)); return true; } + self.touched.lock().unwrap().remove(name); } report.deleted.push(name.clone()); true @@ -1821,6 +1905,7 @@ mod tests { Source::Continue { manifest: published, base: Some(image), + first_touch: vec![], }, None, ) @@ -1841,6 +1926,7 @@ mod tests { Source::Continue { manifest: published, base: None, + first_touch: vec![], }, None, ) @@ -1890,6 +1976,7 @@ mod tests { Source::Continue { manifest: published, base: None, + first_touch: vec![], }, None, Some(Arc::new(peer)), @@ -2031,6 +2118,7 @@ mod tests { Some(Source::Continue { manifest: ManifestRef::new(), base: None, + first_touch: vec![], }), ) .await @@ -2093,6 +2181,7 @@ mod tests { Source::Continue { manifest: v3, base: Some(image), + first_touch: vec![], }, None, ) @@ -2450,6 +2539,7 @@ mod tests { Source::Continue { manifest: published, base: Some(image), + first_touch: vec![], }, None, ) @@ -2491,6 +2581,7 @@ mod tests { Source::Continue { manifest: published, base: Some(image), + first_touch: vec![], }, None, ) @@ -2535,6 +2626,7 @@ mod tests { Source::Continue { manifest: next, base: Some(image), + first_touch: vec![], }, None, ) @@ -2608,6 +2700,7 @@ mod tests { Source::Continue { manifest: published, base: Some(image), + first_touch: vec![], }, None, ) @@ -2671,6 +2764,7 @@ mod tests { Source::Continue { manifest: published, base: Some(image), + first_touch: vec![], }, None, ) @@ -2758,6 +2852,7 @@ mod tests { Source::Continue { manifest: wanted, base: Some(image), + first_touch: vec![], }, None, ) @@ -2772,6 +2867,11 @@ mod tests { wanted, "the live volume holds the destination's manifest" ); + assert_eq!( + r.volumes.first_touch(resumed, DiskRole::Root), + vec![1, 3], + "the source reports what the guest read first (the fake: the delta)" + ); write_byte(&p.root, 2, 7).await; write_byte(&p.root, 4, 8).await; @@ -2804,6 +2904,11 @@ mod tests { settled(&r, resumed, DiskRole::Root).await; assert_eq!(bytes_at(&p.root).await, vec![1, 9, 7, 4, 8]); + assert_eq!( + r.volumes.first_touch(resumed, DiskRole::Root), + vec![1, 3], + "the record outlives the source until the volume goes" + ); let after = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); assert_eq!(after.version, mid.version + 1); assert_eq!(manifest_bytes(&r, after).await, vec![1, 9, 7, 4, 8]); @@ -2820,6 +2925,7 @@ mod tests { "a settled role has the normal layout" ); r.volumes.release(resumed).await.unwrap(); + assert!(r.volumes.first_touch(resumed, DiskRole::Root).is_empty()); } /// The first byte of each chunk of a published manifest; 0 for a @@ -2858,6 +2964,7 @@ mod tests { Source::Continue { manifest: next, base: Some(image), + first_touch: vec![], }, None, ) @@ -2897,6 +3004,7 @@ mod tests { Source::Continue { manifest: next, base: Some(image), + first_touch: vec![], }, None, ) @@ -2942,6 +3050,7 @@ mod tests { Source::Continue { manifest: image, base: None, + first_touch: vec![], }, None, ) diff --git a/crates/engram-host-core/src/test_support.rs b/crates/engram-host-core/src/test_support.rs index 29c704b9b..1b2bc69b3 100644 --- a/crates/engram-host-core/src/test_support.rs +++ b/crates/engram-host-core/src/test_support.rs @@ -219,6 +219,8 @@ struct FileSource { path: PathBuf, holders: Arc>>, stopped: Arc, + /// Reported as read first: every delta chunk, in order. + delta: Vec, } #[async_trait::async_trait] @@ -227,7 +229,7 @@ impl SourceDevice for FileSource { &self.path } fn touched(&self) -> Vec { - Vec::new() + self.delta.clone() } async fn stop(self: Box) -> io::Result<()> { if self.holders.lock().unwrap().contains(&self.path) { @@ -273,6 +275,7 @@ impl SourceServer for FileSources { path, holders: self.holders.clone(), stopped: self.stopped.clone(), + delta: delta.iter().map(|c| *c as u32).collect(), })) } @@ -281,7 +284,7 @@ impl SourceServer for FileSources { path: &Path, _fill: FillEngine, _manifest: &Manifest, - _delta: Vec, + delta: Vec, _region_bytes: u64, ) -> io::Result> { std::fs::metadata(path)?; @@ -289,6 +292,7 @@ impl SourceServer for FileSources { path: path.to_path_buf(), holders: self.holders.clone(), stopped: self.stopped.clone(), + delta: delta.iter().map(|c| *c as u32).collect(), })) } } From 302cf07e6a251205c6e7df98c979699ec722de2c Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 13:20:18 -0800 Subject: [PATCH 08/35] host-core, host-agent: a hydrating source moves by snapshot A live move carries the disk in three tiers: the destination continues the advertised checkpoint and pulls the blocks changed since it, before and under the pause. The destination's volume now hydrates while those blocks land, which is correct as it stands: a block the pull writes changes the mapping the way a guest write does, and the destination's first checkpoint lays it over the advertised disk by the same rule. The source side has no such reference while it hydrates, or after it settles before its first checkpoint: there is no checkpoint volume, so "changed since the checkpoint" would be the whole copy plus the guest's writes, not a live move's small difference. `take_precopy` refuses with `Unsupported`, the presetup maps that to the coordinator's snapshot-move fallback, and the session moves by checkpoint and resume until a checkpoint makes the normal layout. Before this the diff failed on a missing volume and the coordinator rolled the move back. Tests: the refusal while hydrating and after the settle, and the live move possible again after a checkpoint. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .../engram-host-agent/src/pooled_backend.rs | 8 +- .../engram-host-core/src/session_volumes.rs | 78 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index e0c47a94c..08238d7d1 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -2203,7 +2203,13 @@ impl PooledBackend { .take_precopy(id, role) .instrument(tracing::info_span!("presetup.precopy_snapshot", %role)) .await - .map_err(|e| SandboxError::Snapshot(format!("{role} disk precopy: {e}")))?; + .map_err(|e| match e.kind() { + // The coordinator moves the session by snapshot. + std::io::ErrorKind::Unsupported => SandboxError::InvalidSpec(format!( + "{role} disk precopy: {e} — use snapshot-rehome" + )), + _ => SandboxError::Snapshot(format!("{role} disk precopy: {e}")), + })?; taken.push((role, pending)); let pending = &taken.last().expect("just pushed").1; let changed = volumes diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index d41593101..8c31ed6d7 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -1112,12 +1112,32 @@ impl SessionVolumes { /// move to carry before the pause. It is independent of the /// transient snapshot [`Self::take`] makes, so the capture can take /// its own beside it. A precopy left by an earlier move is replaced. + /// + /// `Unsupported` for a role with no checkpoint volume: a hydrating + /// device, or a settled one before its first checkpoint. The blocks + /// it changed since its manifest are the whole copy plus the guest's + /// writes, which is not a live move's small difference; the session + /// moves by snapshot until a checkpoint makes the normal layout. pub async fn take_precopy( &self, sandbox: SandboxId, role: DiskRole, ) -> io::Result { let previous = self.checkpoint_of(sandbox, role).await?; + match self + .disk + .open(&checkpoint_name(sandbox, role, previous)) + .await + { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("{role} disk of {sandbox} has no checkpoint volume yet"), + )) + } + Err(e) => return Err(e), + } let name = precopy_name(sandbox, role); self.delete_if_present(&name).await?; let live = self.live(sandbox, role).await?; @@ -2928,6 +2948,64 @@ mod tests { assert!(r.volumes.first_touch(resumed, DiskRole::Root).is_empty()); } + /// A live move carries the blocks changed since the checkpoint + /// volume; a role without one (hydrating, or settled before its + /// first checkpoint) refuses the precopy, and the first checkpoint + /// makes the live move possible again. + #[tokio::test] + async fn a_role_without_a_checkpoint_volume_refuses_a_live_move() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + write_byte(&p.root, 1, 9).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + r.volumes + .provision( + resumed, + Source::Continue { + manifest: wanted, + base: Some(image), + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + let refused = |e: io::Error| assert_eq!(e.kind(), io::ErrorKind::Unsupported, "{e}"); + refused( + r.volumes + .take_precopy(resumed, DiskRole::Root) + .await + .unwrap_err(), + ); + settled(&r, resumed, DiskRole::Root).await; + refused( + r.volumes + .take_precopy(resumed, DiskRole::Root) + .await + .unwrap_err(), + ); + r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + let precopy = r + .volumes + .take_precopy(resumed, DiskRole::Root) + .await + .unwrap(); + assert!(r + .volumes + .changed_blocks(&precopy, CHUNK) + .await + .unwrap() + .is_empty()); + r.volumes.discard(precopy).await.unwrap(); + r.volumes.release(resumed).await.unwrap(); + } + /// The first byte of each chunk of a published manifest; 0 for a /// hole. async fn manifest_bytes(r: &Rig, manifest: ManifestRef) -> Vec { From 11d72e691763e2f9b8a139252433c8ce284fbe80 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 13:20:43 -0800 Subject: [PATCH 09/35] docs: ADR 0125 amended for the hydrating resume; DESIGN.md session disk ADR 0125 said the dirty chunks are written before the VM starts and nothing is faulted lazily on the disk path. Decision 3 and the Resume lifecycle now describe the dm-clone device, its ublk source served from the fill engine, the discard that marks held regions copied, the settle, the checkpoint rule in the window and the restart that serves a volume again from the pool's names. The staging tradeoff no longer claims there is no lazy fallback; a new tradeoff names the userspace process on the cold-region path and the ublk stop order. A dated divergence entry carries the dev-cluster numbers (the tuned hydration rate against the kernel's defaults, the copy's start lag, udev's open after a create) and the decisions the plan left open: the reattach generation, the source that moves by snapshot, the block tiers that stay. The facts list and the DESIGN.md fact page name the tests that guard each new fact. Node-prep's `modprobe ublk_drv` landed with the restore. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- DESIGN.md | 22 ++++- docs/adr/0125-kernel-owned-session-disk.md | 107 ++++++++++++++++++--- 2 files changed, 115 insertions(+), 14 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 2d1e94f57..c3ba52db5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -230,9 +230,29 @@ the test named beside it. `thin_delta_reports_exactly_what_the_full_compare_reports`). Under a guest writing at full rate the sync and snapshot stay under two seconds however much it wrote (`the_disk_pause_is_bounded_while_the_guest_writes_at_full_rate`). +- A resume boots before its disk is copied in. The live volume is a dm-clone device over + a snapshot of the nearest base the host holds (the kept checkpoint volume, else the image + volume, else an empty volume); regions the base holds in final form are marked copied by + a discard, and the kernel copies the rest in behind the running guest from a ublk source + the host agent serves out of the fill engine. A guest read of a region the copy has not + reached goes through the source at `Demand`; the chunks the guest read first on the + session's last window are fetched at `Prefix` ahead of the copy. When every region is + copied the device is reloaded to a plain mapping of its destination and the source + stops; the path Firecracker opened never changes + (`a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination`, + `a_lazy_resume_reads_cold_chunks_through_the_source_and_settles`). A checkpoint in the + window reads nothing through the source and loses no write: a block whose mapping + changed since the base is the snapshot's, any other is the manifest's + (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, + `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`). A + host-agent restart serves every hydrating volume again from the pool's names, and never + deletes one it does not serve (`a_killed_process_leaves_a_volume_the_next_one_serves_again`). + A node needs `ublk_drv`; node-prep loads it beside `dm_thin_pool`. - A live move carries the disk in 64 KiB blocks in three tiers: the checkpoint the source advertises, the blocks a snapshot taken while the guest runs changed since it, and the - blocks the snapshot under the pause changed since that one. The pause carries only what + blocks the snapshot under the pause changed since that one. A source with no checkpoint + volume (hydrating, or settled before its first checkpoint) moves by snapshot + (`a_role_without_a_checkpoint_volume_refuses_a_live_move`). The pause carries only what the guest wrote during the precopy (`two_host_teleport_seals_only_the_residual_under_steady_writes`, `a_precopy_snapshot_lives_beside_the_one_the_capture_takes`), and the destination's diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index a74030d71..0b855c337 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -2,7 +2,9 @@ - Status: Accepted (2026-11-04). This ADR supersedes ADR 0110 (honest writes) and Decision 3 of ADR 0090 (the quarantined survivor channel), - and retires ADR 0049 (the NBD slot allocator). + and retires ADR 0049 (the NBD slot allocator). Amended 2026-11-09: + a resume hydrates the disk behind the running guest (Decision 3, the + Resume lifecycle, and the dated divergence entry). - Date: 2026-11-03 - Commit chain (branch `disk/session-disk-seam`, one PR): step 1 `ed6186b7c`..`08403960d` (the seam, `FileDisk`, `CrashDisk`, the two @@ -180,11 +182,17 @@ thin-provisioning tools it calls at run time (`thin_dump`, readiness. 3. **A session disk is a thin snapshot.** A fresh session's root disk is a snapshot of its image volume. A resumed session's root disk is - a snapshot of its base image volume with the session's dirty chunks - written in. Guest swap (ADR 0112) starts as a fresh thin volume - with no origin and is checkpointed with the root disk from then on - (see the divergences). Every volume a session owns is recorded in - its `sandbox.json`. + a snapshot of the nearest base this host holds (its kept checkpoint + volume, else the base image volume, else an empty volume) under a + dm-clone device that copies the difference to its manifest in + behind the running guest; the guest starts at once and reads a + region the copy has not reached through the device's source, a + ublk device the host-agent serves from the fill engine. Once every + region is copied the device is reloaded to a plain mapping of its + destination and the source goes. Guest swap (ADR 0112) starts as a + fresh thin volume with no origin and is checkpointed with the root + disk from then on (see the divergences). The pool's volume names + are the record of what a session owns. 4. **Firecracker opens the device path.** The guest's virtio-blk drive is a device node the host-agent makes itself under `/dev/engram/` (`mknod` from the kernel's major:minor, so no udev rule is in the @@ -251,7 +259,10 @@ thin-provisioning tools it calls at run time (`thin_dump`, /dev/engram/engram-pool.sb--root session root disk (snapshot of an image volume) /dev/engram/engram-pool.sb--swap session swap (a fresh volume at first; checkpointed like root) /dev/engram/engram-pool.sb--ckpt- a checkpoint snapshot, transient -//sandbox.json records the volume names +/dev/engram/engram-pool.sb---dest- the destination a hydrating live volume copies into +/dev/engram/engram-pool.sb---base- the base that destination is a snapshot of +/dev/ublkb the source a hydrating live volume reads cold regions from +//sandbox.json records the sandbox; the volume names are the pool's ``` The pool's metadata is the only record of which thins exist and the @@ -312,9 +323,29 @@ space falls under twice the placement floor. The eviction finalize (`eviction_finalize.rs`) keeps its shape: a host-owned job over durable artifacts, now the checkpoint snapshot instead of a frozen dirty file. -**Resume.** Decision 3. The dirty chunks are written before the VM -starts; nothing is faulted lazily from the chunk store on the disk path. -The memory path keeps its lazy base (ADR 0092). +**Resume.** Decision 3. The host snapshots the nearest base it holds +under the `dest` name, makes the live volume a dm-clone device over it +with hydration off and discard passdown off, discards every region the +base already holds in final form and every region the manifest leaves +out (a discard on the clone marks the region hydrated without a copy; +the guest never discards, so a discard always means "already correct"), +starts the source and turns hydration on with a tuned batch and +threshold, and boots the VM. The source answers a guest read of a cold +region with a ranged `Demand` read and the kernel's ascending copy +with whole-chunk `Hydrate` reads; the chunks the guest read first on +the session's last window are fetched at `Prefix` before the copy +reaches them. A settle task polls the clone's status and, when every +region is copied, suspends the device, reloads it to `linear` over the +destination, resumes it and stops the source: the path Firecracker +opened never changes. A checkpoint in the window diffs the snapshot's +mapping against the base: a block whose mapping changed (copied or +written) is the snapshot's, any other is the manifest's, so nothing is +read through the source and no write is lost; the published manifest +takes the destination's name until the settle, after which the next +checkpoint makes the normal layout. A live volume that hydrates and +whose source this process does not serve is never deleted; the next +host-agent serves it again from the names alone. The memory path keeps +its lazy base (ADR 0092). **Teleport (ADR 0123).** A live move carries the disk in three tiers. The target stages the checkpoint the source advertises at presetup. The @@ -510,11 +541,20 @@ that adds code without retiring the row it replaces is incomplete. repair use `thin_check`, `thin_dump` and `thin_repair`. `thin_check` on a 750 GiB pool takes about 4 s. Operators lose `ls` on a dirty file and gain `dmsetup status`. -- **Staging precedes the first boot on a node.** The prefetch +- **Staging an image precedes the first boot on a node.** The prefetch supervisor already downloads and pins the full base before marking an image ready, so this is the same cost moved from a cache directory to - a volume. It is listed here because it is now structural: there is no - lazy fallback. + a volume. A session's own difference from the image is not staged: + it hydrates behind the running guest. +- **A hydrating window has a userspace process on the cold-region + path.** The host-agent serves the clone's source; a guest read of a + region the copy has not reached waits on it, and on a host-agent + restart waits until the next process serves the volume again (the + kernel queues the request, under ublk user recovery). The window is + seconds long and bounded by the copy rate; after it the kernel alone + is in the path. Stopping a ublk device with a request queued and no + server blocks in the kernel until reboot, so a source is stopped only + after the device that mapped onto it is gone. ## Testing @@ -648,6 +688,31 @@ thin-pool host and the reverse. pick mirrors the Postgres in-transaction pick, which gates on RAM and CPU; the pool floor is a coordinator-side candidate filter, like the work-dir disk floor, and is covered by the placement unit test. +- **2026-11-09: the resume hydrates behind the running guest.** The + eager stage made a cold resume of an old session a function of its + difference from the staged image: 27 to 29 s of a 39 s wake for a + 31 GB root disk that shared nothing with the hourly image, and the + internal deployment's bakes move at least 40% of the disk's blocks + every hour, so a session older than an hour is the normal case. The + live volume is now a dm-clone device over a snapshot of the nearest + base, with a ublk source the host-agent serves from the fill engine + (Decision 3). Measured on a dev-cluster c3 host: dm-clone hydrates at + about 370 MiB/s with `hydration_batch_size 16` and + `hydration_threshold 128` against about 50 MiB/s with the kernel's + defaults of 1; the copy starts about 1 s after hydration is turned + on; a `dmsetup remove` right after a create meets udev's open for + about 0.2 s and is retried. Libublk's control commands run on a + thread-local ring, so every thread that stops, recovers or deletes a + device makes one first. The reattach generation goes to 3: a + pre-upgrade drain evicts every session, so no node carries a + hydrating volume across the roll in either direction. A source in + its window, or settled before its first checkpoint, moves by + snapshot: it has no checkpoint volume for a live move's block diff, + and the presetup says so. The live move's three block tiers stay; the + destination's volume hydrates while the tiers land, which the + checkpoint rule covers like a guest write. The heartbeat gains no + field and `sandbox.json` records no device: the pool's names are the + record, and a restart serves every hydrating volume again from them. ## Facts this ADR establishes @@ -679,6 +744,22 @@ names the test that guards each one. As landed, the guards are: (`a_first_checkpoint_uploads_every_nonzero_chunk_and_stage_reproduces_it`, `a_checkpoint_over_the_pool_reproduces_through_stage`, `a_checkpoint_reproduces_the_guest_disk_on_another_disk`) +- A resume boots before its disk is copied in; the guest reads a cold + region through the source and the copy settles onto the destination + under the path Firecracker opened. + (`a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination`, + `a_lazy_resume_reads_cold_chunks_through_the_source_and_settles`, + `a_lazy_resume_starts_on_a_hydrating_volume_and_settles_onto_it`) +- A checkpoint in the window reads nothing through the source and + loses no write; no checkpoint volume exists before the settle. + (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, + `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`) +- A host-agent restart serves a hydrating volume again from the pool's + names; a volume nothing serves is never deleted. + (`a_killed_process_leaves_a_volume_the_next_one_serves_again`, + `a_restart_serves_a_hydrating_volume_again`) +- A role without a checkpoint volume moves by snapshot. + (`a_role_without_a_checkpoint_volume_refuses_a_live_move`) - The heartbeat carries observed facts only; no field asks the coordinator to take an action. From 9164184a2705088cdbf48ff0232bd745c9d7e99a Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 13:55:43 -0800 Subject: [PATCH 10/35] host-core: reattach serves a role once; one failed sandbox does not stop the pass A second `reattach` of a role this process already serves asked the kernel to recover a device with a live server, which failed after libublk's 30 s recovery retry. The role is skipped while its source is in `serving`. `reattach_all` logs a sandbox whose reattach fails and goes on to the rest: its volumes stay, unserved and undeletable, and the other sandboxes come up. Found on the dev cluster by the root test that kills a process mid-window. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .../engram-host-core/src/session_volumes.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 8c31ed6d7..16122cdb0 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -925,6 +925,9 @@ impl SessionVolumes { let mut roles = Vec::new(); for role in self.lazy_roles(sandbox).await? { let live = live_name(sandbox, role); + if self.serving.lock().unwrap().contains_key(&live) { + continue; + } let Hydration::Hydrating { source, .. } = self.disk.hydration(&live).await? else { continue; }; @@ -957,7 +960,10 @@ impl SessionVolumes { /// [`Self::reattach`] for every sandbox with a hydrating role, live /// or not: a volume whose source nothing serves cannot be deleted, - /// so this runs before the startup reconcile. + /// so this runs before the startup reconcile. A role this process + /// already serves is skipped. A sandbox whose reattach fails is + /// logged and left as it is (its volumes stay, unserved and + /// undeletable) so the rest of the host comes up. pub async fn reattach_all(&self) -> io::Result> { let mut sandboxes: Vec = Vec::new(); for p in self.disk.list().await?.iter().filter_map(parse) { @@ -967,8 +973,11 @@ impl SessionVolumes { } let mut reattached = Vec::new(); for sandbox in sandboxes { - for role in self.reattach(sandbox).await? { - reattached.push((sandbox, role)); + match self.reattach(sandbox).await { + Ok(roles) => reattached.extend(roles.into_iter().map(|role| (sandbox, role))), + Err(error) => { + tracing::error!(sandbox_id = %sandbox, %error, "hydrating volume not served again"); + } } } Ok(reattached) @@ -3104,6 +3113,10 @@ mod tests { restarted.reattach(sandbox).await.unwrap(), vec![DiskRole::Root] ); + assert!( + restarted.reattach_all().await.unwrap().is_empty(), + "a role this process serves is not served twice" + ); loop { if let Hydration::Settled = restarted.settle(sandbox, DiskRole::Root).await.unwrap() { break; From 23cc8643654d9c29862c16d33cb179f847c865e2 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 13:55:43 -0800 Subject: [PATCH 11/35] disk-thin, host-agent: root tests read off the runtime and name the metadata thin The `blk_source` serve test read the device on the runtime that dispatches the source's requests; on one thread the read waits for itself, and the device keeps the read queued under user recovery, so the test process could not be killed. The reads run on a blocking thread now. The clone test expected the metadata thin as `t-clone-io-md-`; it is `-md`. Both found on the dev cluster. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-disk-thin/tests/clone.rs | 21 ++++++++++++++------ crates/engram-host-agent/tests/blk_source.rs | 15 +++++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs index 87f620ae9..e6b0bacf1 100644 --- a/crates/engram-disk-thin/tests/clone.rs +++ b/crates/engram-disk-thin/tests/clone.rs @@ -99,7 +99,11 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati .await .unwrap(); let listed = pool.list().await.unwrap(); - assert!(listed.contains(&name("io")) && listed.contains(&name("io-md"))); + let metadata = VolumeName::new(format!("{}-md", name("io"))).unwrap(); + assert!( + listed.contains(&name("io")) && listed.contains(&metadata), + "the device and its metadata thin are volumes of the pool: {listed:?}" + ); assert_eq!( pool.open(&name("io")).await.unwrap().size_bytes, device.size_bytes @@ -136,8 +140,9 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati vec![1, 3, 5] ); assert_eq!(mid.read_chunk(ChunkIndex(5)).await.unwrap(), fill(5)); - let mid = mid.name.clone(); - pool.delete(&mid).await.unwrap(); + let mid_name = mid.name.clone(); + drop(mid); + pool.delete(&mid_name).await.unwrap(); pool.settle(&device.name, &dest).await.unwrap(); assert_eq!( @@ -146,9 +151,13 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati ); let after = pool.snapshot(&device, &name("after")).await.unwrap(); assert_eq!(after.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); - let after = after.name.clone(); - pool.delete(&after).await.unwrap(); - assert!(!pool.list().await.unwrap().contains(&name("io-md"))); + let after_name = after.name.clone(); + drop(after); + pool.delete(&after_name).await.unwrap(); + assert!( + !pool.list().await.unwrap().contains(&metadata), + "the settle deleted the metadata thin" + ); assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); assert_eq!(device.read_chunk(ChunkIndex(5)).await.unwrap(), fill(5)); diff --git a/crates/engram-host-agent/tests/blk_source.rs b/crates/engram-host-agent/tests/blk_source.rs index eb33d3c30..79286c9d2 100644 --- a/crates/engram-host-agent/tests/blk_source.rs +++ b/crates/engram-host-agent/tests/blk_source.rs @@ -76,6 +76,15 @@ fn read_at(device: &File, offset: u64, len: usize) -> Vec { buf.as_slice().to_vec() } +/// A read off the runtime: the source's dispatch runs on it, so a read +/// that blocks a runtime thread waits for itself. +async fn read_off_runtime(device: &Arc, offset: u64, len: usize) -> Vec { + let device = device.clone(); + tokio::task::spawn_blocking(move || read_at(&device, offset, len)) + .await + .unwrap() +} + #[tokio::test] #[ignore = "requires root and the ublk_drv module"] async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { @@ -88,13 +97,13 @@ async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { .await .unwrap(); let path = source.path().to_path_buf(); - let device = open_direct(&path); + let device = Arc::new(open_direct(&path)); let image = image(); // A guest read far from where the copy starts: served, and recorded. let at = 3 * CHUNK + 8192; assert_eq!( - read_at(&device, at, 4096), + read_off_runtime(&device, at, 4096).await, image[at as usize..at as usize + 4096] ); assert_eq!(source.touched(), vec![3]); @@ -102,7 +111,7 @@ async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { // The copy's stream: the whole device, region by region, ascending. for at in (0..CHUNKS * CHUNK).step_by(REGION as usize) { assert_eq!( - read_at(&device, at, REGION as usize), + read_off_runtime(&device, at, REGION as usize).await, image[at as usize..(at + REGION) as usize], "region at {at}" ); From dc359780842146a2a96a024bbf0be38d2e22e4c1 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 16:21:36 -0800 Subject: [PATCH 12/35] disk-thin, host-core: a settled volume is a thin of its own The settle reloaded the hydrated device as `linear` over its destination, so the live device mapped the destination for the rest of its life. The first checkpoint after the settle publishes, then deletes the destination and base names to reach the normal layout, and that delete failed with `Resource busy`: every first post-settle checkpoint failed once, and a teleport that needed one aborted. Found on the dev cluster with real guests; no data was lost, the retry recovered. The settle now snapshots the destination's thin under the device's suspend (the destination is suspended for its snapshot, as the pool requires of an active origin) and reloads the device as that snapshot. The live volume is then a thin like any other, shares only blocks with the destination, and the destination deletes cleanly. A failed reload resumes the clone table and deletes the snapshot; a crash between the two leaves a thin with no device, which the pool open already deletes. The `linear` paths go: `thin_under`, `open`, `delete`, `list`, `hydration` and `parse::linear_device` know thin and clone only. The snapshot and the settle share `create_snap`. The fake hydrating disk stops sharing the destination's file at the settle, so a delete of a destination the live volume depends on fails in the unit tests too. The root test writes to the settled device, reads the destination unchanged, and deletes the destination under the open device. The settle log's `regions` is what the copy still had to bring in when the driver first looked, not the device's total. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- DESIGN.md | 2 +- crates/engram-disk-thin/src/disk.rs | 75 +++++++++++++------ crates/engram-disk-thin/src/parse.rs | 18 +---- crates/engram-disk-thin/tests/clone.rs | 19 ++++- .../engram-host-agent/src/pooled_backend.rs | 12 ++- crates/engram-host-core/src/disk.rs | 3 +- .../engram-host-core/src/session_volumes.rs | 6 +- crates/engram-host-core/src/test_support.rs | 5 ++ docs/adr/0125-kernel-owned-session-disk.md | 4 +- 9 files changed, 88 insertions(+), 56 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index c3ba52db5..7b14a3cd9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -237,7 +237,7 @@ the test named beside it. the host agent serves out of the fill engine. A guest read of a region the copy has not reached goes through the source at `Demand`; the chunks the guest read first on the session's last window are fetched at `Prefix` ahead of the copy. When every region is - copied the device is reloaded to a plain mapping of its destination and the source + copied the device is reloaded as a thin snapshot of its destination and the source stops; the path Firecracker opened never changes (`a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination`, `a_lazy_resume_reads_cold_chunks_through_the_source_and_settles`). A checkpoint in the diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 6ff15412a..c6cb3a0d7 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -154,14 +154,13 @@ impl ThinPoolDisk { } /// The thin a volume's data lives on, and its size: the volume - /// itself for a thin, the destination for a hydrating or settled - /// device. A snapshot of the volume is a snapshot of that thin. + /// itself for a thin, the destination for a hydrating device. A + /// snapshot of the volume is a snapshot of that thin. async fn thin_under(&self, name: &VolumeName) -> io::Result<(ThinParams, u64)> { let target = self.target_of(name).await?; let devno = match target.kind.as_str() { "thin" => return self.thin_of(name).await, "clone" => parse::clone_dest(&target.params)?, - "linear" => parse::linear_device(&target.params)?, kind => { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -397,6 +396,21 @@ fn single_target(targets: &[dm::Target], kind: &str) -> io::Result { } } +/// A snapshot of thin `origin` under the next free id. The caller holds +/// the origin's device suspended. +async fn create_snap(dm: &dm::Dm, pool: &str, next_id: &AtomicU32, origin: u32) -> io::Result { + loop { + let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; + match dm + .message(pool, &format!("create_snap {id} {origin}")) + .await + { + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + created => return created.map(|()| id), + } + } +} + #[async_trait] impl SessionDisk for ThinPoolDisk { async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { @@ -421,16 +435,7 @@ impl SessionDisk for ThinPoolDisk { let next_id = self.next_id.clone(); let (created, resumed) = self .while_suspended(self.device_name(&origin.name), move |dm| async move { - loop { - let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; - match dm - .message(&pool, &format!("create_snap {id} {}", thin.id)) - .await - { - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, - created => return created.map(|()| id), - } - } + create_snap(&dm, &pool, &next_id, thin.id).await }) .await?; let id = match (created, resumed) { @@ -451,7 +456,7 @@ impl SessionDisk for ThinPoolDisk { let target = self.target_of(name).await?; let sectors = match target.kind.as_str() { "thin" => self.thin_of(name).await?.1, - "clone" | "linear" => target.length_sectors, + "clone" => target.length_sectors, kind => { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -463,12 +468,12 @@ impl SessionDisk for ThinPoolDisk { } /// Waits a bounded time for the last holder to close the device. A - /// hydrating or settled device takes its metadata thin with it. + /// hydrating device takes its metadata thin with it. async fn delete(&self, name: &VolumeName) -> io::Result<()> { let target = self.target_of(name).await?; match target.kind.as_str() { "thin" => self.delete_thin(name).await, - "clone" | "linear" => { + "clone" => { self.remove_device(name).await?; self.delete_metadata(name).await } @@ -494,7 +499,7 @@ impl SessionDisk for ThinPoolDisk { match self.dm.table(&device).await { Ok(targets) => { if let [target] = targets.as_slice() { - if ["thin", "clone", "linear"].contains(&target.kind.as_str()) { + if ["thin", "clone"].contains(&target.kind.as_str()) { names.push(name); } } @@ -598,7 +603,7 @@ impl SessionDisk for ThinPoolDisk { source: dm::device_path(parse::clone_source(&table)?).await?, }) } - [target] if target.kind == "linear" || target.kind == "thin" => Ok(Hydration::Settled), + [target] if target.kind == "thin" => Ok(Hydration::Settled), _ => Err(io::Error::new( io::ErrorKind::InvalidInput, format!("{name} is not a session volume: {status:?}"), @@ -606,9 +611,13 @@ impl SessionDisk for ThinPoolDisk { } } - /// Reloads the device as `linear` over `dest` under a suspend. A - /// failed reload resumes the clone table. The metadata thin goes - /// last: the kernel closes it when the clone table is destroyed. + /// Under the device's suspend, snapshots `dest`'s thin and reloads + /// the device as that snapshot: a thin like any other, which shares + /// blocks with `dest` and nothing else. `dest` is suspended for its + /// snapshot, as the pool requires of an active origin. A failed + /// reload resumes the clone table and deletes the snapshot. The + /// metadata thin goes last: the kernel closes it when the clone + /// table is destroyed. async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { match self.hydration(name).await? { Hydration::Hydrating { @@ -620,12 +629,30 @@ impl SessionDisk for ThinPoolDisk { )); } Hydration::Hydrating { .. } => { - let (_, sectors) = self.thin_of(&dest.name).await?; - let params = format!("{} 0", dm::devno_of(&dest.path)?); + let (thin, sectors) = self.thin_of(&dest.name).await?; let device = self.device_name(name); + let dest_device = self.device_name(&dest.name); + let (pool, pool_dev, next_id) = + (self.pool.clone(), self.pool_dev, self.next_id.clone()); let (loaded, resumed) = self .while_suspended(device.clone(), move |dm| async move { - dm.load(&device, sectors, "linear", ¶ms).await + dm.suspend(&dest_device).await?; + let created = create_snap(&dm, &pool, &next_id, thin.id).await; + let dest_resumed = dm.resume(&dest_device).await; + let id = created?; + let loaded = match dest_resumed { + Ok(()) => { + dm.load(&device, sectors, "thin", &format!("{pool_dev} {id}")) + .await + } + Err(e) => Err(e), + }; + if loaded.is_err() { + if let Err(error) = dm.message(&pool, &format!("delete {id}")).await { + tracing::warn!(%pool, id, %error, "thin left behind by a failed settle"); + } + } + loaded }) .await?; loaded.and(resumed)?; diff --git a/crates/engram-disk-thin/src/parse.rs b/crates/engram-disk-thin/src/parse.rs index ae44a60e0..f00c78a23 100644 --- a/crates/engram-disk-thin/src/parse.rs +++ b/crates/engram-disk-thin/src/parse.rs @@ -156,14 +156,6 @@ pub fn clone_dest(params: &str) -> io::Result { } } -/// The device of a `linear` table: ` `. -pub fn linear_device(params: &str) -> io::Result { - match params.split_whitespace().collect::>().as_slice() { - [device, _] => DevNo::parse(device), - _ => Err(invalid(format!("not linear params: {params:?}"))), - } -} - /// The `DEVNAME=` line of a sysfs `uevent` file. pub fn devname(uevent: &str) -> io::Result { uevent @@ -297,7 +289,7 @@ mod tests { } #[test] - fn clone_dest_is_the_second_device_and_linear_the_only_one() { + fn clone_dest_is_the_second_device() { let clone = "252:3 252:4 259:0 128 2 no_hydration no_discard_passdown 0"; assert_eq!( clone_dest(clone).unwrap(), @@ -306,14 +298,6 @@ mod tests { minor: 4 } ); - assert_eq!( - linear_device("252:4 0").unwrap(), - DevNo { - major: 252, - minor: 4 - } - ); - assert!(linear_device("252:4").is_err()); } #[test] diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs index e6b0bacf1..0875f9cd4 100644 --- a/crates/engram-disk-thin/tests/clone.rs +++ b/crates/engram-disk-thin/tests/clone.rs @@ -174,12 +174,23 @@ async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destinati "the skipped chunk 0 still shares its blocks with the origin" ); + // The settled device is a volume of its own: a write to it leaves + // the destination as it was, and the destination goes while the + // device stays open and readable. + device.write_chunk(ChunkIndex(6), fill(9)).await.unwrap(); + device.sync().await.unwrap(); + assert_eq!(dest.read_chunk(ChunkIndex(6)).await.unwrap(), fill(0)); + let dest_name = dest.name.clone(); + drop(dest); + pool.delete(&dest_name).await.unwrap(); + assert_eq!(device.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); + assert_eq!(device.read_chunk(ChunkIndex(6)).await.unwrap(), fill(9)); + // A volume holds its device open; a delete needs it closed. - let names = (device.name.clone(), dest.name.clone(), base.name.clone()); - drop((device, dest, base)); - let (device, dest, base) = names; + let names = (device.name.clone(), base.name.clone()); + drop((device, base)); + let (device, base) = names; pool.delete(&device).await.unwrap(); - pool.delete(&dest).await.unwrap(); pool.delete(&base).await.unwrap(); assert!(!pool .list() diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index 08238d7d1..41ada5bfc 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -1060,21 +1060,25 @@ const SETTLE_RETRY: std::time::Duration = std::time::Duration::from_secs(5); async fn settle_disks(volumes: Arc, id: SandboxId, roles: Vec) { for role in roles { let started = crate::time_source::metrics_now(); - let mut regions = 0; + // The regions the copy still had to bring in when this driver + // first looked; none when the volume was complete by then. + let mut regions: Option = None; loop { match volumes.settle(id, role).await { Ok(engram_host_core::Hydration::Settled) => { tracing::info!( sandbox_id = %id, role = %role, - regions, + regions = regions.unwrap_or(0), elapsed_ms = started.elapsed().as_millis() as u64, "disk settled", ); break; } - Ok(engram_host_core::Hydration::Hydrating { total, .. }) => { - regions = total; + Ok(engram_host_core::Hydration::Hydrating { + hydrated, total, .. + }) => { + regions.get_or_insert(total - hydrated); tokio::time::sleep(SETTLE_POLL).await; } Err(error) => { diff --git a/crates/engram-host-core/src/disk.rs b/crates/engram-host-core/src/disk.rs index d0c34629b..b9a44f966 100644 --- a/crates/engram-host-core/src/disk.rs +++ b/crates/engram-host-core/src/disk.rs @@ -469,7 +469,8 @@ pub trait SessionDisk: Send + Sync { Ok(Hydration::Settled) } - /// Map a fully hydrated device straight onto `dest`. Every I/O in + /// Make a fully hydrated device a volume of its own with `dest`'s + /// content; `dest` stays, and is the caller's to delete. Every I/O in /// flight completes first; open handles keep working; `source` is /// not read again. async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 16122cdb0..357f911d3 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -882,9 +882,9 @@ impl SessionVolumes { } /// One step of a hydrating role's way to a settled volume: its - /// progress, and once every region is in, the cutover onto the - /// destination and the end of its source. `Settled` when the live - /// volume maps straight onto its destination. + /// progress, and once every region is in, the cutover to a volume + /// of its own and the end of its source. `Settled` when the live + /// volume no longer hydrates. pub async fn settle(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { let live = live_name(sandbox, role); match self.disk.hydration(&live).await? { diff --git a/crates/engram-host-core/src/test_support.rs b/crates/engram-host-core/src/test_support.rs index 1b2bc69b3..febf2d140 100644 --- a/crates/engram-host-core/src/test_support.rs +++ b/crates/engram-host-core/src/test_support.rs @@ -199,6 +199,11 @@ impl SessionDisk for HydratingDisk { format!("{name}: {} chunks are not copied", clone.pending.len()), )); } + // The device stops sharing the destination's file: a volume of + // its own, as the pool makes it. + let own = clone.device.with_extension("settling"); + std::fs::copy(&clone.device, &own)?; + std::fs::rename(&own, &clone.device)?; clone.settled = true; self.holders.lock().unwrap().remove(&clone.source); Ok(()) diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index 0b855c337..d4861f52d 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -188,7 +188,7 @@ thin-provisioning tools it calls at run time (`thin_dump`, behind the running guest; the guest starts at once and reads a region the copy has not reached through the device's source, a ublk device the host-agent serves from the fill engine. Once every - region is copied the device is reloaded to a plain mapping of its + region is copied the device is reloaded as a thin snapshot of its destination and the source goes. Guest swap (ADR 0112) starts as a fresh thin volume with no origin and is checkpointed with the root disk from then on (see the divergences). The pool's volume names @@ -335,7 +335,7 @@ region with a ranged `Demand` read and the kernel's ascending copy with whole-chunk `Hydrate` reads; the chunks the guest read first on the session's last window are fetched at `Prefix` before the copy reaches them. A settle task polls the clone's status and, when every -region is copied, suspends the device, reloads it to `linear` over the +region is copied, suspends the device, reloads it as a thin snapshot of the destination, resumes it and stops the source: the path Firecracker opened never changes. A checkpoint in the window diffs the snapshot's mapping against the base: a block whose mapping changed (copied or From ddbbab1c20927c5938ff2b2c9243b40a68856e5f Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 16:21:36 -0800 Subject: [PATCH 13/35] nix: the dev shell tracks nixos-26.05, which has Node 26 #1736 raised the shell's Node to `nodejs_26`, but the locked nixpkgs (nixos-unstable, 2026-04-22) has no such package, so `nix develop` failed to evaluate in every fresh checkout. Today's nixos-unstable has it but ships `sqlx-cli` 0.9.0, which has no cached macOS build and fails to link from source. nixos-26.05 has Node 26.13 and `sqlx-cli` 0.8.6, the workspace's sqlx version, from the binary cache. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 115c23098..3d459bc4b 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1776877367, - "narHash": "sha256-EHq1/OX139R1RvBzOJ0aMRT3xnWyqtHBRUBuO1gFzjI=", + "lastModified": 1793865203, + "narHash": "sha256-/cpdPHaySDIyCgZI6EWngiM6jK3YLUNzv1b7lTzy6Bk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0726a0ecb6d4e08f6adced58726b95db924cef57", + "rev": "c56a87d8e41ce69bd2af57eb35dcc117a9dfca33", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a50d52b3c..eeaee1035 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "Engram — pinned dev toolchain (Rust + just + jq + sqlx-cli + cargo-watch + cargo-nextest)."; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; rust-overlay = { url = "github:oxalica/rust-overlay"; From 060d2dba032405451de2e8d44c3caff83924b8ae Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 16:55:21 -0800 Subject: [PATCH 14/35] host-core: a restore picks its base by what it shares, not by name A restore that did not find the session's kept checkpoint started from the image version the snapshot named, and from nothing when the host did not stage that exact version. With an image whose tag moves every hour that is the normal case: the host holds a newer version that shares most of its chunks with the session's disk, and the whole disk became the copy. `nearest_base` compares the wanted manifest with the lineage's kept checkpoint and every staged image version and takes the volume that holds the most chunks in place, the same hash at the same offset (`shared_chunks`; the kept checkpoint wins a tie). No volume of the manifest's shape that shares a chunk means no base. The base hint goes from `Source::Fork` and `Source::Continue`; the restore no longer reads the snapshot's `base_disk_manifest` for the disk. The eager path on a host without a hydrating device takes the same base. Tests: a session that began on one image version resumes from a moved one and copies only what that version lacks; `shared_chunks` counts by offset and refuses a different shape. The "without a base" test now runs on a host that holds no volume of the image. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- DESIGN.md | 7 +- .../engram-host-agent/src/pooled_backend.rs | 7 +- .../engram-host-agent/tests/lazy_restore.rs | 2 - crates/engram-host-core/src/disk_flows.rs | 30 ++ .../engram-host-core/src/session_volumes.rs | 271 ++++++++---------- docs/adr/0125-kernel-owned-session-disk.md | 15 +- 6 files changed, 174 insertions(+), 158 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 7b14a3cd9..de24cf5ef 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -231,8 +231,11 @@ the test named beside it. full rate the sync and snapshot stay under two seconds however much it wrote (`the_disk_pause_is_bounded_while_the_guest_writes_at_full_rate`). - A resume boots before its disk is copied in. The live volume is a dm-clone device over - a snapshot of the nearest base the host holds (the kept checkpoint volume, else the image - volume, else an empty volume); regions the base holds in final form are marked copied by + a snapshot of the nearest base the host holds: the kept checkpoint volume or staged image + version that shares the most chunks with the session's manifest at the same offset, so an + image whose tag moved still supplies the blocks it shares + (`a_resume_starts_from_the_staged_volume_that_shares_the_most`), or an empty volume when + none shares any; regions the base holds in final form are marked copied by a discard, and the kernel copies the rest in behind the running guest from a ublk source the host agent serves out of the fill engine. A guest read of a region the copy has not reached goes through the source at `Demand`; the chunks the guest read first on the diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index 41ada5bfc..391094d49 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -2050,9 +2050,8 @@ impl PooledBackend { let manifest = metadata.disk_manifest.ok_or_else(|| { SandboxError::MemoryImageUnusable("snapshot has no disk manifest".into()) })?; - let base = metadata.base_disk_manifest; let root = if fork_disk { - Source::Fork { manifest, base } + Source::Fork { manifest } } else { let first_touch = match (read_trace_lineage(src).await, self.chunk_store.as_ref()) { (Some(session), Some(store)) => { @@ -2063,7 +2062,6 @@ impl PooledBackend { }; Source::Continue { manifest, - base, first_touch, } }; @@ -2082,7 +2080,6 @@ impl PooledBackend { } (Some(manifest), _) => Some(Source::Continue { manifest, - base: None, first_touch: Vec::new(), }), (None, _) if !fresh && !sidecar.base_capture => { @@ -4739,7 +4736,6 @@ impl SandboxBackend for PooledBackend { let root = match spec.rootfs_manifest { Some(manifest) => Source::Continue { manifest, - base: None, first_touch: Vec::new(), }, None => { @@ -4767,7 +4763,6 @@ impl SandboxBackend for PooledBackend { let swap = match (spec.swap_manifest, spec.swap_mib.filter(|m| *m > 0)) { (Some(manifest), _) => Some(Source::Continue { manifest, - base: None, first_touch: Vec::new(), }), (None, Some(mib)) => Some(Source::Empty { diff --git a/crates/engram-host-agent/tests/lazy_restore.rs b/crates/engram-host-agent/tests/lazy_restore.rs index a04f10eff..845d0b80c 100644 --- a/crates/engram-host-agent/tests/lazy_restore.rs +++ b/crates/engram-host-agent/tests/lazy_restore.rs @@ -136,7 +136,6 @@ async fn a_lazy_resume_reads_cold_chunks_through_the_source_and_settles() { sandbox, Source::Continue { manifest, - base: None, first_touch: vec![], }, None, @@ -190,7 +189,6 @@ async fn child_provisions_and_waits() { sandbox, Source::Continue { manifest, - base: None, first_touch: vec![], }, None, diff --git a/crates/engram-host-core/src/disk_flows.rs b/crates/engram-host-core/src/disk_flows.rs index a5df6b246..033bcf9ae 100644 --- a/crates/engram-host-core/src/disk_flows.rs +++ b/crates/engram-host-core/src/disk_flows.rs @@ -230,6 +230,14 @@ pub fn hydration_plan(wanted: &Manifest, base: Option<&Manifest>) -> io::Result< }) } +/// How many chunks of `wanted` a volume holding `base` already has in +/// place: the same hash at the same offset. `None` when the two differ +/// in shape, so the volume cannot be a base at all. +pub(crate) fn shared_chunks(wanted: &Manifest, base: &Manifest) -> Option { + let delta = wanted.delta_from(base).ok()?; + Some(wanted.chunks.len() - delta.write.len()) +} + /// The chunks of `first_touch` (guest reads of a hydrating volume, in /// order) that `delta` copies, as `wanted`'s hashes: what a restore /// fetches at `Prefix` before the copy reaches them. Chunks the base @@ -661,6 +669,28 @@ mod tests { ); } + #[test] + fn shared_chunks_counts_equal_hashes_at_equal_offsets() { + let disk = |bytes: &[u8]| Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: bytes + .iter() + .enumerate() + .filter(|(_, b)| **b != 0) + .map(|(i, b)| ChunkRef { + offset: i as u64 * CHUNK, + hash: ChunkHash::of(&fill(*b)), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, CHUNK * bytes.len() as u64) + }; + let wanted = disk(&[1, 2, 0, 4]); + assert_eq!(shared_chunks(&wanted, &disk(&[1, 2, 3, 9])), Some(2)); + // The same bytes at another offset are not in place. + assert_eq!(shared_chunks(&wanted, &disk(&[2, 1, 0, 0])), Some(0)); + assert_eq!(shared_chunks(&wanted, &disk(&[1, 2, 0])), None); + } + /// Reads of chunks 3, 0 and 4 came first; the copy brings 1 and 3, /// so only chunk 3 is fetched ahead. #[test] diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 357f911d3..f498f18b2 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -44,7 +44,9 @@ use uuid::Uuid; use crate::disk::{ Capacity, Hydration, SessionDisk, SourceDevice, SourceServer, Volume, VolumeName, }; -use crate::disk_flows::{checkpoint_split, hydration_plan, prefix_hashes, shape_matches, stage}; +use crate::disk_flows::{ + checkpoint_split, hydration_plan, prefix_hashes, shape_matches, shared_chunks, stage, +}; use crate::image_volumes::{self, ImageVolumes}; const LIVE_PREFIX: &str = "sb-"; @@ -239,17 +241,13 @@ pub enum Source { Image { manifest: ManifestRef }, /// A manifest many sandboxes share: the volume starts a new lineage /// that holds a copy of it. - Fork { - manifest: ManifestRef, - base: Option, - }, + Fork { manifest: ManifestRef }, /// The sandbox's own manifest: the volume continues its lineage. /// `first_touch` lists the chunks the guest read before the copy /// reached them on its last hydrating restore, in order; the ones /// this restore must copy are fetched first. Continue { manifest: ManifestRef, - base: Option, first_touch: Vec, }, /// Zeros, in a new lineage. @@ -486,7 +484,7 @@ impl SessionVolumes { let live = self.commit_live(&tmp, sandbox, role, lineage).await?; (live, lineage, StagePath::ImageFork) } - Source::Fork { manifest, base } => { + Source::Fork { manifest } => { let wanted = self.manifest(manifest).await?; let lineage = self.new_lineage(&wanted).await?; match self.staged(manifest).await { @@ -496,7 +494,7 @@ impl SessionVolumes { (live, lineage, StagePath::StagedSnapshot) } None => { - let base = self.image_base(base).await; + let base = self.nearest_base(&wanted, None).await?; let (live, path) = self .bring_up( fill, @@ -516,7 +514,6 @@ impl SessionVolumes { } Source::Continue { manifest, - base, first_touch, } => match self.kept(sandbox, role, manifest).await? { Some((owner, volume, _, held)) if held == manifest => { @@ -526,19 +523,7 @@ impl SessionVolumes { } kept => { let wanted = self.manifest(manifest).await?; - let base = match kept { - Some((owner, volume, held_manifest, held)) - if volume.size_bytes == wanted.total_bytes => - { - Some(Base { - volume, - manifest: held_manifest, - manifest_ref: held, - kept_by: Some(owner), - }) - } - _ => self.image_base(base).await, - }; + let base = self.nearest_base(&wanted, kept).await?; let (live, path) = self .bring_up( fill, @@ -796,16 +781,50 @@ impl SessionVolumes { Ok(live) } - /// The staged volume of `image` as a base, when this host has it. - async fn image_base(&self, image: Option) -> Option { - let image = image?; - let (volume, manifest) = self.staged(image).await?; - Some(Base { - volume, - manifest, - manifest_ref: image, - kept_by: None, - }) + /// The volume on this host that holds the most of `wanted` in + /// place, as a base: the lineage's `kept` checkpoint or a staged + /// image version, whichever shares more chunks with `wanted` at the + /// same offset (the kept one on a tie). No snapshot names it: an + /// image whose tag moved since the session began is as good a base + /// as the chunks it still shares. `None` when no volume of + /// `wanted`'s shape shares a chunk. + async fn nearest_base( + &self, + wanted: &Manifest, + kept: Option<(SandboxId, Volume, Manifest, ManifestRef)>, + ) -> io::Result> { + let mut candidates: Vec = kept + .into_iter() + .map(|(owner, volume, manifest, manifest_ref)| Base { + volume, + manifest, + manifest_ref, + kept_by: Some(owner), + }) + .collect(); + for image in image_volumes::staged(&self.disk.list().await?) { + if let Some((volume, manifest)) = self.staged(image).await { + candidates.push(Base { + volume, + manifest, + manifest_ref: image, + kept_by: None, + }); + } + } + let mut best: Option<(usize, Base)> = None; + for base in candidates { + if base.volume.size_bytes != wanted.total_bytes { + continue; + } + let Some(shared) = shared_chunks(wanted, &base.manifest) else { + continue; + }; + if shared > best.as_ref().map_or(0, |(most, _)| *most) { + best = Some((shared, base)); + } + } + Ok(best.map(|(_, base)| base)) } /// The newest checkpoint volume of `role` that `sandbox` or a released @@ -1666,10 +1685,7 @@ mod tests { } fn fork(image: ManifestRef) -> Source { - Source::Fork { - manifest: image, - base: None, - } + Source::Fork { manifest: image } } /// Publish an image whose chunk `i` holds byte `pattern[i]` (0 is a @@ -1866,14 +1882,7 @@ mod tests { let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); assert_eq!(bytes_at(&p.root).await, vec![1, 0, 3]); @@ -1912,14 +1921,7 @@ mod tests { let first = SandboxId::new(); let p = r .volumes - .provision( - first, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(first, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 1, 9).await; @@ -1933,7 +1935,6 @@ mod tests { second, Source::Continue { manifest: published, - base: Some(image), first_touch: vec![], }, None, @@ -1954,7 +1955,6 @@ mod tests { third, Source::Continue { manifest: published, - base: None, first_touch: vec![], }, None, @@ -2004,7 +2004,6 @@ mod tests { second, Source::Continue { manifest: published, - base: None, first_touch: vec![], }, None, @@ -2023,14 +2022,7 @@ mod tests { let first = SandboxId::new(); let p = r .volumes - .provision( - first, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(first, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 1, 9).await; @@ -2045,7 +2037,6 @@ mod tests { sandbox, Source::Fork { manifest: published, - base: Some(image), }, None, ) @@ -2074,14 +2065,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 0, 7).await; @@ -2105,10 +2089,7 @@ mod tests { .volumes .provision( sandbox, - Source::Fork { - manifest: image, - base: None, - }, + Source::Fork { manifest: image }, Some(Source::Empty { size_bytes: CHUNK * 2, }), @@ -2140,13 +2121,9 @@ mod tests { .volumes .provision( sandbox, - Source::Fork { - manifest: image, - base: None, - }, + Source::Fork { manifest: image }, Some(Source::Continue { manifest: ManifestRef::new(), - base: None, first_touch: vec![], }), ) @@ -2166,14 +2143,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); let lineage = checkpoints_of(&r, sandbox, DiskRole::Root).await[0]; @@ -2209,7 +2179,6 @@ mod tests { resumed, Source::Continue { manifest: v3, - base: Some(image), first_touch: vec![], }, None, @@ -2226,14 +2195,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 0, 5).await; @@ -2255,14 +2217,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); let half = CHUNK / 2; @@ -2321,14 +2276,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); assert_eq!( @@ -2376,14 +2324,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); let lineage = checkpoints_of(&r, sandbox, DiskRole::Root).await[0]; @@ -2421,23 +2362,13 @@ mod tests { r.volumes .provision( live, - Source::Fork { - manifest: image, - base: None, - }, + Source::Fork { manifest: image }, Some(Source::Empty { size_bytes: CHUNK }), ) .await .unwrap(); r.volumes - .provision( - dead, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(dead, Source::Fork { manifest: image }, None) .await .unwrap(); let lineage = checkpoints_of(&r, live, DiskRole::Root).await[0]; @@ -2567,7 +2498,6 @@ mod tests { resumed, Source::Continue { manifest: published, - base: Some(image), first_touch: vec![], }, None, @@ -2609,7 +2539,6 @@ mod tests { destination, Source::Continue { manifest: published, - base: Some(image), first_touch: vec![], }, None, @@ -2654,7 +2583,6 @@ mod tests { resumed, Source::Continue { manifest: next, - base: Some(image), first_touch: vec![], }, None, @@ -2728,7 +2656,6 @@ mod tests { sandbox, Source::Continue { manifest: published, - base: Some(image), first_touch: vec![], }, None, @@ -2792,7 +2719,6 @@ mod tests { resumed, Source::Continue { manifest: published, - base: Some(image), first_touch: vec![], }, None, @@ -2880,7 +2806,6 @@ mod tests { resumed, Source::Continue { manifest: wanted, - base: Some(image), first_touch: vec![], }, None, @@ -2978,7 +2903,6 @@ mod tests { resumed, Source::Continue { manifest: wanted, - base: Some(image), first_touch: vec![], }, None, @@ -3050,7 +2974,6 @@ mod tests { sandbox, Source::Continue { manifest: next, - base: Some(image), first_touch: vec![], }, None, @@ -3090,7 +3013,6 @@ mod tests { sandbox, Source::Continue { manifest: next, - base: Some(image), first_touch: vec![], }, None, @@ -3127,12 +3049,70 @@ mod tests { assert!(restarted.reattach(sandbox).await.unwrap().is_empty()); } - /// Without a base on the host, the destination starts from zeros and - /// every listed chunk is the delta. + /// The session began on image [1, 2, 3, 4, 5] and holds + /// [1, 2, 3, 9, 5]. The image's tag moved: the host stages only + /// [1, 2, 3, 4, 6] now, and a second image that shares one chunk. + /// The resume starts from the moved image, which holds three of the + /// session's chunks in place, and copies the other two. + #[tokio::test] + async fn a_resume_starts_from_the_staged_volume_that_shares_the_most() { + let r = lazy_rig(); + let began_on = staged_image(&r, &[1, 2, 3, 4, 5]).await; + let first = SandboxId::new(); + let p = r + .volumes + .provision(first, fork(began_on), None) + .await + .unwrap(); + write_byte(&p.root, 3, 9).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let other = staged_image(&r, &[1, 7, 7, 7, 7]).await; + let moved = staged_image(&r, &[1, 2, 3, 4, 6]).await; + r.volumes.images().reconcile(&[other, moved]).await.unwrap(); + assert!(!names(&r) + .await + .contains(&image_volumes::volume_name(began_on).to_string())); + + let resumed = SandboxId::new(); + let p = r + .volumes + .provision( + resumed, + Source::Continue { + manifest: wanted, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert!( + names(&r) + .await + .contains(&base_name(resumed, DiskRole::Root, Some(moved)).to_string()), + "the base is the staged image that shares the most chunks" + ); + assert_eq!( + r.volumes.first_touch(resumed, DiskRole::Root), + vec![3, 4], + "the copy brings only the chunks the moved image lacks (the fake reports the delta)" + ); + settled(&r, resumed, DiskRole::Root).await; + assert_eq!(bytes_at(&p.root).await, vec![1, 2, 3, 9, 5]); + r.volumes.release(resumed).await.unwrap(); + } + + /// With no volume on the host that shares a chunk, the destination + /// starts from zeros and every listed chunk is the delta. #[tokio::test] async fn a_lazy_resume_without_a_base_starts_from_zeros() { let r = lazy_rig(); let image = staged_image(&r, &[1, 0, 3]).await; + r.volumes.images().reconcile(&[]).await.unwrap(); let sandbox = SandboxId::new(); let p = r .volumes @@ -3140,7 +3120,6 @@ mod tests { sandbox, Source::Continue { manifest: image, - base: None, first_touch: vec![], }, None, diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index d4861f52d..10f192c02 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -182,8 +182,10 @@ thin-provisioning tools it calls at run time (`thin_dump`, readiness. 3. **A session disk is a thin snapshot.** A fresh session's root disk is a snapshot of its image volume. A resumed session's root disk is - a snapshot of the nearest base this host holds (its kept checkpoint - volume, else the base image volume, else an empty volume) under a + a snapshot of the nearest base this host holds (the kept checkpoint + volume or staged image version that shares the most chunks with the + session's manifest at the same offset; an empty volume when none + shares any) under a dm-clone device that copies the difference to its manifest in behind the running guest; the guest starts at once and reads a region the copy has not reached through the device's source, a @@ -713,6 +715,12 @@ thin-pool host and the reverse. checkpoint rule covers like a guest write. The heartbeat gains no field and `sandbox.json` records no device: the pool's names are the record, and a restart serves every hydrating volume again from them. + The base is chosen by content, not by name: the restore compares the + session's manifest with the kept checkpoint and every staged image + version and snapshots the one that holds the most chunks in place, + so an image whose tag moved since the session began still supplies + the blocks it shares. The snapshot's `base_disk_manifest` no longer + steers the disk. ## Facts this ADR establishes @@ -760,6 +768,9 @@ names the test that guards each one. As landed, the guards are: `a_restart_serves_a_hydrating_volume_again`) - A role without a checkpoint volume moves by snapshot. (`a_role_without_a_checkpoint_volume_refuses_a_live_move`) +- A resume starts from the volume on the host that shares the most + chunks with its manifest, whichever image version that is. + (`a_resume_starts_from_the_staged_volume_that_shares_the_most`) - The heartbeat carries observed facts only; no field asks the coordinator to take an action. From 0c78bb3f92cc16e6d71db6241de5e150df2692f5 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:03:07 -0800 Subject: [PATCH 15/35] dst-cosim: the simulated host forks without a base hint `Source::Fork` lost its `base` field with the base chosen by content; the co-simulation host still passed one. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-dst-cosim/src/host.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/engram-dst-cosim/src/host.rs b/crates/engram-dst-cosim/src/host.rs index 6ea64cc0b..49d862c1a 100644 --- a/crates/engram-dst-cosim/src/host.rs +++ b/crates/engram-dst-cosim/src/host.rs @@ -129,14 +129,7 @@ pub(crate) async fn build_base_sandbox( .await .expect("seed base manifest"); volumes - .provision( - sandbox_id, - Source::Fork { - manifest: base_ref, - base: None, - }, - None, - ) + .provision(sandbox_id, Source::Fork { manifest: base_ref }, None) .await .expect("provision the root volume"); sandbox_id From c41df2e26bc040f7e89981b07944e445476656f8 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:55:00 -0800 Subject: [PATCH 16/35] host-core: only a snapshot taken after the settle becomes a checkpoint volume A checkpoint taken while the volume hydrated and published after the settle became the role's checkpoint volume: `publish` read the live device's state, saw it settled, and renamed the earlier snapshot. That snapshot holds base bytes in every region the copy had not reached. Its manifest was right, but a resume of the lineage on the same host boots the kept volume without a fetch, and `nearest_base` can pick it. `take` now records whether the live volume was settled before the snapshot, and `publish` decides by that. A partial snapshot is deleted and the destination takes the published name, as during the window; the next checkpoint makes the checkpoint volume. A pending checkpoint found by name after a crash counts as partial. Test: a snapshot taken with one of two chunks copied, a settle, then the publish; no checkpoint volume appears until the next checkpoint, whose volume holds the manifest's bytes. Found by two independent reviews of the branch. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .../engram-host-core/src/session_volumes.rs | 110 ++++++++++++++---- 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index f498f18b2..db9f77dc4 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -287,6 +287,11 @@ pub struct PendingCheckpoint { role: DiskRole, snapshot: Volume, previous: ManifestRef, + /// The live volume held its whole manifest when the snapshot was + /// taken: it was not hydrating. Only such a snapshot can become a + /// checkpoint volume; a snapshot of a hydrating volume still shows + /// the base where the copy had not reached. + whole: bool, } impl PendingCheckpoint { @@ -1109,18 +1114,25 @@ impl SessionVolumes { let tmp = transient_name(sandbox, role); self.delete_if_present(&tmp).await?; let live = self.live(sandbox, role).await?; + // Read before the snapshot: a settle between the two only makes + // a whole snapshot count as partial, which costs one more + // checkpoint before the role has its checkpoint volume. + let whole = self.disk.hydration(&live.name).await? == Hydration::Settled; let snapshot = checkpoint_split::take(self.disk.as_ref(), &live, &tmp).await?; Ok(PendingCheckpoint { sandbox, role, snapshot, previous, + whole, }) } /// The checkpoint a `take` left unpublished, found by name: a new /// process redrives the publish after a crash. `NotFound` when - /// there is none. + /// there is none. Whether the volume was hydrating at the take is + /// not recorded, so the snapshot never becomes the first checkpoint + /// volume of a role brought up on a hydrating device. pub async fn pending( &self, sandbox: SandboxId, @@ -1133,6 +1145,7 @@ impl SessionVolumes { role, snapshot, previous, + whole: false, }) } @@ -1175,6 +1188,7 @@ impl SessionVolumes { role, snapshot, previous, + whole: true, }) } @@ -1220,18 +1234,20 @@ impl SessionVolumes { /// A role with no checkpoint volume is on a hydrating device, and /// `previous` is its destination's manifest: the upload lays the /// blocks that changed since the base over it - /// ([`checkpoint_split::upload_hydrating`]). While the device still - /// hydrates, the destination takes the published manifest's name - /// and the snapshot goes, so the names stay the record and no - /// checkpoint volume exists before the settle; once settled, the - /// snapshot becomes the checkpoint volume and the destination and - /// base names go with the layout they described. + /// ([`checkpoint_split::upload_hydrating`]). A snapshot taken while + /// the device hydrated holds base bytes where the copy had not + /// reached, so it goes, and the destination takes the published + /// manifest's name: the names stay the record and no checkpoint + /// volume exists yet. A snapshot taken after the settle becomes the + /// checkpoint volume, and the destination and base names go with + /// the layout they described. pub async fn publish(&self, pending: PendingCheckpoint) -> io::Result { let PendingCheckpoint { sandbox, role, snapshot, previous, + whole, } = pending; let previous_manifest = self.manifest(previous).await?; let previous_name = checkpoint_name(sandbox, role, previous); @@ -1301,19 +1317,17 @@ impl SessionVolumes { self.disk.rename(&tmp, &published).await?; self.disk.delete(&previous_name).await?; } - Some((dest, base)) => match self.disk.hydration(&live_name(sandbox, role)).await? { - Hydration::Hydrating { .. } => { - self.disk - .rename(&dest, &dest_name(sandbox, role, next)) - .await?; - self.disk.delete(&tmp).await?; - } - Hydration::Settled => { - self.disk.rename(&tmp, &published).await?; - self.disk.delete(&dest).await?; - self.disk.delete(&base).await?; - } - }, + Some((dest, base)) if whole => { + self.disk.rename(&tmp, &published).await?; + self.disk.delete(&dest).await?; + self.disk.delete(&base).await?; + } + Some((dest, _)) => { + self.disk + .rename(&dest, &dest_name(sandbox, role, next)) + .await?; + self.disk.delete(&tmp).await?; + } } Ok(next) } @@ -2882,6 +2896,62 @@ mod tests { assert!(r.volumes.first_touch(resumed, DiskRole::Root).is_empty()); } + /// Base [1, 2, 3], wanted [1, 9, 8]. A snapshot taken when only + /// chunk 1 is copied shows the base's 3 in chunk 2. The volume + /// settles before the publish: the manifest is the wanted disk, and + /// the snapshot does not become the checkpoint volume a resume on + /// this host would boot. The next checkpoint's snapshot does. + #[tokio::test] + async fn a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + write_byte(&p.root, 1, 9).await; + write_byte(&p.root, 2, 8).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + let source = Source::Continue { + manifest: wanted, + first_touch: vec![], + }; + r.volumes.provision(resumed, source, None).await.unwrap(); + let pending = r.volumes.take(resumed, DiskRole::Root).await.unwrap(); + assert_eq!( + bytes_at(&pending.snapshot().path).await, + vec![1, 9, 3], + "the snapshot holds the base where the copy has not reached" + ); + settled(&r, resumed, DiskRole::Root).await; + + let mid = r.volumes.publish(pending).await.unwrap(); + assert_eq!(manifest_bytes(&r, mid).await, vec![1, 9, 8]); + assert_eq!( + checkpoints_of(&r, resumed, DiskRole::Root).await, + vec![], + "no checkpoint volume from a partial snapshot" + ); + assert!(names(&r) + .await + .contains(&dest_name(resumed, DiskRole::Root, mid).to_string())); + + let after = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + assert_eq!( + checkpoints_of(&r, resumed, DiskRole::Root).await, + vec![after] + ); + let kept = r + .volumes + .disk() + .open(&checkpoint_name(resumed, DiskRole::Root, after)) + .await + .unwrap(); + assert_eq!(bytes_at(&kept.path).await, vec![1, 9, 8]); + } + /// A live move carries the blocks changed since the checkpoint /// volume; a role without one (hydrating, or settled before its /// first checkpoint) refuses the precopy, and the first checkpoint From 2b693634810afdc5e68227d6106b292de9e09b9e Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:55:01 -0800 Subject: [PATCH 17/35] disk-thin, host-core: a failed hydrate and an interrupted settle leave no metadata volume Three ways a clone's metadata thin stayed in the pool, under a name no sweep parses: - `hydrate` kept its handle of the metadata device open across its own cleanup, so the delete after a failed create, discard or enable met a busy device; a failed device-number lookup skipped the cleanup. - A process that died between the cutover and the metadata delete left it: the next `settle` saw a settled device and did nothing. - A metadata delete that failed after the cutover was never tried again, for the same reason. `hydrate` keeps only the path and builds the table inside the guarded block. `SessionVolumes::settle` runs the disk's settle, which is the cleanup on a settled device, while the role still has its destination; `reattach` does the same for a role it finds settled. The skip discard also refused a volume whose last chunk is short; it now clips the discard to the volume. The metadata size comment gave the bitset of a 2 million region volume; a 300 GiB root has 4.9 million. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-disk-thin/src/disk.rs | 30 +++++++++++-------- .../engram-host-core/src/session_volumes.rs | 14 ++++++++- crates/engram-host-core/src/test_support.rs | 12 ++++---- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index c6cb3a0d7..6256ee241 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -22,8 +22,8 @@ const ID_MASK: u32 = 0x00FF_FFFF; const REMOVE_ATTEMPTS: u32 = 10; const REMOVE_RETRY: Duration = Duration::from_millis(100); /// A dm-clone metadata volume: a superblock and one bit per region in -/// 4 KiB blocks. 8 MiB holds the bitset of a 2 million region volume -/// with room for its transaction shadows. +/// 4 KiB blocks. A 300 GiB volume has 4.9 million 64 KiB regions, a +/// 600 KiB bitset; 8 MiB leaves room for its transaction shadows. const CLONE_METADATA_BYTES: u64 = 8 << 20; /// Regions per background copy and copies in flight: 1 MiB bios, 8 MiB /// in flight. The kernel's defaults of 1 and 1 copy at a few MiB/s. @@ -371,13 +371,14 @@ async fn skip_regions(device: &Volume, skip: &[ChunkIndex]) -> io::Result<()> { last += 1; } let offset = first * chunk; - let end = (last + 1) * chunk; - if end > size { + if last * chunk >= size { return Err(io::Error::new( io::ErrorKind::InvalidInput, format!("skip chunk {last} lies beyond the {size}-byte volume"), )); } + // The last chunk of a volume can be short. + let end = ((last + 1) * chunk).min(size); punch_hole(&file, offset, end - offset)?; } Ok(()) @@ -558,19 +559,22 @@ impl SessionDisk for ThinPoolDisk { format!("{}: {sectors} sectors is not whole pool blocks", dest.name), )); } + // Only the path is kept: an open handle of ours would keep a + // failed hydrate from deleting the metadata volume. let metadata = self .create(&metadata_name(name)?, CLONE_METADATA_BYTES) - .await?; - let params = format!( - "{} {} {} {region_sectors} 2 no_hydration no_discard_passdown \ - 4 hydration_threshold {HYDRATION_THRESHOLD_REGIONS} \ - hydration_batch_size {HYDRATION_BATCH_REGIONS}", - dm::devno_of(&metadata.path)?, - dm::devno_of(&dest.path)?, - dm::devno_of(source)?, - ); + .await? + .path; let device = self.device_name(name); let hydrating = async { + let params = format!( + "{} {} {} {region_sectors} 2 no_hydration no_discard_passdown \ + 4 hydration_threshold {HYDRATION_THRESHOLD_REGIONS} \ + hydration_batch_size {HYDRATION_BATCH_REGIONS}", + dm::devno_of(&metadata)?, + dm::devno_of(&dest.path)?, + dm::devno_of(source)?, + ); self.dm.create(&device, sectors, "clone", ¶ms).await?; let volume = self.volume(name, sectors).await?; skip_regions(&volume, skip).await?; diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index db9f77dc4..6b46d64a3 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -920,7 +920,17 @@ impl SessionVolumes { let dest = self.disk.open(&dest).await?; self.disk.settle(&live, &dest).await?; } - Hydration::Settled => {} + // A settle that stopped after the cutover left the device's + // metadata volume; the disk's settle removes it. No + // destination: the first checkpoint ended the layout. + Hydration::Settled => match self.lazy_layout(sandbox, role).await { + Ok((dest, _)) => { + let dest = self.disk.open(&dest).await?; + self.disk.settle(&live, &dest).await?; + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + }, } self.stop_source(&live).await?; Ok(Hydration::Settled) @@ -953,6 +963,8 @@ impl SessionVolumes { continue; } let Hydration::Hydrating { source, .. } = self.disk.hydration(&live).await? else { + // The process died between the cutover and its cleanup. + self.settle(sandbox, role).await?; continue; }; let (dest, base) = self.lazy_layout(sandbox, role).await?; diff --git a/crates/engram-host-core/src/test_support.rs b/crates/engram-host-core/src/test_support.rs index febf2d140..9942146e7 100644 --- a/crates/engram-host-core/src/test_support.rs +++ b/crates/engram-host-core/src/test_support.rs @@ -65,7 +65,8 @@ impl BlobStorage for CountingBlob { /// kernel: `hydrate` links `name` to the destination file, so a write /// to either lands in both, and each `hydration` call copies one chunk /// the plan did not skip from the source file. `settle` needs every -/// chunk copied and changes nothing: the link is direct already. +/// chunk copied and gives the device a file of its own; on a device +/// that does not hydrate it does nothing. pub(crate) struct HydratingDisk { inner: FileDisk, chunk_size: u64, @@ -187,12 +188,9 @@ impl SessionDisk for HydratingDisk { } async fn settle(&self, name: &VolumeName, _dest: &Volume) -> io::Result<()> { let mut clones = self.clones.lock().unwrap(); - let clone = clones.get_mut(name).ok_or_else(|| { - io::Error::new( - io::ErrorKind::Unsupported, - format!("{name} is not hydrating"), - ) - })?; + let Some(clone) = clones.get_mut(name).filter(|c| !c.settled) else { + return Ok(()); + }; if !clone.pending.is_empty() { return Err(io::Error::new( io::ErrorKind::InvalidInput, From 5efba729ffba16beebff7d8e017891c9d4f41655 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:55:01 -0800 Subject: [PATCH 18/35] host-agent: the settle driver serves a hydrating disk first, for every live sandbox A reattach that failed at startup (a store error on either manifest) was logged once and never tried again: the guest's reads of cold regions and the kernel's copy stayed queued until the next restart. A create from an explicit manifest hydrates like a resume and had no settle driver at all, so its clone, source and base never went. The driver now starts with `reattach`, retried like its other steps, and takes no role list: it runs after a restore, after a create, and for every live sandbox at startup. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .../engram-host-agent/src/pooled_backend.rs | 66 ++++++++++--------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index 391094d49..1dc6e3b9e 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -1053,11 +1053,26 @@ fn record_warm_hook_failure_metric(kind: engram_core::types::CaptureFailureKind) const SETTLE_POLL: std::time::Duration = std::time::Duration::from_millis(250); const SETTLE_RETRY: std::time::Duration = std::time::Duration::from_secs(5); -/// Settle each of the sandbox's hydrating disks: poll until the kernel +/// Settle each of the sandbox's hydrating disks: serve the sources this +/// process does not serve yet (after a restart), poll until the kernel /// has every region, then cut the device over and stop its source -/// (`SessionVolumes::settle` is the step). A failed step is retried; a +/// (`SessionVolumes::settle` is the step). A failed step is retried, so +/// a store outage at startup does not leave a guest's disk unserved; a /// destroy ends the driver by dropping its task. -async fn settle_disks(volumes: Arc, id: SandboxId, roles: Vec) { +async fn settle_disks(volumes: Arc, id: SandboxId) { + let roles = loop { + let hydrating = async { + volumes.reattach(id).await?; + volumes.hydrating(id).await + }; + match hydrating.await { + Ok(roles) => break roles, + Err(error) => { + tracing::warn!(sandbox_id = %id, %error, "hydrating disks not served; retrying"); + tokio::time::sleep(SETTLE_RETRY).await; + } + } + }; for role in roles { let started = crate::time_source::metrics_now(); // The regions the copy still had to bring in when this driver @@ -1997,28 +2012,20 @@ impl PooledBackend { } } } - (Ok(()), Some(volumes)) => match volumes.hydrating(id).await { - Ok(roles) => self.settle_disks(id, roles), - Err(error) => { - tracing::warn!(sandbox_id = %id, %error, "the hydrating disks of a restore are unknown; none settles") - } - }, + (Ok(()), Some(_)) => self.settle_disks(id), (Ok(()), None) => {} } restore } - /// Drive `roles`, the sandbox's hydrating disks, to settled volumes - /// in the background. - fn settle_disks(&self, id: SandboxId, roles: Vec) { + /// Drive the sandbox's hydrating disks to settled volumes in the + /// background. + fn settle_disks(&self, id: SandboxId) { let Some(volumes) = self.volumes.clone() else { return; }; - if roles.is_empty() { - return; - } let task = tokio::spawn( - settle_disks(volumes, id, roles) + settle_disks(volumes, id) .instrument(tracing::info_span!("disk.settle", sandbox_id = %id)), ); self.disk_settles.insert(id, AbortTaskOnDrop(task)); @@ -2559,21 +2566,16 @@ impl PooledBackend { } }; match volumes.reattach_all().await { + Ok(reattached) if reattached.is_empty() => {} Ok(reattached) => { - for id in live.iter().copied() { - let roles: Vec = reattached - .iter() - .filter(|(sandbox, _)| *sandbox == id) - .map(|(_, role)| *role) - .collect(); - self.settle_disks(id, roles); - } - if !reattached.is_empty() { - tracing::info!(volumes = reattached.len(), "hydrating disks served again"); - } + tracing::info!(volumes = reattached.len(), "hydrating disks served again"); } Err(error) => tracing::warn!(%error, "startup reattach of hydrating disks failed"), } + // Each driver serves what the pass above could not. + for id in live.iter().copied() { + self.settle_disks(id); + } let finalizer = self.eviction_finalizer(); match crate::eviction_finalize::reconcile_volumes(volumes, finalizer.as_ref(), live).await { Ok(report) => { @@ -4780,9 +4782,13 @@ impl SandboxBackend for PooledBackend { spec.swap_source = provisioned.swap; spec.swap_manifest = provisioned.swap_manifest; let created = self.inner.create(id, spec).await; - if created.is_err() { - if let Err(error) = volumes.release(id).await { - tracing::warn!(sandbox_id = %id, %error, "release after a failed create failed"); + match &created { + // A create from a manifest hydrates like a resume. + Ok(()) => self.settle_disks(id), + Err(_) => { + if let Err(error) = volumes.release(id).await { + tracing::warn!(sandbox_id = %id, %error, "release after a failed create failed"); + } } } created From 4e99bce139ba89bed452a30a14f1d18cf6c5be05 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:55:01 -0800 Subject: [PATCH 19/35] host-agent: the source stalls on a store error, and a guest read does not wait at Hydrate - A fetch error became an I/O error on the guest's root disk. The source now tries again, from 100 ms doubling to 5 s, until the device stops; only a missing chunk, a hash mismatch and an out-of-range read fail. - A read inside the stream window waited on the window's fetch at `Hydrate`, behind every other session's background work. A read that is not whole regions is the guest's: it goes to a ranged `Demand` read unless the chunk is already in. A window fetch that has waited 1 s is asked again at `Demand`. - The host-agent set the ublk source on every Linux host, so a node without `ublk_drv` failed every cold resume. It probes `/dev/ublk-control` at startup; without it, it logs an error and writes each volume in full before the guest starts. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- crates/engram-host-agent/src/blk_source.rs | 108 ++++++++++++++++++--- crates/engram-host-agent/src/lib.rs | 12 ++- 2 files changed, 103 insertions(+), 17 deletions(-) diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index a3de554f7..2defbb28e 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -13,7 +13,12 @@ //! out region by region, so the store sees one GET per chunk and the //! copy runs at the engine's rate. A read anywhere else is the guest //! waiting (or a write the kernel first copies the region for): it gets -//! a ranged GET at `Demand`, and its chunk is recorded as touched. +//! a ranged GET at `Demand`, and its chunk is recorded as touched. So +//! does a read inside the window that is not whole regions, which only +//! the guest sends, when the window's chunk is not in yet. +//! +//! A store error that can pass stalls the read until it does: an I/O +//! error on a guest's root disk outlives the outage that caused it. use std::collections::{HashMap, VecDeque}; use std::fs::File; @@ -26,7 +31,8 @@ use std::time::Duration; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; -use engram_chunk_store::{ChunkHash, Manifest}; +use engram_chunk_store::{ChunkHash, ChunkStoreError, Manifest}; +use engram_core::error::BlobError; use engram_fill::{Class, FillEngine}; use engram_host_core::{SourceDevice, SourceServer}; use io_uring::{opcode, types}; @@ -42,6 +48,13 @@ const WINDOW_AHEAD: usize = 4; /// Region-shaped reads at ascending delta chunks, outside the window, /// that move the stream there (a host-agent restart loses the cursor). const RESYNC_READS: u32 = 4; +/// A stream chunk's fetch still waiting at `Hydrate` after this long is +/// asked for again at `Demand`: a guest read of whole regions inside the +/// window waits on it, and `Hydrate` can wait behind other sessions. +const STREAM_PATIENCE: Duration = Duration::from_secs(1); +/// The wait after a failed fetch: from the first, doubling to the last. +const RETRY_FIRST: Duration = Duration::from_millis(100); +const RETRY_LAST: Duration = Duration::from_secs(5); /// The tag the eventfd read carries; real tags are below the depth. const EVENT_TAG: u16 = u16::MAX; const QUEUE_DEPTH: u16 = 64; @@ -371,20 +384,30 @@ impl Shared { match self.hashes[chunk as usize] { None => out.resize(out.len() + piece as usize, 0), Some(hash) => { + let regions = at.is_multiple_of(self.region_bytes) + && piece.is_multiple_of(self.region_bytes); let bytes = match self.classify(chunk, at, piece) { - Serve::Stream(cell) => { - let whole = cell - .get_or_try_init(|| self.engine.read(hash, Class::Hydrate)) - .await - .map_err(|e| io::Error::other(e.to_string()))?; + Serve::Stream(cell) if regions || cell.initialized() => { + let whole = self + .until_fetched(chunk, || async { + cell.get_or_try_init(|| stream_chunk(&self.engine, hash)) + .await + .cloned() + }) + .await?; whole.slice(in_chunk as usize..(in_chunk + piece) as usize) } - Serve::Demand => { + Serve::Stream(_) | Serve::Demand => { self.touch(chunk); - self.engine - .read_range(hash, in_chunk, piece as usize, Class::Demand) - .await - .map_err(|e| io::Error::other(e.to_string()))? + self.until_fetched(chunk, || { + self.engine.read_range( + hash, + in_chunk, + piece as usize, + Class::Demand, + ) + }) + .await? } }; if bytes.len() as u64 != piece { @@ -404,6 +427,35 @@ impl Shared { Ok(out.freeze()) } + /// `fetch`, again after each error that can pass, until the device + /// stops. A chunk the store does not hold, or holds with other + /// bytes, fails at once. + async fn until_fetched(&self, chunk: u64, fetch: F) -> io::Result + where + F: Fn() -> Fut, + Fut: std::future::Future>, + { + let mut wait = RETRY_FIRST; + loop { + let error = match fetch().await { + Ok(bytes) => return Ok(bytes), + Err(error) => error, + }; + let lasting = matches!( + error, + ChunkStoreError::Blob(BlobError::NotFound | BlobError::RangeOutOfBounds { .. }) + | ChunkStoreError::HashMismatch { .. } + | ChunkStoreError::RangeOutOfBounds { .. } + ); + if lasting || self.stopping.load(Ordering::Relaxed) { + return Err(io::Error::other(error.to_string())); + } + tracing::warn!(chunk, %error, retry_ms = wait.as_millis() as u64, "ublk source: fetch failed; retrying"); + tokio::time::sleep(wait).await; + wait = (wait * 2).min(RETRY_LAST); + } + } + /// Stream or demand, by position against the window. A stream read /// at the chunk after the cursor moves the cursor; the chunk before /// it still serves, since the copy's reads of one chunk and the next @@ -477,10 +529,7 @@ impl Shared { let cell = slot.insert(Arc::default()).clone(); let engine = self.engine.clone(); tokio::spawn(async move { - if let Err(e) = cell - .get_or_try_init(|| engine.read(hash, Class::Hydrate)) - .await - { + if let Err(e) = cell.get_or_try_init(|| stream_chunk(&engine, hash)).await { tracing::debug!(chunk, error = %e, "stream prefetch failed; the read retries it"); } }); @@ -496,6 +545,33 @@ impl Shared { } } +/// A whole verified chunk for the stream window: at `Hydrate`, and at +/// `Demand` too once that has waited `STREAM_PATIENCE`. +async fn stream_chunk(engine: &FillEngine, hash: ChunkHash) -> Result { + let hydrate = engine.read(hash, Class::Hydrate); + tokio::pin!(hydrate); + tokio::select! { + biased; + read = &mut hydrate => return read, + () = tokio::time::sleep(STREAM_PATIENCE) => {} + } + tokio::select! { + read = &mut hydrate => read, + read = engine.read(hash, Class::Demand) => read, + } +} + +/// Whether this host can serve a source: the ublk control device is +/// there and opens. Without it a session volume is written in full +/// before its guest starts. +pub fn available() -> io::Result<()> { + File::options() + .read(true) + .write(true) + .open("/dev/ublk-control") + .map(drop) +} + struct Request { tag: u16, offset: u64, diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index ea9d89a66..d7e0f452b 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -306,7 +306,17 @@ impl HostAgent { if let Some(disk) = self.session_disk.clone() { let volumes = engram_host_core::SessionVolumes::new(disk, fill.clone()); #[cfg(target_os = "linux")] - let volumes = volumes.with_sources(Arc::new(blk_source::UblkSources)); + let volumes = match blk_source::available() { + Ok(()) => volumes.with_sources(Arc::new(blk_source::UblkSources)), + Err(error) => { + tracing::error!( + %error, + "no ublk control device (is ublk_drv loaded?): every resume \ + writes its whole disk before the guest starts" + ); + volumes + } + }; p = p.with_session_volumes(Arc::new(volumes)); } p = p.with_fill(fill); From 1062145e2d7c13b650e39f80ba09257f8e3033e7 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:55:01 -0800 Subject: [PATCH 20/35] host-core: disk.hydrate says where its time goes `disk.hydrate` took 0.7 s of a cold resume on the dev cluster with no span under it. The zeroing, the source's start and the device's create each get one, and the first two run together: neither touches what the other does. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .../engram-host-core/src/session_volumes.rs | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 6b46d64a3..8f25d279c 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -742,17 +742,29 @@ impl SessionVolumes { .await?; drop(base_volume); shape_matches(&dest, wanted)?; - for chunk in &plan.zero { - dest.zero_chunk(*chunk).await?; + // The zeroing and the source's start do not touch each other. + let zero = async { + for chunk in &plan.zero { + dest.zero_chunk(*chunk).await?; + } + Ok::<_, io::Error>(()) } - let source = sources + .instrument(tracing::info_span!("disk.hydrate.zero")); + let serve = sources .serve(fill.clone(), wanted, plan.delta.clone(), region_bytes) - .await?; - let live = match self - .disk - .hydrate(&dest, source.path(), &live_name(sandbox, role), &plan.skip) - .await - { + .instrument(tracing::info_span!("disk.hydrate.source")); + let (zeroed, source) = tokio::join!(zero, serve); + let source = source?; + let live = match zeroed { + Ok(()) => { + self.disk + .hydrate(&dest, source.path(), &live_name(sandbox, role), &plan.skip) + .instrument(tracing::info_span!("disk.hydrate.device")) + .await + } + Err(e) => Err(e), + }; + let live = match live { Ok(live) => live, Err(e) => { if let Err(error) = source.stop().await { From e044932b3179776e52898669c84e99b6ed1ff92b Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 17:55:01 -0800 Subject: [PATCH 21/35] docs: the lazy resume's upgrade note, rollback rule and review facts - CHANGELOG: the upgrade note for the hydrating resume (`ublk_drv`, generation 3, drain before a rollback). - ADR 0125 said no node carries a hydrating volume across the roll "in either direction". The drain acts only on hosts below the target's generation, so a rollback needs a drain by hand, as for generations 1 and 2. The ADR and DESIGN.md also gain the facts the review fixes established and the memory a source holds. - The root test's comments claimed a cold read and a death mid-copy; they say what the test proves. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- CHANGELOG.md | 8 +++++++ DESIGN.md | 8 +++++-- .../engram-host-agent/tests/lazy_restore.rs | 18 +++++++------- docs/adr/0125-kernel-owned-session-disk.md | 24 ++++++++++++++----- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a572b3fc7..85b484b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,14 @@ starts with **Upgrade notes**: read them before you upgrade. The upgrade procedu above. Nothing ends and no data is lost: each session resumes on the new build the next time it is used. With `upgradeDrain.enabled: false`, evict every session before the upgrade. To roll back past this release, drain the hosts first the same way. +- **A resume copies its disk in behind the running VM, which needs `ublk_drv`.** The host + agent serves the part of a session's disk the host does not hold as a ublk block device, + so the node needs the `ublk_drv` module; node-prep loads it. On a node without + `/dev/ublk-control` the host agent logs an error at startup and writes each disk in full + before its VM starts, as before. The reattach generation goes to 3, so this upgrade + drains every host as described above. To roll back past this release, drain the hosts + first: an earlier host agent cannot serve a disk that is still copying, and its VM would + hang. ### Changed diff --git a/DESIGN.md b/DESIGN.md index de24cf5ef..cc48ee97a 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -247,10 +247,14 @@ the test named beside it. window reads nothing through the source and loses no write: a block whose mapping changed since the base is the snapshot's, any other is the manifest's (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, - `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`). A + `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`). Only a + snapshot taken after the settle becomes a checkpoint volume + (`a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume`). A host-agent restart serves every hydrating volume again from the pool's names, and never deletes one it does not serve (`a_killed_process_leaves_a_volume_the_next_one_serves_again`). - A node needs `ublk_drv`; node-prep loads it beside `dm_thin_pool`. + A store error that can pass stalls a read of the source; it does not fail it. + A node needs `ublk_drv`; node-prep loads it beside `dm_thin_pool`. Without the ublk + control device a host writes each volume in full before its guest starts. - A live move carries the disk in 64 KiB blocks in three tiers: the checkpoint the source advertises, the blocks a snapshot taken while the guest runs changed since it, and the blocks the snapshot under the pause changed since that one. A source with no checkpoint diff --git a/crates/engram-host-agent/tests/lazy_restore.rs b/crates/engram-host-agent/tests/lazy_restore.rs index 845d0b80c..d3ccd06e4 100644 --- a/crates/engram-host-agent/tests/lazy_restore.rs +++ b/crates/engram-host-agent/tests/lazy_restore.rs @@ -1,8 +1,9 @@ //! A lazy resume over the dm-thin pool with a ublk source, as root: the -//! device the guest would open answers a cold read with the manifest's -//! bytes while the kernel copies, the volume settles onto its -//! destination, and a process that dies mid-copy leaves a volume the -//! next process must serve again before it can delete it. Needs +//! device the guest would open answers a read with the manifest's bytes +//! whether the kernel has copied the region or not, the volume settles +//! onto its destination, and a process that dies before the settle +//! leaves a volume the next process must serve again before it can +//! delete it. A read queued on a dead server is the `blk_source` test's. Needs //! `ENGRAM_TEST_THIN_POOL` and `ENGRAM_TEST_UBLK` (the CI root lane); //! unset, the tests skip. // tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) @@ -145,8 +146,8 @@ async fn a_lazy_resume_reads_cold_chunks_through_the_source_and_settles() { assert_eq!(provisioned.root_stage, StagePath::Lazy); assert_eq!(volumes.sources().len(), 1, "one source serves the root"); - // The last chunk is the last the ascending copy reaches: read now, it - // comes through the source. A write to a hole lands on the device. + // The last chunk is the last the ascending copy reaches; read now, it + // is right from either side. A write to a hole lands on the device. let live = volumes.live(sandbox, DiskRole::Root).await.unwrap(); assert_eq!(live.read_chunk(ChunkIndex(7)).await.unwrap(), fill(8)); live.write_chunk(ChunkIndex(2), fill(9)).await.unwrap(); @@ -201,8 +202,9 @@ async fn child_provisions_and_waits() { std::future::pending::<()>().await; } -/// A host-agent killed mid-copy: the next host-agent cannot delete the -/// hydrating volume until it serves the source again, then settles it. +/// A host-agent killed before the settle: the next host-agent cannot +/// delete the hydrating volume until it serves the source again, then +/// settles it. #[tokio::test] #[ignore = "requires root, a dm-thin pool and the ublk_drv module"] async fn a_killed_process_leaves_a_volume_the_next_one_serves_again() { diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index 10f192c02..5ab83a17d 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -343,11 +343,16 @@ opened never changes. A checkpoint in the window diffs the snapshot's mapping against the base: a block whose mapping changed (copied or written) is the snapshot's, any other is the manifest's, so nothing is read through the source and no write is lost; the published manifest -takes the destination's name until the settle, after which the next -checkpoint makes the normal layout. A live volume that hydrates and +takes the destination's name, and only a snapshot taken after the +settle becomes a checkpoint volume: one taken in the window holds base +bytes where the copy had not reached. A live volume that hydrates and whose source this process does not serve is never deleted; the next -host-agent serves it again from the names alone. The memory path keeps -its lazy base (ADR 0092). +host-agent serves it again from the names alone, and retries until the +store answers. A store error that can pass stalls a read of the source +instead of failing it, so the guest never sees an I/O error for an +outage. A host without the ublk control device writes each volume in +full before its guest starts and logs an error at startup. The memory +path keeps its lazy base (ADR 0092). **Teleport (ADR 0123).** A live move carries the disk in three tiers. The target stages the checkpoint the source advertises at presetup. The @@ -705,9 +710,14 @@ thin-pool host and the reverse. on; a `dmsetup remove` right after a create meets udev's open for about 0.2 s and is retried. Libublk's control commands run on a thread-local ring, so every thread that stops, recovers or deletes a - device makes one first. The reattach generation goes to 3: a + device makes one first. The reattach generation goes to 3: the pre-upgrade drain evicts every session, so no node carries a - hydrating volume across the roll in either direction. A source in + hydrating volume into the roll. The drain acts on hosts below the + target's generation only, so a rollback past this change needs the + hosts drained by hand first, as for generations 1 and 2: the earlier + build serves no source, and a guest on a hydrating volume would wait + on its cold regions for ever. A source holds up to six chunks in + memory (96 MiB at 16 MiB chunks) for each hydrating volume. A source in its window, or settled before its first checkpoint, moves by snapshot: it has no checkpoint volume for a live move's block diff, and the presetup says so. The live move's three block tiers stay; the @@ -762,6 +772,8 @@ names the test that guards each one. As landed, the guards are: loses no write; no checkpoint volume exists before the settle. (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`) +- Only a snapshot taken after the settle becomes a checkpoint volume. + (`a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume`) - A host-agent restart serves a hydrating volume again from the pool's names; a volume nothing serves is never deleted. (`a_killed_process_leaves_a_volume_the_next_one_serves_again`, From bdce02571a1f265f9292d45ec6ca5e0f9614c1e1 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 18:18:43 -0800 Subject: [PATCH 22/35] ci: the root lane accepts a built-in ublk driver and says what the kernel has The runner's kernel has no `ublk_drv` module, so `modprobe` failed the lane before any test. A kernel can also have the driver built in: the step now needs `/dev/ublk-control`, not the module, and prints the kernel's ublk and dm-clone configuration when the device is missing. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_017XkqDqWbQdPXdy94AdFV5e --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 202247375..337ce8418 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1178,7 +1178,16 @@ jobs: sudo apt-get install -y -qq --no-install-recommends thin-provisioning-tools sudo env POOL_DIR=/var/lib/engram/thin POOL_DATA_SIZE=4G POOL_META_SIZE=128M \ sh deploy/helm/engram-host-fleet/files/thin-pool.sh - sudo modprobe ublk_drv + # ublk is a module on some kernels and built in on others. + sudo modprobe ublk_drv || true + if [ ! -e /dev/ublk-control ]; then + echo "this runner's kernel has no ublk driver:" + uname -r + grep -h -E 'CONFIG_BLK_DEV_UBLK|CONFIG_DM_CLONE' "/boot/config-$(uname -r)" || true + zcat /proc/config.gz 2>/dev/null | grep -E 'CONFIG_BLK_DEV_UBLK|CONFIG_DM_CLONE' || true + ls "/lib/modules/$(uname -r)" || true + exit 1 + fi # Every state node prep can find: a used disk, a disk without room, # a crash before the first activation, a restart. From 6815718cb81a5152fe2d4dd9111647472cdef83d Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:01:26 -0800 Subject: [PATCH 23/35] ci: the ublk source's tests run on a kernel that has ublk The Blacksmith runner's kernel is built with `CONFIG_DM_CLONE=y` and without `CONFIG_BLK_DEV_UBLK`, and has no module tree, so no step can load the driver there. `blk_source` and `lazy_restore` move to a new lane, `test-ublk-root`, on a GitHub-hosted Ubuntu runner, which ships `ublk_drv` in `linux-modules-extra`. They boot no guest and need no KVM. The dm-clone test stays in `test-fc-root`. The new lane is in the gate's `needs` and runs under the Firecracker lanes' detector flag. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- .github/workflows/ci.yml | 68 +++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 337ce8418..0de534e66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1178,16 +1178,6 @@ jobs: sudo apt-get install -y -qq --no-install-recommends thin-provisioning-tools sudo env POOL_DIR=/var/lib/engram/thin POOL_DATA_SIZE=4G POOL_META_SIZE=128M \ sh deploy/helm/engram-host-fleet/files/thin-pool.sh - # ublk is a module on some kernels and built in on others. - sudo modprobe ublk_drv || true - if [ ! -e /dev/ublk-control ]; then - echo "this runner's kernel has no ublk driver:" - uname -r - grep -h -E 'CONFIG_BLK_DEV_UBLK|CONFIG_DM_CLONE' "/boot/config-$(uname -r)" || true - zcat /proc/config.gz 2>/dev/null | grep -E 'CONFIG_BLK_DEV_UBLK|CONFIG_DM_CLONE' || true - ls "/lib/modules/$(uname -r)" || true - exit 1 - fi # Every state node prep can find: a used disk, a disk without room, # a crash before the first activation, a restart. @@ -1195,12 +1185,12 @@ jobs: run: sudo sh scripts/thin-pool-states-test.sh # The disk over the pool, then the session disk under a real guest - # over the same pool (`session_disk` reads ENGRAM_TEST_THIN_POOL) - # and the ublk source a hydrating volume reads from. + # over the same pool (`session_disk` reads ENGRAM_TEST_THIN_POOL). + # The ublk source's tests run in `test-ublk-root`: this runner's + # kernel has no ublk driver. - name: dm-thin session-disk tests (root) env: ENGRAM_TEST_THIN_POOL: engram-pool - ENGRAM_TEST_UBLK: "1" run: | sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-disk-thin \ @@ -1212,8 +1202,6 @@ jobs: sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-host-agent \ --test session_disk \ - --test blk_source \ - --test lazy_restore \ --run-ignored ignored-only \ --no-fail-fast \ --test-threads=1 @@ -1263,6 +1251,55 @@ jobs: run: | sudo chown -R "$USER":"$USER" target/ ~/.cargo ~/.docker 2>/dev/null || true + # ------------------------------------------------------------------ + # The ublk source of a hydrating session volume (ADR 0125), as root + # over a dm-thin pool. No guest boots, so no KVM. It runs on a + # GitHub-hosted runner because the Blacksmith kernel is built without + # `CONFIG_BLK_DEV_UBLK`; Ubuntu ships `ublk_drv` in + # `linux-modules-extra`, as the production node image ships it as a + # module. + # ------------------------------------------------------------------ + test-ublk-root: + name: tests (ublk source, root) + needs: detect + if: needs.detect.outputs.test_fc == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: Install protoc, the pool tools and the kernel's extra modules + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends \ + protobuf-compiler thin-provisioning-tools "linux-modules-extra-$(uname -r)" + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ublk-root + - uses: taiki-e/install-action@nextest + - name: Build the dm-thin pool and load ublk + run: | + sudo env POOL_DIR=/var/lib/engram/thin POOL_DATA_SIZE=4G POOL_META_SIZE=128M \ + sh deploy/helm/engram-host-fleet/files/thin-pool.sh + sudo modprobe ublk_drv + test -e /dev/ublk-control + - name: ublk source and lazy restore tests (root) + env: + ENGRAM_TEST_THIN_POOL: engram-pool + ENGRAM_TEST_UBLK: "1" + run: | + sudo -E env "PATH=$PATH" cargo nextest run \ + -p engram-host-agent \ + --test blk_source \ + --test lazy_restore \ + --run-ignored ignored-only \ + --no-fail-fast \ + --test-threads=1 + - name: Restore ownership after sudo tests + if: always() + run: | + sudo chown -R "$USER":"$USER" target/ ~/.cargo 2>/dev/null || true + test-fc-e2e: name: tests (firecracker true-e2e, Linux + KVM) needs: detect @@ -2465,6 +2502,7 @@ jobs: - test-fc-unprivileged - test-fc-pooled - test-fc-root + - test-ublk-root - test-fc-e2e - vz - test-macos From ba93d5d7e17da12e720b2c23bdff6f40d87801fe Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:10:20 -0800 Subject: [PATCH 24/35] host-agent: a read that meets a shutting-down host-agent waits for the next one On SIGTERM the runtime drops the source's dispatch task, and the queue thread, still alive, answered each new read with EIO. On the dev cluster a restart during the copy gave the guest I/O errors on its root disk and made dm-clone fail the copies in flight. The queue thread now leaves such a read unanswered. The device has `UBLK_F_USER_RECOVERY_REISSUE`, so the kernel gives it to the server the next host-agent starts. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-host-agent/src/blk_source.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 2defbb28e..3ebe99564 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -704,9 +704,11 @@ fn queue_loop( offset: iod.start_sector << 9, len: iod.nr_sectors << 9, }; - if requests.send(request).is_err() { - complete(queue, tag, buf.as_slice(), -libc::EIO); - } + // No receiver: the runtime is shutting down with the process. The + // read stays unanswered, and the kernel gives it to the next + // server (`UBLK_F_USER_RECOVERY_REISSUE`); an error here would + // reach the guest. + let _ = requests.send(request); }); } From 87253ad658542ca6852bd19314f5e8176956b87e Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:10:20 -0800 Subject: [PATCH 25/35] host-agent: only a read outside the window is a first touch A read inside the stream window that is not whole regions goes to `Demand` since 4e99bce13, and was also recorded as touched. The host's own probe of a new block device reads sector 0, which is in the window of a new stream, so every resume recorded chunk 0, and the root test `a_source_serves_the_manifest_and_records_the_chunks_read_cold` failed on a real kernel. A chunk in the window arrives without a prefix fetch; only a read outside it is recorded. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-host-agent/src/blk_source.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 3ebe99564..1b2a3cfd9 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -397,8 +397,14 @@ impl Shared { .await?; whole.slice(in_chunk as usize..(in_chunk + piece) as usize) } - Serve::Stream(_) | Serve::Demand => { - self.touch(chunk); + served => { + // A chunk in the window arrives without help, + // and the host's own probe of a new device + // reads there: only a read outside it counts + // as the guest's first touch. + if matches!(served, Serve::Demand) { + self.touch(chunk); + } self.until_fetched(chunk, || { self.engine.read_range( hash, From 3c9c5476a80f7431d7cb30bd75c86512dd049bc4 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:10:20 -0800 Subject: [PATCH 26/35] host-agent, disk-thin: the copy is limited by the pool's disk, not by its own window Measured on the dev cluster, a 31.7 GiB difference beside a 13 GiB memory fill: with 4 chunks fetched ahead and 8 MiB of kernel copies in flight the copy averaged 264 to 376 MiB/s and settled in 86 to 123 s; the samples show the kernel's copies waiting on the source or on their own in-flight limit while the host had idle CPU and link. With 12 chunks ahead and 32 MiB in flight it ran at 760 to 1017 MiB/s steady (608 MiB/s average, settled in 53 s) with the pool's backing disk at 85 to 100% utilisation. The host-agent's peak RSS did not grow, and the guest's read gaps and prompt timings were not worse. The ADR records the numbers, the 224 MiB a source can hold per hydrating volume, and two limits that stay: dm-clone starts no background copy while guest I/O is in flight on the device, and a checkpoint in the window reads every copied block back from the same disk, which nearly stops the copy while it runs. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-disk-thin/src/disk.rs | 7 ++++--- crates/engram-host-agent/src/blk_source.rs | 7 ++++--- docs/adr/0125-kernel-owned-session-disk.md | 15 ++++++++++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 6256ee241..6492b301b 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -25,10 +25,11 @@ const REMOVE_RETRY: Duration = Duration::from_millis(100); /// 4 KiB blocks. A 300 GiB volume has 4.9 million 64 KiB regions, a /// 600 KiB bitset; 8 MiB leaves room for its transaction shadows. const CLONE_METADATA_BYTES: u64 = 8 << 20; -/// Regions per background copy and copies in flight: 1 MiB bios, 8 MiB -/// in flight. The kernel's defaults of 1 and 1 copy at a few MiB/s. +/// Regions per background copy and copies in flight: 1 MiB bios, 32 MiB +/// in flight. The kernel's defaults of 1 and 1 copy at a few MiB/s; at +/// 8 MiB in flight the copies wait on the destination's writes. const HYDRATION_BATCH_REGIONS: u64 = 16; -const HYDRATION_THRESHOLD_REGIONS: u64 = 128; +const HYDRATION_THRESHOLD_REGIONS: u64 = 512; pub struct ThinPoolDisk { dm: dm::Dm, diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 1b2a3cfd9..2407c9b3b 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -42,9 +42,10 @@ use libublk::{sys, BufDesc, BufDescList, UblkError, UblkFlags, UblkIORes}; use tokio::sync::{mpsc, oneshot, OnceCell}; /// Chunks fetched ahead of the stream chunk. With 16 MiB chunks this is -/// 64 MiB in flight per device, which keeps the copy near the engine's -/// rate; the window holds at most this many plus two chunks. -const WINDOW_AHEAD: usize = 4; +/// 192 MiB in flight per device, enough that the pool's disk limits the +/// copy and not the fetches; the window holds at most this many plus +/// two chunks. +const WINDOW_AHEAD: usize = 12; /// Region-shaped reads at ascending delta chunks, outside the window, /// that move the stream there (a host-agent restart loses the cursor). const RESYNC_READS: u32 = 4; diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index 5ab83a17d..d9a2f6dbe 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -706,7 +706,16 @@ thin-pool host and the reverse. (Decision 3). Measured on a dev-cluster c3 host: dm-clone hydrates at about 370 MiB/s with `hydration_batch_size 16` and `hydration_threshold 128` against about 50 MiB/s with the kernel's - defaults of 1; the copy starts about 1 s after hydration is turned + defaults of 1. With a 31.7 GiB difference beside a 13 GiB memory + fill, that setting and a source window of 4 chunks left the copies + waiting on the source or on their own limit; `hydration_threshold + 512` and a window of 12 chunks copy at 760 to 1017 MiB/s, where the + pool's backing disk is the limit, and settle in 53 s. Two limits + stay: dm-clone starts no background copy while guest I/O is in flight + on the device, and a checkpoint in the window reads every copied + block back from the same disk (a copied block and a written block + look the same in the pool's mapping), which nearly stops the copy + while it runs. The copy starts about 1 s after hydration is turned on; a `dmsetup remove` right after a create meets udev's open for about 0.2 s and is retried. Libublk's control commands run on a thread-local ring, so every thread that stops, recovers or deletes a @@ -716,8 +725,8 @@ thin-pool host and the reverse. target's generation only, so a rollback past this change needs the hosts drained by hand first, as for generations 1 and 2: the earlier build serves no source, and a guest on a hydrating volume would wait - on its cold regions for ever. A source holds up to six chunks in - memory (96 MiB at 16 MiB chunks) for each hydrating volume. A source in + on its cold regions for ever. A source holds up to 14 chunks in + memory (224 MiB at 16 MiB chunks) for each hydrating volume. A source in its window, or settled before its first checkpoint, moves by snapshot: it has no checkpoint volume for a live move's block diff, and the presetup says so. The live move's three block tiers stay; the From bee8b6498466de1e5b90a3ab3ff03bb30e982eb2 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:11:42 -0800 Subject: [PATCH 27/35] ci: the ublk lane loads the module before it looks for a package The GitHub runner's kernel (7.0, azure) has no `linux-modules-extra` package, so the install failed before the lane tried the module. The lane loads `ublk_drv` first and installs the extra package only when that fails. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- .github/workflows/ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0de534e66..62a3d9908 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1255,9 +1255,8 @@ jobs: # The ublk source of a hydrating session volume (ADR 0125), as root # over a dm-thin pool. No guest boots, so no KVM. It runs on a # GitHub-hosted runner because the Blacksmith kernel is built without - # `CONFIG_BLK_DEV_UBLK`; Ubuntu ships `ublk_drv` in - # `linux-modules-extra`, as the production node image ships it as a - # module. + # `CONFIG_BLK_DEV_UBLK`; Ubuntu's kernel has `ublk_drv` as a module, + # as the production node image does. # ------------------------------------------------------------------ test-ublk-root: name: tests (ublk source, root) @@ -1268,11 +1267,11 @@ jobs: steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - - name: Install protoc, the pool tools and the kernel's extra modules + - name: Install protoc and the pool tools run: | sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends \ - protobuf-compiler thin-provisioning-tools "linux-modules-extra-$(uname -r)" + protobuf-compiler thin-provisioning-tools - uses: Swatinem/rust-cache@v2 with: shared-key: ublk-root @@ -1281,7 +1280,11 @@ jobs: run: | sudo env POOL_DIR=/var/lib/engram/thin POOL_DATA_SIZE=4G POOL_META_SIZE=128M \ sh deploy/helm/engram-host-fleet/files/thin-pool.sh - sudo modprobe ublk_drv + # Some Ubuntu kernels keep `ublk_drv` in a separate package. + if ! sudo modprobe ublk_drv; then + sudo apt-get install -y -qq --no-install-recommends "linux-modules-extra-$(uname -r)" + sudo modprobe ublk_drv + fi test -e /dev/ublk-control - name: ublk source and lazy restore tests (root) env: From fd710d469faa83a066b5be96eb9d592be505fdba Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:32:36 -0800 Subject: [PATCH 28/35] host-agent, disk-thin, helm: the two hydration limits are settings The source's window (whole chunks fetched ahead of the kernel's copy) and dm-clone's `hydration_threshold` (regions the kernel copies at one time) decide how fast a resumed disk copies in and how much memory and pool I/O it takes. The right values depend on the node's disk, so the host-fleet chart sets them (`thinPool.hydration.windowChunks`, `thinPool.hydration.inFlightRegions`) through `ENGRAM_DISK_HYDRATION_WINDOW_CHUNKS` and `ENGRAM_DISK_HYDRATION_IN_FLIGHT_REGIONS`. Unset, the host-agent uses 12 and 512 as before. The batch size stays a constant. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-disk-thin/src/disk.rs | 20 ++++++-- crates/engram-host-agent/src/blk_source.rs | 50 ++++++++++++++----- crates/engram-host-agent/src/config.rs | 8 +++ crates/engram-host-agent/src/lib.rs | 6 ++- crates/engram-host-agent/src/main.rs | 27 ++++++++++ crates/engram-host-agent/tests/blk_source.rs | 6 ++- .../engram-host-agent/tests/lazy_restore.rs | 2 +- .../templates/host-agent.daemonset.yaml | 4 ++ deploy/helm/engram-host-fleet/values.yaml | 8 +++ 9 files changed, 108 insertions(+), 23 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 6492b301b..347c5c495 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -25,11 +25,12 @@ const REMOVE_RETRY: Duration = Duration::from_millis(100); /// 4 KiB blocks. A 300 GiB volume has 4.9 million 64 KiB regions, a /// 600 KiB bitset; 8 MiB leaves room for its transaction shadows. const CLONE_METADATA_BYTES: u64 = 8 << 20; -/// Regions per background copy and copies in flight: 1 MiB bios, 32 MiB -/// in flight. The kernel's defaults of 1 and 1 copy at a few MiB/s; at -/// 8 MiB in flight the copies wait on the destination's writes. +/// Regions per background copy: 1 MiB bios. The kernel's default of 1 +/// copies at a few MiB/s. const HYDRATION_BATCH_REGIONS: u64 = 16; -const HYDRATION_THRESHOLD_REGIONS: u64 = 512; +/// Regions the kernel copies at one time: 32 MiB in flight. At 8 MiB +/// the copies wait on the destination's writes. +const HYDRATION_IN_FLIGHT_REGIONS: u64 = 512; pub struct ThinPoolDisk { dm: dm::Dm, @@ -39,6 +40,7 @@ pub struct ThinPoolDisk { nodes_dir: PathBuf, chunk_size: u64, block_bytes: u64, + hydration_in_flight: u64, /// Seeded from the pool at `open`. The kernel refuses a duplicate id /// and `create` and `snapshot` move on, so a wrap of the 24-bit space /// is safe. @@ -73,6 +75,7 @@ impl ThinPoolDisk { nodes_dir: nodes_dir.to_path_buf(), chunk_size, block_bytes, + hydration_in_flight: HYDRATION_IN_FLIGHT_REGIONS, next_id: Arc::new(AtomicU32::new(1)), metadata_snap: tokio::sync::Mutex::new(()), }; @@ -97,6 +100,12 @@ impl ThinPoolDisk { Ok(disk) } + /// Regions the kernel copies at one time into each hydrating volume. + pub fn with_hydration_in_flight(mut self, regions: u64) -> Self { + self.hydration_in_flight = regions; + self + } + pub fn pool(&self) -> &str { &self.pool } @@ -570,8 +579,9 @@ impl SessionDisk for ThinPoolDisk { let hydrating = async { let params = format!( "{} {} {} {region_sectors} 2 no_hydration no_discard_passdown \ - 4 hydration_threshold {HYDRATION_THRESHOLD_REGIONS} \ + 4 hydration_threshold {} \ hydration_batch_size {HYDRATION_BATCH_REGIONS}", + self.hydration_in_flight, dm::devno_of(&metadata)?, dm::devno_of(&dest.path)?, dm::devno_of(source)?, diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 2407c9b3b..69e682636 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -41,11 +41,6 @@ use libublk::io::{UblkDev, UblkIOCtx, UblkQueue}; use libublk::{sys, BufDesc, BufDescList, UblkError, UblkFlags, UblkIORes}; use tokio::sync::{mpsc, oneshot, OnceCell}; -/// Chunks fetched ahead of the stream chunk. With 16 MiB chunks this is -/// 192 MiB in flight per device, enough that the pool's disk limits the -/// copy and not the fetches; the window holds at most this many plus -/// two chunks. -const WINDOW_AHEAD: usize = 12; /// Region-shaped reads at ascending delta chunks, outside the window, /// that move the stream there (a host-agent restart loses the cursor). const RESYNC_READS: u32 = 4; @@ -78,13 +73,15 @@ impl BlkSource { /// Serve `manifest` through a new device. `delta` lists, ascending, /// the chunks the kernel will copy (the ones not skipped), which is /// where the stream runs; `region_bytes` is the kernel's copy unit. + /// The stream fetches `window_ahead` chunks ahead of that copy. pub async fn serve( engine: FillEngine, manifest: &Manifest, delta: Vec, region_bytes: u64, + window_ahead: usize, ) -> io::Result { - Self::start(engine, manifest, delta, region_bytes, None).await + Self::start(engine, manifest, delta, region_bytes, window_ahead, None).await } /// Take over device `dev_id` from a host-agent that died: the reads it @@ -95,8 +92,17 @@ impl BlkSource { manifest: &Manifest, delta: Vec, region_bytes: u64, + window_ahead: usize, ) -> io::Result { - Self::start(engine, manifest, delta, region_bytes, Some(dev_id)).await + Self::start( + engine, + manifest, + delta, + region_bytes, + window_ahead, + Some(dev_id), + ) + .await } async fn start( @@ -104,6 +110,7 @@ impl BlkSource { manifest: &Manifest, delta: Vec, region_bytes: u64, + window_ahead: usize, recover: Option, ) -> io::Result { let shared = Arc::new(Shared::new( @@ -111,6 +118,7 @@ impl BlkSource { manifest, delta, region_bytes, + window_ahead, recover.is_none(), )?); let builder = UblkCtrlBuilder::default() @@ -266,6 +274,9 @@ struct Shared { region_bytes: u64, /// Ascending chunks the kernel copies. delta: Vec, + /// Chunks fetched ahead of the stream chunk; the window holds at + /// most this many plus two. + window_ahead: usize, window: Mutex, touched: Mutex, stopping: AtomicBool, @@ -297,6 +308,7 @@ impl Shared { manifest: &Manifest, delta: Vec, region_bytes: u64, + window_ahead: usize, fresh: bool, ) -> io::Result { let chunk_size = manifest.chunk_size.0; @@ -343,6 +355,7 @@ impl Shared { size_bytes: manifest.total_bytes, region_bytes, delta, + window_ahead, window: Mutex::new(Window { cursor: None, chunks: HashMap::new(), @@ -519,7 +532,7 @@ impl Shared { } /// Put the cursor at `pos`, drop the chunks behind the one before it, - /// and fetch the chunks up to `WINDOW_AHEAD` past it that are not in + /// and fetch the chunks up to `window_ahead` past it that are not in /// the window yet. fn advance(&self, window: &mut Window, pos: usize) { if window.cursor != Some(pos) { @@ -527,7 +540,7 @@ impl Shared { let keep_from = pos.checked_sub(1).map_or(0, |p| self.delta[p]); window.chunks.retain(|chunk, _| *chunk >= keep_from); } - let last = (pos + WINDOW_AHEAD).min(self.delta.len() - 1); + let last = (pos + self.window_ahead).min(self.delta.len() - 1); for chunk in self.delta[pos..=last].iter().copied() { let Some(hash) = self.hashes[chunk as usize] else { continue; @@ -782,8 +795,11 @@ fn with_ctrl_ring(f: impl FnOnce() -> Result) -> io::Result f().map_err(ublk_err) } -/// The host's [`SourceServer`]: every source is a [`BlkSource`]. -pub struct UblkSources; +/// The host's [`SourceServer`]: every source is a [`BlkSource`] that +/// fetches `window_ahead` chunks ahead of the kernel's copy. +pub struct UblkSources { + pub window_ahead: usize, +} #[async_trait] impl SourceServer for UblkSources { @@ -795,7 +811,7 @@ impl SourceServer for UblkSources { region_bytes: u64, ) -> io::Result> { Ok(Box::new( - BlkSource::serve(fill, manifest, delta, region_bytes).await?, + BlkSource::serve(fill, manifest, delta, region_bytes, self.window_ahead).await?, )) } @@ -808,7 +824,15 @@ impl SourceServer for UblkSources { region_bytes: u64, ) -> io::Result> { Ok(Box::new( - BlkSource::recover(dev_id_of(path)?, fill, manifest, delta, region_bytes).await?, + BlkSource::recover( + dev_id_of(path)?, + fill, + manifest, + delta, + region_bytes, + self.window_ahead, + ) + .await?, )) } } diff --git a/crates/engram-host-agent/src/config.rs b/crates/engram-host-agent/src/config.rs index 8be524e15..ea54b3879 100644 --- a/crates/engram-host-agent/src/config.rs +++ b/crates/engram-host-agent/src/config.rs @@ -53,8 +53,15 @@ pub struct HostAgentConfig { /// Whether an idle-eviction capture keeps the in-guest harness alive /// or stops it first. pub idle_eviction_harness: crate::host_client::IdleEvictionHarness, + /// Whole chunks a hydrating disk's source fetches ahead of the + /// kernel's copy. + pub disk_hydration_window_chunks: usize, } +/// With 16 MiB chunks, 192 MiB in flight per hydrating disk: enough that +/// the pool's disk limits the copy and not the fetches. +pub const DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS: usize = 12; + impl Default for HostAgentConfig { fn default() -> Self { Self { @@ -71,6 +78,7 @@ impl Default for HostAgentConfig { migrate_peer_listen_addr: None, backend_name: "process".to_string(), idle_eviction_harness: Default::default(), + disk_hydration_window_chunks: DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS, } } } diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index d7e0f452b..d5bd41b5a 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -110,7 +110,7 @@ mod time_source; pub mod util; pub mod warm_progress; -pub use config::HostAgentConfig; +pub use config::{HostAgentConfig, DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS}; pub struct HostAgent { pub cfg: HostAgentConfig, @@ -307,7 +307,9 @@ impl HostAgent { let volumes = engram_host_core::SessionVolumes::new(disk, fill.clone()); #[cfg(target_os = "linux")] let volumes = match blk_source::available() { - Ok(()) => volumes.with_sources(Arc::new(blk_source::UblkSources)), + Ok(()) => volumes.with_sources(Arc::new(blk_source::UblkSources { + window_ahead: self.cfg.disk_hydration_window_chunks, + })), Err(error) => { tracing::error!( %error, diff --git a/crates/engram-host-agent/src/main.rs b/crates/engram-host-agent/src/main.rs index b809afdf0..317d4aa73 100644 --- a/crates/engram-host-agent/src/main.rs +++ b/crates/engram-host-agent/src/main.rs @@ -62,6 +62,25 @@ struct Cli { #[arg(long, env = "ENGRAM_THIN_POOL")] thin_pool: Option, + /// Whole chunks a resumed disk's copy fetches ahead of the kernel, + /// per disk. Each chunk is held in memory until the kernel has + /// copied it. Unset, 12. + #[arg( + long, + env = "ENGRAM_DISK_HYDRATION_WINDOW_CHUNKS", + value_parser = clap::value_parser!(u32).range(1..) + )] + disk_hydration_window_chunks: Option, + + /// Pool blocks the kernel copies at one time into a resumed disk, + /// per disk. Unset, 512. + #[arg( + long, + env = "ENGRAM_DISK_HYDRATION_IN_FLIGHT_REGIONS", + value_parser = clap::value_parser!(u64).range(1..) + )] + disk_hydration_in_flight_regions: Option, + /// Coordinator endpoint to dial via WebSocket. #[arg(long, env = "ENGRAM_COORDINATOR_ENDPOINT")] coordinator: Option, @@ -333,6 +352,10 @@ async fn main() -> Result<(), HostAgentError> { BackendChoice::Vz => "vz".to_string(), }, idle_eviction_harness: cli.idle_eviction_harness, + disk_hydration_window_chunks: cli.disk_hydration_window_chunks.map_or( + engram_host_agent::DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS, + |chunks| chunks as usize, + ), ..HostAgentConfig::default() }; @@ -543,6 +566,10 @@ async fn main() -> Result<(), HostAgentError> { ) .await .map_err(|e| HostAgentError::Config(format!("thin pool {pool}: {e}")))?; + let disk = match cli.disk_hydration_in_flight_regions { + Some(regions) => disk.with_hydration_in_flight(regions), + None => disk, + }; let capacity = engram_host_core::SessionDisk::capacity(&disk) .await .map_err(|e| HostAgentError::Config(format!("thin pool {pool} capacity: {e}")))?; diff --git a/crates/engram-host-agent/tests/blk_source.rs b/crates/engram-host-agent/tests/blk_source.rs index 79286c9d2..e67a72162 100644 --- a/crates/engram-host-agent/tests/blk_source.rs +++ b/crates/engram-host-agent/tests/blk_source.rs @@ -23,6 +23,7 @@ const CHUNKS: u64 = 4; /// Chunks 0, 1 and 3 hold bytes; chunk 2 is a hole. const HELD: [(u64, u8); 3] = [(0, 0x11), (1, 0x22), (3, 0x33)]; const DELTA: [u64; 3] = [0, 1, 3]; +const WINDOW: usize = 12; fn gate() -> bool { if std::env::var_os("ENGRAM_TEST_UBLK").is_none() { @@ -93,7 +94,7 @@ async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { } let dir = tempfile::tempdir().unwrap(); let (engine, manifest) = engine_and_manifest(dir.path()).await; - let source = BlkSource::serve(engine, &manifest, DELTA.to_vec(), REGION) + let source = BlkSource::serve(engine, &manifest, DELTA.to_vec(), REGION, WINDOW) .await .unwrap(); let path = source.path().to_path_buf(); @@ -140,6 +141,7 @@ async fn child_serves_until_killed() { &manifest, DELTA.to_vec(), REGION, + WINDOW, ) .await .unwrap(); @@ -207,7 +209,7 @@ async fn a_source_killed_with_a_queued_read_is_recovered_by_a_new_server() { nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGKILL).unwrap(); child.wait().unwrap(); - let recovered = BlkSource::recover(dev_id, engine, &manifest, DELTA.to_vec(), REGION) + let recovered = BlkSource::recover(dev_id, engine, &manifest, DELTA.to_vec(), REGION, WINDOW) .await .unwrap(); let bytes = tokio::time::timeout( diff --git a/crates/engram-host-agent/tests/lazy_restore.rs b/crates/engram-host-agent/tests/lazy_restore.rs index d3ccd06e4..21668dd23 100644 --- a/crates/engram-host-agent/tests/lazy_restore.rs +++ b/crates/engram-host-agent/tests/lazy_restore.rs @@ -55,7 +55,7 @@ async fn volumes(pool: &str, store: ChunkStore) -> SessionVolumes { .await .expect("open pool"); SessionVolumes::new(Arc::new(disk), FillEngine::new(store, Default::default())) - .with_sources(Arc::new(UblkSources)) + .with_sources(Arc::new(UblkSources { window_ahead: 12 })) } /// `PATTERN` as a manifest in `store`. diff --git a/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml b/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml index 26eafcc0a..bdaac9a6b 100644 --- a/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml +++ b/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml @@ -274,6 +274,10 @@ spec: value: /opt/engram/firecracker - name: ENGRAM_THIN_POOL value: {{ .Values.thinPool.name | quote }} + - name: ENGRAM_DISK_HYDRATION_WINDOW_CHUNKS + value: {{ .Values.thinPool.hydration.windowChunks | quote }} + - name: ENGRAM_DISK_HYDRATION_IN_FLIGHT_REGIONS + value: {{ .Values.thinPool.hydration.inFlightRegions | quote }} {{- if .Values.coordinator.tokenSecretName }} - name: ENGRAM_COORDINATOR_TOKEN valueFrom: diff --git a/deploy/helm/engram-host-fleet/values.yaml b/deploy/helm/engram-host-fleet/values.yaml index 46a5cec05..ed80e250d 100644 --- a/deploy/helm/engram-host-fleet/values.yaml +++ b/deploy/helm/engram-host-fleet/values.yaml @@ -233,6 +233,14 @@ thinPool: blockSizeSectors: 128 # Blocks; 32768 = 2 GiB at 64 KiB. lowWaterMarkBlocks: 32768 + # A resumed session's disk is copied in behind the running VM. Larger + # values copy faster until the pool's disk is the limit. + hydration: + # Whole chunks (16 MiB) fetched ahead of the kernel's copy, per disk. + # The host-agent holds up to this many plus two in memory per disk. + windowChunks: 12 + # Pool blocks the kernel copies at one time, per disk. + inFlightRegions: 512 hostAgent: grpcPort: 9101 From 9f8c1253f56ba0d132f4b5be42ee279cd9159628 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:32:37 -0800 Subject: [PATCH 29/35] fill: a whole chunk read for a volume is not hashed `FillEngine::read` hashed every chunk it fetched from the store. In a profile of a resume that copies 31.7 GiB, SHA-256 was 9 to 13 percent of the host-agent's CPU. A chunk is checked when it is written to the store or to the chunk cache; a read does not check it again. This changes every caller of `read`: the hydration stream, the eager stage of a volume, the checkpoint of a partly changed chunk of a hydrating volume and the substrate server's whole-chunk disk reads. `get` (a cache fill) checks as before. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-fill/src/engine.rs | 18 ++++++++------- crates/engram-fill/src/engine/tests.rs | 26 +++++++++++++++------- crates/engram-host-agent/src/blk_source.rs | 2 +- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/crates/engram-fill/src/engine.rs b/crates/engram-fill/src/engine.rs index 9e53904ac..4dd511bc3 100644 --- a/crates/engram-fill/src/engine.rs +++ b/crates/engram-fill/src/engine.rs @@ -357,15 +357,17 @@ impl FillEngine { } /// The chunk's bytes for a caller that keeps them itself (a volume - /// write): a cache hit is served, a miss is fetched and verified but - /// not stored in the cache, so the bytes are written to disk once. + /// write): a cache hit is served, a miss is fetched but not stored in + /// the cache, so the bytes are written to disk once. The fetched + /// bytes are not hashed: a chunk is checked when it is written to the + /// store or to the cache, not when it is read. pub async fn read(&self, hash: ChunkHash, class: Class) -> Result { if let Some(cache) = self.store().chunk_cache() { if let Some(bytes) = cache.read_resident(hash).await? { return Ok(bytes); } } - self.fetch_verified(hash, class, None).await + Ok(self.fetch_unverified(hash, class, None).await?.0) } /// `len` bytes at `offset` of a chunk, for a reader that needs part of @@ -381,10 +383,9 @@ impl FillEngine { /// `Hydrate` starts beside it (one flight with every other fill of /// the chunk). Without a cache there is no fill. /// - /// The ranged bytes are not hash-checked: only a whole chunk can be. - /// They go to the caller only, never to the cache; the cache stores - /// the chunk only from the whole-chunk fill, after its hash check. - /// A caller that needs checked bytes uses [`Self::get`]. + /// The ranged bytes are not hashed and go to the caller only; the + /// cache stores the chunk only from the whole-chunk fill, after its + /// hash check. pub async fn read_range( &self, hash: ChunkHash, @@ -530,7 +531,8 @@ impl FillEngine { /// against a peer that has not answered after `peer_race_after`. The /// slot is freed when the bytes are in hand (before the cache writes /// them); a peer answer frees it without a report to the adaptive - /// limit. The caller hashes what came from the store. + /// limit. A caller that stores the bytes under their hash checks + /// what came from the store. async fn fetch_unverified( &self, hash: ChunkHash, diff --git a/crates/engram-fill/src/engine/tests.rs b/crates/engram-fill/src/engine/tests.rs index 45dac75f5..28812d24d 100644 --- a/crates/engram-fill/src/engine/tests.rs +++ b/crates/engram-fill/src/engine/tests.rs @@ -284,22 +284,32 @@ async fn fill_returns_the_first_error_after_the_other_fetches_finish() { } #[tokio::test] -async fn read_verifies_and_leaves_the_cache_alone() { +async fn read_leaves_the_cache_alone() { let f = fixture(); - let good = put(&f, b"volume bytes").await; - let bad = ChunkHash::of(b"expected"); + let hash = put(&f, b"volume bytes").await; + let engine = engine_over(&f, counting(&f, None), Config::default()); + + assert_eq!( + &engine.read(hash, Class::Demand).await.unwrap()[..], + b"volume bytes" + ); + assert!(!f.cache.contains_on_disk(hash)); +} + +#[tokio::test] +async fn read_does_not_hash_what_the_store_returns() { + let f = fixture(); + let hash = ChunkHash::of(b"expected"); f.blob - .put(&bad.storage_key(), Bytes::from_static(b"tampered")) + .put(&hash.storage_key(), Bytes::from_static(b"other bytes")) .await .unwrap(); let engine = engine_over(&f, counting(&f, None), Config::default()); assert_eq!( - &engine.read(good, Class::Demand).await.unwrap()[..], - b"volume bytes" + &engine.read(hash, Class::Demand).await.unwrap()[..], + b"other bytes" ); - assert!(!f.cache.contains_on_disk(good)); - assert!(engine.read(bad, Class::Demand).await.is_err()); } #[tokio::test] diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 69e682636..c99a04db9 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -565,7 +565,7 @@ impl Shared { } } -/// A whole verified chunk for the stream window: at `Hydrate`, and at +/// A whole chunk for the stream window: at `Hydrate`, and at /// `Demand` too once that has waited `STREAM_PATIENCE`. async fn stream_chunk(engine: &FillEngine, hash: ChunkHash) -> Result { let hydrate = engine.read(hash, Class::Hydrate); From 6f0eaccbc89ed915f67c7e378b131dc591212fbc Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:32:37 -0800 Subject: [PATCH 30/35] host-agent: the periodic checkpoint waits for the disks to settle A checkpoint taken while the kernel copies a disk in reads every copied block back from the pool's disk, because a copied block and a written block look the same in the pool's mapping. On the dev cluster two such checkpoints dropped the copy from about 260 MiB/s to 7.6 MiB/s for 28 s. The periodic driver skips a sandbox whose settle driver still runs. The hold ends 10 minutes after the driver started: a driver that cannot settle runs for ever, and the session still needs checkpoints. An eviction or a snapshot does not wait. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-host-agent/src/checkpoint.rs | 9 +++ .../engram-host-agent/src/pooled_backend.rs | 60 ++++++++++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/crates/engram-host-agent/src/checkpoint.rs b/crates/engram-host-agent/src/checkpoint.rs index c86a5c9a4..d4c52c299 100644 --- a/crates/engram-host-agent/src/checkpoint.rs +++ b/crates/engram-host-agent/src/checkpoint.rs @@ -551,6 +551,15 @@ pub async fn run_checkpoint_pass( ); continue; } + if backend.disks_hydrating(sandbox_id) { + metrics::counter!(crate::metrics::CHECKPOINT_SKIPPED_TOTAL).increment(1); + tracing::debug!( + %sandbox_id, + %session_id, + "skipping periodic checkpoint; the disks still hydrate", + ); + continue; + } // A root span: the driver runs on its own timer, not for a request. let span = tracing::info_span!( parent: None, diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index 1dc6e3b9e..5ca75b66f 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -187,6 +187,25 @@ mod item_assembler_tests { } } +#[cfg(test)] +mod settle_hold_tests { + use super::{settle_holds_checkpoint, SETTLE_HOLDS_CHECKPOINT}; + + #[test] + fn a_running_settle_holds_the_periodic_checkpoint_for_a_bounded_time() { + let started = chrono::DateTime::UNIX_EPOCH; + let just_inside = started + SETTLE_HOLDS_CHECKPOINT - chrono::TimeDelta::seconds(1); + + assert!(settle_holds_checkpoint(false, started, just_inside)); + assert!(!settle_holds_checkpoint(true, started, just_inside)); + assert!(!settle_holds_checkpoint( + false, + started, + started + SETTLE_HOLDS_CHECKPOINT + )); + } +} + #[cfg(test)] mod first_touch_tests { use super::*; @@ -861,7 +880,7 @@ pub struct PooledBackend { /// The driver that settles a sandbox's hydrating disks; dropping the /// entry ends it. Destroy drops it before the VM goes, so no settle /// runs against a volume being released. - disk_settles: DashMap, + disk_settles: DashMap, /// The adaptive checkpoint controller's input: what the last /// capture observed about the memory dirty rate (ADR 0101 B). checkpoint_pacing: Arc>, @@ -1112,6 +1131,24 @@ async fn settle_disks(volumes: Arc, id: SandboxId) { /// `?`-return, since `Drop` runs during unwind too. struct AbortTaskOnDrop(tokio::task::JoinHandle<()>); +struct DiskSettle { + task: AbortTaskOnDrop, + started: chrono::DateTime, +} + +/// How long a settle driver that has not finished holds the periodic +/// checkpoint. A driver that cannot settle (a chunk the store lost) +/// runs for ever, and the session still needs checkpoints. +const SETTLE_HOLDS_CHECKPOINT: chrono::TimeDelta = chrono::TimeDelta::minutes(10); + +fn settle_holds_checkpoint( + finished: bool, + started: chrono::DateTime, + now: chrono::DateTime, +) -> bool { + !finished && now - started < SETTLE_HOLDS_CHECKPOINT +} + impl Drop for AbortTaskOnDrop { fn drop(&mut self) { self.0.abort(); @@ -2028,7 +2065,26 @@ impl PooledBackend { settle_disks(volumes, id) .instrument(tracing::info_span!("disk.settle", sandbox_id = %id)), ); - self.disk_settles.insert(id, AbortTaskOnDrop(task)); + self.disk_settles.insert( + id, + DiskSettle { + task: AbortTaskOnDrop(task), + started: self.clock.now_utc(), + }, + ); + } + + /// Whether the kernel still copies the sandbox's disks in. A + /// checkpoint taken now reads every copied block back from the pool's + /// disk, which nearly stops the copy, so the periodic driver waits. + pub fn disks_hydrating(&self, id: SandboxId) -> bool { + self.disk_settles.get(&id).is_some_and(|settle| { + settle_holds_checkpoint( + settle.task.0.is_finished(), + settle.started, + self.clock.now_utc(), + ) + }) } /// End a restore's memory fetches: cancel its background prefetch and From ef3b469a2501765ecf0975bce2ff50b5a9bf8609 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:32:37 -0800 Subject: [PATCH 31/35] docs: the hydration settings, the unhashed read and the checkpoint hold Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- DESIGN.md | 6 +++++- docs/adr/0125-kernel-owned-session-disk.md | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index cc48ee97a..9d0600dd8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -249,7 +249,11 @@ the test named beside it. (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`). Only a snapshot taken after the settle becomes a checkpoint volume - (`a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume`). A + (`a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume`). The periodic + checkpoint waits for the settle, for at most 10 minutes + (`a_running_settle_holds_the_periodic_checkpoint_for_a_bounded_time`); an eviction or + a snapshot does not wait. A whole chunk read for a volume is not hashed + (`read_does_not_hash_what_the_store_returns`). A host-agent restart serves every hydrating volume again from the pool's names, and never deletes one it does not serve (`a_killed_process_leaves_a_volume_the_next_one_serves_again`). A store error that can pass stalls a read of the source; it does not fail it. diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index d9a2f6dbe..b1d75face 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -715,7 +715,12 @@ thin-pool host and the reverse. on the device, and a checkpoint in the window reads every copied block back from the same disk (a copied block and a written block look the same in the pool's mapping), which nearly stops the copy - while it runs. The copy starts about 1 s after hydration is turned + while it runs. So the periodic checkpoint waits for the settle, for + at most 10 minutes; an eviction or a snapshot does not wait. The two + values are host-fleet chart values (`thinPool.hydration`). A whole + chunk read for a volume is not hashed: a chunk is checked when it is + written to the store or to the cache. The copy starts about 1 s after + hydration is turned on; a `dmsetup remove` right after a create meets udev's open for about 0.2 s and is retried. Libublk's control commands run on a thread-local ring, so every thread that stops, recovers or deletes a From b864319ce906c398a068f903532160e52009cd9f Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 19:44:05 -0800 Subject: [PATCH 32/35] disk-thin: the dm-clone table names its arguments `fd710d469` put the in-flight limit first in the positional list, so the table named the limit as the metadata device and the kernel refused it (ENOENT). The `clone` and `lazy_restore` root tests caught it. The format string names each argument. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-disk-thin/src/disk.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 347c5c495..a8c99b95b 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -578,13 +578,13 @@ impl SessionDisk for ThinPoolDisk { let device = self.device_name(name); let hydrating = async { let params = format!( - "{} {} {} {region_sectors} 2 no_hydration no_discard_passdown \ - 4 hydration_threshold {} \ + "{metadata} {dest} {source} {region_sectors} 2 no_hydration \ + no_discard_passdown 4 hydration_threshold {in_flight} \ hydration_batch_size {HYDRATION_BATCH_REGIONS}", - self.hydration_in_flight, - dm::devno_of(&metadata)?, - dm::devno_of(&dest.path)?, - dm::devno_of(source)?, + metadata = dm::devno_of(&metadata)?, + dest = dm::devno_of(&dest.path)?, + source = dm::devno_of(source)?, + in_flight = self.hydration_in_flight, ); self.dm.create(&device, sectors, "clone", ¶ms).await?; let volume = self.volume(name, sectors).await?; From 67f05f925a16487dee82156bb45478588112f5c2 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 20:29:21 -0800 Subject: [PATCH 33/35] ci: the ublk lane shows where a hung test waits On the GitHub runner's kernel (7.0) `a_source_serves_the_manifest_and_records_the_chunks_read_cold` did not end, the signal from nextest's slow timeout did not end it either, and the job ran to its 30-minute limit with no output. A thread blocked in the kernel on a ublk device cannot be killed. After 10 minutes the lane prints each test thread's kernel stack, the ublk devices' in-flight counts and the kernel log, then fails. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62a3d9908..f391d3fd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1297,7 +1297,35 @@ jobs: --test lazy_restore \ --run-ignored ignored-only \ --no-fail-fast \ - --test-threads=1 + --test-threads=1 & + run=$! + # A thread blocked in the kernel on a ublk device cannot be + # killed, so a hung test never ends: show where it waits. + for _ in $(seq 1 120); do + kill -0 "$run" 2>/dev/null || break + sleep 5 + done + if kill -0 "$run" 2>/dev/null; then + echo "::error::the ublk tests did not end in 10 minutes" + for exe in /proc/[0-9]*/exe; do + case "$(sudo readlink "$exe" 2>/dev/null)" in + */deps/blk_source-*|*/deps/lazy_restore-*) ;; + *) continue ;; + esac + pid=${exe#/proc/}; pid=${pid%/exe} + for task in /proc/"$pid"/task/*; do + echo "== pid $pid task ${task##*/} $(cat "$task/comm") $(cut -d' ' -f3 "$task/stat")" + sudo cat "$task/stack" 2>/dev/null || true + done + done + for dev in /sys/block/ublkb*; do + [ -e "$dev" ] && echo "$dev inflight: $(cat "$dev/inflight") holders: $(ls "$dev/holders")" + done + sudo dmsetup ls || true + sudo dmesg | tail -60 + exit 1 + fi + wait "$run" - name: Restore ownership after sudo tests if: always() run: | From 357b39f736daa12451cee5df3d246a501a22fa2c Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 21:05:00 -0800 Subject: [PATCH 34/35] host-agent: a source device gets no partition scan Kernel 7.0 scans a new ublk device for partitions in a work item, and an open of the device waits for that scan. The scan reads the device, so the source must answer while the open waits. The `blk_source` test opened the device on its one runtime thread, which is the thread that answers, and never ended on the CI runner (kernel log: the test blocked on a mutex owned by `ublk_partition_scan_work`, one read in flight). The source sets `UBLK_F_NO_AUTO_PART_SCAN`. Its bytes are a guest's disk, and the host has no use for a partition table a guest wrote. An older kernel does not know the flag and scans as before. The test opens the device off the runtime, as it already reads it. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-host-agent/src/blk_source.rs | 8 +++++++- crates/engram-host-agent/tests/blk_source.rs | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index c99a04db9..6a7e09c49 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -126,7 +126,13 @@ impl BlkSource { .nr_queues(1) .depth(QUEUE_DEPTH) .io_buf_bytes(IO_BUF_BYTES) - .ctrl_flags((sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64); + // No partition scan: the bytes are a guest's, and the host + // has no use for partitions of them. A kernel that does not + // know the flag drops it. + .ctrl_flags( + (sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64 + | sys::UBLK_F_NO_AUTO_PART_SCAN as u64, + ); let builder = match recover { None => builder.dev_flags(UblkFlags::UBLK_DEV_F_ADD_DEV), Some(id) => builder diff --git a/crates/engram-host-agent/tests/blk_source.rs b/crates/engram-host-agent/tests/blk_source.rs index e67a72162..aa36de687 100644 --- a/crates/engram-host-agent/tests/blk_source.rs +++ b/crates/engram-host-agent/tests/blk_source.rs @@ -77,6 +77,15 @@ fn read_at(device: &File, offset: u64, len: usize) -> Vec { buf.as_slice().to_vec() } +/// An open off the runtime: the source's dispatch runs on it, and an +/// open waits for the kernel's partition scan, which reads the device. +async fn open_off_runtime(path: &Path) -> Arc { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || Arc::new(open_direct(&path))) + .await + .unwrap() +} + /// A read off the runtime: the source's dispatch runs on it, so a read /// that blocks a runtime thread waits for itself. async fn read_off_runtime(device: &Arc, offset: u64, len: usize) -> Vec { @@ -98,7 +107,7 @@ async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { .await .unwrap(); let path = source.path().to_path_buf(); - let device = Arc::new(open_direct(&path)); + let device = open_off_runtime(&path).await; let image = image(); // A guest read far from where the copy starts: served, and recorded. From 4317b0c3559bf4dc1cea5e0264a9fcb2101f238c Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Mon, 9 Nov 2026 21:11:04 -0800 Subject: [PATCH 35/35] host-agent: the source does not set the no-partition-scan flag libublk 0.4.8 checks the control flags against its own list, which does not hold `UBLK_F_NO_AUTO_PART_SCAN`, and refuses the device before the kernel sees it. Every serve failed with "Invalid input". The flag goes; the test's open off the runtime thread, which is what the hang on kernel 7.0 needed, stays. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01HNhpmz4QPYaVCE7tFna54D --- crates/engram-host-agent/src/blk_source.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs index 6a7e09c49..c99a04db9 100644 --- a/crates/engram-host-agent/src/blk_source.rs +++ b/crates/engram-host-agent/src/blk_source.rs @@ -126,13 +126,7 @@ impl BlkSource { .nr_queues(1) .depth(QUEUE_DEPTH) .io_buf_bytes(IO_BUF_BYTES) - // No partition scan: the bytes are a guest's, and the host - // has no use for partitions of them. A kernel that does not - // know the flag drops it. - .ctrl_flags( - (sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64 - | sys::UBLK_F_NO_AUTO_PART_SCAN as u64, - ); + .ctrl_flags((sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64); let builder = match recover { None => builder.dev_flags(UblkFlags::UBLK_DEV_F_ADD_DEV), Some(id) => builder