diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5062b5b71..f391d3fd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1186,6 +1186,8 @@ jobs: # The disk over the pool, then the session disk under a real guest # over the same pool (`session_disk` reads ENGRAM_TEST_THIN_POOL). + # The ublk source's tests run in `test-ublk-root`: this runner's + # kernel has no ublk driver. - name: dm-thin session-disk tests (root) env: ENGRAM_TEST_THIN_POOL: engram-pool @@ -1193,6 +1195,7 @@ jobs: sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-disk-thin \ --test thin_pool \ + --test clone \ --run-ignored ignored-only \ --no-fail-fast \ --test-threads=1 @@ -1248,6 +1251,86 @@ jobs: run: | sudo chown -R "$USER":"$USER" target/ ~/.cargo ~/.docker 2>/dev/null || true + # ------------------------------------------------------------------ + # The ublk source of a hydrating session volume (ADR 0125), as root + # over a dm-thin pool. No guest boots, so no KVM. It runs on a + # GitHub-hosted runner because the Blacksmith kernel is built without + # `CONFIG_BLK_DEV_UBLK`; Ubuntu's kernel has `ublk_drv` as a module, + # as the production node image does. + # ------------------------------------------------------------------ + test-ublk-root: + name: tests (ublk source, root) + needs: detect + if: needs.detect.outputs.test_fc == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: Install protoc and the pool tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends \ + protobuf-compiler thin-provisioning-tools + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ublk-root + - uses: taiki-e/install-action@nextest + - name: Build the dm-thin pool and load ublk + run: | + sudo env POOL_DIR=/var/lib/engram/thin POOL_DATA_SIZE=4G POOL_META_SIZE=128M \ + sh deploy/helm/engram-host-fleet/files/thin-pool.sh + # Some Ubuntu kernels keep `ublk_drv` in a separate package. + if ! sudo modprobe ublk_drv; then + sudo apt-get install -y -qq --no-install-recommends "linux-modules-extra-$(uname -r)" + sudo modprobe ublk_drv + fi + test -e /dev/ublk-control + - name: ublk source and lazy restore tests (root) + env: + ENGRAM_TEST_THIN_POOL: engram-pool + ENGRAM_TEST_UBLK: "1" + run: | + sudo -E env "PATH=$PATH" cargo nextest run \ + -p engram-host-agent \ + --test blk_source \ + --test lazy_restore \ + --run-ignored ignored-only \ + --no-fail-fast \ + --test-threads=1 & + run=$! + # A thread blocked in the kernel on a ublk device cannot be + # killed, so a hung test never ends: show where it waits. + for _ in $(seq 1 120); do + kill -0 "$run" 2>/dev/null || break + sleep 5 + done + if kill -0 "$run" 2>/dev/null; then + echo "::error::the ublk tests did not end in 10 minutes" + for exe in /proc/[0-9]*/exe; do + case "$(sudo readlink "$exe" 2>/dev/null)" in + */deps/blk_source-*|*/deps/lazy_restore-*) ;; + *) continue ;; + esac + pid=${exe#/proc/}; pid=${pid%/exe} + for task in /proc/"$pid"/task/*; do + echo "== pid $pid task ${task##*/} $(cat "$task/comm") $(cut -d' ' -f3 "$task/stat")" + sudo cat "$task/stack" 2>/dev/null || true + done + done + for dev in /sys/block/ublkb*; do + [ -e "$dev" ] && echo "$dev inflight: $(cat "$dev/inflight") holders: $(ls "$dev/holders")" + done + sudo dmsetup ls || true + sudo dmesg | tail -60 + exit 1 + fi + wait "$run" + - name: Restore ownership after sudo tests + if: always() + run: | + sudo chown -R "$USER":"$USER" target/ ~/.cargo 2>/dev/null || true + test-fc-e2e: name: tests (firecracker true-e2e, Linux + KVM) needs: detect @@ -2450,6 +2533,7 @@ jobs: - test-fc-unprivileged - test-fc-pooled - test-fc-root + - test-ublk-root - test-fc-e2e - vz - test-macos diff --git a/CHANGELOG.md b/CHANGELOG.md index c7cc6ce8b..85b484b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,13 +62,23 @@ starts with **Upgrade notes**: read them before you upgrade. The upgrade procedu above. Nothing ends and no data is lost: each session resumes on the new build the next time it is used. With `upgradeDrain.enabled: false`, evict every session before the upgrade. To roll back past this release, drain the hosts first the same way. +- **A resume copies its disk in behind the running VM, which needs `ublk_drv`.** The host + agent serves the part of a session's disk the host does not hold as a ublk block device, + so the node needs the `ublk_drv` module; node-prep loads it. On a node without + `/dev/ublk-control` the host agent logs an error at startup and writes each disk in full + before its VM starts, as before. The reattach generation goes to 3, so this upgrade + drains every host as described above. To roll back past this release, drain the hosts + first: an earlier host agent cannot serve a disk that is still copying, and its VM would + hang. ### Changed - **The kernel owns the session disk.** A VM's disk is a thin volume in a device-mapper pool; no host process is in the data path, and a host agent restart does not touch a running VM's disk. A base image is staged into the pool once per host and every session's volume - is a snapshot of it, so a create writes nothing. A checkpoint pauses the VM for tens of + is a snapshot of it, so a create writes nothing. A resume starts the VM on a snapshot of + the nearest base the host holds and copies the difference in behind it, so the time to + the first prompt does not grow with the size of the session's disk. A checkpoint pauses the VM for tens of milliseconds however much it wrote and uploads only the blocks that changed. The disk daemon, the copy-on-write view, the flush pipeline, the durability-rollback path, and the quarantined-survivor eviction are gone with it. diff --git a/Cargo.lock b/Cargo.lock index 59e4b459c..e91ce87b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,6 +247,43 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + [[package]] name = "async-graphql-parser" version = "7.2.1" @@ -271,6 +308,82 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.6", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.6", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.6", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -293,6 +406,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.92" @@ -989,12 +1108,15 @@ dependencies = [ "itertools 0.12.1", "lazy_static", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex 1.3.0", "syn 2.0.117", + "which", ] [[package]] @@ -1055,6 +1177,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitmaps" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" + [[package]] name = "blake3" version = "1.8.7" @@ -1095,6 +1223,19 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "borsh" version = "1.8.1" @@ -1549,7 +1690,7 @@ dependencies = [ "futures-core", "mio", "parking_lot 0.12.5", - "rustix", + "rustix 1.1.6", "signal-hook", "signal-hook-mio", "winapi", @@ -1662,6 +1803,16 @@ dependencies = [ "darling_macro 0.20.11", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + [[package]] name = "darling" version = "0.24.1" @@ -1686,6 +1837,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + [[package]] name = "darling_core" version = "0.24.1" @@ -1710,6 +1875,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.24.1" @@ -1883,6 +2059,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_setters" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e6f6fa1f03c14ae082120b84b3c7fbd7b8588d924cf2d7c3daf9afd49df8b9" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -2692,8 +2880,10 @@ dependencies = [ "futures", "futures-util", "hex", + "io-uring", "jemalloc_pprof", "libc", + "libublk", "metrics", "metrics-exporter-prometheus", "nix 0.31.3", @@ -3259,7 +3449,7 @@ dependencies = [ "proptest", "pulldown-cmark", "ratatui", - "rustix", + "rustix 1.1.6", "serde", "serde_json", "syntect", @@ -3599,6 +3789,19 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.34" @@ -3622,6 +3825,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.34" @@ -4356,6 +4565,18 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "io-uring" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878c26bc5e45fe089ce2ca25689260c152a87ce1d7de2ca86101efeea6973932" +dependencies = [ + "bindgen", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.12.1" @@ -4829,6 +5050,41 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libublk" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212246283957aba80cbd6f04586d19eb57cf94aa49232c2a8a6b25b72c356239" +dependencies = [ + "async-lock", + "bitflags 2.13.2", + "bitmaps", + "derive_setters", + "futures-timer", + "io-uring", + "libc", + "libublk-rs-sys", + "log", + "serde", + "serde_json", + "slab", + "smol", + "thiserror 1.0.69", +] + +[[package]] +name = "libublk-rs-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd981fd7cf597c782453c2520f0d8f511dd6f66fc184b68577d7c7cf7108e44" +dependencies = [ + "anyhow", + "bindgen", + "libc", + "regex", + "serde", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -4838,6 +5094,12 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -5948,6 +6210,17 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -5981,6 +6254,20 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.6", + "windows-sys 0.61.2", +] + [[package]] name = "polyval" version = "0.7.3" @@ -6708,6 +6995,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.6" @@ -6717,7 +7017,7 @@ dependencies = [ "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -7240,6 +7540,23 @@ dependencies = [ "serde", ] +[[package]] +name = "smol" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33bd3e260892199c3ccfc487c88b2da2265080acb316cd920da72fdfd7c599f" +dependencies = [ + "async-channel", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-net", + "async-process", + "blocking", + "futures-lite", +] + [[package]] name = "socket2" version = "0.6.5" @@ -7638,7 +7955,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.6", "windows-sys 0.61.2", ] @@ -7650,7 +7967,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.2", "parking_lot 0.12.5", - "rustix", + "rustix 1.1.6", "signal-hook", "windows-sys 0.61.2", ] @@ -8879,6 +9196,18 @@ dependencies = [ "wezterm-dynamic", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "1.6.1" @@ -9268,10 +9597,12 @@ dependencies = [ "aws-smithy-types", "axum", "axum-core", + "bindgen", "bitflags 2.13.2", "bytes", "cc", "chrono", + "concurrent-queue", "crc32fast", "crossbeam-utils", "crypto-common 0.1.7", @@ -9284,6 +9615,7 @@ dependencies = [ "displaydoc", "either", "errno", + "event-listener-strategy", "flate2", "form_urlencoded", "futures-channel", @@ -9307,6 +9639,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "libc", + "linux-raw-sys 0.12.1", "log", "memchr", "mio", @@ -9321,6 +9654,7 @@ dependencies = [ "percent-encoding", "phf_shared", "ppv-lite86", + "prettyplease", "proc-macro2", "prost", "prost-build", @@ -9335,7 +9669,7 @@ dependencies = [ "regex-syntax", "reqwest", "rsa", - "rustix", + "rustix 1.1.6", "rustls", "serde", "serde_core", @@ -9404,7 +9738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08" dependencies = [ "libc", - "rustix", + "rustix 1.1.6", ] [[package]] diff --git a/DESIGN.md b/DESIGN.md index 2d1e94f57..9d0600dd8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -230,9 +230,40 @@ the test named beside it. `thin_delta_reports_exactly_what_the_full_compare_reports`). Under a guest writing at full rate the sync and snapshot stay under two seconds however much it wrote (`the_disk_pause_is_bounded_while_the_guest_writes_at_full_rate`). +- A resume boots before its disk is copied in. The live volume is a dm-clone device over + a snapshot of the nearest base the host holds: the kept checkpoint volume or staged image + version that shares the most chunks with the session's manifest at the same offset, so an + image whose tag moved still supplies the blocks it shares + (`a_resume_starts_from_the_staged_volume_that_shares_the_most`), or an empty volume when + none shares any; regions the base holds in final form are marked copied by + a discard, and the kernel copies the rest in behind the running guest from a ublk source + the host agent serves out of the fill engine. A guest read of a region the copy has not + reached goes through the source at `Demand`; the chunks the guest read first on the + session's last window are fetched at `Prefix` ahead of the copy. When every region is + copied the device is reloaded as a thin snapshot of its destination and the source + stops; the path Firecracker opened never changes + (`a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination`, + `a_lazy_resume_reads_cold_chunks_through_the_source_and_settles`). A checkpoint in the + window reads nothing through the source and loses no write: a block whose mapping + changed since the base is the snapshot's, any other is the manifest's + (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, + `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`). Only a + snapshot taken after the settle becomes a checkpoint volume + (`a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume`). The periodic + checkpoint waits for the settle, for at most 10 minutes + (`a_running_settle_holds_the_periodic_checkpoint_for_a_bounded_time`); an eviction or + a snapshot does not wait. A whole chunk read for a volume is not hashed + (`read_does_not_hash_what_the_store_returns`). A + host-agent restart serves every hydrating volume again from the pool's names, and never + deletes one it does not serve (`a_killed_process_leaves_a_volume_the_next_one_serves_again`). + A store error that can pass stalls a read of the source; it does not fail it. + A node needs `ublk_drv`; node-prep loads it beside `dm_thin_pool`. Without the ublk + control device a host writes each volume in full before its guest starts. - A live move carries the disk in 64 KiB blocks in three tiers: the checkpoint the source advertises, the blocks a snapshot taken while the guest runs changed since it, and the - blocks the snapshot under the pause changed since that one. The pause carries only what + blocks the snapshot under the pause changed since that one. A source with no checkpoint + volume (hydrating, or settled before its first checkpoint) moves by snapshot + (`a_role_without_a_checkpoint_volume_refuses_a_live_move`). The pause carries only what the guest wrote during the precopy (`two_host_teleport_seals_only_the_residual_under_steady_writes`, `a_precopy_snapshot_lives_beside_the_one_the_capture_takes`), and the destination's diff --git a/crates/engram-chunk-store/src/first_touch.rs b/crates/engram-chunk-store/src/first_touch.rs index 09d4c4fb2..008a761c8 100644 --- a/crates/engram-chunk-store/src/first_touch.rs +++ b/crates/engram-chunk-store/src/first_touch.rs @@ -34,6 +34,13 @@ impl FirstTouch { format!("traces/{session}/first-touch-rest.v1") } + /// The store key of the root disk chunks a session read before the + /// copy of a hydrating restore reached them, in order, encoded as for + /// [`Self::key`]. The next restore fetches the ones it must copy first. + pub fn disk_key(session: Uuid) -> String { + format!("traces/{session}/disk-first-touch.v1") + } + /// The indices touched up to and including `until_ms`, in order. pub fn until(&self, until_ms: i64) -> Vec { self.entries diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 906241e9d..a8c99b95b 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -1,5 +1,6 @@ //! The disk over the ioctl layer; Linux only. +use std::fs::File; use std::io; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; @@ -9,7 +10,9 @@ use std::time::Duration; use crate::dm; use crate::parse::{self, DevNo, ThinParams, METADATA_BLOCK_BYTES}; use async_trait::async_trait; -use engram_host_core::disk::{Capacity, SessionDisk, Volume, VolumeName}; +use engram_host_core::disk::{ + punch_hole, Capacity, ChunkIndex, Hydration, SessionDisk, Volume, VolumeName, +}; const SECTOR_BYTES: u64 = 512; /// dm-thin device ids are 24 bits wide. @@ -18,6 +21,16 @@ const ID_MASK: u32 = 0x00FF_FFFF; /// to close it. const REMOVE_ATTEMPTS: u32 = 10; const REMOVE_RETRY: Duration = Duration::from_millis(100); +/// A dm-clone metadata volume: a superblock and one bit per region in +/// 4 KiB blocks. A 300 GiB volume has 4.9 million 64 KiB regions, a +/// 600 KiB bitset; 8 MiB leaves room for its transaction shadows. +const CLONE_METADATA_BYTES: u64 = 8 << 20; +/// Regions per background copy: 1 MiB bios. The kernel's default of 1 +/// copies at a few MiB/s. +const HYDRATION_BATCH_REGIONS: u64 = 16; +/// Regions the kernel copies at one time: 32 MiB in flight. At 8 MiB +/// the copies wait on the destination's writes. +const HYDRATION_IN_FLIGHT_REGIONS: u64 = 512; pub struct ThinPoolDisk { dm: dm::Dm, @@ -27,6 +40,7 @@ pub struct ThinPoolDisk { nodes_dir: PathBuf, chunk_size: u64, block_bytes: u64, + hydration_in_flight: u64, /// Seeded from the pool at `open`. The kernel refuses a duplicate id /// and `create` and `snapshot` move on, so a wrap of the 24-bit space /// is safe. @@ -61,9 +75,20 @@ impl ThinPoolDisk { nodes_dir: nodes_dir.to_path_buf(), chunk_size, block_bytes, + hydration_in_flight: HYDRATION_IN_FLIGHT_REGIONS, next_id: Arc::new(AtomicU32::new(1)), metadata_snap: tokio::sync::Mutex::new(()), }; + // A host-agent that died inside `while_suspended` left a device + // holding every I/O; its inactive table, if any, is the one it + // meant to make live. + let prefix = format!("{pool}."); + for (device, _) in disk.dm.list().await? { + if device.starts_with(&prefix) && disk.dm.suspended(&device).await? { + tracing::warn!(device, "resuming a device left suspended"); + disk.dm.resume(&device).await?; + } + } let in_pool = disk.thin_ids().await?; let active = disk.active_ids().await?; for id in in_pool.iter().filter(|id| !active.contains(id)) { @@ -75,6 +100,12 @@ impl ThinPoolDisk { Ok(disk) } + /// Regions the kernel copies at one time into each hydrating volume. + pub fn with_hydration_in_flight(mut self, regions: u64) -> Self { + self.hydration_in_flight = regions; + self + } + pub fn pool(&self) -> &str { &self.pool } @@ -96,20 +127,11 @@ impl ThinPoolDisk { self.nodes_dir.join(self.device_name(name)) } + /// The id of every thin on this pool that has a device, from one + /// device list and one table read per device. A device that goes + /// away between the two is not on the pool. async fn active_ids(&self) -> io::Result> { - Ok(self - .thins() - .await? - .into_iter() - .map(|t| t.params.id) - .collect()) - } - - /// Every thin device on this pool, from one device list and one - /// table read per device. A device that goes away between the two - /// is not on the pool. - async fn thins(&self) -> io::Result> { - let mut thins = Vec::new(); + let mut ids = Vec::new(); for (name, _) in self.dm.list().await? { let targets = match self.dm.table(&name).await { Ok(targets) => targets, @@ -124,17 +146,66 @@ impl ThinPoolDisk { } let params = parse::thin_params(&target.params)?; if params.pool == self.pool_dev { - thins.push(Thin { name, params }); + ids.push(params.id); } } - Ok(thins) + Ok(ids) + } + + /// The one target of the device behind `name`. + async fn target_of(&self, name: &VolumeName) -> io::Result { + match self.dm.table(&self.device_name(name)).await?.as_slice() { + [target] => Ok(target.clone()), + targets => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{name} has {} targets, not one", targets.len()), + )), + } + } + + /// The thin a volume's data lives on, and its size: the volume + /// itself for a thin, the destination for a hydrating device. A + /// snapshot of the volume is a snapshot of that thin. + async fn thin_under(&self, name: &VolumeName) -> io::Result<(ThinParams, u64)> { + let target = self.target_of(name).await?; + let devno = match target.kind.as_str() { + "thin" => return self.thin_of(name).await, + "clone" => parse::clone_dest(&target.params)?, + kind => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {kind} device, which this disk did not make"), + )) + } + }; + let prefix = format!("{}.", self.pool); + let dest = self + .dm + .list() + .await? + .into_iter() + .find(|(_, d)| *d == devno) + .and_then(|(device, _)| VolumeName::new(device.strip_prefix(&prefix)?.to_string()).ok()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("{name} maps onto {devno}, which is not a volume of this pool"), + ) + })?; + self.thin_of(&dest).await } /// The thin behind `name` and its size, refused when it belongs to /// another pool. async fn thin_of(&self, name: &VolumeName) -> io::Result<(ThinParams, u64)> { - let targets = self.dm.table(&self.device_name(name)).await?; - let params = parse::thin_params(&single_target(&targets, "thin")?)?; + let target = self.target_of(name).await?; + if target.kind != "thin" { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {} device, not a thin", target.kind), + )); + } + let params = parse::thin_params(&target.params)?; if params.pool != self.pool_dev { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -144,7 +215,67 @@ impl ThinPoolDisk { ), )); } - Ok((params, targets[0].length_sectors)) + Ok((params, target.length_sectors)) + } + + /// Run `op` while `device` is suspended, then resume it, as one owned + /// task: a suspended device holds every I/O, so the resume runs + /// whatever happens to the caller. Both outcomes come back; what a + /// failed resume costs is the caller's to decide. + async fn while_suspended( + &self, + device: String, + op: impl FnOnce(dm::Dm) -> Fut + Send + 'static, + ) -> io::Result<(io::Result, io::Result<()>)> + where + T: Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + let dm = self.dm.clone(); + tokio::spawn(async move { + dm.suspend(&device).await?; + let done = op(dm.clone()).await; + let resumed = dm.resume(&device).await; + Ok((done, resumed)) + }) + .await + .map_err(io::Error::other)? + } + + /// Remove the device, waiting a bounded time for the last holder to + /// close it, then its node. + async fn remove_device(&self, name: &VolumeName) -> io::Result<()> { + let mut attempt = 1; + loop { + match self.dm.remove(&self.device_name(name)).await { + Err(e) if e.kind() == io::ErrorKind::ResourceBusy && attempt < REMOVE_ATTEMPTS => { + attempt += 1; + tokio::time::sleep(REMOVE_RETRY).await; + } + removed => break removed?, + } + } + match tokio::fs::remove_file(self.node_of(name)).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } + } + + async fn delete_thin(&self, name: &VolumeName) -> io::Result<()> { + let (thin, _) = self.thin_of(name).await?; + self.remove_device(name).await?; + self.dm + .message(&self.pool, &format!("delete {}", thin.id)) + .await + } + + /// Delete the metadata thin of the hydrating device `name`, if a + /// settle or an earlier delete has not. + async fn delete_metadata(&self, name: &VolumeName) -> io::Result<()> { + match self.delete_thin(&metadata_name(name)?).await { + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + deleted => deleted, + } } /// A device over thin `id`. On failure the thin is deleted from the @@ -211,11 +342,6 @@ impl ThinPoolDisk { } } -struct Thin { - name: String, - params: ThinParams, -} - /// A reserved metadata snapshot; released on drop. struct MetadataSnap<'a> { dm: &'a dm::Dm, @@ -230,6 +356,47 @@ impl Drop for MetadataSnap<'_> { } } +/// The metadata volume of a hydrating device. +fn metadata_name(name: &VolumeName) -> io::Result { + VolumeName::new(format!("{name}-md")) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) +} + +/// Mark the `skip` chunks of a hydrating device as copied: under +/// `no_discard_passdown`, a discard of regions the kernel has not copied +/// marks them copied and leaves the destination's blocks as they are. +/// Consecutive chunks go in one discard; a skip list is most of a +/// volume, in long runs. +async fn skip_regions(device: &Volume, skip: &[ChunkIndex]) -> io::Result<()> { + let file = File::options().read(true).write(true).open(&device.path)?; + let (chunk, size) = (device.chunk_size, device.size_bytes); + let mut chunks: Vec = skip.iter().map(|c| c.0).collect(); + chunks.sort_unstable(); + chunks.dedup(); + tokio::task::spawn_blocking(move || { + let mut chunks = chunks.into_iter().peekable(); + while let Some(first) = chunks.next() { + let mut last = first; + while chunks.next_if_eq(&(last + 1)).is_some() { + last += 1; + } + let offset = first * chunk; + if last * chunk >= size { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("skip chunk {last} lies beyond the {size}-byte volume"), + )); + } + // The last chunk of a volume can be short. + let end = ((last + 1) * chunk).min(size); + punch_hole(&file, offset, end - offset)?; + } + Ok(()) + }) + .await + .map_err(io::Error::other)? +} + fn single_target(targets: &[dm::Target], kind: &str) -> io::Result { match targets { [target] if target.kind == kind => Ok(target.params.clone()), @@ -240,6 +407,21 @@ fn single_target(targets: &[dm::Target], kind: &str) -> io::Result { } } +/// A snapshot of thin `origin` under the next free id. The caller holds +/// the origin's device suspended. +async fn create_snap(dm: &dm::Dm, pool: &str, next_id: &AtomicU32, origin: u32) -> io::Result { + loop { + let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; + match dm + .message(pool, &format!("create_snap {id} {origin}")) + .await + { + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + created => return created.map(|()| id), + } + } +} + #[async_trait] impl SessionDisk for ThinPoolDisk { async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { @@ -259,81 +441,85 @@ impl SessionDisk for ThinPoolDisk { } async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { - let (thin, sectors) = self.thin_of(&origin.name).await?; - let dm = self.dm.clone(); + let (thin, sectors) = self.thin_under(&origin.name).await?; let pool = self.pool.clone(); let next_id = self.next_id.clone(); - let origin_name = self.device_name(&origin.name); - // The origin holds all I/O while it is suspended, so the resume - // must run whatever happens to the caller: the calls are one owned - // task. - let id = tokio::spawn(async move { - dm.suspend(&origin_name).await?; - let created = loop { - let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; - match dm - .message(&pool, &format!("create_snap {id} {}", thin.id)) - .await - { - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, - created => break created.map(|()| id), - } - }; - let resumed = dm.resume(&origin_name).await; - match (created, resumed) { - (Ok(id), Ok(())) => Ok(id), - (Ok(id), Err(e)) => { - // The snapshot exists but nothing will activate it. - if let Err(error) = dm.message(&pool, &format!("delete {id}")).await { - tracing::warn!(pool = %pool, id, %error, "thin left behind by a failed resume"); - } - Err(e) + let (created, resumed) = self + .while_suspended(self.device_name(&origin.name), move |dm| async move { + create_snap(&dm, &pool, &next_id, thin.id).await + }) + .await?; + let id = match (created, resumed) { + (Ok(id), Ok(())) => id, + (Ok(id), Err(e)) => { + // The snapshot exists but nothing will activate it. + if let Err(error) = self.dm.message(&self.pool, &format!("delete {id}")).await { + tracing::warn!(pool = %self.pool, id, %error, "thin left behind by a failed resume"); } - (Err(e), _) => Err(e), + return Err(e); } - }) - .await - .map_err(io::Error::other)??; + (Err(e), _) => return Err(e), + }; self.activate(name, id, sectors).await } async fn open(&self, name: &VolumeName) -> io::Result { - let (_, sectors) = self.thin_of(name).await?; + let target = self.target_of(name).await?; + let sectors = match target.kind.as_str() { + "thin" => self.thin_of(name).await?.1, + "clone" => target.length_sectors, + kind => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {kind} device, which this disk did not make"), + )) + } + }; self.volume(name, sectors).await } - /// Waits a bounded time for the last holder to close the device. + /// Waits a bounded time for the last holder to close the device. A + /// hydrating device takes its metadata thin with it. async fn delete(&self, name: &VolumeName) -> io::Result<()> { - let (thin, _) = self.thin_of(name).await?; - let mut attempt = 1; - loop { - match self.dm.remove(&self.device_name(name)).await { - Err(e) if e.kind() == io::ErrorKind::ResourceBusy && attempt < REMOVE_ATTEMPTS => { - attempt += 1; - tokio::time::sleep(REMOVE_RETRY).await; - } - removed => break removed?, + let target = self.target_of(name).await?; + match target.kind.as_str() { + "thin" => self.delete_thin(name).await, + "clone" => { + self.remove_device(name).await?; + self.delete_metadata(name).await } + kind => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is a {kind} device, which this disk did not make"), + )), } - match tokio::fs::remove_file(self.node_of(name)).await { - Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), - _ => {} - } - self.dm - .message(&self.pool, &format!("delete {}", thin.id)) - .await } + /// The pool's thins and the hydrating and settled devices over them, + /// all under this pool's name. async fn list(&self) -> io::Result> { - Ok(self - .thins() - .await? - .into_iter() - .filter_map(|t| { - let volume = t.name.strip_prefix(&self.pool)?.strip_prefix('.')?; - VolumeName::new(volume.to_string()).ok() - }) - .collect()) + let prefix = format!("{}.", self.pool); + let mut names = Vec::new(); + for (device, _) in self.dm.list().await? { + let Some(volume) = device.strip_prefix(&prefix) else { + continue; + }; + let Ok(name) = VolumeName::new(volume.to_string()) else { + continue; + }; + match self.dm.table(&device).await { + Ok(targets) => { + if let [target] = targets.as_slice() { + if ["thin", "clone"].contains(&target.kind.as_str()) { + names.push(name); + } + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + } + } + Ok(names) } async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { @@ -359,6 +545,138 @@ impl SessionDisk for ThinPoolDisk { }) } + fn region_bytes(&self) -> Option { + Some(self.block_bytes) + } + + /// A `clone` target over `dest` with `source` as its source and a + /// region per pool block (the unit the pool shares and `dirty_blocks` + /// sees; a guest write to a cold region waits for one region's copy), + /// created with hydration off so the skip discards land before any + /// copy; then hydration on. A failure leaves nothing. + async fn hydrate( + &self, + dest: &Volume, + source: &Path, + name: &VolumeName, + skip: &[ChunkIndex], + ) -> io::Result { + let (_, sectors) = self.thin_of(&dest.name).await?; + let region_sectors = self.block_bytes / SECTOR_BYTES; + if !sectors.is_multiple_of(region_sectors) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{}: {sectors} sectors is not whole pool blocks", dest.name), + )); + } + // Only the path is kept: an open handle of ours would keep a + // failed hydrate from deleting the metadata volume. + let metadata = self + .create(&metadata_name(name)?, CLONE_METADATA_BYTES) + .await? + .path; + let device = self.device_name(name); + let hydrating = async { + let params = format!( + "{metadata} {dest} {source} {region_sectors} 2 no_hydration \ + no_discard_passdown 4 hydration_threshold {in_flight} \ + hydration_batch_size {HYDRATION_BATCH_REGIONS}", + metadata = dm::devno_of(&metadata)?, + dest = dm::devno_of(&dest.path)?, + source = dm::devno_of(source)?, + in_flight = self.hydration_in_flight, + ); + self.dm.create(&device, sectors, "clone", ¶ms).await?; + let volume = self.volume(name, sectors).await?; + skip_regions(&volume, skip).await?; + self.dm.message(&device, "enable_hydration").await?; + Ok(volume) + } + .await; + if hydrating.is_err() { + let removed = match self.remove_device(name).await { + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + removed => removed, + }; + if let Err(error) = removed.and(self.delete_metadata(name).await) { + tracing::warn!(%name, %error, "hydrating device left behind by a failed hydrate"); + } + } + hydrating + } + + async fn hydration(&self, name: &VolumeName) -> io::Result { + let device = self.device_name(name); + let status = self.dm.status(&device).await?; + match status.as_slice() { + [target] if target.kind == "clone" => { + let status = parse::clone_status(&target.params)?; + let table = single_target(&self.dm.table(&device).await?, "clone")?; + Ok(Hydration::Hydrating { + hydrated: status.hydrated_regions, + total: status.total_regions, + source: dm::device_path(parse::clone_source(&table)?).await?, + }) + } + [target] if target.kind == "thin" => Ok(Hydration::Settled), + _ => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is not a session volume: {status:?}"), + )), + } + } + + /// Under the device's suspend, snapshots `dest`'s thin and reloads + /// the device as that snapshot: a thin like any other, which shares + /// blocks with `dest` and nothing else. `dest` is suspended for its + /// snapshot, as the pool requires of an active origin. A failed + /// reload resumes the clone table and deletes the snapshot. The + /// metadata thin goes last: the kernel closes it when the clone + /// table is destroyed. + async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { + match self.hydration(name).await? { + Hydration::Hydrating { + hydrated, total, .. + } if hydrated < total => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name}: {hydrated} of {total} regions are hydrated"), + )); + } + Hydration::Hydrating { .. } => { + let (thin, sectors) = self.thin_of(&dest.name).await?; + let device = self.device_name(name); + let dest_device = self.device_name(&dest.name); + let (pool, pool_dev, next_id) = + (self.pool.clone(), self.pool_dev, self.next_id.clone()); + let (loaded, resumed) = self + .while_suspended(device.clone(), move |dm| async move { + dm.suspend(&dest_device).await?; + let created = create_snap(&dm, &pool, &next_id, thin.id).await; + let dest_resumed = dm.resume(&dest_device).await; + let id = created?; + let loaded = match dest_resumed { + Ok(()) => { + dm.load(&device, sectors, "thin", &format!("{pool_dev} {id}")) + .await + } + Err(e) => Err(e), + }; + if loaded.is_err() { + if let Err(error) = dm.message(&pool, &format!("delete {id}")).await { + tracing::warn!(%pool, id, %error, "thin left behind by a failed settle"); + } + } + loaded + }) + .await?; + loaded.and(resumed)?; + } + Hydration::Settled => {} + } + self.delete_metadata(name).await + } + /// Reads the pool's mapping tree instead of the volumes. A block is /// dirty when any pool block inside it differs; `block_bytes` is a /// multiple of the pool's block size. diff --git a/crates/engram-disk-thin/src/dm.rs b/crates/engram-disk-thin/src/dm.rs index 073caaa5b..ece66ae86 100644 --- a/crates/engram-disk-thin/src/dm.rs +++ b/crates/engram-disk-thin/src/dm.rs @@ -352,6 +352,28 @@ impl Dm { ready } + /// Load a new single-target table as the inactive table of `name`; + /// `resume` makes it live. A failed load leaves the active table. + pub async fn load( + &self, + name: &str, + length_sectors: u64, + kind: &str, + params: &str, + ) -> io::Result<()> { + let mut load = Request::new(name, 0)?; + load.push_target(length_sectors, kind, params); + self.call( + "load", + name, + load, + DM_TABLE_LOAD_CMD, + io::ErrorKind::ResourceBusy, + ) + .await + .map(drop) + } + pub async fn remove(&self, name: &str) -> io::Result<()> { self.call( "remove", @@ -398,6 +420,21 @@ impl Dm { .map(drop) } + /// Whether `name` is suspended: the kernel sets `DM_SUSPEND_FLAG` in + /// every reply about a suspended device. + pub async fn suspended(&self, name: &str) -> io::Result { + let reply = self + .call( + "status", + name, + Request::new(name, 0)?, + DM_DEV_STATUS_CMD, + io::ErrorKind::ResourceBusy, + ) + .await?; + Ok(reply.flags() & DM_SUSPEND_FLAG != 0) + } + pub async fn resume(&self, name: &str) -> io::Result<()> { self.call( "resume", @@ -486,6 +523,23 @@ pub async fn device_path(devno: DevNo) -> io::Result { Ok(PathBuf::from("/dev").join(parse::devname(&uevent)?)) } +/// The device number of the block device node at `path`. +pub fn devno_of(path: &Path) -> io::Result { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + let meta = std::fs::metadata(path)?; + if !meta.file_type().is_block_device() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a block device", path.display()), + )); + } + let rdev = meta.rdev(); + Ok(DevNo { + major: libc::major(rdev), + minor: libc::minor(rdev), + }) +} + /// Create a block device node, replacing one that points elsewhere. #[allow(unsafe_code)] pub fn make_node(path: &Path, devno: DevNo) -> io::Result<()> { diff --git a/crates/engram-disk-thin/src/parse.rs b/crates/engram-disk-thin/src/parse.rs index 1fa11c43c..f00c78a23 100644 --- a/crates/engram-disk-thin/src/parse.rs +++ b/crates/engram-disk-thin/src/parse.rs @@ -110,6 +110,52 @@ pub fn pool_status(status: &str) -> io::Result { } } +/// `clone` status: ` / +/// / +/// ... ... `. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CloneStatus { + pub hydrated_regions: u64, + pub total_regions: u64, + pub hydrating_regions: u64, +} + +/// A clone whose metadata is not read-write (`ro`, or the single word +/// `Fail`) is an error: it copies nothing more and must not be settled. +pub fn clone_status(status: &str) -> io::Result { + let fields: Vec<&str> = status.split_whitespace().collect(); + match fields.as_slice() { + ["Fail" | "Error"] => Err(invalid(format!("clone is {}", fields[0]))), + [.., mode] if *mode != "rw" => Err(invalid(format!("clone is {mode}"))), + [_, _, _, regions, hydrating, ..] => { + let (hydrated_regions, total_regions) = fraction(regions)?; + Ok(CloneStatus { + hydrated_regions, + total_regions, + hydrating_regions: number(hydrating)?, + }) + } + _ => Err(invalid(format!("not clone status: {status:?}"))), + } +} + +/// The source device of a `clone` table: +/// ` ...`. +pub fn clone_source(params: &str) -> io::Result { + match params.split_whitespace().collect::>().as_slice() { + [_, _, source, ..] => DevNo::parse(source), + _ => Err(invalid(format!("not clone params: {params:?}"))), + } +} + +/// The destination device of a `clone` table. +pub fn clone_dest(params: &str) -> io::Result { + match params.split_whitespace().collect::>().as_slice() { + [_, dest, ..] => DevNo::parse(dest), + _ => Err(invalid(format!("not clone params: {params:?}"))), + } +} + /// The `DEVNAME=` line of a sysfs `uevent` file. pub fn devname(uevent: &str) -> io::Result { uevent @@ -242,6 +288,50 @@ mod tests { } } + #[test] + fn clone_dest_is_the_second_device() { + let clone = "252:3 252:4 259:0 128 2 no_hydration no_discard_passdown 0"; + assert_eq!( + clone_dest(clone).unwrap(), + DevNo { + major: 252, + minor: 4 + } + ); + } + + #[test] + fn clone_source_is_the_third_device() { + assert_eq!( + clone_source("252:3 252:4 259:0 128 2 no_hydration no_discard_passdown 0").unwrap(), + DevNo { + major: 259, + minor: 0 + } + ); + assert!(clone_source("252:3 252:4").is_err()); + } + + #[test] + fn clone_status_reads_the_region_counts() { + let status = clone_status( + "8 25/2048 128 1500/4096 16 2 no_hydration no_discard_passdown 4 hydration_threshold 128 hydration_batch_size 16 rw", + ) + .unwrap(); + assert_eq!( + status, + CloneStatus { + hydrated_regions: 1500, + total_regions: 4096, + hydrating_regions: 16, + } + ); + for bad in ["8 25/2048 128 4096/4096 0 0 0 ro", "Fail"] { + let err = clone_status(bad).unwrap_err().to_string(); + assert!(err.starts_with("clone is "), "{bad}: {err}"); + } + } + #[test] fn devname_finds_the_node_name() { assert_eq!( diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs new file mode 100644 index 000000000..0875f9cd4 --- /dev/null +++ b/crates/engram-disk-thin/tests/clone.rs @@ -0,0 +1,247 @@ +#![cfg(target_os = "linux")] +//! A hydrating device over the pool, as root, with a loop device as the +//! source. `ENGRAM_TEST_THIN_POOL` names the pool as in `thin_pool.rs`; +//! unset, the test skips. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use engram_disk_thin::ThinPoolDisk; +use engram_host_core::disk::{ChunkIndex, Hydration, SessionDisk, VolumeName}; + +/// Two pool blocks per chunk: a skip discard spans regions, and the +/// kernel copies a chunk as more than one region. +const CHUNK: u64 = 2 * 65536; +const CHUNKS: u64 = 8; +const REGIONS: u64 = CHUNKS * 2; + +macro_rules! pool_or_skip { + () => { + match std::env::var("ENGRAM_TEST_THIN_POOL") { + Ok(pool) => ThinPoolDisk::open(&pool, CHUNK, Path::new("/dev/engram-test")) + .await + .expect("open pool"), + Err(_) => { + eprintln!("SKIP: ENGRAM_TEST_THIN_POOL is not set"); + return; + } + } + }; +} + +fn name(role: &str) -> VolumeName { + VolumeName::new(format!("t-clone-{role}-{}", std::process::id())).unwrap() +} + +fn fill(byte: u8) -> Vec { + vec![byte; CHUNK as usize] +} + +/// A loop device over a file, detached on drop. +struct Loop(PathBuf); + +impl Loop { + async fn over(file: &Path) -> Self { + let output = tokio::process::Command::new("losetup") + .args(["--find", "--show"]) + .arg(file) + .output() + .await + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + Self(PathBuf::from( + String::from_utf8(output.stdout).unwrap().trim(), + )) + } +} + +impl Drop for Loop { + fn drop(&mut self) { + let _ = std::process::Command::new("losetup") + .arg("-d") + .arg(&self.0) + .status(); + } +} + +/// The source holds chunks 0, 1 and 3; the destination's origin holds +/// chunks 0 and 1 with chunk 0 equal to the source's. Skipped: chunk 0 +/// (held) and the zero chunks. The kernel copies 1 and 3; the guest +/// writes 5 through the device. After the settle the destination reads +/// as the source plus the write, and chunk 0 still shares its blocks +/// with the origin. +#[tokio::test] +#[ignore] +async fn a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination() { + let pool = pool_or_skip!(); + let base = pool.create(&name("base"), CHUNKS * CHUNK).await.unwrap(); + base.write_chunk(ChunkIndex(0), fill(1)).await.unwrap(); + base.write_chunk(ChunkIndex(1), fill(2)).await.unwrap(); + let dest = pool.snapshot(&base, &name("l")).await.unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let source_file = dir.path().join("source"); + let mut source = vec![0u8; (CHUNKS * CHUNK) as usize]; + for (chunk, byte) in [(0, 1u8), (1, 3), (3, 4)] { + let at = (chunk * CHUNK) as usize; + source[at..at + CHUNK as usize].fill(byte); + } + std::fs::write(&source_file, &source).unwrap(); + let source = Loop::over(&source_file).await; + + let skip: Vec = [0, 2, 4, 5, 6, 7].map(ChunkIndex).to_vec(); + let device = pool + .hydrate(&dest, &source.0, &name("io"), &skip) + .await + .unwrap(); + let listed = pool.list().await.unwrap(); + let metadata = VolumeName::new(format!("{}-md", name("io"))).unwrap(); + assert!( + listed.contains(&name("io")) && listed.contains(&metadata), + "the device and its metadata thin are volumes of the pool: {listed:?}" + ); + assert_eq!( + pool.open(&name("io")).await.unwrap().size_bytes, + device.size_bytes + ); + + assert_eq!(device.read_chunk(ChunkIndex(0)).await.unwrap(), fill(1)); + assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); + assert_eq!(device.read_chunk(ChunkIndex(2)).await.unwrap(), fill(0)); + assert_eq!(device.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); + device.write_chunk(ChunkIndex(5), fill(5)).await.unwrap(); + + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match pool.hydration(&device.name).await.unwrap() { + Hydration::Hydrating { + hydrated, + total, + source: from, + } if hydrated == total => { + assert_eq!((total, from), (REGIONS, source.0.clone())); + break; + } + Hydration::Hydrating { .. } if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + state => panic!("hydration did not finish: {state:?}"), + } + } + // A snapshot of the hydrating device is a snapshot of its + // destination, under the device's suspend; it diffs against the base. + let mid = pool.snapshot(&device, &name("mid")).await.unwrap(); + assert_eq!( + pool.dirty_blocks(&base, &mid, CHUNK).await.unwrap(), + vec![1, 3, 5] + ); + assert_eq!(mid.read_chunk(ChunkIndex(5)).await.unwrap(), fill(5)); + let mid_name = mid.name.clone(); + drop(mid); + pool.delete(&mid_name).await.unwrap(); + + pool.settle(&device.name, &dest).await.unwrap(); + assert_eq!( + pool.hydration(&device.name).await.unwrap(), + Hydration::Settled + ); + let after = pool.snapshot(&device, &name("after")).await.unwrap(); + assert_eq!(after.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); + let after_name = after.name.clone(); + drop(after); + pool.delete(&after_name).await.unwrap(); + assert!( + !pool.list().await.unwrap().contains(&metadata), + "the settle deleted the metadata thin" + ); + + assert_eq!(device.read_chunk(ChunkIndex(1)).await.unwrap(), fill(3)); + assert_eq!(device.read_chunk(ChunkIndex(5)).await.unwrap(), fill(5)); + for (chunk, byte) in [(0, 1u8), (1, 3), (2, 0), (3, 4), (5, 5)] { + assert_eq!( + dest.read_chunk(ChunkIndex(chunk)).await.unwrap(), + fill(byte), + "chunk {chunk}" + ); + } + assert_eq!( + pool.dirty_blocks(&base, &dest, CHUNK).await.unwrap(), + vec![1, 3, 5], + "the skipped chunk 0 still shares its blocks with the origin" + ); + + // The settled device is a volume of its own: a write to it leaves + // the destination as it was, and the destination goes while the + // device stays open and readable. + device.write_chunk(ChunkIndex(6), fill(9)).await.unwrap(); + device.sync().await.unwrap(); + assert_eq!(dest.read_chunk(ChunkIndex(6)).await.unwrap(), fill(0)); + let dest_name = dest.name.clone(); + drop(dest); + pool.delete(&dest_name).await.unwrap(); + assert_eq!(device.read_chunk(ChunkIndex(3)).await.unwrap(), fill(4)); + assert_eq!(device.read_chunk(ChunkIndex(6)).await.unwrap(), fill(9)); + + // A volume holds its device open; a delete needs it closed. + let names = (device.name.clone(), base.name.clone()); + drop((device, base)); + let (device, base) = names; + pool.delete(&device).await.unwrap(); + pool.delete(&base).await.unwrap(); + assert!(!pool + .list() + .await + .unwrap() + .iter() + .any(|v| v.as_str().starts_with("t-clone-"))); +} + +/// A settle before the copy is done is refused and changes nothing. +#[tokio::test] +#[ignore] +async fn settle_refuses_a_device_that_is_still_hydrating() { + let pool = pool_or_skip!(); + let dest = pool.create(&name("early-l"), CHUNKS * CHUNK).await.unwrap(); + let dir = tempfile::tempdir().unwrap(); + let source_file = dir.path().join("source"); + std::fs::write(&source_file, vec![7u8; (CHUNKS * CHUNK) as usize]).unwrap(); + let source = Loop::over(&source_file).await; + // A suspended destination takes no write, so the copy cannot finish. + dmsetup(&["suspend", &pool.device_name(&dest.name)]).await; + let device = pool + .hydrate(&dest, &source.0, &name("early-io"), &[]) + .await + .unwrap(); + let err = pool.settle(&device.name, &dest).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput, "{err}"); + assert!(matches!( + pool.hydration(&device.name).await.unwrap(), + Hydration::Hydrating { .. } + )); + dmsetup(&["resume", &pool.device_name(&dest.name)]).await; + // A delete while the kernel is still copying leaks nothing. + let names = (device.name.clone(), dest.name.clone()); + drop((device, dest)); + let (device, dest) = names; + pool.delete(&device).await.unwrap(); + pool.delete(&dest).await.unwrap(); + assert!(!pool + .list() + .await + .unwrap() + .iter() + .any(|v| v.as_str().starts_with("t-clone-early"))); +} + +async fn dmsetup(args: &[&str]) { + let status = tokio::process::Command::new("dmsetup") + .args(args) + .status() + .await + .unwrap(); + assert!(status.success(), "dmsetup {args:?}"); +} diff --git a/crates/engram-disk-thin/tests/thin_pool.rs b/crates/engram-disk-thin/tests/thin_pool.rs index 4a5ce09e1..67df8bdf4 100644 --- a/crates/engram-disk-thin/tests/thin_pool.rs +++ b/crates/engram-disk-thin/tests/thin_pool.rs @@ -318,6 +318,37 @@ async fn open_deletes_a_thin_whose_device_is_gone_and_create_skips_a_taken_id() assert!(deleted.success()); } +/// A host-agent that died between a suspend and its resume left the +/// device holding every I/O; the next `open` resumes it. +#[tokio::test] +#[ignore] +async fn open_resumes_a_device_left_suspended() { + let disk = pool_or_skip!(); + let held = name("suspended", "v"); + let v = disk.create(&held, CHUNK).await.unwrap(); + let suspended = tokio::process::Command::new("dmsetup") + .args(["suspend", &disk.device_name(&held)]) + .status() + .await + .unwrap(); + assert!(suspended.success()); + + let disk = ThinPoolDisk::open(disk.pool(), CHUNK, Path::new("/dev/engram-test")) + .await + .unwrap(); + let read = tokio::time::timeout( + std::time::Duration::from_secs(5), + v.read_chunk(ChunkIndex(0)), + ) + .await + .expect("the device was resumed") + .unwrap(); + assert_eq!(read, fill(0)); + + drop(v); + disk.delete(&held).await.unwrap(); +} + /// A reserve left behind by a holder that died: the next read releases /// it, reserves its own and releases that one too. #[tokio::test] diff --git a/crates/engram-dst-cosim/src/host.rs b/crates/engram-dst-cosim/src/host.rs index 6ea64cc0b..49d862c1a 100644 --- a/crates/engram-dst-cosim/src/host.rs +++ b/crates/engram-dst-cosim/src/host.rs @@ -129,14 +129,7 @@ pub(crate) async fn build_base_sandbox( .await .expect("seed base manifest"); volumes - .provision( - sandbox_id, - Source::Fork { - manifest: base_ref, - base: None, - }, - None, - ) + .provision(sandbox_id, Source::Fork { manifest: base_ref }, None) .await .expect("provision the root volume"); sandbox_id diff --git a/crates/engram-fill/src/engine.rs b/crates/engram-fill/src/engine.rs index 9e53904ac..4dd511bc3 100644 --- a/crates/engram-fill/src/engine.rs +++ b/crates/engram-fill/src/engine.rs @@ -357,15 +357,17 @@ impl FillEngine { } /// The chunk's bytes for a caller that keeps them itself (a volume - /// write): a cache hit is served, a miss is fetched and verified but - /// not stored in the cache, so the bytes are written to disk once. + /// write): a cache hit is served, a miss is fetched but not stored in + /// the cache, so the bytes are written to disk once. The fetched + /// bytes are not hashed: a chunk is checked when it is written to the + /// store or to the cache, not when it is read. pub async fn read(&self, hash: ChunkHash, class: Class) -> Result { if let Some(cache) = self.store().chunk_cache() { if let Some(bytes) = cache.read_resident(hash).await? { return Ok(bytes); } } - self.fetch_verified(hash, class, None).await + Ok(self.fetch_unverified(hash, class, None).await?.0) } /// `len` bytes at `offset` of a chunk, for a reader that needs part of @@ -381,10 +383,9 @@ impl FillEngine { /// `Hydrate` starts beside it (one flight with every other fill of /// the chunk). Without a cache there is no fill. /// - /// The ranged bytes are not hash-checked: only a whole chunk can be. - /// They go to the caller only, never to the cache; the cache stores - /// the chunk only from the whole-chunk fill, after its hash check. - /// A caller that needs checked bytes uses [`Self::get`]. + /// The ranged bytes are not hashed and go to the caller only; the + /// cache stores the chunk only from the whole-chunk fill, after its + /// hash check. pub async fn read_range( &self, hash: ChunkHash, @@ -530,7 +531,8 @@ impl FillEngine { /// against a peer that has not answered after `peer_race_after`. The /// slot is freed when the bytes are in hand (before the cache writes /// them); a peer answer frees it without a report to the adaptive - /// limit. The caller hashes what came from the store. + /// limit. A caller that stores the bytes under their hash checks + /// what came from the store. async fn fetch_unverified( &self, hash: ChunkHash, diff --git a/crates/engram-fill/src/engine/tests.rs b/crates/engram-fill/src/engine/tests.rs index 45dac75f5..28812d24d 100644 --- a/crates/engram-fill/src/engine/tests.rs +++ b/crates/engram-fill/src/engine/tests.rs @@ -284,22 +284,32 @@ async fn fill_returns_the_first_error_after_the_other_fetches_finish() { } #[tokio::test] -async fn read_verifies_and_leaves_the_cache_alone() { +async fn read_leaves_the_cache_alone() { let f = fixture(); - let good = put(&f, b"volume bytes").await; - let bad = ChunkHash::of(b"expected"); + let hash = put(&f, b"volume bytes").await; + let engine = engine_over(&f, counting(&f, None), Config::default()); + + assert_eq!( + &engine.read(hash, Class::Demand).await.unwrap()[..], + b"volume bytes" + ); + assert!(!f.cache.contains_on_disk(hash)); +} + +#[tokio::test] +async fn read_does_not_hash_what_the_store_returns() { + let f = fixture(); + let hash = ChunkHash::of(b"expected"); f.blob - .put(&bad.storage_key(), Bytes::from_static(b"tampered")) + .put(&hash.storage_key(), Bytes::from_static(b"other bytes")) .await .unwrap(); let engine = engine_over(&f, counting(&f, None), Config::default()); assert_eq!( - &engine.read(good, Class::Demand).await.unwrap()[..], - b"volume bytes" + &engine.read(hash, Class::Demand).await.unwrap()[..], + b"other bytes" ); - assert!(!f.cache.contains_on_disk(good)); - assert!(engine.read(bad, Class::Demand).await.is_err()); } #[tokio::test] diff --git a/crates/engram-host-agent/Cargo.toml b/crates/engram-host-agent/Cargo.toml index 2c3accc95..b63f126aa 100644 --- a/crates/engram-host-agent/Cargo.toml +++ b/crates/engram-host-agent/Cargo.toml @@ -106,6 +106,10 @@ workspace-hack = { version = "0.1", path = "../../workspace-hack" } # linux5_13: `RegisterMode::MINOR` — the exact ioctl mode the probe # exercises. userfaultfd = { version = "0.9", features = ["linux5_13"] } +# The ublk device `blk_source` serves; `io-uring` for the eventfd read +# that wakes its queue thread. +libublk = "0.4.8" +io-uring = "0.7" # #1003: jemalloc as the global allocator (prod is musl-static; musl # mallocng retained 23 GB of ~48 MiB groups under chunk-buffer churn) # + the sampled heap profiler (SIGUSR2 → pprof dump) + allocator diff --git a/crates/engram-host-agent/src/blk_source.rs b/crates/engram-host-agent/src/blk_source.rs new file mode 100644 index 000000000..c99a04db9 --- /dev/null +++ b/crates/engram-host-agent/src/blk_source.rs @@ -0,0 +1,903 @@ +//! A read-only block device served from the fill engine: the source a +//! hydrating session volume copies its cold regions from +//! (`SessionDisk::hydrate`). The device is a ublk device the host-agent +//! serves in process; the kernel reads it for the regions the guest +//! touches before they are copied and for its background copy, and +//! nothing else opens it. +//! +//! Two kinds of read arrive and the source cannot be told which is +//! which, so it tells them apart by position. The background copy walks +//! the volume's cold regions in ascending order, one region or a batch +//! at a time; the source keeps a window of whole chunks around that +//! stream, fetched at `Hydrate` and a few chunks ahead, and hands them +//! out region by region, so the store sees one GET per chunk and the +//! copy runs at the engine's rate. A read anywhere else is the guest +//! waiting (or a write the kernel first copies the region for): it gets +//! a ranged GET at `Demand`, and its chunk is recorded as touched. So +//! does a read inside the window that is not whole regions, which only +//! the guest sends, when the window's chunk is not in yet. +//! +//! A store error that can pass stalls the read until it does: an I/O +//! error on a guest's root disk outlives the outage that caused it. + +use std::collections::{HashMap, VecDeque}; +use std::fs::File; +use std::io::{self, Write}; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::{Bytes, BytesMut}; +use engram_chunk_store::{ChunkHash, ChunkStoreError, Manifest}; +use engram_core::error::BlobError; +use engram_fill::{Class, FillEngine}; +use engram_host_core::{SourceDevice, SourceServer}; +use io_uring::{opcode, types}; +use libublk::ctrl::{UblkCtrl, UblkCtrlBuilder}; +use libublk::io::{UblkDev, UblkIOCtx, UblkQueue}; +use libublk::{sys, BufDesc, BufDescList, UblkError, UblkFlags, UblkIORes}; +use tokio::sync::{mpsc, oneshot, OnceCell}; + +/// Region-shaped reads at ascending delta chunks, outside the window, +/// that move the stream there (a host-agent restart loses the cursor). +const RESYNC_READS: u32 = 4; +/// A stream chunk's fetch still waiting at `Hydrate` after this long is +/// asked for again at `Demand`: a guest read of whole regions inside the +/// window waits on it, and `Hydrate` can wait behind other sessions. +const STREAM_PATIENCE: Duration = Duration::from_secs(1); +/// The wait after a failed fetch: from the first, doubling to the last. +const RETRY_FIRST: Duration = Duration::from_millis(100); +const RETRY_LAST: Duration = Duration::from_secs(5); +/// The tag the eventfd read carries; real tags are below the depth. +const EVENT_TAG: u16 = u16::MAX; +const QUEUE_DEPTH: u16 = 64; +/// The largest read the kernel sends; the block layer splits above it. +const IO_BUF_BYTES: u32 = 512 << 10; +const DEVICE_NODE_WAIT: Duration = Duration::from_secs(5); +/// Entries in a thread's ublk control ring; commands go one at a time. +const CTRL_RING_DEPTH: u32 = 16; + +/// A served device. [`Self::stop`] ends it; a dropped one keeps serving +/// until the process exits, after which [`Self::recover`] takes it over. +pub struct BlkSource { + ctrl: Arc, + path: PathBuf, + shared: Arc, + server: Option>, +} + +impl BlkSource { + /// Serve `manifest` through a new device. `delta` lists, ascending, + /// the chunks the kernel will copy (the ones not skipped), which is + /// where the stream runs; `region_bytes` is the kernel's copy unit. + /// The stream fetches `window_ahead` chunks ahead of that copy. + pub async fn serve( + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + window_ahead: usize, + ) -> io::Result { + Self::start(engine, manifest, delta, region_bytes, window_ahead, None).await + } + + /// Take over device `dev_id` from a host-agent that died: the reads it + /// left queued are reissued to this server. + pub async fn recover( + dev_id: u32, + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + window_ahead: usize, + ) -> io::Result { + Self::start( + engine, + manifest, + delta, + region_bytes, + window_ahead, + Some(dev_id), + ) + .await + } + + async fn start( + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + window_ahead: usize, + recover: Option, + ) -> io::Result { + let shared = Arc::new(Shared::new( + engine, + manifest, + delta, + region_bytes, + window_ahead, + recover.is_none(), + )?); + let builder = UblkCtrlBuilder::default() + .name("engram-source") + .nr_queues(1) + .depth(QUEUE_DEPTH) + .io_buf_bytes(IO_BUF_BYTES) + .ctrl_flags((sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64); + let builder = match recover { + None => builder.dev_flags(UblkFlags::UBLK_DEV_F_ADD_DEV), + Some(id) => builder + .id(id as i32) + .dev_flags(UblkFlags::UBLK_DEV_F_RECOVER_DEV), + }; + let ctrl = Arc::new(builder.build().map_err(ublk_err)?); + if recover.is_some() { + let ctrl = ctrl.clone(); + tokio::task::spawn_blocking(move || with_ctrl_ring(|| ctrl.start_user_recover())) + .await + .map_err(io::Error::other)??; + } + let (requests, queue) = mpsc::unbounded_channel(); + let completions = Arc::new(Completions::new()?); + tokio::spawn(dispatch(shared.clone(), queue, completions.clone())); + + let (ready, started) = oneshot::channel(); + let server = std::thread::Builder::new() + .name("engram-source".into()) + .spawn({ + let ctrl = ctrl.clone(); + let shared = shared.clone(); + move || { + let size_bytes = shared.size_bytes; + let ready = Arc::new(Mutex::new(Some(ready))); + let queue_fn = { + let shared = shared.clone(); + move |qid: u16, dev: &UblkDev| { + queue_loop(qid, dev, &shared, &requests, &completions) + } + }; + let run = with_ctrl_ring(|| { + ctrl.run_target( + |dev| { + dev.set_default_params(size_bytes); + dev.tgt.params.basic.attrs |= sys::UBLK_ATTR_READ_ONLY; + Ok(()) + }, + queue_fn, + { + let ready = ready.clone(); + move |_| { + if let Some(ready) = ready.lock().unwrap().take() { + let _ = ready.send(Ok(())); + } + } + }, + ) + }); + if let Err(e) = run { + match ready.lock().unwrap().take() { + Some(ready) => { + let _ = ready.send(Err(e)); + } + None => { + tracing::warn!(error = %e, "ublk source server ended with an error") + } + } + } + } + })?; + started.await.map_err(|_| { + io::Error::other("the ublk server thread ended before the device started") + })??; + let path = PathBuf::from(ctrl.get_bdev_path()); + wait_for_node(&path).await?; + Ok(Self { + ctrl, + path, + shared, + server: Some(server), + }) + } + + /// The block device: `/dev/ublkb`. + pub fn path(&self) -> &Path { + &self.path + } + + pub fn dev_id(&self) -> u32 { + self.ctrl.dev_info().dev_id + } + + /// Chunks the guest read before the copy reached them, in first-read + /// order. + pub fn touched(&self) -> Vec { + self.shared.touched.lock().unwrap().order.clone() + } + + /// Stop serving and delete the device. Refused while a device still + /// maps onto it: a device with reads queued and no server to answer + /// them holds the kernel until reboot, so the mapping goes first. + pub async fn stop(mut self) -> io::Result<()> { + let holders = self + .path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| format!("/sys/block/{n}/holders")); + if let Some(holders) = holders { + let held = std::fs::read_dir(&holders)?.count(); + if held > 0 { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{} is still the source of {held} device(s)", + self.path.display() + ), + )); + } + } + self.shared.stopping.store(true, Ordering::Relaxed); + let ctrl = self.ctrl.clone(); + let server = self.server.take(); + tokio::task::spawn_blocking(move || -> io::Result<()> { + with_ctrl_ring(|| ctrl.kill_dev())?; + if let Some(server) = server { + server + .join() + .map_err(|_| io::Error::other("the ublk server thread panicked"))?; + } + with_ctrl_ring(|| ctrl.del_dev())?; + Ok(()) + }) + .await + .map_err(io::Error::other)? + } +} + +impl Drop for BlkSource { + fn drop(&mut self) { + if self.server.is_some() { + tracing::warn!(path = %self.path.display(), "ublk source dropped while serving; it serves until the process exits"); + } + } +} + +/// What the engine side and the queue thread share. +struct Shared { + engine: FillEngine, + /// The manifest's hash per chunk; `None` reads as zeros. + hashes: Vec>, + chunk_size: u64, + size_bytes: u64, + region_bytes: u64, + /// Ascending chunks the kernel copies. + delta: Vec, + /// Chunks fetched ahead of the stream chunk; the window holds at + /// most this many plus two. + window_ahead: usize, + window: Mutex, + touched: Mutex, + stopping: AtomicBool, +} + +struct Window { + /// Position in `delta` of the chunk the kernel's copy is in; `None` + /// until the stream shows itself. + cursor: Option, + chunks: HashMap>>, + /// The last region-shaped read outside the window and how many came + /// in ascending order. + resync: Option<(u64, u32)>, +} + +struct Touched { + order: Vec, + seen: Vec, +} + +enum Serve { + Stream(Arc>), + Demand, +} + +impl Shared { + fn new( + engine: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + window_ahead: usize, + fresh: bool, + ) -> io::Result { + let chunk_size = manifest.chunk_size.0; + if region_bytes == 0 + || chunk_size == 0 + || !chunk_size.is_multiple_of(region_bytes) + || !manifest.total_bytes.is_multiple_of(512) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "manifest of {} bytes in {chunk_size}-byte chunks does not fit {region_bytes}-byte regions", + manifest.total_bytes + ), + )); + } + let count = manifest.expected_chunk_count() as usize; + let mut hashes = vec![None; count]; + for chunk in &manifest.chunks { + let index = (chunk.offset / chunk_size) as usize; + match hashes.get_mut(index) { + Some(slot) if chunk.offset.is_multiple_of(chunk_size) => *slot = Some(chunk.hash), + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "manifest chunk at offset {} lies outside the volume", + chunk.offset + ), + )) + } + } + } + if !delta.is_sorted() || delta.last().is_some_and(|last| *last as usize >= count) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "delta chunks are not ascending indices of the volume", + )); + } + let shared = Self { + engine, + hashes, + chunk_size, + size_bytes: manifest.total_bytes, + region_bytes, + delta, + window_ahead, + window: Mutex::new(Window { + cursor: None, + chunks: HashMap::new(), + resync: None, + }), + touched: Mutex::new(Touched { + order: Vec::new(), + seen: vec![false; count], + }), + stopping: AtomicBool::new(false), + }; + // A new device's copy starts at the first delta chunk; fetch it + // and the chunks after it before the kernel asks. + if fresh && !shared.delta.is_empty() { + let mut window = shared.window.lock().unwrap(); + shared.advance(&mut window, 0); + } + Ok(shared) + } + + async fn read(&self, offset: u64, len: u64) -> io::Result { + let end = offset + .checked_add(len) + .filter(|end| *end <= self.size_bytes) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "read of {len} at {offset} lies outside the {}-byte device", + self.size_bytes + ), + ) + })?; + let mut out = BytesMut::with_capacity(len as usize); + let mut at = offset; + while at < end { + let chunk = at / self.chunk_size; + let in_chunk = at % self.chunk_size; + let piece = (self.chunk_size - in_chunk).min(end - at); + match self.hashes[chunk as usize] { + None => out.resize(out.len() + piece as usize, 0), + Some(hash) => { + let regions = at.is_multiple_of(self.region_bytes) + && piece.is_multiple_of(self.region_bytes); + let bytes = match self.classify(chunk, at, piece) { + Serve::Stream(cell) if regions || cell.initialized() => { + let whole = self + .until_fetched(chunk, || async { + cell.get_or_try_init(|| stream_chunk(&self.engine, hash)) + .await + .cloned() + }) + .await?; + whole.slice(in_chunk as usize..(in_chunk + piece) as usize) + } + served => { + // A chunk in the window arrives without help, + // and the host's own probe of a new device + // reads there: only a read outside it counts + // as the guest's first touch. + if matches!(served, Serve::Demand) { + self.touch(chunk); + } + self.until_fetched(chunk, || { + self.engine.read_range( + hash, + in_chunk, + piece as usize, + Class::Demand, + ) + }) + .await? + } + }; + if bytes.len() as u64 != piece { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "chunk {chunk} gave {} bytes for a {piece}-byte read", + bytes.len() + ), + )); + } + out.extend_from_slice(&bytes); + } + } + at += piece; + } + Ok(out.freeze()) + } + + /// `fetch`, again after each error that can pass, until the device + /// stops. A chunk the store does not hold, or holds with other + /// bytes, fails at once. + async fn until_fetched(&self, chunk: u64, fetch: F) -> io::Result + where + F: Fn() -> Fut, + Fut: std::future::Future>, + { + let mut wait = RETRY_FIRST; + loop { + let error = match fetch().await { + Ok(bytes) => return Ok(bytes), + Err(error) => error, + }; + let lasting = matches!( + error, + ChunkStoreError::Blob(BlobError::NotFound | BlobError::RangeOutOfBounds { .. }) + | ChunkStoreError::HashMismatch { .. } + | ChunkStoreError::RangeOutOfBounds { .. } + ); + if lasting || self.stopping.load(Ordering::Relaxed) { + return Err(io::Error::other(error.to_string())); + } + tracing::warn!(chunk, %error, retry_ms = wait.as_millis() as u64, "ublk source: fetch failed; retrying"); + tokio::time::sleep(wait).await; + wait = (wait * 2).min(RETRY_LAST); + } + } + + /// Stream or demand, by position against the window. A stream read + /// at the chunk after the cursor moves the cursor; the chunk before + /// it still serves, since the copy's reads of one chunk and the next + /// overlap in flight. + fn classify(&self, chunk: u64, at: u64, len: u64) -> Serve { + let mut window = self.window.lock().unwrap(); + let near = window.cursor.and_then(|pos| { + let before = pos.checked_sub(1).map(|p| self.delta[p]); + if chunk == self.delta[pos] || Some(chunk) == before { + Some(pos) + } else if self.delta.get(pos + 1) == Some(&chunk) { + Some(pos + 1) + } else { + None + } + }); + let pos = match near { + Some(pos) => { + window.resync = None; + pos + } + None => { + let shaped = + at.is_multiple_of(self.region_bytes) && len.is_multiple_of(self.region_bytes); + match self.resync(&mut window, chunk, shaped) { + Some(pos) => pos, + None => return Serve::Demand, + } + } + }; + self.advance(&mut window, pos); + Serve::Stream(window.chunks.entry(chunk).or_default().clone()) + } + + /// A region-shaped read of a delta chunk outside the window: the + /// first one places a stream the device has not seen yet; otherwise + /// `RESYNC_READS` of them in ascending order move the stream there. + fn resync(&self, window: &mut Window, chunk: u64, shaped: bool) -> Option { + if !shaped { + return None; + } + let pos = self.delta.binary_search(&chunk).ok()?; + let run = match window.resync { + Some((last, run)) if chunk >= last => run + 1, + _ => 1, + }; + window.resync = Some((chunk, run)); + if window.cursor.is_none() || run >= RESYNC_READS { + window.resync = None; + window.chunks.clear(); + return Some(pos); + } + None + } + + /// Put the cursor at `pos`, drop the chunks behind the one before it, + /// and fetch the chunks up to `window_ahead` past it that are not in + /// the window yet. + fn advance(&self, window: &mut Window, pos: usize) { + if window.cursor != Some(pos) { + window.cursor = Some(pos); + let keep_from = pos.checked_sub(1).map_or(0, |p| self.delta[p]); + window.chunks.retain(|chunk, _| *chunk >= keep_from); + } + let last = (pos + self.window_ahead).min(self.delta.len() - 1); + for chunk in self.delta[pos..=last].iter().copied() { + let Some(hash) = self.hashes[chunk as usize] else { + continue; + }; + if let std::collections::hash_map::Entry::Vacant(slot) = window.chunks.entry(chunk) { + let cell = slot.insert(Arc::default()).clone(); + let engine = self.engine.clone(); + tokio::spawn(async move { + if let Err(e) = cell.get_or_try_init(|| stream_chunk(&engine, hash)).await { + tracing::debug!(chunk, error = %e, "stream prefetch failed; the read retries it"); + } + }); + } + } + } + + fn touch(&self, chunk: u64) { + let mut touched = self.touched.lock().unwrap(); + if !std::mem::replace(&mut touched.seen[chunk as usize], true) { + touched.order.push(chunk as u32); + } + } +} + +/// A whole chunk for the stream window: at `Hydrate`, and at +/// `Demand` too once that has waited `STREAM_PATIENCE`. +async fn stream_chunk(engine: &FillEngine, hash: ChunkHash) -> Result { + let hydrate = engine.read(hash, Class::Hydrate); + tokio::pin!(hydrate); + tokio::select! { + biased; + read = &mut hydrate => return read, + () = tokio::time::sleep(STREAM_PATIENCE) => {} + } + tokio::select! { + read = &mut hydrate => read, + read = engine.read(hash, Class::Demand) => read, + } +} + +/// Whether this host can serve a source: the ublk control device is +/// there and opens. Without it a session volume is written in full +/// before its guest starts. +pub fn available() -> io::Result<()> { + File::options() + .read(true) + .write(true) + .open("/dev/ublk-control") + .map(drop) +} + +struct Request { + tag: u16, + offset: u64, + len: u32, +} + +/// Finished reads on their way back to the queue thread, which sleeps +/// in `io_uring_enter`; the eventfd wakes it. +struct Completions { + done: Mutex)>>, + event: File, +} + +impl Completions { + fn new() -> io::Result { + // SAFETY: eventfd takes two integers and returns a new descriptor + // or -1; nothing is borrowed. + let fd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: `fd` is a fresh descriptor this process owns and nothing + // else closes. + let event = File::from(unsafe { OwnedFd::from_raw_fd(fd) }); + Ok(Self { + done: Mutex::new(VecDeque::new()), + event, + }) + } + + fn push(&self, tag: u16, result: io::Result) { + self.done.lock().unwrap().push_back((tag, result)); + if let Err(e) = (&self.event).write_all(&1u64.to_ne_bytes()) { + tracing::error!(error = %e, "ublk source: eventfd write failed; a read stays unanswered"); + } + } + + fn drain(&self) -> Vec<(u16, io::Result)> { + self.done.lock().unwrap().drain(..).collect() + } +} + +async fn dispatch( + shared: Arc, + mut requests: mpsc::UnboundedReceiver, + completions: Arc, +) { + while let Some(Request { tag, offset, len }) = requests.recv().await { + let shared = shared.clone(); + let completions = completions.clone(); + tokio::spawn(async move { + let result = shared.read(offset, len as u64).await; + completions.push(tag, result); + }); + } +} + +/// The queue thread: hands each read to the engine side and completes +/// it when the eventfd says its bytes are back. Runs until the device +/// stops. +fn queue_loop( + qid: u16, + dev: &UblkDev, + shared: &Shared, + requests: &mpsc::UnboundedSender, + completions: &Completions, +) { + let bufs = dev.alloc_queue_io_bufs(); + let queue = match UblkQueue::new(qid, dev) + .and_then(|q| q.submit_fetch_commands_unified(BufDescList::Slices(Some(&bufs)))) + { + Ok(queue) => queue, + Err(e) => { + tracing::error!(qid, error = %e, "ublk source queue did not start"); + return; + } + }; + let mut counter = 0u64; + arm_event(&queue, &completions.event, &mut counter); + queue.wait_and_handle_io(|queue, tag, io| { + if io.is_tgt_io() { + for (tag, result) in completions.drain() { + let buf = &bufs[tag as usize]; + let res = match result { + Ok(bytes) if bytes.len() <= buf.len() => { + // SAFETY: the buffer is `buf.len()` bytes; between + // this request's arrival and its commit below, this + // thread is its only writer and the kernel does not + // read it. + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + buf.as_mut_ptr(), + bytes.len(), + ); + } + bytes.len() as i32 + } + Ok(bytes) => { + tracing::error!( + tag, + len = bytes.len(), + "ublk source: read answered with more than its buffer" + ); + -libc::EIO + } + Err(e) => { + tracing::warn!(tag, error = %e, "ublk source: read failed"); + -libc::EIO + } + }; + complete(queue, tag, buf.as_slice(), res); + } + arm_event(queue, &completions.event, &mut counter); + return; + } + let iod = queue.get_iod(tag); + let buf = &bufs[tag as usize]; + let op = iod.op_flags & 0xff; + if op != sys::UBLK_IO_OP_READ { + complete(queue, tag, buf.as_slice(), -libc::EOPNOTSUPP); + return; + } + if shared.stopping.load(Ordering::Relaxed) { + complete(queue, tag, buf.as_slice(), -libc::EIO); + return; + } + let request = Request { + tag, + offset: iod.start_sector << 9, + len: iod.nr_sectors << 9, + }; + // No receiver: the runtime is shutting down with the process. The + // read stays unanswered, and the kernel gives it to the next + // server (`UBLK_F_USER_RECOVERY_REISSUE`); an error here would + // reach the guest. + let _ = requests.send(request); + }); +} + +fn complete(queue: &UblkQueue, tag: u16, buf: &[u8], res: i32) { + if let Err(e) = + queue.complete_io_cmd_unified(tag, BufDesc::Slice(buf), Ok(UblkIORes::Result(res))) + { + tracing::error!(tag, error = %e, "ublk source: completion not submitted"); + } +} + +/// Queue a read of the eventfd so a completion wakes the queue thread. +fn arm_event(queue: &UblkQueue, event: &File, counter: &mut u64) { + let sqe = opcode::Read::new( + types::Fd(event.as_raw_fd()), + std::ptr::from_mut(counter).cast::(), + std::mem::size_of::() as u32, + ) + .build() + .user_data(UblkIOCtx::build_user_data(EVENT_TAG, 0, 0, true)); + if let Err(e) = queue.ublk_submit_sqe_sync(sqe) { + tracing::error!(error = %e, "ublk source: eventfd read not submitted; completions stall"); + } +} + +async fn wait_for_node(path: &Path) -> io::Result<()> { + let appears = async { + while tokio::fs::metadata(path).await.is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + tokio::time::timeout(DEVICE_NODE_WAIT, appears) + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::NotFound, + format!("{} did not appear", path.display()), + ) + }) +} + +fn ublk_err(e: UblkError) -> io::Error { + match e { + UblkError::IOError(e) => e, + other => io::Error::other(other.to_string()), + } +} + +/// Run `f`, a libublk control command, on this thread. libublk sends +/// control commands over a thread-local ring that only the thread which +/// built the `UblkCtrl` has; every other thread makes its own first. +fn with_ctrl_ring(f: impl FnOnce() -> Result) -> io::Result { + libublk::ublk_init_ctrl_task_ring(|ring| { + if ring.is_none() { + *ring = Some( + io_uring::IoUring::::builder() + .build(CTRL_RING_DEPTH) + .map_err(UblkError::IOError)?, + ); + } + Ok(()) + }) + .map_err(ublk_err)?; + f().map_err(ublk_err) +} + +/// The host's [`SourceServer`]: every source is a [`BlkSource`] that +/// fetches `window_ahead` chunks ahead of the kernel's copy. +pub struct UblkSources { + pub window_ahead: usize, +} + +#[async_trait] +impl SourceServer for UblkSources { + async fn serve( + &self, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result> { + Ok(Box::new( + BlkSource::serve(fill, manifest, delta, region_bytes, self.window_ahead).await?, + )) + } + + async fn recover( + &self, + path: &Path, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result> { + Ok(Box::new( + BlkSource::recover( + dev_id_of(path)?, + fill, + manifest, + delta, + region_bytes, + self.window_ahead, + ) + .await?, + )) + } +} + +#[async_trait] +impl SourceDevice for BlkSource { + fn path(&self) -> &Path { + BlkSource::path(self) + } + + fn touched(&self) -> Vec { + BlkSource::touched(self) + } + + async fn stop(self: Box) -> io::Result<()> { + BlkSource::stop(*self).await + } +} + +/// The id in `/dev/ublkb`. +fn dev_id_of(path: &Path) -> io::Result { + path.file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_prefix("ublkb")) + .and_then(|id| id.parse().ok()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a ublk block device", path.display()), + ) + }) +} + +/// Delete every ublk device nothing maps onto and no server in this +/// process serves (`served` lists their paths): what a process left +/// when it died between making a device and mapping a volume onto it. +/// A device a volume still maps onto stays, with a warning: deleting it +/// would wait on reads nothing answers. +pub async fn reap_orphans(served: &[PathBuf]) -> io::Result> { + let mut reaped = Vec::new(); + let mut entries = match tokio::fs::read_dir("/dev").await { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(reaped), + Err(e) => return Err(e), + }; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let Ok(id) = dev_id_of(&path) else { continue }; + if served.contains(&path) { + continue; + } + let held = std::fs::read_dir(format!("/sys/block/ublkb{id}/holders")) + .map(|holders| holders.count()) + .unwrap_or(0); + if held > 0 { + tracing::warn!(device = %path.display(), holders = held, "an unserved ublk device is still mapped; it is not deleted"); + continue; + } + tokio::task::spawn_blocking(move || { + with_ctrl_ring(|| UblkCtrl::new_simple(id as i32)?.del_dev()) + }) + .await + .map_err(io::Error::other)??; + tracing::info!(device = %path.display(), "deleted an orphaned ublk device"); + reaped.push(id); + } + Ok(reaped) +} diff --git a/crates/engram-host-agent/src/checkpoint.rs b/crates/engram-host-agent/src/checkpoint.rs index c86a5c9a4..d4c52c299 100644 --- a/crates/engram-host-agent/src/checkpoint.rs +++ b/crates/engram-host-agent/src/checkpoint.rs @@ -551,6 +551,15 @@ pub async fn run_checkpoint_pass( ); continue; } + if backend.disks_hydrating(sandbox_id) { + metrics::counter!(crate::metrics::CHECKPOINT_SKIPPED_TOTAL).increment(1); + tracing::debug!( + %sandbox_id, + %session_id, + "skipping periodic checkpoint; the disks still hydrate", + ); + continue; + } // A root span: the driver runs on its own timer, not for a request. let span = tracing::info_span!( parent: None, diff --git a/crates/engram-host-agent/src/config.rs b/crates/engram-host-agent/src/config.rs index 8be524e15..ea54b3879 100644 --- a/crates/engram-host-agent/src/config.rs +++ b/crates/engram-host-agent/src/config.rs @@ -53,8 +53,15 @@ pub struct HostAgentConfig { /// Whether an idle-eviction capture keeps the in-guest harness alive /// or stops it first. pub idle_eviction_harness: crate::host_client::IdleEvictionHarness, + /// Whole chunks a hydrating disk's source fetches ahead of the + /// kernel's copy. + pub disk_hydration_window_chunks: usize, } +/// With 16 MiB chunks, 192 MiB in flight per hydrating disk: enough that +/// the pool's disk limits the copy and not the fetches. +pub const DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS: usize = 12; + impl Default for HostAgentConfig { fn default() -> Self { Self { @@ -71,6 +78,7 @@ impl Default for HostAgentConfig { migrate_peer_listen_addr: None, backend_name: "process".to_string(), idle_eviction_harness: Default::default(), + disk_hydration_window_chunks: DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS, } } } diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index b0406445f..d5bd41b5a 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -87,6 +87,8 @@ pub(crate) mod snapshot_post; pub mod substrate_server; pub use host_client::{IdleEvictionHarness, LocalHostClient}; #[cfg(target_os = "linux")] +pub mod blk_source; +#[cfg(target_os = "linux")] pub mod heap_profile; pub mod heartbeat; pub mod idle_evictor; @@ -108,7 +110,7 @@ mod time_source; pub mod util; pub mod warm_progress; -pub use config::HostAgentConfig; +pub use config::{HostAgentConfig, DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS}; pub struct HostAgent { pub cfg: HostAgentConfig, @@ -302,9 +304,22 @@ impl HostAgent { } if let Some(fill) = self.fill.clone() { if let Some(disk) = self.session_disk.clone() { - p = p.with_session_volumes(Arc::new( - engram_host_core::SessionVolumes::new(disk, fill.clone()), - )); + let volumes = engram_host_core::SessionVolumes::new(disk, fill.clone()); + #[cfg(target_os = "linux")] + let volumes = match blk_source::available() { + Ok(()) => volumes.with_sources(Arc::new(blk_source::UblkSources { + window_ahead: self.cfg.disk_hydration_window_chunks, + })), + Err(error) => { + tracing::error!( + %error, + "no ublk control device (is ublk_drv loaded?): every resume \ + writes its whole disk before the guest starts" + ); + volumes + } + }; + p = p.with_session_volumes(Arc::new(volumes)); } p = p.with_fill(fill); // ADR 0080 §C: the MaterializeImage scratch root — diff --git a/crates/engram-host-agent/src/main.rs b/crates/engram-host-agent/src/main.rs index b809afdf0..317d4aa73 100644 --- a/crates/engram-host-agent/src/main.rs +++ b/crates/engram-host-agent/src/main.rs @@ -62,6 +62,25 @@ struct Cli { #[arg(long, env = "ENGRAM_THIN_POOL")] thin_pool: Option, + /// Whole chunks a resumed disk's copy fetches ahead of the kernel, + /// per disk. Each chunk is held in memory until the kernel has + /// copied it. Unset, 12. + #[arg( + long, + env = "ENGRAM_DISK_HYDRATION_WINDOW_CHUNKS", + value_parser = clap::value_parser!(u32).range(1..) + )] + disk_hydration_window_chunks: Option, + + /// Pool blocks the kernel copies at one time into a resumed disk, + /// per disk. Unset, 512. + #[arg( + long, + env = "ENGRAM_DISK_HYDRATION_IN_FLIGHT_REGIONS", + value_parser = clap::value_parser!(u64).range(1..) + )] + disk_hydration_in_flight_regions: Option, + /// Coordinator endpoint to dial via WebSocket. #[arg(long, env = "ENGRAM_COORDINATOR_ENDPOINT")] coordinator: Option, @@ -333,6 +352,10 @@ async fn main() -> Result<(), HostAgentError> { BackendChoice::Vz => "vz".to_string(), }, idle_eviction_harness: cli.idle_eviction_harness, + disk_hydration_window_chunks: cli.disk_hydration_window_chunks.map_or( + engram_host_agent::DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS, + |chunks| chunks as usize, + ), ..HostAgentConfig::default() }; @@ -543,6 +566,10 @@ async fn main() -> Result<(), HostAgentError> { ) .await .map_err(|e| HostAgentError::Config(format!("thin pool {pool}: {e}")))?; + let disk = match cli.disk_hydration_in_flight_regions { + Some(regions) => disk.with_hydration_in_flight(regions), + None => disk, + }; let capacity = engram_host_core::SessionDisk::capacity(&disk) .await .map_err(|e| HostAgentError::Config(format!("thin pool {pool} capacity: {e}")))?; diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index b9a43d489..5ca75b66f 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -187,6 +187,25 @@ mod item_assembler_tests { } } +#[cfg(test)] +mod settle_hold_tests { + use super::{settle_holds_checkpoint, SETTLE_HOLDS_CHECKPOINT}; + + #[test] + fn a_running_settle_holds_the_periodic_checkpoint_for_a_bounded_time() { + let started = chrono::DateTime::UNIX_EPOCH; + let just_inside = started + SETTLE_HOLDS_CHECKPOINT - chrono::TimeDelta::seconds(1); + + assert!(settle_holds_checkpoint(false, started, just_inside)); + assert!(!settle_holds_checkpoint(true, started, just_inside)); + assert!(!settle_holds_checkpoint( + false, + started, + started + SETTLE_HOLDS_CHECKPOINT + )); + } +} + #[cfg(test)] mod first_touch_tests { use super::*; @@ -266,7 +285,16 @@ mod first_touch_tests { entries: vec![(2, 10), (3, 20), (1, 30)], }; std::fs::write(&file, record.encode()).unwrap(); - publish_first_touch(&plain, session, &file, Some(10)).await; + publish_first_touch(&plain, session, &file, Some(10), vec![5, 2]).await; + assert_eq!( + first_touch_indices( + &plain, + session, + &engram_chunk_store::FirstTouch::disk_key(session.as_uuid()) + ) + .await, + vec![5, 2] + ); let cache = engram_chunk_store::cache::ChunkCache::new( engram_chunk_store::cache::ChunkCacheConfig { @@ -350,6 +378,19 @@ async fn read_trace_lineage(src: &Path) -> Option { serde_json::from_value(value.get("trace_lineage_id")?.clone()).ok() } +/// The chunk indices published under `key` for `session`, in touch +/// order; empty when none were, or the record is unreadable. +async fn first_touch_indices(store: &ChunkStore, session: SessionId, key: &str) -> Vec { + match store.blob_storage().get(key).await { + Ok(bytes) => engram_chunk_store::FirstTouch::decode_indices(&bytes).unwrap_or_default(), + Err(engram_core::BlobError::NotFound) => Vec::new(), + Err(e) => { + tracing::warn!(%session, key, error = %e, "first-touch prefix unreadable"); + Vec::new() + } + } +} + /// The chunk indices under `key` as this manifest's chunk hashes, in /// order. Positions the manifest leaves out are zero pages and need no /// fetch. @@ -359,14 +400,7 @@ async fn first_touch_hashes( key: &str, manifest: &engram_chunk_store::Manifest, ) -> Vec { - let indices = match store.blob_storage().get(key).await { - Ok(bytes) => engram_chunk_store::FirstTouch::decode_indices(&bytes).unwrap_or_default(), - Err(engram_core::BlobError::NotFound) => return Vec::new(), - Err(e) => { - tracing::warn!(%session, error = %e, "first-touch prefix unreadable"); - return Vec::new(); - } - }; + let indices = first_touch_indices(store, session, key).await; let chunk_size = manifest.chunk_size.as_u64(); let by_offset: std::collections::HashMap = manifest.chunks.iter().map(|c| (c.offset, c.hash)).collect(); @@ -426,15 +460,32 @@ fn split_first_touch(record: &engram_chunk_store::FirstTouch, cut_ms: i64) -> (V (prefix, rest) } -/// Publish the chunks the sandbox touched up to its first completed run -/// (all of them before one completes), for the session's next restore to -/// fetch first. Best-effort. +/// Publish the memory chunks the sandbox touched up to its first +/// completed run (all of them before one completes) and the root disk +/// chunks the guest read before a hydrating restore's copy reached them +/// (`disk`, at most [`FIRST_TOUCH_PREFIX_CAP`]), for the session's next +/// restore to fetch first. Best-effort. async fn publish_first_touch( store: &ChunkStore, session: SessionId, file: &Path, first_run_completed: Option, + mut disk: Vec, ) { + let blob = store.blob_storage(); + let encode = engram_chunk_store::FirstTouch::encode_indices; + disk.truncate(FIRST_TOUCH_PREFIX_CAP); + if !disk.is_empty() { + let key = engram_chunk_store::FirstTouch::disk_key(session.as_uuid()); + match blob.put(&key, encode(&disk).into()).await { + Ok(_) => { + tracing::info!(%session, chunks = disk.len(), "published disk first-touch prefix") + } + Err(e) => { + tracing::warn!(%session, error = %e, "publish disk first-touch prefix failed") + } + } + } let Ok(bytes) = tokio::fs::read(file).await else { return; }; @@ -446,8 +497,6 @@ async fn publish_first_touch( return; } let (prefix, rest) = split_first_touch(&record, first_run_completed.unwrap_or(i64::MAX)); - let encode = engram_chunk_store::FirstTouch::encode_indices; - let blob = store.blob_storage(); let key = engram_chunk_store::FirstTouch::key(session.as_uuid()); let rest_key = engram_chunk_store::FirstTouch::rest_key(session.as_uuid()); match tokio::try_join!( @@ -828,6 +877,10 @@ pub struct PooledBackend { /// A lazy restore's background memory prefetch; dropping the entry /// cancels it. memory_prefetches: DashMap, + /// The driver that settles a sandbox's hydrating disks; dropping the + /// entry ends it. Destroy drops it before the VM goes, so no settle + /// runs against a volume being released. + disk_settles: DashMap, /// The adaptive checkpoint controller's input: what the last /// capture observed about the memory dirty rate (ADR 0101 B). checkpoint_pacing: Arc>, @@ -1014,6 +1067,63 @@ fn record_warm_hook_failure_metric(kind: engram_core::types::CaptureFailureKind) .increment(1); } +/// How often a settle driver reads a hydrating disk's progress, and how +/// long it waits after a step fails. +const SETTLE_POLL: std::time::Duration = std::time::Duration::from_millis(250); +const SETTLE_RETRY: std::time::Duration = std::time::Duration::from_secs(5); + +/// Settle each of the sandbox's hydrating disks: serve the sources this +/// process does not serve yet (after a restart), poll until the kernel +/// has every region, then cut the device over and stop its source +/// (`SessionVolumes::settle` is the step). A failed step is retried, so +/// a store outage at startup does not leave a guest's disk unserved; a +/// destroy ends the driver by dropping its task. +async fn settle_disks(volumes: Arc, id: SandboxId) { + let roles = loop { + let hydrating = async { + volumes.reattach(id).await?; + volumes.hydrating(id).await + }; + match hydrating.await { + Ok(roles) => break roles, + Err(error) => { + tracing::warn!(sandbox_id = %id, %error, "hydrating disks not served; retrying"); + tokio::time::sleep(SETTLE_RETRY).await; + } + } + }; + for role in roles { + let started = crate::time_source::metrics_now(); + // The regions the copy still had to bring in when this driver + // first looked; none when the volume was complete by then. + let mut regions: Option = None; + loop { + match volumes.settle(id, role).await { + Ok(engram_host_core::Hydration::Settled) => { + tracing::info!( + sandbox_id = %id, + role = %role, + regions = regions.unwrap_or(0), + elapsed_ms = started.elapsed().as_millis() as u64, + "disk settled", + ); + break; + } + Ok(engram_host_core::Hydration::Hydrating { + hydrated, total, .. + }) => { + regions.get_or_insert(total - hydrated); + tokio::time::sleep(SETTLE_POLL).await; + } + Err(error) => { + tracing::warn!(sandbox_id = %id, role = %role, %error, "disk settle step failed; retrying"); + tokio::time::sleep(SETTLE_RETRY).await; + } + } + } + } +} + /// Abort the wrapped task on drop, including hint polling after DrainDone. /// A leg wrapped by /// [`spawn_leg_keepalive`] stops resending stale progress the moment @@ -1021,6 +1131,24 @@ fn record_warm_hook_failure_metric(kind: engram_core::types::CaptureFailureKind) /// `?`-return, since `Drop` runs during unwind too. struct AbortTaskOnDrop(tokio::task::JoinHandle<()>); +struct DiskSettle { + task: AbortTaskOnDrop, + started: chrono::DateTime, +} + +/// How long a settle driver that has not finished holds the periodic +/// checkpoint. A driver that cannot settle (a chunk the store lost) +/// runs for ever, and the session still needs checkpoints. +const SETTLE_HOLDS_CHECKPOINT: chrono::TimeDelta = chrono::TimeDelta::minutes(10); + +fn settle_holds_checkpoint( + finished: bool, + started: chrono::DateTime, + now: chrono::DateTime, +) -> bool { + !finished && now - started < SETTLE_HOLDS_CHECKPOINT +} + impl Drop for AbortTaskOnDrop { fn drop(&mut self) { self.0.abort(); @@ -1911,17 +2039,54 @@ impl PooledBackend { } .await; remove_staged_manifests(staged_manifests, &src, id, &staged_refs).await; - if restore.is_err() { - self.drop_failed_restore(id); - if let Some(volumes) = self.volumes.as_ref() { - if let Err(error) = volumes.release(id).await { - tracing::warn!(sandbox_id = %id, %error, "release after a failed restore failed"); + let volumes = self.volumes.as_ref(); + match (&restore, volumes) { + (Err(_), _) => { + self.drop_failed_restore(id); + if let Some(volumes) = volumes { + if let Err(error) = volumes.release(id).await { + tracing::warn!(sandbox_id = %id, %error, "release after a failed restore failed"); + } } } + (Ok(()), Some(_)) => self.settle_disks(id), + (Ok(()), None) => {} } restore } + /// Drive the sandbox's hydrating disks to settled volumes in the + /// background. + fn settle_disks(&self, id: SandboxId) { + let Some(volumes) = self.volumes.clone() else { + return; + }; + let task = tokio::spawn( + settle_disks(volumes, id) + .instrument(tracing::info_span!("disk.settle", sandbox_id = %id)), + ); + self.disk_settles.insert( + id, + DiskSettle { + task: AbortTaskOnDrop(task), + started: self.clock.now_utc(), + }, + ); + } + + /// Whether the kernel still copies the sandbox's disks in. A + /// checkpoint taken now reads every copied block back from the pool's + /// disk, which nearly stops the copy, so the periodic driver waits. + pub fn disks_hydrating(&self, id: SandboxId) -> bool { + self.disk_settles.get(&id).is_some_and(|settle| { + settle_holds_checkpoint( + settle.task.0.is_finished(), + settle.started, + self.clock.now_utc(), + ) + }) + } + /// End a restore's memory fetches: cancel its background prefetch and /// drop its peer registration, which ends the peer's batching task. fn drop_restore_fetches(&self, id: SandboxId) { @@ -1948,11 +2113,20 @@ impl PooledBackend { let manifest = metadata.disk_manifest.ok_or_else(|| { SandboxError::MemoryImageUnusable("snapshot has no disk manifest".into()) })?; - let base = metadata.base_disk_manifest; let root = if fork_disk { - Source::Fork { manifest, base } + Source::Fork { manifest } } else { - Source::Continue { manifest, base } + let first_touch = match (read_trace_lineage(src).await, self.chunk_store.as_ref()) { + (Some(session), Some(store)) => { + let key = engram_chunk_store::FirstTouch::disk_key(session.as_uuid()); + first_touch_indices(store, session, &key).await + } + _ => Vec::new(), + }; + Source::Continue { + manifest, + first_touch, + } }; let sidecar = read_sidecar_swap(src).await?; let swap = match (metadata.swap_manifest, sidecar.swap_mib) { @@ -1969,7 +2143,7 @@ impl PooledBackend { } (Some(manifest), _) => Some(Source::Continue { manifest, - base: None, + first_touch: Vec::new(), }), (None, _) if !fresh && !sidecar.base_capture => { return Err(SandboxError::MemoryImageUnusable( @@ -2039,6 +2213,7 @@ impl PooledBackend { peer_registrations: Arc::new(DashMap::new()), sandbox_registrations: Arc::new(DashMap::new()), memory_prefetches: DashMap::new(), + disk_settles: DashMap::new(), checkpoint_pacing: Arc::new(DashMap::new()), checkpoint_dir: None, checkpoint_recorded: Arc::new(tokio::sync::Notify::new()), @@ -2092,7 +2267,13 @@ impl PooledBackend { .take_precopy(id, role) .instrument(tracing::info_span!("presetup.precopy_snapshot", %role)) .await - .map_err(|e| SandboxError::Snapshot(format!("{role} disk precopy: {e}")))?; + .map_err(|e| match e.kind() { + // The coordinator moves the session by snapshot. + std::io::ErrorKind::Unsupported => SandboxError::InvalidSpec(format!( + "{role} disk precopy: {e} — use snapshot-rehome" + )), + _ => SandboxError::Snapshot(format!("{role} disk precopy: {e}")), + })?; taken.push((role, pending)); let pending = &taken.last().expect("just pushed").1; let changed = volumes @@ -2424,10 +2605,11 @@ impl PooledBackend { )) } - /// Delete the volumes of every sandbox that is neither alive nor - /// awaiting an eviction finalize. Runs once at startup, after the - /// reattach pass fixes the live set and before anything can - /// provision a sandbox. + /// Serve again the sources of every hydrating volume the previous + /// process left, settle the live sandboxes' ones, and delete the + /// volumes of every sandbox that is neither alive nor awaiting an + /// eviction finalize. Runs once at startup, after the reattach pass + /// fixes the live set and before anything can provision a sandbox. pub async fn reconcile_volumes(&self) { let Some(volumes) = &self.volumes else { return; @@ -2439,6 +2621,17 @@ impl PooledBackend { return; } }; + match volumes.reattach_all().await { + Ok(reattached) if reattached.is_empty() => {} + Ok(reattached) => { + tracing::info!(volumes = reattached.len(), "hydrating disks served again"); + } + Err(error) => tracing::warn!(%error, "startup reattach of hydrating disks failed"), + } + // Each driver serves what the pass above could not. + for id in live.iter().copied() { + self.settle_disks(id); + } let finalizer = self.eviction_finalizer(); match crate::eviction_finalize::reconcile_volumes(volumes, finalizer.as_ref(), live).await { Ok(report) => { @@ -2455,6 +2648,10 @@ impl PooledBackend { } Err(error) => tracing::warn!(%error, "startup volume reconcile failed"), } + #[cfg(target_os = "linux")] + if let Err(error) = crate::blk_source::reap_orphans(&volumes.sources()).await { + tracing::warn!(%error, "orphaned ublk devices were not all deleted"); + } } /// Issue #529: re-drive every un-acked eviction finalize record at @@ -2526,8 +2723,13 @@ impl PooledBackend { Some(FirstRun::Completed { at_ms }) => Some(*at_ms), _ => None, }; + let disk = self + .volumes + .as_ref() + .map(|v| v.first_touch(id, DiskRole::Root)) + .unwrap_or_default(); tokio::spawn(async move { - publish_first_touch(&chunk_store, session_id, &file, first_run_completed).await; + publish_first_touch(&chunk_store, session_id, &file, first_run_completed, disk).await; }); } @@ -4592,7 +4794,7 @@ impl SandboxBackend for PooledBackend { let root = match spec.rootfs_manifest { Some(manifest) => Source::Continue { manifest, - base: None, + first_touch: Vec::new(), }, None => { let Some((uri, cached)) = cached.as_ref() else { @@ -4619,7 +4821,7 @@ impl SandboxBackend for PooledBackend { let swap = match (spec.swap_manifest, spec.swap_mib.filter(|m| *m > 0)) { (Some(manifest), _) => Some(Source::Continue { manifest, - base: None, + first_touch: Vec::new(), }), (None, Some(mib)) => Some(Source::Empty { size_bytes: u64::from(mib) << 20, @@ -4636,9 +4838,13 @@ impl SandboxBackend for PooledBackend { spec.swap_source = provisioned.swap; spec.swap_manifest = provisioned.swap_manifest; let created = self.inner.create(id, spec).await; - if created.is_err() { - if let Err(error) = volumes.release(id).await { - tracing::warn!(sandbox_id = %id, %error, "release after a failed create failed"); + match &created { + // A create from a manifest hydrates like a resume. + Ok(()) => self.settle_disks(id), + Err(_) => { + if let Err(error) = volumes.release(id).await { + tracing::warn!(sandbox_id = %id, %error, "release after a failed create failed"); + } } } created @@ -5861,6 +6067,7 @@ impl SandboxBackend for PooledBackend { self.held_snapshots.remove(&id); self.hint_pollers.remove(&id); self.memory_prefetches.remove(&id); + self.disk_settles.remove(&id); self.teardowns .run_or_join(id, || { let session_bindings = self.session_bindings.clone(); diff --git a/crates/engram-host-agent/tests/blk_source.rs b/crates/engram-host-agent/tests/blk_source.rs new file mode 100644 index 000000000..aa36de687 --- /dev/null +++ b/crates/engram-host-agent/tests/blk_source.rs @@ -0,0 +1,235 @@ +//! The ublk source under a real kernel, as root, with `ublk_drv` loaded. +//! `ENGRAM_TEST_UBLK` set runs the tests (the CI root lane); unset, they +//! skip. +// tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) +#![allow(clippy::disallowed_methods)] +#![cfg(target_os = "linux")] + +use std::fs::File; +use std::os::unix::fs::{FileExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind}; +use engram_fill::FillEngine; +use engram_host_agent::blk_source::BlkSource; +use engram_storage_local::LocalBlobStorage; +use libublk::helpers::IoBuf; + +const CHUNK: u64 = 256 << 10; +const REGION: u64 = 64 << 10; +const CHUNKS: u64 = 4; +/// Chunks 0, 1 and 3 hold bytes; chunk 2 is a hole. +const HELD: [(u64, u8); 3] = [(0, 0x11), (1, 0x22), (3, 0x33)]; +const DELTA: [u64; 3] = [0, 1, 3]; +const WINDOW: usize = 12; + +fn gate() -> bool { + if std::env::var_os("ENGRAM_TEST_UBLK").is_none() { + eprintln!("SKIP: ENGRAM_TEST_UBLK is not set"); + return false; + } + true +} + +fn image() -> Vec { + let mut image = vec![0u8; (CHUNKS * CHUNK) as usize]; + for (chunk, byte) in HELD { + let at = (chunk * CHUNK) as usize; + image[at..at + CHUNK as usize].fill(byte); + } + image +} + +/// A store holding the image's chunks under `dir`, and its manifest. +async fn engine_and_manifest(dir: &Path) -> (FillEngine, Manifest) { + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.join("blobs")))); + let mut chunks = Vec::new(); + for (chunk, byte) in HELD { + let hash = store.put_chunk(&vec![byte; CHUNK as usize]).await.unwrap(); + chunks.push(ChunkRef { + offset: chunk * CHUNK, + hash, + }); + } + let manifest = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks, + ..Manifest::empty(ManifestKind::Disk, CHUNKS * CHUNK) + }; + (FillEngine::new(store, Default::default()), manifest) +} + +/// The device opened for direct I/O, so every read reaches the source +/// with the offset and length the test gives it. +fn open_direct(path: &Path) -> File { + File::options() + .read(true) + .custom_flags(libc::O_DIRECT) + .open(path) + .unwrap() +} + +fn read_at(device: &File, offset: u64, len: usize) -> Vec { + let mut buf = IoBuf::::new(len); + device.read_exact_at(buf.as_mut_slice(), offset).unwrap(); + buf.as_slice().to_vec() +} + +/// An open off the runtime: the source's dispatch runs on it, and an +/// open waits for the kernel's partition scan, which reads the device. +async fn open_off_runtime(path: &Path) -> Arc { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || Arc::new(open_direct(&path))) + .await + .unwrap() +} + +/// A read off the runtime: the source's dispatch runs on it, so a read +/// that blocks a runtime thread waits for itself. +async fn read_off_runtime(device: &Arc, offset: u64, len: usize) -> Vec { + let device = device.clone(); + tokio::task::spawn_blocking(move || read_at(&device, offset, len)) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires root and the ublk_drv module"] +async fn a_source_serves_the_manifest_and_records_the_chunks_read_cold() { + if !gate() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let (engine, manifest) = engine_and_manifest(dir.path()).await; + let source = BlkSource::serve(engine, &manifest, DELTA.to_vec(), REGION, WINDOW) + .await + .unwrap(); + let path = source.path().to_path_buf(); + let device = open_off_runtime(&path).await; + let image = image(); + + // A guest read far from where the copy starts: served, and recorded. + let at = 3 * CHUNK + 8192; + assert_eq!( + read_off_runtime(&device, at, 4096).await, + image[at as usize..at as usize + 4096] + ); + assert_eq!(source.touched(), vec![3]); + + // The copy's stream: the whole device, region by region, ascending. + for at in (0..CHUNKS * CHUNK).step_by(REGION as usize) { + assert_eq!( + read_off_runtime(&device, at, REGION as usize).await, + image[at as usize..(at + REGION) as usize], + "region at {at}" + ); + } + assert_eq!(source.touched(), vec![3], "the stream is not a guest read"); + + drop(device); + source.stop().await.unwrap(); + assert!(!path.exists(), "{} was deleted", path.display()); +} + +/// The server half of the recovery test: serves a source from the +/// directory `ENGRAM_UBLK_CHILD_DIR` names until it is killed. +#[tokio::test] +#[ignore = "a helper the recovery test runs as a child process"] +async fn child_serves_until_killed() { + let Some(dir) = std::env::var_os("ENGRAM_UBLK_CHILD_DIR") else { + return; + }; + let dir = PathBuf::from(dir); + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.join("blobs")))); + let manifest: Manifest = + serde_json::from_slice(&std::fs::read(dir.join("manifest.json")).unwrap()).unwrap(); + let source = BlkSource::serve( + FillEngine::new(store, Default::default()), + &manifest, + DELTA.to_vec(), + REGION, + WINDOW, + ) + .await + .unwrap(); + std::fs::write(dir.join("dev_id"), source.dev_id().to_string()).unwrap(); + std::future::pending::<()>().await; +} + +/// A host-agent killed with a read queued at its source: the device +/// stays, the read waits, and the next host-agent's server answers it. +#[tokio::test] +#[ignore = "requires root and the ublk_drv module"] +async fn a_source_killed_with_a_queued_read_is_recovered_by_a_new_server() { + if !gate() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let (engine, manifest) = engine_and_manifest(dir.path()).await; + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "child_serves_until_killed", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("ENGRAM_UBLK_CHILD_DIR", dir.path()) + .spawn() + .unwrap(); + let dev_id_file = dir.path().join("dev_id"); + let deadline = Instant::now() + Duration::from_secs(20); + while !dev_id_file.exists() { + assert!( + Instant::now() < deadline, + "the child did not serve a device" + ); + assert!(child.try_wait().unwrap().is_none(), "the child exited"); + std::thread::sleep(Duration::from_millis(50)); + } + let dev_id: u32 = std::fs::read_to_string(&dev_id_file) + .unwrap() + .trim() + .parse() + .unwrap(); + let path = PathBuf::from(format!("/dev/ublkb{dev_id}")); + let image = image(); + assert_eq!( + read_at(&open_direct(&path), 0, REGION as usize), + image[..REGION as usize] + ); + + // Freeze the server, queue a read it cannot answer, then kill it. + let pid = nix::unistd::Pid::from_raw(child.id() as i32); + nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGSTOP).unwrap(); + let at = 3 * CHUNK + REGION; + let queued = std::thread::spawn({ + let path = path.clone(); + move || read_at(&open_direct(&path), at, REGION as usize) + }); + std::thread::sleep(Duration::from_millis(300)); + assert!(!queued.is_finished(), "the read waits on the frozen server"); + nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGKILL).unwrap(); + child.wait().unwrap(); + + let recovered = BlkSource::recover(dev_id, engine, &manifest, DELTA.to_vec(), REGION, WINDOW) + .await + .unwrap(); + let bytes = tokio::time::timeout( + Duration::from_secs(10), + tokio::task::spawn_blocking(move || queued.join().unwrap()), + ) + .await + .expect("the queued read completes after recovery") + .unwrap(); + assert_eq!(bytes, image[at as usize..(at + REGION) as usize]); + + recovered.stop().await.unwrap(); + assert!(!path.exists(), "{} was deleted", path.display()); +} diff --git a/crates/engram-host-agent/tests/lazy_restore.rs b/crates/engram-host-agent/tests/lazy_restore.rs new file mode 100644 index 000000000..21668dd23 --- /dev/null +++ b/crates/engram-host-agent/tests/lazy_restore.rs @@ -0,0 +1,267 @@ +//! A lazy resume over the dm-thin pool with a ublk source, as root: the +//! device the guest would open answers a read with the manifest's bytes +//! whether the kernel has copied the region or not, the volume settles +//! onto its destination, and a process that dies before the settle +//! leaves a volume the next process must serve again before it can +//! delete it. A read queued on a dead server is the `blk_source` test's. Needs +//! `ENGRAM_TEST_THIN_POOL` and `ENGRAM_TEST_UBLK` (the CI root lane); +//! unset, the tests skip. +// tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) +#![allow(clippy::disallowed_methods)] +#![cfg(target_os = "linux")] + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind}; +use engram_core::types::manifest::ManifestRef; +use engram_core::types::snapshot::DiskRole; +use engram_core::SandboxId; +use engram_disk_thin::ThinPoolDisk; +use engram_fill::FillEngine; +use engram_host_agent::blk_source::UblkSources; +use engram_host_core::disk::ChunkIndex; +use engram_host_core::session_volumes::{live_name, Source, StagePath}; +use engram_host_core::{Hydration, SessionVolumes}; +use engram_storage_local::LocalBlobStorage; + +/// Two pool blocks per chunk, as in the disk-thin clone test. +const CHUNK: u64 = 2 * 65536; +/// Chunk `i` holds byte `PATTERN[i]`; zero is a hole. +const PATTERN: [u8; 8] = [1, 2, 0, 4, 0, 0, 7, 8]; +const SETTLE_DEADLINE: Duration = Duration::from_secs(30); + +fn gate() -> Option { + let pool = std::env::var("ENGRAM_TEST_THIN_POOL").ok(); + if pool.is_none() || std::env::var_os("ENGRAM_TEST_UBLK").is_none() { + eprintln!("SKIP: ENGRAM_TEST_THIN_POOL or ENGRAM_TEST_UBLK is not set"); + return None; + } + pool +} + +fn fill(byte: u8) -> Vec { + vec![byte; CHUNK as usize] +} + +fn store(dir: &Path) -> ChunkStore { + ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.join("blobs")))) +} + +/// Session volumes over the pool, with the host's ublk sources. +async fn volumes(pool: &str, store: ChunkStore) -> SessionVolumes { + let disk = ThinPoolDisk::open(pool, CHUNK, Path::new("/dev/engram-test")) + .await + .expect("open pool"); + SessionVolumes::new(Arc::new(disk), FillEngine::new(store, Default::default())) + .with_sources(Arc::new(UblkSources { window_ahead: 12 })) +} + +/// `PATTERN` as a manifest in `store`. +async fn manifest_in(store: &ChunkStore) -> ManifestRef { + let mut chunks = Vec::new(); + for (i, byte) in PATTERN.iter().enumerate() { + if *byte != 0 { + chunks.push(ChunkRef { + offset: i as u64 * CHUNK, + hash: store.put_chunk(&fill(*byte)).await.unwrap(), + }); + } + } + let manifest = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks, + ..Manifest::empty(ManifestKind::Disk, PATTERN.len() as u64 * CHUNK) + }; + let manifest_ref = ManifestRef::new(); + store.put_manifest(manifest_ref, &manifest).await.unwrap(); + manifest_ref +} + +async fn settled(volumes: &SessionVolumes, sandbox: SandboxId) { + let deadline = Instant::now() + SETTLE_DEADLINE; + loop { + match volumes.settle(sandbox, DiskRole::Root).await.unwrap() { + Hydration::Settled => return, + Hydration::Hydrating { + hydrated, total, .. + } => { + assert!( + Instant::now() < deadline, + "the volume did not settle: {hydrated} of {total} regions" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + } +} + +async fn assert_disk(volumes: &SessionVolumes, sandbox: SandboxId, expected: &[u8]) { + let live = volumes.live(sandbox, DiskRole::Root).await.unwrap(); + for (i, byte) in expected.iter().enumerate() { + assert_eq!( + live.read_chunk(ChunkIndex(i as u64)).await.unwrap(), + fill(*byte), + "chunk {i}" + ); + } +} + +async fn dmsetup(args: &[&str]) { + let status = tokio::process::Command::new("dmsetup") + .args(args) + .status() + .await + .unwrap(); + assert!(status.success(), "dmsetup {args:?}"); +} + +/// The device name of the sandbox's live root volume in the pool. +fn live_device(pool: &str, sandbox: SandboxId) -> String { + format!("{pool}.{}", live_name(sandbox, DiskRole::Root)) +} + +#[tokio::test] +#[ignore = "requires root, a dm-thin pool and the ublk_drv module"] +async fn a_lazy_resume_reads_cold_chunks_through_the_source_and_settles() { + let Some(pool) = gate() else { return }; + let dir = tempfile::tempdir().unwrap(); + let store = store(dir.path()); + let manifest = manifest_in(&store).await; + let volumes = volumes(&pool, store).await; + let sandbox = SandboxId::new(); + + let provisioned = volumes + .provision( + sandbox, + Source::Continue { + manifest, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(provisioned.root_stage, StagePath::Lazy); + assert_eq!(volumes.sources().len(), 1, "one source serves the root"); + + // The last chunk is the last the ascending copy reaches; read now, it + // is right from either side. A write to a hole lands on the device. + let live = volumes.live(sandbox, DiskRole::Root).await.unwrap(); + assert_eq!(live.read_chunk(ChunkIndex(7)).await.unwrap(), fill(8)); + live.write_chunk(ChunkIndex(2), fill(9)).await.unwrap(); + drop(live); + + settled(&volumes, sandbox).await; + assert!(volumes.hydrating(sandbox).await.unwrap().is_empty()); + assert!(volumes.sources().is_empty(), "the source stopped"); + assert_disk(&volumes, sandbox, &[1, 2, 9, 4, 0, 0, 7, 8]).await; + + volumes.release(sandbox).await.unwrap(); + let prefix = format!("sb-{sandbox}-"); + assert!(!volumes + .disk() + .list() + .await + .unwrap() + .iter() + .any(|n| n.as_str().starts_with(&prefix))); +} + +/// The first half of the recovery test, in a child process: provisions +/// the sandbox lazily, suspends its live device so the copy halts, and +/// waits to be killed. +#[tokio::test] +#[ignore = "a helper the recovery test runs as a child process"] +async fn child_provisions_and_waits() { + let Some(dir) = std::env::var_os("ENGRAM_LAZY_CHILD_DIR") else { + return; + }; + let dir = PathBuf::from(dir); + let pool = std::env::var("ENGRAM_TEST_THIN_POOL").unwrap(); + let store = store(&dir); + let manifest: ManifestRef = + serde_json::from_slice(&std::fs::read(dir.join("manifest.json")).unwrap()).unwrap(); + let volumes = volumes(&pool, store).await; + let sandbox = SandboxId::new(); + let provisioned = volumes + .provision( + sandbox, + Source::Continue { + manifest, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(provisioned.root_stage, StagePath::Lazy); + dmsetup(&["suspend", &live_device(&pool, sandbox)]).await; + std::fs::write(dir.join("sandbox"), sandbox.to_string()).unwrap(); + std::future::pending::<()>().await; +} + +/// A host-agent killed before the settle: the next host-agent cannot +/// delete the hydrating volume until it serves the source again, then +/// settles it. +#[tokio::test] +#[ignore = "requires root, a dm-thin pool and the ublk_drv module"] +async fn a_killed_process_leaves_a_volume_the_next_one_serves_again() { + let Some(pool) = gate() else { return }; + let dir = tempfile::tempdir().unwrap(); + let store = store(dir.path()); + let manifest = manifest_in(&store).await; + std::fs::write( + dir.path().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "child_provisions_and_waits", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("ENGRAM_LAZY_CHILD_DIR", dir.path()) + .spawn() + .unwrap(); + let sandbox_file = dir.path().join("sandbox"); + let deadline = Instant::now() + Duration::from_secs(30); + while !sandbox_file.exists() { + assert!(Instant::now() < deadline, "the child did not provision"); + assert!(child.try_wait().unwrap().is_none(), "the child exited"); + std::thread::sleep(Duration::from_millis(50)); + } + let sandbox: SandboxId = std::fs::read_to_string(&sandbox_file) + .unwrap() + .trim() + .parse() + .unwrap(); + nix::sys::signal::kill( + nix::unistd::Pid::from_raw(child.id() as i32), + nix::sys::signal::Signal::SIGKILL, + ) + .unwrap(); + child.wait().unwrap(); + + let volumes = volumes(&pool, store).await; + let refused = volumes.release(sandbox).await.unwrap_err(); + assert_eq!( + refused.kind(), + std::io::ErrorKind::ResourceBusy, + "{refused}" + ); + assert_eq!( + volumes.reattach_all().await.unwrap(), + vec![(sandbox, DiskRole::Root)] + ); + assert!(volumes.reattach_all().await.unwrap().is_empty()); + dmsetup(&["resume", &live_device(&pool, sandbox)]).await; + + settled(&volumes, sandbox).await; + assert_disk(&volumes, sandbox, &PATTERN).await; + assert!(volumes.sources().is_empty()); + volumes.release(sandbox).await.unwrap(); +} diff --git a/crates/engram-host-core/src/disk.rs b/crates/engram-host-core/src/disk.rs index ea8c8a51d..b9a44f966 100644 --- a/crates/engram-host-core/src/disk.rs +++ b/crates/engram-host-core/src/disk.rs @@ -15,6 +15,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use async_trait::async_trait; +use engram_chunk_store::Manifest; +use engram_fill::FillEngine; /// The name a volume is created and reopened by. #[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -72,6 +74,21 @@ impl Capacity { } } +/// How far the kernel has copied a hydrating volume's regions in. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Hydration { + /// `hydrated` of `total` regions are in the destination; the rest + /// still read from the block device at `source`. + Hydrating { + hydrated: u64, + total: u64, + source: PathBuf, + }, + /// Every region is in the destination and the device maps onto it + /// directly ([`SessionDisk::settle`] ran). + Settled, +} + /// An open volume. Clones share the file handle. #[derive(Clone, Debug)] pub struct Volume { @@ -416,6 +433,54 @@ pub trait SessionDisk: Send + Sync { async fn capacity(&self) -> io::Result; + /// The unit [`Self::hydrate`] copies in, which divides the chunk + /// size; `None` on a platform without a hydrating device, where a + /// volume is written in full before the guest starts. + fn region_bytes(&self) -> Option { + None + } + + /// A device named `name` over `dest` that a guest can open before + /// `dest` holds its data: a read of a region not yet in `dest` is + /// served from the read-only block device at `source`, a write goes + /// to `dest` (after the kernel copies the rest of its region in), + /// and the kernel copies every other region from `source` into + /// `dest` in the background. `skip` lists the chunks `dest` already + /// holds in their final form; they are never copied, so blocks + /// `dest` shares with its origin stay shared. + async fn hydrate( + &self, + dest: &Volume, + source: &Path, + name: &VolumeName, + skip: &[ChunkIndex], + ) -> io::Result { + let _ = (dest, source, skip); + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("{name}: this platform has no hydrating device"), + )) + } + + /// Progress of a device [`Self::hydrate`] made; `Settled` for a volume + /// that never hydrated or has settled. + async fn hydration(&self, name: &VolumeName) -> io::Result { + let _ = name; + Ok(Hydration::Settled) + } + + /// Make a fully hydrated device a volume of its own with `dest`'s + /// content; `dest` stays, and is the caller's to delete. Every I/O in + /// flight completes first; open handles keep working; `source` is + /// not read again. + async fn settle(&self, name: &VolumeName, dest: &Volume) -> io::Result<()> { + let _ = dest; + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("{name} is not a hydrating device on this platform"), + )) + } + /// Ascending indices of the `block_bytes`-sized blocks that differ /// between two snapshots of one origin; `block_bytes` divides the /// chunk size. The default reads both in full and is the oracle a @@ -460,6 +525,45 @@ pub trait SessionDisk: Send + Sync { } } +/// A read-only block device that serves a manifest from the fill +/// engine: what a hydrating volume copies its cold regions from. +#[async_trait] +pub trait SourceDevice: Send + Sync { + fn path(&self) -> &Path; + + /// Chunks a guest read before the copy reached them, in first-read + /// order. + fn touched(&self) -> Vec; + + /// Stop serving and delete the device. Refused while a device still + /// maps onto it. + async fn stop(self: Box) -> io::Result<()>; +} + +/// Makes [`SourceDevice`]s. `delta` lists, ascending, the chunks the kernel +/// copies from the device; `region_bytes` is its copy unit. +#[async_trait] +pub trait SourceServer: Send + Sync { + async fn serve( + &self, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result>; + + /// Take over the device at `path` from a process that died serving + /// it; the reads it left queued are answered. + async fn recover( + &self, + path: &Path, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + region_bytes: u64, + ) -> io::Result>; +} + /// The data extents of `file` as `(offset, len)` ranges, from /// `SEEK_DATA`/`SEEK_HOLE`. Holes read as zeros and are not listed. A /// filesystem may coalesce extents, which only widens a range. diff --git a/crates/engram-host-core/src/disk_flows.rs b/crates/engram-host-core/src/disk_flows.rs index 0a7baeb62..033bcf9ae 100644 --- a/crates/engram-host-core/src/disk_flows.rs +++ b/crates/engram-host-core/src/disk_flows.rs @@ -1,15 +1,19 @@ //! Chunks between a [`Volume`] and the chunk store: [`stage`] turns a -//! manifest into a volume, [`checkpoint`] turns a volume into a manifest. +//! manifest into a volume, [`hydration_plan`] says what a kernel copy +//! does instead, [`checkpoint`] turns a volume into a manifest. +use std::collections::{BTreeMap, HashMap}; use std::io; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestDelta, ManifestKind}; +use engram_chunk_store::{ + ChunkHash, ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestDelta, ManifestKind, +}; use engram_fill::{Class, FillEngine}; -use crate::disk::{is_zero_chunk, ChunkIndex, SessionDisk, Volume, VolumeName}; +use crate::disk::{blocks_per_chunk, is_zero_chunk, ChunkIndex, SessionDisk, Volume, VolumeName}; use tracing::Instrument as _; /// Store calls in flight; peak buffered memory is this many chunks. @@ -23,7 +27,7 @@ fn store_err(e: engram_chunk_store::ChunkStoreError) -> io::Error { io::Error::other(e) } -fn shape_matches(volume: &Volume, manifest: &Manifest) -> io::Result<()> { +pub(crate) fn shape_matches(volume: &Volume, manifest: &Manifest) -> io::Result<()> { if volume.chunk_size != manifest.chunk_size.as_u64() || volume.size_bytes != manifest.total_bytes { @@ -185,6 +189,75 @@ pub async fn stage_at( Ok(volume) } +/// What brings a snapshot of `base`'s volume (an empty volume when +/// `None`) to `wanted` when the kernel copies the difference in behind +/// a running guest ([`SessionDisk::hydrate`]). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HydrationPlan { + /// The chunks the kernel copies from the source, ascending: listed + /// in `wanted` with a hash the base does not hold at that offset. + pub delta: Vec, + /// Chunks to zero on the destination before the copy starts: the + /// base holds them, `wanted` does not. + pub zero: Vec, + /// Every chunk not in `delta`: the destination holds it already. + pub skip: Vec, +} + +pub fn hydration_plan(wanted: &Manifest, base: Option<&Manifest>) -> io::Result { + wanted.validate().map_err(store_err)?; + let chunk_size = wanted.chunk_size.as_u64(); + let (write, zero) = match base { + Some(base) => { + let delta = wanted.delta_from(base).map_err(store_err)?; + (delta.write, delta.zero) + } + None => (wanted.chunks.clone(), Vec::new()), + }; + let mut delta: Vec = write.iter().map(|c| c.offset / chunk_size).collect(); + delta.sort_unstable(); + let skip = (0..wanted.expected_chunk_count()) + .filter(|chunk| delta.binary_search(chunk).is_err()) + .map(ChunkIndex) + .collect(); + Ok(HydrationPlan { + delta, + zero: zero + .iter() + .map(|offset| ChunkIndex(offset / chunk_size)) + .collect(), + skip, + }) +} + +/// How many chunks of `wanted` a volume holding `base` already has in +/// place: the same hash at the same offset. `None` when the two differ +/// in shape, so the volume cannot be a base at all. +pub(crate) fn shared_chunks(wanted: &Manifest, base: &Manifest) -> Option { + let delta = wanted.delta_from(base).ok()?; + Some(wanted.chunks.len() - delta.write.len()) +} + +/// The chunks of `first_touch` (guest reads of a hydrating volume, in +/// order) that `delta` copies, as `wanted`'s hashes: what a restore +/// fetches at `Prefix` before the copy reaches them. Chunks the base +/// already holds and chunks `wanted` leaves out need no fetch. +pub(crate) fn prefix_hashes( + first_touch: &[u32], + delta: &[u64], + wanted: &Manifest, +) -> Vec { + let chunk_size = wanted.chunk_size.as_u64(); + let by_offset: HashMap = + wanted.chunks.iter().map(|c| (c.offset, c.hash)).collect(); + first_touch + .iter() + .map(|i| u64::from(*i)) + .filter(|i| delta.binary_search(i).is_ok()) + .filter_map(|i| by_offset.get(&(i * chunk_size)).copied()) + .collect() +} + /// A snapshot of `live` and the manifest describing it. With `previous` /// (the last snapshot and its manifest) only the chunks that differ are /// uploaded. The guest must be paused across [`checkpoint_split::take`]; @@ -240,20 +313,96 @@ pub mod checkpoint_split { } }; + let chunks = indices.into_iter().map(|index| { + let snapshot = snapshot.clone(); + (index, async move { snapshot.read_chunk(index).await }) + }); + let delta = put_chunks(store, snapshot.chunk_size, chunks).await?; + Ok(base_manifest.apply(&delta)) + } + + /// Outside the pause, for a snapshot of a hydrating volume. The + /// volume logically holds `wanted`; a block whose mapping still + /// equals `base`'s is one the copy has not reached, and holds the + /// base's bytes where `wanted`'s belong. So a block whose mapping + /// changed since `base` (hydrated or written) is read from the + /// snapshot and any other block is `wanted`'s: a chunk with no + /// changed block keeps `wanted`'s entry unread, one changed + /// throughout is read whole, and a mixed one is `wanted`'s chunk + /// with the changed blocks laid over it. `block_bytes` is the + /// mapping unit; it divides the chunk size. + pub async fn upload_hydrating( + disk: &dyn SessionDisk, + fill: &FillEngine, + snapshot: &Volume, + base: &Volume, + wanted: &Manifest, + block_bytes: u64, + ) -> io::Result { + shape_matches(snapshot, wanted)?; + let per_chunk = blocks_per_chunk(snapshot, block_bytes)?; let chunk_size = snapshot.chunk_size; - // One task per chunk, so the 16 MiB read, the zero check, the - // SHA-256 and the PUT's TLS spread over every core. + let mut changed: BTreeMap> = BTreeMap::new(); + for block in disk.dirty_blocks(base, snapshot, block_bytes).await? { + changed + .entry(block / per_chunk) + .or_default() + .push((block % per_chunk) * block_bytes); + } + let wanted_hashes: HashMap = + wanted.chunks.iter().map(|c| (c.offset, c.hash)).collect(); + let chunks = changed.into_iter().map(|(chunk, blocks)| { + let index = ChunkIndex(chunk); + let offset = chunk * chunk_size; + let (snapshot, fill) = (snapshot.clone(), fill.clone()); + let hash = wanted_hashes.get(&offset).copied(); + (index, async move { + if blocks.len() as u64 == per_chunk { + return snapshot.read_chunk(index).await; + } + let mut bytes = match hash { + Some(hash) => fill + .read(hash, Class::Hydrate) + .await + .map_err(store_err)? + .to_vec(), + None => vec![0; snapshot.read_chunk(index).await?.len()], + }; + for at in blocks { + let end = (at + block_bytes).min(bytes.len() as u64); + let part = snapshot.read_range(offset + at, end - at).await?; + bytes[at as usize..end as usize].copy_from_slice(&part); + } + Ok(bytes) + }) + }); + let delta = put_chunks(fill.store(), chunk_size, chunks).await?; + Ok(wanted.apply(&delta)) + } + + /// Put the chunks `chunks` yields, each a chunk index and the future + /// of its bytes, and return the delta they make: a chunk of zeros is + /// a hole. One task per chunk, so the 16 MiB read, the zero check, + /// the SHA-256 and the PUT's TLS spread over every core. + async fn put_chunks( + store: &ChunkStore, + chunk_size: u64, + chunks: impl Iterator, + ) -> io::Result + where + F: std::future::Future>> + Send + 'static, + { let mut tasks = tokio::task::JoinSet::new(); - let mut results: Vec<(u64, Option)> = Vec::with_capacity(indices.len()); - for index in indices { + let mut results: Vec<(u64, Option)> = Vec::new(); + for (index, bytes) in chunks { while tasks.len() >= CONCURRENCY { let done = tasks.join_next().await.expect("tasks is not empty"); results.push(done.map_err(io::Error::other)??); } - let (snapshot, store) = (snapshot.clone(), store.clone()); + let store = store.clone(); tasks.spawn(async move { let offset = index.0 * chunk_size; - let bytes = snapshot.read_chunk(index).await?; + let bytes = bytes.await?; if is_zero_chunk(&bytes) { return Ok::<_, io::Error>((offset, None)); } @@ -267,7 +416,6 @@ pub mod checkpoint_split { while let Some(done) = tasks.join_next().await { results.push(done.map_err(io::Error::other)??); } - let mut delta = ManifestDelta::default(); for (offset, written) in results { match written { @@ -275,7 +423,7 @@ pub mod checkpoint_split { None => delta.zero.push(offset), } } - Ok(base_manifest.apply(&delta)) + Ok(delta) } } @@ -414,6 +562,159 @@ mod tests { assert_eq!(bytes_of(&over).await, bytes_of(&c2).await); } + /// Base [1, 2, 3, 0, 5]; wanted [1, 9, 0, 4, 5]: chunk 1 changed, + /// chunk 2 dropped, chunk 3 new, the rest held. + #[test] + fn a_hydration_plan_copies_the_changed_chunks_zeroes_the_dropped_and_skips_the_rest() { + let manifest = |pattern: &[u8]| Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: pattern + .iter() + .enumerate() + .filter(|(_, byte)| **byte != 0) + .map(|(i, byte)| ChunkRef { + offset: i as u64 * CHUNK, + hash: engram_chunk_store::ChunkHash::of(&fill(*byte)), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, CHUNK * pattern.len() as u64) + }; + let base = manifest(&[1, 2, 3, 0, 5]); + let wanted = manifest(&[1, 9, 0, 4, 5]); + assert_eq!( + hydration_plan(&wanted, Some(&base)).unwrap(), + HydrationPlan { + delta: vec![1, 3], + zero: vec![ChunkIndex(2)], + skip: [0, 2, 4].map(ChunkIndex).to_vec(), + } + ); + assert_eq!( + hydration_plan(&wanted, None).unwrap(), + HydrationPlan { + delta: vec![0, 1, 3, 4], + zero: vec![], + skip: vec![ChunkIndex(2)], + } + ); + let other_shape = manifest(&[1, 2]); + assert!(hydration_plan(&wanted, Some(&other_shape)).is_err()); + } + + /// Base [1, 2, 3, 0]; wanted [7, 8, 3, 9]. The snapshot of the + /// hydrating volume holds chunk 0 hydrated (7), the first half of + /// chunk 1 written (5) with the rest not yet copied (still 2), chunk + /// 2 held, and chunk 3 not yet copied (still 0). The published disk + /// is the wanted one with the write laid over it. + #[tokio::test] + async fn a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk() { + let r = rig(); + let fill_engine = engram_fill::FillEngine::new(r.store.clone(), Default::default()); + let base = r.disk.create(&name("base"), CHUNK * 4).await.unwrap(); + for (i, byte) in [1u8, 2, 3].into_iter().enumerate() { + base.write_chunk(ChunkIndex(i as u64), fill(byte)) + .await + .unwrap(); + } + let mut wanted_chunks = Vec::new(); + for (i, byte) in [7u8, 8, 3, 9].into_iter().enumerate() { + wanted_chunks.push(ChunkRef { + offset: i as u64 * CHUNK, + hash: r.store.put_chunk(&fill(byte)).await.unwrap(), + }); + } + let wanted = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: wanted_chunks, + ..Manifest::empty(ManifestKind::Disk, CHUNK * 4) + }; + let snapshot = r.disk.snapshot(&base, &name("snap")).await.unwrap(); + snapshot.write_chunk(ChunkIndex(0), fill(7)).await.unwrap(); + snapshot + .write_range(CHUNK, vec![5u8; (CHUNK / 2) as usize]) + .await + .unwrap(); + + let published = checkpoint_split::upload_hydrating( + &r.disk, + &fill_engine, + &snapshot, + &base, + &wanted, + CHUNK / 2, + ) + .await + .unwrap(); + let mut chunk_1 = vec![5u8; (CHUNK / 2) as usize]; + chunk_1.extend(vec![8u8; (CHUNK / 2) as usize]); + let expected = [fill(7), chunk_1, fill(3), fill(9)]; + assert_eq!(published.chunks.len(), 4); + for (i, bytes) in expected.iter().enumerate() { + assert_eq!( + published.chunks[i], + ChunkRef { + offset: i as u64 * CHUNK, + hash: ChunkHash::of(bytes) + }, + "chunk {i}" + ); + } + assert_eq!( + fill_engine + .read(published.chunks[1].hash, Class::Hydrate) + .await + .unwrap() + .to_vec(), + expected[1] + ); + } + + #[test] + fn shared_chunks_counts_equal_hashes_at_equal_offsets() { + let disk = |bytes: &[u8]| Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: bytes + .iter() + .enumerate() + .filter(|(_, b)| **b != 0) + .map(|(i, b)| ChunkRef { + offset: i as u64 * CHUNK, + hash: ChunkHash::of(&fill(*b)), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, CHUNK * bytes.len() as u64) + }; + let wanted = disk(&[1, 2, 0, 4]); + assert_eq!(shared_chunks(&wanted, &disk(&[1, 2, 3, 9])), Some(2)); + // The same bytes at another offset are not in place. + assert_eq!(shared_chunks(&wanted, &disk(&[2, 1, 0, 0])), Some(0)); + assert_eq!(shared_chunks(&wanted, &disk(&[1, 2, 0])), None); + } + + /// Reads of chunks 3, 0 and 4 came first; the copy brings 1 and 3, + /// so only chunk 3 is fetched ahead. + #[test] + fn the_prefix_is_the_first_touch_in_order_restricted_to_the_delta() { + let wanted = Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: [1u8, 9, 0, 4, 5] + .iter() + .enumerate() + .filter(|(_, b)| **b != 0) + .map(|(i, b)| ChunkRef { + offset: i as u64 * CHUNK, + hash: ChunkHash::of(&fill(*b)), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, CHUNK * 5) + }; + assert_eq!( + prefix_hashes(&[3, 0, 4], &[1, 3], &wanted), + vec![ChunkHash::of(&fill(4))] + ); + assert!(prefix_hashes(&[], &[1, 3], &wanted).is_empty()); + } + #[tokio::test] async fn stage_refuses_a_manifest_whose_shape_differs_from_its_base() { let r = rig(); diff --git a/crates/engram-host-core/src/lib.rs b/crates/engram-host-core/src/lib.rs index c3cf5b005..989f148c1 100644 --- a/crates/engram-host-core/src/lib.rs +++ b/crates/engram-host-core/src/lib.rs @@ -41,9 +41,10 @@ mod test_support; pub use checkpoint::checkpoint_tail_admits_publish; pub use coord::{CoordControlPlane, CoordError}; pub use disk::{ - data_extents, is_zero_chunk, Capacity, ChunkIndex, FileDisk, SessionDisk, Volume, VolumeName, + data_extents, is_zero_chunk, Capacity, ChunkIndex, FileDisk, Hydration, SessionDisk, + SourceDevice, SourceServer, Volume, VolumeName, }; -pub use disk_flows::{checkpoint, checkpoint_split, stage}; +pub use disk_flows::{checkpoint, checkpoint_split, hydration_plan, stage, HydrationPlan}; pub use finalize::{plan_finalize_retry, FinalizeRetry, FinalizeStage}; pub use fs::{HostFs, TokioFs}; pub use session_volumes::SessionVolumes; diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 9d5f32dc6..8f25d279c 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -6,6 +6,8 @@ //! sb-- live //! sb---ckpt--v the last checkpoint; holds that manifest //! tmp-sb-- a stage or a checkpoint in flight +//! sb---dest--v a hydrating live volume's destination; becomes that manifest +//! sb---base--v what the destination was snapshotted from; holds that manifest //! ``` //! //! A checkpoint volume is named after a manifest the chunk store holds, @@ -14,12 +16,19 @@ //! and the next checkpoint publishes under it, so no caller tracks a //! manifest ref between calls. //! +//! A role whose manifest is not on the host starts on a hydrating +//! device: the live volume reads the base's blocks and a source the +//! fill engine serves, and the kernel copies the difference in behind +//! the guest ([`SessionDisk::hydrate`]). Until that device settles there +//! is no checkpoint volume; the `dest` and `base` names record what +//! the live data converges to and what it is measured against. +//! //! A released sandbox keeps its newest checkpoint volume per role, so a //! resume on this host is a snapshot of it instead of a stage from blob //! storage. [`SessionVolumes::reap`] deletes kept checkpoints, oldest //! release first, when the pool needs the space. -use std::collections::VecDeque; +use std::collections::{HashMap, HashSet, VecDeque}; use std::io; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -32,13 +41,21 @@ use engram_fill::{Class, FillEngine, PeerSource}; use tracing::Instrument as _; use uuid::Uuid; -use crate::disk::{Capacity, SessionDisk, Volume, VolumeName}; -use crate::disk_flows::{checkpoint_split, stage}; +use crate::disk::{ + Capacity, Hydration, SessionDisk, SourceDevice, SourceServer, Volume, VolumeName, +}; +use crate::disk_flows::{ + checkpoint_split, hydration_plan, prefix_hashes, shape_matches, shared_chunks, stage, +}; use crate::image_volumes::{self, ImageVolumes}; const LIVE_PREFIX: &str = "sb-"; const TRANSIENT_PREFIX: &str = "tmp-sb-"; const CHECKPOINT_INFIX: &str = "-ckpt-"; +const DEST_INFIX: &str = "-dest-"; +const BASE_INFIX: &str = "-base-"; +/// The base of a hydration that started from zeros. +const BASE_NONE: &str = "none"; const PRECOPY_SUFFIX: &str = "-pre"; /// What a volume name says. @@ -47,6 +64,11 @@ pub enum Kind { Live, Checkpoint(ManifestRef), Transient, + /// The destination of a hydrating live volume; holds the manifest + /// once the hydration settles. + Dest(ManifestRef), + /// What a hydration's destination was snapshotted from. + Base(Option), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -77,27 +99,49 @@ pub fn live_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { } pub fn checkpoint_name(sandbox: SandboxId, role: DiskRole, manifest: ManifestRef) -> VolumeName { + named_after(sandbox, role, CHECKPOINT_INFIX, &manifest_suffix(manifest)) +} + +fn dest_name(sandbox: SandboxId, role: DiskRole, manifest: ManifestRef) -> VolumeName { + named_after(sandbox, role, DEST_INFIX, &manifest_suffix(manifest)) +} + +fn base_name(sandbox: SandboxId, role: DiskRole, manifest: Option) -> VolumeName { + let suffix = manifest.map_or(BASE_NONE.to_string(), manifest_suffix); + named_after(sandbox, role, BASE_INFIX, &suffix) +} + +fn named_after(sandbox: SandboxId, role: DiskRole, infix: &str, suffix: &str) -> VolumeName { VolumeName::new(format!( - "{LIVE_PREFIX}{}-{}{CHECKPOINT_INFIX}{}-v{}", + "{LIVE_PREFIX}{}-{role}{infix}{suffix}", sandbox.as_uuid().simple(), - role, - manifest.manifest_id.simple(), - manifest.version )) - .expect("a checkpoint volume name is at most 100 bytes of [a-z0-9-]") + .expect("a named volume is at most 100 bytes of [a-z0-9-]") +} + +fn manifest_suffix(manifest: ManifestRef) -> String { + format!("{}-v{}", manifest.manifest_id.simple(), manifest.version) +} + +fn parse_manifest_suffix(suffix: &str) -> Option { + let (id, version) = suffix.split_once("-v")?; + Some(ManifestRef { + manifest_id: Uuid::try_parse(id).ok()?, + version: version.parse().ok()?, + }) } /// How [`SessionVolumes`] brought up a role's disk. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StagePath { /// The role's checkpoint was already on this host. Kept, - /// Only the chunks that changed, written onto a kept checkpoint. - OntoKept, - /// The delta from the image, written onto its staged volume. - FromImage, - /// Every chunk, written into an empty volume: the image's volume is - /// not on this host. - Whole, + /// A hydrating device over a snapshot of the nearest base this host + /// holds: the guest starts at once and the kernel copies the + /// difference in behind it. + Lazy, + /// The difference from the nearest base written in full before the + /// guest starts: this platform has no hydrating device. + Staged, /// A snapshot of a manifest this host had staged. StagedSnapshot, /// A fork of the image's volume. @@ -110,9 +154,8 @@ impl StagePath { pub fn as_str(self) -> &'static str { match self { StagePath::Kept => "kept", - StagePath::OntoKept => "onto_kept", - StagePath::FromImage => "from_image", - StagePath::Whole => "whole", + StagePath::Lazy => "lazy", + StagePath::Staged => "staged", StagePath::StagedSnapshot => "staged_snapshot", StagePath::ImageFork => "image_fork", StagePath::Empty => "empty", @@ -148,39 +191,49 @@ pub fn parse(name: &VolumeName) -> Option { }; let (sandbox, rest) = rest.split_once('-')?; let sandbox = SandboxId::from(Uuid::try_parse(sandbox).ok()?); - let (role, rest) = match rest.split_once(CHECKPOINT_INFIX) { - Some((role, manifest)) => (role, Some(manifest)), - None => (rest, None), - }; - let role = match (transient, role.strip_suffix(PRECOPY_SUFFIX)) { - (true, Some(role)) => role, - _ => role, - }; - let role: DiskRole = role.parse().ok()?; - let kind = match (transient, rest) { - (true, None) => Kind::Transient, - (false, None) => Kind::Live, - (false, Some(manifest)) => { - let (id, version) = manifest.split_once("-v")?; - Kind::Checkpoint(ManifestRef { - manifest_id: Uuid::try_parse(id).ok()?, - version: version.parse().ok()?, - }) + let infixed = [CHECKPOINT_INFIX, DEST_INFIX, BASE_INFIX] + .into_iter() + .find_map(|infix| { + rest.split_once(infix) + .map(|(role, suffix)| (role, infix, suffix)) + }); + let (role, kind) = match (transient, infixed) { + (true, None) => ( + rest.strip_suffix(PRECOPY_SUFFIX).unwrap_or(rest), + Kind::Transient, + ), + (false, None) => (rest, Kind::Live), + (false, Some((role, CHECKPOINT_INFIX, suffix))) => { + (role, Kind::Checkpoint(parse_manifest_suffix(suffix)?)) + } + (false, Some((role, DEST_INFIX, suffix))) => { + (role, Kind::Dest(parse_manifest_suffix(suffix)?)) } - (true, Some(_)) => return None, + (false, Some((role, BASE_INFIX, BASE_NONE))) => (role, Kind::Base(None)), + (false, Some((role, BASE_INFIX, suffix))) => { + (role, Kind::Base(Some(parse_manifest_suffix(suffix)?))) + } + _ => return None, }; Some(Parsed { sandbox, - role, + role: role.parse().ok()?, kind, }) } -/// Where a live volume's contents come from. A manifest is written in -/// full before the sandbox starts; when `base` names an image staged on -/// this host, that volume is the origin and only the difference is -/// written. A manifest that is itself a staged image is snapshotted. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// Live volumes first: a hydrating one holds its destination open, so +/// the destination can go only after it. +fn live_first(names: &mut [(VolumeName, Parsed)]) { + names.sort_by_key(|(_, p)| p.kind != Kind::Live); +} + +/// Where a live volume's contents come from. A manifest that is itself a +/// staged image is snapshotted. Otherwise, when `base` names an image +/// staged on this host, the volume starts as a snapshot of it and only +/// the difference comes in: behind the running guest where the platform +/// has a hydrating device, in full before it starts elsewhere. +#[derive(Clone, Debug, PartialEq, Eq)] pub enum Source { /// An image's disk: the volume starts a new lineage as a snapshot of /// the image's volume, which is staged first when the host has none @@ -188,14 +241,14 @@ pub enum Source { Image { manifest: ManifestRef }, /// A manifest many sandboxes share: the volume starts a new lineage /// that holds a copy of it. - Fork { - manifest: ManifestRef, - base: Option, - }, + Fork { manifest: ManifestRef }, /// The sandbox's own manifest: the volume continues its lineage. + /// `first_touch` lists the chunks the guest read before the copy + /// reached them on its last hydrating restore, in order; the ones + /// this restore must copy are fetched first. Continue { manifest: ManifestRef, - base: Option, + first_touch: Vec, }, /// Zeros, in a new lineage. Empty { size_bytes: u64 }, @@ -234,6 +287,11 @@ pub struct PendingCheckpoint { role: DiskRole, snapshot: Volume, previous: ManifestRef, + /// The live volume held its whole manifest when the snapshot was + /// taken: it was not hydrating. Only such a snapshot can become a + /// checkpoint volume; a snapshot of a hydrating volume still shows + /// the base where the copy had not reached. + whole: bool, } impl PendingCheckpoint { @@ -264,6 +322,27 @@ pub struct Report { pub kept: Vec, } +/// A volume on this host that holds a manifest near the one wanted: +/// what a role's volume starts from. +struct Base { + volume: Volume, + manifest: Manifest, + manifest_ref: ManifestRef, + /// The sandbox whose kept checkpoint volume this is; a hydration + /// takes the volume over as its base. + kept_by: Option, +} + +/// What a live volume this host does not hold is brought up to: the +/// manifest, the lineage it continues, the base to start from, and the +/// chunks the guest read first on the last hydrating restore. +struct Wanted<'a> { + manifest: &'a Manifest, + lineage: ManifestRef, + base: Option, + first_touch: &'a [u32], +} + /// The owner of every `sb-` volume on a host. pub struct SessionVolumes { disk: Arc, @@ -273,6 +352,14 @@ pub struct SessionVolumes { /// first. A sandbox `reconcile` finds with no live volume joins at /// the front: its age is unknown, so it goes first. kept: Mutex>, + /// Serves the sources hydrating volumes copy from; without it every + /// volume is written in full before its guest starts. + sources: Option>, + /// The source of each hydrating live volume this process serves. + serving: Mutex>>, + /// What each stopped source recorded as read first, kept until the + /// live volume goes. + touched: Mutex>>, } impl SessionVolumes { @@ -282,9 +369,24 @@ impl SessionVolumes { disk, fill, kept: Mutex::new(VecDeque::new()), + sources: None, + serving: Mutex::new(HashMap::new()), + touched: Mutex::new(HashMap::new()), } } + /// Start guests on hydrating volumes, with `sources` serving what + /// the kernel copies from. + pub fn with_sources(mut self, sources: Arc) -> Self { + self.sources = Some(sources); + self + } + + /// The source server and the copy unit, when this host hydrates. + fn hydration_support(&self) -> Option<(&dyn SourceServer, u64)> { + Some((self.sources.as_deref()?, self.disk.region_bytes()?)) + } + pub fn disk(&self) -> &dyn SessionDisk { self.disk.as_ref() } @@ -358,11 +460,10 @@ impl SessionVolumes { } } - /// The checkpoint volume first, under the transient name until its - /// manifest is in the store; then the live volume as its snapshot. - /// A `Continue` of a checkpoint this host kept for the lineage skips - /// the stage; one of a newer manifest stages only the delta onto it. - /// A stage fetches through `fill`. + /// The live volume of a role. A manifest this host holds in full (a + /// staged image, a kept checkpoint of it) is snapshotted; a + /// `Continue` or `Fork` of one it does not hold starts from the + /// nearest base it has ([`Self::bring_up`]). #[tracing::instrument( name = "disk.provision", skip_all, @@ -377,122 +478,98 @@ impl SessionVolumes { ) -> io::Result<(PathBuf, ManifestRef, StagePath)> { let tmp = transient_name(sandbox, role); self.delete_if_present(&tmp).await?; - let ((checkpoint, lineage), path) = match source { + let (live, lineage, path) = match source { Source::Image { manifest } => { - let wanted = self - .fill - .store() - .get_manifest(manifest) - .instrument(tracing::info_span!("disk.manifest_get")) - .await - .map_err(store_err)?; + let wanted = self.manifest(manifest).await?; self.images .fork(manifest, &tmp) .instrument(tracing::info_span!("disk.image_fork")) .await?; - ( - self.new_lineage(&tmp, sandbox, role, &wanted).await?, - StagePath::ImageFork, - ) + let lineage = self.new_lineage(&wanted).await?; + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + (live, lineage, StagePath::ImageFork) } - Source::Fork { manifest, base } => { - let wanted = self - .fill - .store() - .get_manifest(manifest) - .instrument(tracing::info_span!("disk.manifest_get")) - .await - .map_err(store_err)?; - let path = match self.staged(manifest).await { + Source::Fork { manifest } => { + let wanted = self.manifest(manifest).await?; + let lineage = self.new_lineage(&wanted).await?; + match self.staged(manifest).await { Some((origin, _)) => { self.disk.snapshot(&origin, &tmp).await?; - StagePath::StagedSnapshot + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + (live, lineage, StagePath::StagedSnapshot) } None => { - self.stage_into(fill, &tmp, &wanted, base) - .instrument(tracing::info_span!("disk.stage")) - .await? - .1 + let base = self.nearest_base(&wanted, None).await?; + let (live, path) = self + .bring_up( + fill, + sandbox, + role, + Wanted { + manifest: &wanted, + lineage, + base, + first_touch: &[], + }, + ) + .await?; + (live, lineage, path) } - }; - (self.new_lineage(&tmp, sandbox, role, &wanted).await?, path) + } } - Source::Continue { manifest, base } => { - match self.kept(sandbox, role, manifest).await? { - Some((owner, volume, _, held)) if held == manifest => ( - ( - self.adopt(owner, volume, sandbox, role, held).await?, - manifest, - ), - StagePath::Kept, - ), - kept => { - let wanted = self - .fill - .store() - .get_manifest(manifest) - .instrument(tracing::info_span!("disk.manifest_get")) - .await - .map_err(store_err)?; - let path = match kept { - Some((_, volume, held_manifest, _)) - if volume.size_bytes == wanted.total_bytes => - { - stage( - self.disk.as_ref(), - fill, - Class::Demand, - &tmp, - &wanted, - Some((&volume, &held_manifest)), - ) - .instrument(tracing::info_span!("disk.stage", onto_kept = true)) - .await?; - StagePath::OntoKept - } - _ => { - self.stage_into(fill, &tmp, &wanted, base) - .instrument(tracing::info_span!("disk.stage")) - .await? - .1 - } - }; - ( - (self.commit(&tmp, sandbox, role, manifest).await?, manifest), - path, + Source::Continue { + manifest, + first_touch, + } => match self.kept(sandbox, role, manifest).await? { + Some((owner, volume, _, held)) if held == manifest => { + let checkpoint = self.adopt(owner, volume, sandbox, role, held).await?; + let live = self.live_snapshot(&checkpoint, sandbox, role).await?; + (live, manifest, StagePath::Kept) + } + kept => { + let wanted = self.manifest(manifest).await?; + let base = self.nearest_base(&wanted, kept).await?; + let (live, path) = self + .bring_up( + fill, + sandbox, + role, + Wanted { + manifest: &wanted, + lineage: manifest, + base, + first_touch: &first_touch, + }, ) - } + .await?; + (live, manifest, path) } - } + }, Source::Empty { size_bytes } => { let volume = self.disk.create(&tmp, size_bytes).await?; let manifest = Manifest { chunk_size: ChunkSize::bytes(volume.chunk_size), ..Manifest::empty(ManifestKind::Disk, size_bytes) }; - ( - self.new_lineage(&tmp, sandbox, role, &manifest).await?, - StagePath::Empty, - ) + let lineage = self.new_lineage(&manifest).await?; + let live = self.commit_live(&tmp, sandbox, role, lineage).await?; + (live, lineage, StagePath::Empty) } }; - let live = self - .disk - .snapshot(&checkpoint, &live_name(sandbox, role)) - .instrument(tracing::info_span!("disk.live_snapshot")) - .await?; Ok((live.path, lineage, path)) } - /// Start a lineage that holds `manifest`, whose bytes `tmp` holds, - /// and commit `tmp` as its first checkpoint. - async fn new_lineage( - &self, - tmp: &VolumeName, - sandbox: SandboxId, - role: DiskRole, - manifest: &Manifest, - ) -> io::Result<(Volume, ManifestRef)> { + async fn manifest(&self, manifest: ManifestRef) -> io::Result { + self.fill + .store() + .get_manifest(manifest) + .instrument(tracing::info_span!("disk.manifest_get")) + .await + .map_err(store_err) + } + + /// Start a lineage that holds `manifest`. + async fn new_lineage(&self, manifest: &Manifest) -> io::Result { let lineage = ManifestRef::new(); self.fill .store() @@ -500,7 +577,32 @@ impl SessionVolumes { .instrument(tracing::info_span!("disk.manifest_put")) .await .map_err(store_err)?; - Ok((self.commit(tmp, sandbox, role, lineage).await?, lineage)) + Ok(lineage) + } + + /// Commit `tmp`, which holds `lineage`'s manifest, as the role's + /// checkpoint volume, and make the live volume its snapshot. + async fn commit_live( + &self, + tmp: &VolumeName, + sandbox: SandboxId, + role: DiskRole, + lineage: ManifestRef, + ) -> io::Result { + let checkpoint = self.commit(tmp, sandbox, role, lineage).await?; + self.live_snapshot(&checkpoint, sandbox, role).await + } + + async fn live_snapshot( + &self, + checkpoint: &Volume, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result { + self.disk + .snapshot(checkpoint, &live_name(sandbox, role)) + .instrument(tracing::info_span!("disk.live_snapshot")) + .await } /// Give the staged transient volume its checkpoint name and delete @@ -536,6 +638,212 @@ impl SessionVolumes { self.disk.open(&name).await } + /// The live volume of a role whose manifest is not on this host: a + /// hydrating device when this host has one and serves sources, else + /// the difference from the base written in full first. + async fn bring_up( + &self, + fill: &FillEngine, + sandbox: SandboxId, + role: DiskRole, + wanted: Wanted<'_>, + ) -> io::Result<(Volume, StagePath)> { + if self.hydration_support().is_some() { + let live = self + .hydrate(fill, sandbox, role, wanted) + .instrument(tracing::info_span!( + "disk.hydrate", + held = tracing::field::Empty, + delta = tracing::field::Empty, + zero = tracing::field::Empty, + prefix = tracing::field::Empty, + )) + .await?; + return Ok((live, StagePath::Lazy)); + } + let tmp = transient_name(sandbox, role); + stage( + self.disk.as_ref(), + fill, + Class::Demand, + &tmp, + wanted.manifest, + wanted.base.as_ref().map(|b| (&b.volume, &b.manifest)), + ) + .instrument(tracing::info_span!("disk.stage")) + .await?; + let live = self + .commit_live(&tmp, sandbox, role, wanted.lineage) + .await?; + Ok((live, StagePath::Staged)) + } + + /// A hydrating live volume: its destination is a snapshot of the + /// base (a kept checkpoint volume is taken over; an empty volume + /// stands in for no base) under the `dest` name, and its source + /// serves `wanted` through `fill`. Chunks the base holds in final + /// form are skipped; chunks `wanted` leaves out are zeroed first. + /// The `first_touch` chunks the copy must bring in are fetched at + /// `Prefix` without waiting, so the guest's first reads find them + /// resident before the copy reaches them. + /// Records on the current span how many chunks the base `held`, the + /// kernel copies (`delta`), were zeroed (`zero`) and make the `prefix`. + async fn hydrate( + &self, + fill: &FillEngine, + sandbox: SandboxId, + role: DiskRole, + wanted: Wanted<'_>, + ) -> io::Result { + let Wanted { + manifest: wanted, + lineage, + base, + first_touch, + } = wanted; + let (sources, region_bytes) = self.hydration_support().ok_or_else(|| { + io::Error::new(io::ErrorKind::Unsupported, "this host does not hydrate") + })?; + if let Some(base) = &base { + shape_matches(&base.volume, &base.manifest)?; + } + let plan = hydration_plan(wanted, base.as_ref().map(|b| &b.manifest))?; + let base_volume = match base { + Some(Base { + volume, + manifest_ref, + kept_by: Some(_), + .. + }) => { + let name = base_name(sandbox, role, Some(manifest_ref)); + self.disk.rename(&volume.name, &name).await?; + drop(volume); + self.disk.open(&name).await? + } + Some(Base { + volume, + manifest_ref, + kept_by: None, + .. + }) => { + self.disk + .snapshot(&volume, &base_name(sandbox, role, Some(manifest_ref))) + .await? + } + None => { + self.disk + .create(&base_name(sandbox, role, None), wanted.total_bytes) + .await? + } + }; + let dest = self + .disk + .snapshot(&base_volume, &dest_name(sandbox, role, lineage)) + .await?; + drop(base_volume); + shape_matches(&dest, wanted)?; + // The zeroing and the source's start do not touch each other. + let zero = async { + for chunk in &plan.zero { + dest.zero_chunk(*chunk).await?; + } + Ok::<_, io::Error>(()) + } + .instrument(tracing::info_span!("disk.hydrate.zero")); + let serve = sources + .serve(fill.clone(), wanted, plan.delta.clone(), region_bytes) + .instrument(tracing::info_span!("disk.hydrate.source")); + let (zeroed, source) = tokio::join!(zero, serve); + let source = source?; + let live = match zeroed { + Ok(()) => { + self.disk + .hydrate(&dest, source.path(), &live_name(sandbox, role), &plan.skip) + .instrument(tracing::info_span!("disk.hydrate.device")) + .await + } + Err(e) => Err(e), + }; + let live = match live { + Ok(live) => live, + Err(e) => { + if let Err(error) = source.stop().await { + tracing::warn!(sandbox_id = %sandbox, %role, %error, "source of a failed hydration not stopped"); + } + return Err(e); + } + }; + let prefix = prefix_hashes(first_touch, &plan.delta, wanted); + let span = tracing::Span::current(); + span.record("held", (plan.skip.len() - plan.zero.len()) as u64); + span.record("delta", plan.delta.len() as u64); + span.record("zero", plan.zero.len() as u64); + span.record("prefix", prefix.len() as u64); + if !prefix.is_empty() { + let fill = fill.clone(); + tokio::spawn( + async move { + match fill.fill(prefix, Class::Prefix).await { + Ok(stats) => tracing::info!(?stats, "disk first-touch prefix resident"), + Err(error) => tracing::warn!(%error, "disk first-touch prefix failed"), + } + } + .instrument(tracing::info_span!("disk.prefix", sandbox_id = %sandbox, %role)), + ); + } + self.serving + .lock() + .unwrap() + .insert(live.name.clone(), source); + Ok(live) + } + + /// The volume on this host that holds the most of `wanted` in + /// place, as a base: the lineage's `kept` checkpoint or a staged + /// image version, whichever shares more chunks with `wanted` at the + /// same offset (the kept one on a tie). No snapshot names it: an + /// image whose tag moved since the session began is as good a base + /// as the chunks it still shares. `None` when no volume of + /// `wanted`'s shape shares a chunk. + async fn nearest_base( + &self, + wanted: &Manifest, + kept: Option<(SandboxId, Volume, Manifest, ManifestRef)>, + ) -> io::Result> { + let mut candidates: Vec = kept + .into_iter() + .map(|(owner, volume, manifest, manifest_ref)| Base { + volume, + manifest, + manifest_ref, + kept_by: Some(owner), + }) + .collect(); + for image in image_volumes::staged(&self.disk.list().await?) { + if let Some((volume, manifest)) = self.staged(image).await { + candidates.push(Base { + volume, + manifest, + manifest_ref: image, + kept_by: None, + }); + } + } + let mut best: Option<(usize, Base)> = None; + for base in candidates { + if base.volume.size_bytes != wanted.total_bytes { + continue; + } + let Some(shared) = shared_chunks(wanted, &base.manifest) else { + continue; + }; + if shared > best.as_ref().map_or(0, |(most, _)| *most) { + best = Some((shared, base)); + } + } + Ok(best.map(|(_, base)| base)) + } + /// The newest checkpoint volume of `role` that `sandbox` or a released /// sandbox kept for the lineage of `wanted`, no newer than it: the /// sandbox that kept it, the volume, its manifest and ref. A resume @@ -597,36 +905,6 @@ impl SessionVolumes { self.disk.open(&name).await } - /// Write `wanted` into a new volume `name` through `fill`, from the - /// staged volume of `base` when this host has it. Without it, every - /// chunk of `wanted` is written. - async fn stage_into( - &self, - fill: &FillEngine, - name: &VolumeName, - wanted: &Manifest, - base: Option, - ) -> io::Result<(Volume, StagePath)> { - let base = match base { - Some(image) => self.staged(image).await, - None => None, - }; - let path = match base { - Some(_) => StagePath::FromImage, - None => StagePath::Whole, - }; - let volume = stage( - self.disk.as_ref(), - fill, - Class::Demand, - name, - wanted, - base.as_ref().map(|(v, m)| (v, m)), - ) - .await?; - Ok((volume, path)) - } - /// The staged volume of an image and its manifest, when this host /// has both. async fn staged(&self, image: ManifestRef) -> Option<(Volume, Manifest)> { @@ -639,6 +917,202 @@ impl SessionVolumes { } } + /// One step of a hydrating role's way to a settled volume: its + /// progress, and once every region is in, the cutover to a volume + /// of its own and the end of its source. `Settled` when the live + /// volume no longer hydrates. + pub async fn settle(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + let live = live_name(sandbox, role); + match self.disk.hydration(&live).await? { + state @ Hydration::Hydrating { + hydrated, total, .. + } if hydrated < total => return Ok(state), + Hydration::Hydrating { .. } => { + let (dest, _) = self.lazy_layout(sandbox, role).await?; + let dest = self.disk.open(&dest).await?; + self.disk.settle(&live, &dest).await?; + } + // A settle that stopped after the cutover left the device's + // metadata volume; the disk's settle removes it. No + // destination: the first checkpoint ended the layout. + Hydration::Settled => match self.lazy_layout(sandbox, role).await { + Ok((dest, _)) => { + let dest = self.disk.open(&dest).await?; + self.disk.settle(&live, &dest).await?; + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + }, + } + self.stop_source(&live).await?; + Ok(Hydration::Settled) + } + + /// The roles of `sandbox` whose live volume is still hydrating. + pub async fn hydrating(&self, sandbox: SandboxId) -> io::Result> { + let mut roles = Vec::new(); + for role in self.lazy_roles(sandbox).await? { + if let Hydration::Hydrating { .. } = + self.disk.hydration(&live_name(sandbox, role)).await? + { + roles.push(role); + } + } + Ok(roles) + } + + /// After a restart: serve again the sources of `sandbox`'s hydrating + /// roles, which the process that died left to the kernel. Returns + /// those roles. + pub async fn reattach(&self, sandbox: SandboxId) -> io::Result> { + let Some((sources, region_bytes)) = self.hydration_support() else { + return Ok(Vec::new()); + }; + let mut roles = Vec::new(); + for role in self.lazy_roles(sandbox).await? { + let live = live_name(sandbox, role); + if self.serving.lock().unwrap().contains_key(&live) { + continue; + } + let Hydration::Hydrating { source, .. } = self.disk.hydration(&live).await? else { + // The process died between the cutover and its cleanup. + self.settle(sandbox, role).await?; + continue; + }; + let (dest, base) = self.lazy_layout(sandbox, role).await?; + let (Some(Kind::Dest(wanted)), Some(Kind::Base(base))) = + (parse(&dest).map(|p| p.kind), parse(&base).map(|p| p.kind)) + else { + unreachable!("lazy_layout returns a dest and a base name"); + }; + let wanted = self.manifest(wanted).await?; + let base = match base { + Some(base) => Some(self.manifest(base).await?), + None => None, + }; + let plan = hydration_plan(&wanted, base.as_ref())?; + let recovered = sources + .recover( + &source, + self.fill.clone(), + &wanted, + plan.delta, + region_bytes, + ) + .await?; + self.serving.lock().unwrap().insert(live, recovered); + roles.push(role); + } + Ok(roles) + } + + /// [`Self::reattach`] for every sandbox with a hydrating role, live + /// or not: a volume whose source nothing serves cannot be deleted, + /// so this runs before the startup reconcile. A role this process + /// already serves is skipped. A sandbox whose reattach fails is + /// logged and left as it is (its volumes stay, unserved and + /// undeletable) so the rest of the host comes up. + pub async fn reattach_all(&self) -> io::Result> { + let mut sandboxes: Vec = Vec::new(); + for p in self.disk.list().await?.iter().filter_map(parse) { + if matches!(p.kind, Kind::Dest(_)) && !sandboxes.contains(&p.sandbox) { + sandboxes.push(p.sandbox); + } + } + let mut reattached = Vec::new(); + for sandbox in sandboxes { + match self.reattach(sandbox).await { + Ok(roles) => reattached.extend(roles.into_iter().map(|role| (sandbox, role))), + Err(error) => { + tracing::error!(sandbox_id = %sandbox, %error, "hydrating volume not served again"); + } + } + } + Ok(reattached) + } + + /// The chunks the guest read before the copy reached them, in + /// order: from the role's source while it serves, and after its stop + /// until the volume goes. Empty for a role that never hydrated. + pub fn first_touch(&self, sandbox: SandboxId, role: DiskRole) -> Vec { + let live = live_name(sandbox, role); + if let Some(source) = self.serving.lock().unwrap().get(&live) { + return source.touched(); + } + self.touched + .lock() + .unwrap() + .get(&live) + .cloned() + .unwrap_or_default() + } + + /// The devices of the sources this process serves. + pub fn sources(&self) -> Vec { + self.serving + .lock() + .unwrap() + .values() + .map(|s| s.path().to_path_buf()) + .collect() + } + + /// The roles of `sandbox` brought up on a hydrating device whose + /// first checkpoint has not run: they have a `dest` volume. + async fn lazy_roles(&self, sandbox: SandboxId) -> io::Result> { + Ok(self + .disk + .list() + .await? + .iter() + .filter_map(parse) + .filter(|p| p.sandbox == sandbox && matches!(p.kind, Kind::Dest(_))) + .map(|p| p.role) + .collect()) + } + + /// The `dest` and `base` volume names of a lazy role. + async fn lazy_layout( + &self, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result<(VolumeName, VolumeName)> { + let (mut dest, mut base) = (None, None); + for name in self.disk.list().await? { + match parse(&name) { + Some(p) if p.sandbox == sandbox && p.role == role => match p.kind { + Kind::Dest(_) => dest = Some(name), + Kind::Base(_) => base = Some(name), + _ => {} + }, + _ => {} + } + } + match (dest, base) { + (Some(dest), Some(base)) => Ok((dest, base)), + _ => Err(io::Error::new( + io::ErrorKind::NotFound, + format!("sandbox {sandbox} has no hydrating {role} volume"), + )), + } + } + + /// End the source of `live`, when this process serves one. The + /// device that read it must be gone or settled first. + async fn stop_source(&self, live: &VolumeName) -> io::Result<()> { + let source = self.serving.lock().unwrap().remove(live); + match source { + Some(source) => { + self.touched + .lock() + .unwrap() + .insert(live.clone(), source.touched()); + source.stop().await + } + None => Ok(()), + } + } + /// The roles the sandbox has a live volume for. pub async fn roles(&self, sandbox: SandboxId) -> io::Result> { let names = self.disk.list().await?; @@ -664,18 +1138,25 @@ impl SessionVolumes { let tmp = transient_name(sandbox, role); self.delete_if_present(&tmp).await?; let live = self.live(sandbox, role).await?; + // Read before the snapshot: a settle between the two only makes + // a whole snapshot count as partial, which costs one more + // checkpoint before the role has its checkpoint volume. + let whole = self.disk.hydration(&live.name).await? == Hydration::Settled; let snapshot = checkpoint_split::take(self.disk.as_ref(), &live, &tmp).await?; Ok(PendingCheckpoint { sandbox, role, snapshot, previous, + whole, }) } /// The checkpoint a `take` left unpublished, found by name: a new /// process redrives the publish after a crash. `NotFound` when - /// there is none. + /// there is none. Whether the volume was hydrating at the take is + /// not recorded, so the snapshot never becomes the first checkpoint + /// volume of a role brought up on a hydrating device. pub async fn pending( &self, sandbox: SandboxId, @@ -688,6 +1169,7 @@ impl SessionVolumes { role, snapshot, previous, + whole: false, }) } @@ -695,12 +1177,32 @@ impl SessionVolumes { /// move to carry before the pause. It is independent of the /// transient snapshot [`Self::take`] makes, so the capture can take /// its own beside it. A precopy left by an earlier move is replaced. + /// + /// `Unsupported` for a role with no checkpoint volume: a hydrating + /// device, or a settled one before its first checkpoint. The blocks + /// it changed since its manifest are the whole copy plus the guest's + /// writes, which is not a live move's small difference; the session + /// moves by snapshot until a checkpoint makes the normal layout. pub async fn take_precopy( &self, sandbox: SandboxId, role: DiskRole, ) -> io::Result { let previous = self.checkpoint_of(sandbox, role).await?; + match self + .disk + .open(&checkpoint_name(sandbox, role, previous)) + .await + { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("{role} disk of {sandbox} has no checkpoint volume yet"), + )) + } + Err(e) => return Err(e), + } let name = precopy_name(sandbox, role); self.delete_if_present(&name).await?; let live = self.live(sandbox, role).await?; @@ -710,6 +1212,7 @@ impl SessionVolumes { role, snapshot, previous, + whole: true, }) } @@ -751,29 +1254,66 @@ impl SessionVolumes { /// Outside the pause: upload what changed since the last checkpoint, /// publish the manifest at the lineage's next free version, and make /// the snapshot the sandbox's checkpoint volume. + /// + /// A role with no checkpoint volume is on a hydrating device, and + /// `previous` is its destination's manifest: the upload lays the + /// blocks that changed since the base over it + /// ([`checkpoint_split::upload_hydrating`]). A snapshot taken while + /// the device hydrated holds base bytes where the copy had not + /// reached, so it goes, and the destination takes the published + /// manifest's name: the names stay the record and no checkpoint + /// volume exists yet. A snapshot taken after the settle becomes the + /// checkpoint volume, and the destination and base names go with + /// the layout they described. pub async fn publish(&self, pending: PendingCheckpoint) -> io::Result { let PendingCheckpoint { sandbox, role, snapshot, previous, + whole, } = pending; + let previous_manifest = self.manifest(previous).await?; let previous_name = checkpoint_name(sandbox, role, previous); - let previous_volume = self.disk.open(&previous_name).await?; - let previous_manifest = self - .fill - .store() - .get_manifest(previous) - .await - .map_err(store_err)?; - let manifest = checkpoint_split::upload( - self.disk.as_ref(), - self.fill.store(), - &snapshot, - Some((&previous_volume, &previous_manifest)), - ) - .await?; - drop(previous_volume); + let (manifest, lazy) = match self.disk.open(&previous_name).await { + Ok(previous_volume) => { + let manifest = checkpoint_split::upload( + self.disk.as_ref(), + self.fill.store(), + &snapshot, + Some((&previous_volume, &previous_manifest)), + ) + .await?; + (manifest, None) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let (dest, base) = self.lazy_layout(sandbox, role).await?; + if parse(&dest).map(|p| p.kind) != Some(Kind::Dest(previous)) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{dest} is not the destination for {previous}"), + )); + } + let block_bytes = self.disk.region_bytes().ok_or_else(|| { + io::Error::new( + io::ErrorKind::Unsupported, + format!("{dest} exists on a disk without a hydrating device"), + ) + })?; + let base_volume = self.disk.open(&base).await?; + let manifest = checkpoint_split::upload_hydrating( + self.disk.as_ref(), + &self.fill, + &snapshot, + &base_volume, + &previous_manifest, + block_bytes, + ) + .await?; + (manifest, Some((dest, base))) + } + Err(e) => return Err(e), + }; // The store, not the volume name, says which versions exist: a // crash after `put_manifest` and before the rename leaves a // version the name never reached. @@ -795,10 +1335,24 @@ impl SessionVolumes { .map_err(store_err)?; let tmp = snapshot.name.clone(); drop(snapshot); - self.disk - .rename(&tmp, &checkpoint_name(sandbox, role, next)) - .await?; - self.disk.delete(&previous_name).await?; + let published = checkpoint_name(sandbox, role, next); + match lazy { + None => { + self.disk.rename(&tmp, &published).await?; + self.disk.delete(&previous_name).await?; + } + Some((dest, base)) if whole => { + self.disk.rename(&tmp, &published).await?; + self.disk.delete(&dest).await?; + self.disk.delete(&base).await?; + } + Some((dest, _)) => { + self.disk + .rename(&dest, &dest_name(sandbox, role, next)) + .await?; + self.disk.delete(&tmp).await?; + } + } Ok(next) } @@ -836,14 +1390,15 @@ impl SessionVolumes { /// open stays and is reported as failed. pub async fn purge(&self, sandbox: SandboxId) -> io::Result { let mut report = Report::default(); - for name in self.disk.list().await? { - if parse(&name).is_some_and(|p| p.sandbox == sandbox) { - match self.disk.delete(&name).await { - Ok(()) => report.deleted.push(name), - Err(e) => report.failed.push((name, e)), - } - } - } + let names: Vec<(VolumeName, Parsed)> = self + .disk + .list() + .await? + .into_iter() + .filter_map(|name| parse(&name).map(|p| (name, p))) + .filter(|(_, p)| p.sandbox == sandbox) + .collect(); + self.delete_all(names, &mut report).await; if report.failed.is_empty() { self.kept.lock().unwrap().retain(|s| *s != sandbox); } @@ -895,7 +1450,7 @@ impl SessionVolumes { live: &[SandboxId], ) -> io::Result { let mut report = Report::default(); - let mut newest: std::collections::HashMap<(SandboxId, DiskRole), u64> = Default::default(); + let mut newest: HashMap<(SandboxId, DiskRole), u64> = Default::default(); let parsed: Vec<(VolumeName, Parsed)> = self .disk .list() @@ -910,23 +1465,90 @@ impl SessionVolumes { *v = (*v).max(m.version); } } + let mut unwanted = Vec::new(); for (name, p) in parsed { - let unwanted = match p.kind { - Kind::Live | Kind::Transient => !live.contains(&p.sandbox), - Kind::Checkpoint(m) => m.version < newest[&(p.sandbox, p.role)], + let wanted = match p.kind { + Kind::Live | Kind::Transient | Kind::Dest(_) | Kind::Base(_) => { + live.contains(&p.sandbox) + } + Kind::Checkpoint(m) => m.version == newest[&(p.sandbox, p.role)], }; - if !unwanted { - if matches!(p.kind, Kind::Checkpoint(_)) && !report.kept.contains(&p.sandbox) { - report.kept.push(p.sandbox); + match wanted { + true if matches!(p.kind, Kind::Checkpoint(_)) => { + if !report.kept.contains(&p.sandbox) { + report.kept.push(p.sandbox); + } } + true => {} + false => unwanted.push((name, p)), + } + } + self.delete_all(unwanted, &mut report).await; + Ok(report) + } + + /// Delete `names`, each sandbox's live volume first. A live volume + /// that stays keeps the destination and base under it: the kernel + /// copies from them and `reattach` rebuilds its plan from them. + async fn delete_all(&self, mut names: Vec<(VolumeName, Parsed)>, report: &mut Report) { + live_first(&mut names); + let mut live_stays = HashSet::new(); + for (name, p) in names { + if matches!(p.kind, Kind::Dest(_) | Kind::Base(_)) && live_stays.contains(&p.sandbox) { continue; } - match self.disk.delete(&name).await { - Ok(()) => report.deleted.push(name), - Err(e) => report.failed.push((name, e)), + let deleted = self.delete_into(&name, p.kind, report).await; + if p.kind == Kind::Live && !deleted { + live_stays.insert(p.sandbox); + } + } + } + + /// Delete one volume into `report`; whether it is gone. A deleted + /// live volume's source stops with it. A hydrating volume whose + /// source this process does not serve is not deleted: the kernel + /// removes a device only after its reads complete, and nothing + /// answers them, so the delete would hang (`reattach` first). + async fn delete_into(&self, name: &VolumeName, kind: Kind, report: &mut Report) -> bool { + if kind == Kind::Live && !self.serving.lock().unwrap().contains_key(name) { + match self.disk.hydration(name).await { + Ok(Hydration::Hydrating { source, .. }) => { + report.failed.push(( + name.clone(), + io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{name} hydrates from {}, which nothing serves", + source.display() + ), + ), + )); + return false; + } + Ok(Hydration::Settled) => {} + Err(e) => { + report.failed.push((name.clone(), e)); + return false; + } + } + } + match self.disk.delete(name).await { + Ok(()) => { + if kind == Kind::Live { + if let Err(e) = self.stop_source(name).await { + report.failed.push((name.clone(), e)); + return true; + } + self.touched.lock().unwrap().remove(name); + } + report.deleted.push(name.clone()); + true + } + Err(e) => { + report.failed.push((name.clone(), e)); + false } } - Ok(report) } /// The released sandboxes whose kept checkpoints a resume may take. @@ -945,27 +1567,28 @@ impl SessionVolumes { } } - /// The manifest of the sandbox's checkpoint volume: what the last - /// publish made durable. + /// The manifest the role's live volume holds: its checkpoint + /// volume's, or, while it hydrates with no checkpoint volume yet, + /// its destination's. pub async fn checkpoint_of( &self, sandbox: SandboxId, role: DiskRole, ) -> io::Result { - let mut found: Option = None; - for name in self.disk.list().await? { - if let Some(Parsed { - sandbox: s, - role: r, - kind: Kind::Checkpoint(m), - }) = parse(&name) - { - if s == sandbox && r == role && found.is_none_or(|f| m.version > f.version) { - found = Some(m); + let (mut checkpoint, mut dest): (Option, Option) = (None, None); + for p in self.disk.list().await?.iter().filter_map(parse) { + if (p.sandbox, p.role) != (sandbox, role) { + continue; + } + match p.kind { + Kind::Checkpoint(m) if checkpoint.is_none_or(|c| m.version > c.version) => { + checkpoint = Some(m) } + Kind::Dest(m) => dest = Some(m), + _ => {} } } - found.ok_or_else(|| { + checkpoint.or(dest).ok_or_else(|| { io::Error::new( io::ErrorKind::NotFound, format!("sandbox {sandbox} has no {} checkpoint volume", role), @@ -989,7 +1612,8 @@ mod tests { use super::*; use crate::disk::{ChunkIndex, FileDisk}; use crate::disk_flows::checkpoint; - use crate::test_support::CountingBlob; + use crate::test_support::{CountingBlob, FileSources, HydratingDisk}; + use std::sync::atomic::Ordering; const CHUNK: u64 = 4096; @@ -999,6 +1623,8 @@ mod tests { store: ChunkStore, volumes: SessionVolumes, blob: Arc, + /// Set on a rig whose disk hydrates. + sources: Option>, } fn rig() -> Rig { @@ -1007,6 +1633,31 @@ mod tests { rig_on(dir, disk) } + /// A rig whose disk has a hydrating device and whose host serves + /// sources: resumes take the lazy path. + fn lazy_rig() -> Rig { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(HydratingDisk::new(&dir.path().join("volumes"), CHUNK).unwrap()); + let sources = Arc::new(disk.sources(&dir.path().join("sources"))); + let mut r = rig_on(dir, disk); + r.volumes = r.volumes.with_sources(sources.clone()); + r.sources = Some(sources); + r + } + + /// Drive `settle` until the role's volume is settled; the steps it + /// took. + async fn settled(r: &Rig, sandbox: SandboxId, role: DiskRole) -> u64 { + let mut steps = 0; + loop { + steps += 1; + match r.volumes.settle(sandbox, role).await.unwrap() { + Hydration::Settled => return steps, + Hydration::Hydrating { .. } => assert!(steps < 100, "the hydration never ends"), + } + } + } + /// A rig whose pool reports `POOL_TOTAL` bytes with `VOLUME_COST` /// used per volume, so reaping frees space. fn pressured_rig() -> Rig { @@ -1030,6 +1681,7 @@ mod tests { disk, store, blob, + sources: None, _dir: dir, } } @@ -1071,10 +1723,7 @@ mod tests { } fn fork(image: ManifestRef) -> Source { - Source::Fork { - manifest: image, - base: None, - } + Source::Fork { manifest: image } } /// Publish an image whose chunk `i` holds byte `pattern[i]` (0 is a @@ -1198,6 +1847,24 @@ mod tests { }) ); } + assert_eq!( + parse(&dest_name(sandbox, DiskRole::Root, manifest)), + Some(Parsed { + sandbox, + role: DiskRole::Root, + kind: Kind::Dest(manifest) + }) + ); + for base in [Some(manifest), None] { + assert_eq!( + parse(&base_name(sandbox, DiskRole::Swap, base)), + Some(Parsed { + sandbox, + role: DiskRole::Swap, + kind: Kind::Base(base) + }) + ); + } assert_eq!(parse(&image_volumes::volume_name(manifest)), None); assert_eq!(parse(&VolumeName::new("sb-nothex-root").unwrap()), None); assert_eq!( @@ -1253,14 +1920,7 @@ mod tests { let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); assert_eq!(bytes_at(&p.root).await, vec![1, 0, 3]); @@ -1299,14 +1959,7 @@ mod tests { let first = SandboxId::new(); let p = r .volumes - .provision( - first, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(first, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 1, 9).await; @@ -1320,7 +1973,7 @@ mod tests { second, Source::Continue { manifest: published, - base: Some(image), + first_touch: vec![], }, None, ) @@ -1340,7 +1993,7 @@ mod tests { third, Source::Continue { manifest: published, - base: None, + first_touch: vec![], }, None, ) @@ -1389,7 +2042,7 @@ mod tests { second, Source::Continue { manifest: published, - base: None, + first_touch: vec![], }, None, Some(Arc::new(peer)), @@ -1407,14 +2060,7 @@ mod tests { let first = SandboxId::new(); let p = r .volumes - .provision( - first, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(first, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 1, 9).await; @@ -1429,7 +2075,6 @@ mod tests { sandbox, Source::Fork { manifest: published, - base: Some(image), }, None, ) @@ -1458,14 +2103,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 0, 7).await; @@ -1489,10 +2127,7 @@ mod tests { .volumes .provision( sandbox, - Source::Fork { - manifest: image, - base: None, - }, + Source::Fork { manifest: image }, Some(Source::Empty { size_bytes: CHUNK * 2, }), @@ -1524,13 +2159,10 @@ mod tests { .volumes .provision( sandbox, - Source::Fork { - manifest: image, - base: None, - }, + Source::Fork { manifest: image }, Some(Source::Continue { manifest: ManifestRef::new(), - base: None, + first_touch: vec![], }), ) .await @@ -1549,14 +2181,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); let lineage = checkpoints_of(&r, sandbox, DiskRole::Root).await[0]; @@ -1592,7 +2217,7 @@ mod tests { resumed, Source::Continue { manifest: v3, - base: Some(image), + first_touch: vec![], }, None, ) @@ -1608,14 +2233,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); write_byte(&p.root, 0, 5).await; @@ -1637,14 +2255,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); let half = CHUNK / 2; @@ -1703,14 +2314,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); assert_eq!( @@ -1758,14 +2362,7 @@ mod tests { let sandbox = SandboxId::new(); let p = r .volumes - .provision( - sandbox, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(sandbox, Source::Fork { manifest: image }, None) .await .unwrap(); let lineage = checkpoints_of(&r, sandbox, DiskRole::Root).await[0]; @@ -1803,23 +2400,13 @@ mod tests { r.volumes .provision( live, - Source::Fork { - manifest: image, - base: None, - }, + Source::Fork { manifest: image }, Some(Source::Empty { size_bytes: CHUNK }), ) .await .unwrap(); r.volumes - .provision( - dead, - Source::Fork { - manifest: image, - base: None, - }, - None, - ) + .provision(dead, Source::Fork { manifest: image }, None) .await .unwrap(); let lineage = checkpoints_of(&r, live, DiskRole::Root).await[0]; @@ -1949,7 +2536,7 @@ mod tests { resumed, Source::Continue { manifest: published, - base: Some(image), + first_touch: vec![], }, None, ) @@ -1990,7 +2577,7 @@ mod tests { destination, Source::Continue { manifest: published, - base: Some(image), + first_touch: vec![], }, None, ) @@ -2034,7 +2621,7 @@ mod tests { resumed, Source::Continue { manifest: next, - base: Some(image), + first_touch: vec![], }, None, ) @@ -2107,7 +2694,7 @@ mod tests { sandbox, Source::Continue { manifest: published, - base: Some(image), + first_touch: vec![], }, None, ) @@ -2144,4 +2731,505 @@ mod tests { assert_eq!(reaped.deleted.len(), 1); assert!(checkpoints_of(&r, dead, DiskRole::Root).await.is_empty()); } + + /// Image [1, 2, 3]; the session changed chunk 1 and zeroed chunk 2. + /// Its resume on a host that hydrates starts on a device over a + /// snapshot of the image volume: chunk 1 is the whole delta, chunk 2 + /// is zeroed before the copy, chunk 0 is never touched. + #[tokio::test] + async fn a_lazy_resume_starts_on_a_hydrating_volume_and_settles_onto_it() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + assert_eq!(p.root_stage, StagePath::StagedSnapshot); + write_byte(&p.root, 1, 9).await; + write_byte(&p.root, 2, 0).await; + let published = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + let gets = r.blob.gets(); + let p = r + .volumes + .provision( + resumed, + Source::Continue { + manifest: published, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!( + (p.root_stage, p.root_manifest), + (StagePath::Lazy, published) + ); + assert_eq!( + r.blob.gets(), + gets + 1, + "the source serves only the changed chunk" + ); + assert_eq!( + names(&r).await, + vec![ + image_volumes::volume_name(image).to_string(), + live_name(resumed, DiskRole::Root).to_string(), + base_name(resumed, DiskRole::Root, Some(image)).to_string(), + dest_name(resumed, DiskRole::Root, published).to_string(), + ] + .into_iter() + .map(|n| n.to_string()) + .collect::>() + .into_iter() + .collect::>() + ); + // The fake device has no lazy read: until the copy lands, the + // destination shows the base with the dropped chunk zeroed. + assert_eq!(bytes_at(&p.root).await, vec![1, 2, 0]); + assert_eq!( + r.volumes.hydrating(resumed).await.unwrap(), + vec![DiskRole::Root] + ); + + let steps = settled(&r, resumed, DiskRole::Root).await; + assert_eq!( + steps, 1, + "the one chunk copies on the first step, then the cutover" + ); + assert_eq!(bytes_at(&p.root).await, vec![1, 9, 0]); + assert!(r.volumes.hydrating(resumed).await.unwrap().is_empty()); + let sources = r.sources.as_ref().unwrap(); + assert_eq!(sources.stopped.load(Ordering::Relaxed), 1); + assert_eq!( + r.volumes.settle(resumed, DiskRole::Root).await.unwrap(), + Hydration::Settled, + "settle is idempotent" + ); + + r.volumes.release(resumed).await.unwrap(); + assert_eq!( + names(&r).await, + vec![image_volumes::volume_name(image).to_string()], + "a hydrating role keeps no checkpoint volume" + ); + } + + /// Base [1, 2, 3, 0, 5], wanted [1, 9, 0, 4, 5]: chunks 1 and 3 + /// copy in, chunk 2 is zeroed. Before the copy the guest writes 7 + /// over the zeroed chunk and 8 over held chunk 4. A checkpoint then + /// publishes the wanted disk with the writes laid over it, under + /// the destination's name: no checkpoint volume exists until the + /// settle. The next checkpoint, after the settle, makes the normal + /// layout. + #[tokio::test] + async fn a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3, 0, 5]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + write_byte(&p.root, 1, 9).await; + write_byte(&p.root, 2, 0).await; + write_byte(&p.root, 3, 4).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + let p = r + .volumes + .provision( + resumed, + Source::Continue { + manifest: wanted, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert_eq!( + r.volumes + .checkpoint_of(resumed, DiskRole::Root) + .await + .unwrap(), + wanted, + "the live volume holds the destination's manifest" + ); + assert_eq!( + r.volumes.first_touch(resumed, DiskRole::Root), + vec![1, 3], + "the source reports what the guest read first (the fake: the delta)" + ); + write_byte(&p.root, 2, 7).await; + write_byte(&p.root, 4, 8).await; + + let mid = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + assert_eq!( + (mid.manifest_id, mid.version), + (wanted.manifest_id, wanted.version + 1) + ); + assert_eq!(manifest_bytes(&r, mid).await, vec![1, 9, 7, 4, 8]); + let expected: std::collections::BTreeSet = [ + image_volumes::volume_name(image).to_string(), + live_name(resumed, DiskRole::Root).to_string(), + base_name(resumed, DiskRole::Root, Some(image)).to_string(), + dest_name(resumed, DiskRole::Root, mid).to_string(), + ] + .into_iter() + .collect(); + assert_eq!( + names(&r).await, + expected.into_iter().collect::>(), + "the destination carries the published manifest; no checkpoint volume yet" + ); + assert_eq!( + r.volumes + .checkpoint_of(resumed, DiskRole::Root) + .await + .unwrap(), + mid + ); + + settled(&r, resumed, DiskRole::Root).await; + assert_eq!(bytes_at(&p.root).await, vec![1, 9, 7, 4, 8]); + assert_eq!( + r.volumes.first_touch(resumed, DiskRole::Root), + vec![1, 3], + "the record outlives the source until the volume goes" + ); + let after = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + assert_eq!(after.version, mid.version + 1); + assert_eq!(manifest_bytes(&r, after).await, vec![1, 9, 7, 4, 8]); + let expected: std::collections::BTreeSet = [ + image_volumes::volume_name(image).to_string(), + live_name(resumed, DiskRole::Root).to_string(), + checkpoint_name(resumed, DiskRole::Root, after).to_string(), + ] + .into_iter() + .collect(); + assert_eq!( + names(&r).await, + expected.into_iter().collect::>(), + "a settled role has the normal layout" + ); + r.volumes.release(resumed).await.unwrap(); + assert!(r.volumes.first_touch(resumed, DiskRole::Root).is_empty()); + } + + /// Base [1, 2, 3], wanted [1, 9, 8]. A snapshot taken when only + /// chunk 1 is copied shows the base's 3 in chunk 2. The volume + /// settles before the publish: the manifest is the wanted disk, and + /// the snapshot does not become the checkpoint volume a resume on + /// this host would boot. The next checkpoint's snapshot does. + #[tokio::test] + async fn a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + write_byte(&p.root, 1, 9).await; + write_byte(&p.root, 2, 8).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + let source = Source::Continue { + manifest: wanted, + first_touch: vec![], + }; + r.volumes.provision(resumed, source, None).await.unwrap(); + let pending = r.volumes.take(resumed, DiskRole::Root).await.unwrap(); + assert_eq!( + bytes_at(&pending.snapshot().path).await, + vec![1, 9, 3], + "the snapshot holds the base where the copy has not reached" + ); + settled(&r, resumed, DiskRole::Root).await; + + let mid = r.volumes.publish(pending).await.unwrap(); + assert_eq!(manifest_bytes(&r, mid).await, vec![1, 9, 8]); + assert_eq!( + checkpoints_of(&r, resumed, DiskRole::Root).await, + vec![], + "no checkpoint volume from a partial snapshot" + ); + assert!(names(&r) + .await + .contains(&dest_name(resumed, DiskRole::Root, mid).to_string())); + + let after = r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + assert_eq!( + checkpoints_of(&r, resumed, DiskRole::Root).await, + vec![after] + ); + let kept = r + .volumes + .disk() + .open(&checkpoint_name(resumed, DiskRole::Root, after)) + .await + .unwrap(); + assert_eq!(bytes_at(&kept.path).await, vec![1, 9, 8]); + } + + /// A live move carries the blocks changed since the checkpoint + /// volume; a role without one (hydrating, or settled before its + /// first checkpoint) refuses the precopy, and the first checkpoint + /// makes the live move possible again. + #[tokio::test] + async fn a_role_without_a_checkpoint_volume_refuses_a_live_move() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2, 3]).await; + let first = SandboxId::new(); + let p = r.volumes.provision(first, fork(image), None).await.unwrap(); + write_byte(&p.root, 1, 9).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let resumed = SandboxId::new(); + r.volumes + .provision( + resumed, + Source::Continue { + manifest: wanted, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + let refused = |e: io::Error| assert_eq!(e.kind(), io::ErrorKind::Unsupported, "{e}"); + refused( + r.volumes + .take_precopy(resumed, DiskRole::Root) + .await + .unwrap_err(), + ); + settled(&r, resumed, DiskRole::Root).await; + refused( + r.volumes + .take_precopy(resumed, DiskRole::Root) + .await + .unwrap_err(), + ); + r.volumes.checkpoint(resumed, DiskRole::Root).await.unwrap(); + let precopy = r + .volumes + .take_precopy(resumed, DiskRole::Root) + .await + .unwrap(); + assert!(r + .volumes + .changed_blocks(&precopy, CHUNK) + .await + .unwrap() + .is_empty()); + r.volumes.discard(precopy).await.unwrap(); + r.volumes.release(resumed).await.unwrap(); + } + + /// The first byte of each chunk of a published manifest; 0 for a + /// hole. + async fn manifest_bytes(r: &Rig, manifest: ManifestRef) -> Vec { + let fill = engram_fill::FillEngine::new(r.store.clone(), engram_fill::Config::default()); + let manifest = r.store.get_manifest(manifest).await.unwrap(); + let mut out = Vec::new(); + for i in 0..manifest.expected_chunk_count() { + out.push(match manifest.chunk_at(i * CHUNK) { + Some(c) => fill + .read(c.hash, engram_fill::Class::Hydrate) + .await + .unwrap()[0], + None => 0, + }); + } + out + } + + /// A release during the window deletes the device before its + /// source; the source refuses the other order. + #[tokio::test] + async fn a_release_during_hydration_ends_the_device_then_its_source() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2]).await; + let sandbox = SandboxId::new(); + let mut newer = r.store.get_manifest(image).await.unwrap(); + newer.chunks[0].hash = r.store.put_chunk(&vec![7u8; CHUNK as usize]).await.unwrap(); + let next = ManifestRef::new(); + r.store.put_manifest(next, &newer).await.unwrap(); + let p = r + .volumes + .provision( + sandbox, + Source::Continue { + manifest: next, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert_eq!( + r.volumes.hydrating(sandbox).await.unwrap(), + vec![DiskRole::Root] + ); + + r.volumes.release(sandbox).await.unwrap(); + let sources = r.sources.as_ref().unwrap(); + assert_eq!(sources.stopped.load(Ordering::Relaxed), 1); + assert_eq!( + names(&r).await, + vec![image_volumes::volume_name(image).to_string()] + ); + } + + /// A host-agent restart mid-window: the device and its source + /// outlive the process; the next process cannot delete the device + /// before it serves the source again, then settles the volume. + #[tokio::test] + async fn a_restart_serves_a_hydrating_volume_again() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 2]).await; + let sandbox = SandboxId::new(); + let mut newer = r.store.get_manifest(image).await.unwrap(); + newer.chunks[0].hash = r.store.put_chunk(&vec![7u8; CHUNK as usize]).await.unwrap(); + let next = ManifestRef::new(); + r.store.put_manifest(next, &newer).await.unwrap(); + let p = r + .volumes + .provision( + sandbox, + Source::Continue { + manifest: next, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + + let sources = r.sources.clone().unwrap(); + let restarted = SessionVolumes::new( + r.disk.clone(), + engram_fill::FillEngine::new(r.store.clone(), engram_fill::Config::default()), + ) + .with_sources(sources.clone()); + // Until the source is served again, nothing deletes the device: + // its removal would wait on reads nothing answers. + let refused = restarted.release(sandbox).await.unwrap_err(); + assert_eq!(refused.kind(), std::io::ErrorKind::ResourceBusy); + assert_eq!( + restarted.reattach(sandbox).await.unwrap(), + vec![DiskRole::Root] + ); + assert!( + restarted.reattach_all().await.unwrap().is_empty(), + "a role this process serves is not served twice" + ); + loop { + if let Hydration::Settled = restarted.settle(sandbox, DiskRole::Root).await.unwrap() { + break; + } + } + assert_eq!(bytes_at(&p.root).await, vec![7, 2]); + assert_eq!(sources.stopped.load(Ordering::Relaxed), 1); + assert!(restarted.reattach(sandbox).await.unwrap().is_empty()); + } + + /// The session began on image [1, 2, 3, 4, 5] and holds + /// [1, 2, 3, 9, 5]. The image's tag moved: the host stages only + /// [1, 2, 3, 4, 6] now, and a second image that shares one chunk. + /// The resume starts from the moved image, which holds three of the + /// session's chunks in place, and copies the other two. + #[tokio::test] + async fn a_resume_starts_from_the_staged_volume_that_shares_the_most() { + let r = lazy_rig(); + let began_on = staged_image(&r, &[1, 2, 3, 4, 5]).await; + let first = SandboxId::new(); + let p = r + .volumes + .provision(first, fork(began_on), None) + .await + .unwrap(); + write_byte(&p.root, 3, 9).await; + let wanted = r.volumes.checkpoint(first, DiskRole::Root).await.unwrap(); + r.volumes.release(first).await.unwrap(); + r.volumes.purge(first).await.unwrap(); + + let other = staged_image(&r, &[1, 7, 7, 7, 7]).await; + let moved = staged_image(&r, &[1, 2, 3, 4, 6]).await; + r.volumes.images().reconcile(&[other, moved]).await.unwrap(); + assert!(!names(&r) + .await + .contains(&image_volumes::volume_name(began_on).to_string())); + + let resumed = SandboxId::new(); + let p = r + .volumes + .provision( + resumed, + Source::Continue { + manifest: wanted, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert!( + names(&r) + .await + .contains(&base_name(resumed, DiskRole::Root, Some(moved)).to_string()), + "the base is the staged image that shares the most chunks" + ); + assert_eq!( + r.volumes.first_touch(resumed, DiskRole::Root), + vec![3, 4], + "the copy brings only the chunks the moved image lacks (the fake reports the delta)" + ); + settled(&r, resumed, DiskRole::Root).await; + assert_eq!(bytes_at(&p.root).await, vec![1, 2, 3, 9, 5]); + r.volumes.release(resumed).await.unwrap(); + } + + /// With no volume on the host that shares a chunk, the destination + /// starts from zeros and every listed chunk is the delta. + #[tokio::test] + async fn a_lazy_resume_without_a_base_starts_from_zeros() { + let r = lazy_rig(); + let image = staged_image(&r, &[1, 0, 3]).await; + r.volumes.images().reconcile(&[]).await.unwrap(); + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision( + sandbox, + Source::Continue { + manifest: image, + first_touch: vec![], + }, + None, + ) + .await + .unwrap(); + assert_eq!(p.root_stage, StagePath::Lazy); + assert_eq!(bytes_at(&p.root).await, vec![0, 0, 0]); + assert_eq!( + settled(&r, sandbox, DiskRole::Root).await, + 2, + "two chunks to copy" + ); + assert_eq!(bytes_at(&p.root).await, vec![1, 0, 3]); + assert!(names(&r) + .await + .contains(&base_name(sandbox, DiskRole::Root, None).to_string())); + } } diff --git a/crates/engram-host-core/src/test_support.rs b/crates/engram-host-core/src/test_support.rs index 42c6fa17d..9942146e7 100644 --- a/crates/engram-host-core/src/test_support.rs +++ b/crates/engram-host-core/src/test_support.rs @@ -1,12 +1,25 @@ //! Test doubles shared by this crate's unit tests. +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use engram_chunk_store::Manifest; use engram_core::error::BlobError; use engram_core::traits::storage::{BlobObjectMeta, ByteStream}; use engram_core::traits::BlobStorage; +use engram_fill::{Class, FillEngine}; use engram_storage_local::LocalBlobStorage; +use crate::disk::{ + Capacity, ChunkIndex, FileDisk, Hydration, SessionDisk, SourceDevice, SourceServer, Volume, + VolumeName, +}; + /// Counts the chunk GETs a stage makes; manifest reads do not count. pub(crate) struct CountingBlob { inner: LocalBlobStorage, @@ -47,3 +60,242 @@ impl BlobStorage for CountingBlob { self.inner.list_prefix(prefix).await } } + +/// A [`FileDisk`] with a hydrating device, for the lazy path without a +/// kernel: `hydrate` links `name` to the destination file, so a write +/// to either lands in both, and each `hydration` call copies one chunk +/// the plan did not skip from the source file. `settle` needs every +/// chunk copied and gives the device a file of its own; on a device +/// that does not hydrate it does nothing. +pub(crate) struct HydratingDisk { + inner: FileDisk, + chunk_size: u64, + clones: Mutex>, + /// Sources a device still copies from; a stop of one is refused. + holders: Arc>>, +} + +struct Clone { + source: PathBuf, + /// The device's own path, a hard link of the destination: the copy + /// writes through it, as the kernel writes through the destination's + /// device number, so a rename of the destination changes nothing. + device: PathBuf, + pending: Vec, + total: u64, + settled: bool, +} + +impl HydratingDisk { + pub(crate) fn new(root: &Path, chunk_size: u64) -> io::Result { + Ok(Self { + inner: FileDisk::new(root, chunk_size)?, + chunk_size, + clones: Mutex::new(HashMap::new()), + holders: Arc::new(Mutex::new(HashSet::new())), + }) + } + + /// A source server whose stops this disk's devices refuse while + /// they copy. + pub(crate) fn sources(&self, dir: &Path) -> FileSources { + FileSources { + dir: dir.to_path_buf(), + holders: self.holders.clone(), + next: AtomicU64::new(0), + stopped: Arc::new(AtomicU64::new(0)), + } + } +} + +#[async_trait::async_trait] +impl SessionDisk for HydratingDisk { + async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { + self.inner.create(name, size_bytes).await + } + async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { + self.inner.snapshot(origin, name).await + } + async fn open(&self, name: &VolumeName) -> io::Result { + self.inner.open(name).await + } + async fn delete(&self, name: &VolumeName) -> io::Result<()> { + if let Some(clone) = self.clones.lock().unwrap().remove(name) { + self.holders.lock().unwrap().remove(&clone.source); + } + self.inner.delete(name).await + } + async fn list(&self) -> io::Result> { + self.inner.list().await + } + async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { + self.inner.rename(from, to).await + } + async fn capacity(&self) -> io::Result { + self.inner.capacity().await + } + fn region_bytes(&self) -> Option { + Some(self.chunk_size) + } + async fn hydrate( + &self, + dest: &Volume, + source: &Path, + name: &VolumeName, + skip: &[ChunkIndex], + ) -> io::Result { + let path = self.inner.root().join(name.as_str()); + std::fs::hard_link(&dest.path, &path)?; + let pending = (0..dest.chunk_count()) + .map(ChunkIndex) + .filter(|c| !skip.contains(c)) + .collect(); + self.clones.lock().unwrap().insert( + name.clone(), + Clone { + source: source.to_path_buf(), + device: path.clone(), + pending, + total: dest.chunk_count(), + settled: false, + }, + ); + self.holders.lock().unwrap().insert(source.to_path_buf()); + Volume::open(name.clone(), &path, dest.size_bytes, dest.chunk_size) + } + async fn hydration(&self, name: &VolumeName) -> io::Result { + let mut clones = self.clones.lock().unwrap(); + let Some(clone) = clones.get_mut(name) else { + return Ok(Hydration::Settled); + }; + if clone.settled { + return Ok(Hydration::Settled); + } + if let Some(chunk) = clone.pending.first().copied() { + let mut bytes = vec![0u8; self.chunk_size as usize]; + File::open(&clone.source)?.read_exact_at(&mut bytes, chunk.0 * self.chunk_size)?; + File::options() + .write(true) + .open(&clone.device)? + .write_all_at(&bytes, chunk.0 * self.chunk_size)?; + clone.pending.remove(0); + } + Ok(Hydration::Hydrating { + hydrated: clone.total - clone.pending.len() as u64, + total: clone.total, + source: clone.source.clone(), + }) + } + async fn settle(&self, name: &VolumeName, _dest: &Volume) -> io::Result<()> { + let mut clones = self.clones.lock().unwrap(); + let Some(clone) = clones.get_mut(name).filter(|c| !c.settled) else { + return Ok(()); + }; + if !clone.pending.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name}: {} chunks are not copied", clone.pending.len()), + )); + } + // The device stops sharing the destination's file: a volume of + // its own, as the pool makes it. + let own = clone.device.with_extension("settling"); + std::fs::copy(&clone.device, &own)?; + std::fs::rename(&own, &clone.device)?; + clone.settled = true; + self.holders.lock().unwrap().remove(&clone.source); + Ok(()) + } +} + +/// Serves a manifest as a file holding its delta chunks' bytes, written +/// from the fill engine up front; the kernel reads nothing else. +pub(crate) struct FileSources { + dir: PathBuf, + holders: Arc>>, + next: AtomicU64, + /// Sources stopped so far. + pub(crate) stopped: Arc, +} + +struct FileSource { + path: PathBuf, + holders: Arc>>, + stopped: Arc, + /// Reported as read first: every delta chunk, in order. + delta: Vec, +} + +#[async_trait::async_trait] +impl SourceDevice for FileSource { + fn path(&self) -> &Path { + &self.path + } + fn touched(&self) -> Vec { + self.delta.clone() + } + async fn stop(self: Box) -> io::Result<()> { + if self.holders.lock().unwrap().contains(&self.path) { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!("{} is still a device's source", self.path.display()), + )); + } + std::fs::remove_file(&self.path)?; + self.stopped.fetch_add(1, Ordering::Relaxed); + Ok(()) + } +} + +#[async_trait::async_trait] +impl SourceServer for FileSources { + async fn serve( + &self, + fill: FillEngine, + manifest: &Manifest, + delta: Vec, + _region_bytes: u64, + ) -> io::Result> { + std::fs::create_dir_all(&self.dir)?; + let path = self + .dir + .join(format!("src-{}", self.next.fetch_add(1, Ordering::Relaxed))); + let file = File::create_new(&path)?; + file.set_len(manifest.total_bytes)?; + let chunk_size = manifest.chunk_size.as_u64(); + for chunk in manifest + .chunks + .iter() + .filter(|c| delta.contains(&(c.offset / chunk_size))) + { + let bytes = fill + .read(chunk.hash, Class::Hydrate) + .await + .map_err(io::Error::other)?; + file.write_all_at(&bytes, chunk.offset)?; + } + Ok(Box::new(FileSource { + path, + holders: self.holders.clone(), + stopped: self.stopped.clone(), + delta: delta.iter().map(|c| *c as u32).collect(), + })) + } + + async fn recover( + &self, + path: &Path, + _fill: FillEngine, + _manifest: &Manifest, + delta: Vec, + _region_bytes: u64, + ) -> io::Result> { + std::fs::metadata(path)?; + Ok(Box::new(FileSource { + path: path.to_path_buf(), + holders: self.holders.clone(), + stopped: self.stopped.clone(), + delta: delta.iter().map(|c| *c as u32).collect(), + })) + } +} diff --git a/crates/engram-protocol/src/wire.rs b/crates/engram-protocol/src/wire.rs index c76d104ba..e13e5a11b 100644 --- a/crates/engram-protocol/src/wire.rs +++ b/crates/engram-protocol/src/wire.rs @@ -185,8 +185,10 @@ pub const WIRE_VERSION: u32 = 38; /// build that reports nothing counts as 0. 2: the uffd handler speaks only /// substrate protocol version 2 and later, a post-copy `DrainDone` carries /// the hot-set fields, a snapshot create always sends `defer_sync`, and a -/// migration item always carries its hash. -pub const REATTACH_GENERATION: u32 = 2; +/// migration item always carries its hash. 3: a session disk hydrates +/// behind the running guest from a ublk device the host-agent serves, +/// which an earlier build cannot serve again after a restart. +pub const REATTACH_GENERATION: u32 = 3; /// gRPC metadata (header) key carrying the caller's [`WIRE_VERSION`] on /// every coord→host request (issue #229). ASCII, lowercase — tonic diff --git a/deploy/helm/engram-host-fleet/files/thin-pool.sh b/deploy/helm/engram-host-fleet/files/thin-pool.sh index 18ad84993..529e10b6d 100755 --- a/deploy/helm/engram-host-fleet/files/thin-pool.sh +++ b/deploy/helm/engram-host-fleet/files/thin-pool.sh @@ -31,6 +31,10 @@ DATA="$POOL_DIR/data.img" log() { echo "[thin-pool] $*"; } +# The host agent serves the source a resuming session's disk fills from +# as a ublk device; the module is loaded on every start, pool or not. +modprobe ublk_drv 2>/dev/null || true + if dmsetup info "$POOL_NAME" >/dev/null 2>&1; then log "$POOL_NAME is active" dmsetup status "$POOL_NAME" diff --git a/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml b/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml index 26eafcc0a..bdaac9a6b 100644 --- a/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml +++ b/deploy/helm/engram-host-fleet/templates/host-agent.daemonset.yaml @@ -274,6 +274,10 @@ spec: value: /opt/engram/firecracker - name: ENGRAM_THIN_POOL value: {{ .Values.thinPool.name | quote }} + - name: ENGRAM_DISK_HYDRATION_WINDOW_CHUNKS + value: {{ .Values.thinPool.hydration.windowChunks | quote }} + - name: ENGRAM_DISK_HYDRATION_IN_FLIGHT_REGIONS + value: {{ .Values.thinPool.hydration.inFlightRegions | quote }} {{- if .Values.coordinator.tokenSecretName }} - name: ENGRAM_COORDINATOR_TOKEN valueFrom: diff --git a/deploy/helm/engram-host-fleet/values.yaml b/deploy/helm/engram-host-fleet/values.yaml index 46a5cec05..ed80e250d 100644 --- a/deploy/helm/engram-host-fleet/values.yaml +++ b/deploy/helm/engram-host-fleet/values.yaml @@ -233,6 +233,14 @@ thinPool: blockSizeSectors: 128 # Blocks; 32768 = 2 GiB at 64 KiB. lowWaterMarkBlocks: 32768 + # A resumed session's disk is copied in behind the running VM. Larger + # values copy faster until the pool's disk is the limit. + hydration: + # Whole chunks (16 MiB) fetched ahead of the kernel's copy, per disk. + # The host-agent holds up to this many plus two in memory per disk. + windowChunks: 12 + # Pool blocks the kernel copies at one time, per disk. + inFlightRegions: 512 hostAgent: grpcPort: 9101 diff --git a/docs/adr/0125-kernel-owned-session-disk.md b/docs/adr/0125-kernel-owned-session-disk.md index a74030d71..b1d75face 100644 --- a/docs/adr/0125-kernel-owned-session-disk.md +++ b/docs/adr/0125-kernel-owned-session-disk.md @@ -2,7 +2,9 @@ - Status: Accepted (2026-11-04). This ADR supersedes ADR 0110 (honest writes) and Decision 3 of ADR 0090 (the quarantined survivor channel), - and retires ADR 0049 (the NBD slot allocator). + and retires ADR 0049 (the NBD slot allocator). Amended 2026-11-09: + a resume hydrates the disk behind the running guest (Decision 3, the + Resume lifecycle, and the dated divergence entry). - Date: 2026-11-03 - Commit chain (branch `disk/session-disk-seam`, one PR): step 1 `ed6186b7c`..`08403960d` (the seam, `FileDisk`, `CrashDisk`, the two @@ -180,11 +182,19 @@ thin-provisioning tools it calls at run time (`thin_dump`, readiness. 3. **A session disk is a thin snapshot.** A fresh session's root disk is a snapshot of its image volume. A resumed session's root disk is - a snapshot of its base image volume with the session's dirty chunks - written in. Guest swap (ADR 0112) starts as a fresh thin volume - with no origin and is checkpointed with the root disk from then on - (see the divergences). Every volume a session owns is recorded in - its `sandbox.json`. + a snapshot of the nearest base this host holds (the kept checkpoint + volume or staged image version that shares the most chunks with the + session's manifest at the same offset; an empty volume when none + shares any) under a + dm-clone device that copies the difference to its manifest in + behind the running guest; the guest starts at once and reads a + region the copy has not reached through the device's source, a + ublk device the host-agent serves from the fill engine. Once every + region is copied the device is reloaded as a thin snapshot of its + destination and the source goes. Guest swap (ADR 0112) starts as a + fresh thin volume with no origin and is checkpointed with the root + disk from then on (see the divergences). The pool's volume names + are the record of what a session owns. 4. **Firecracker opens the device path.** The guest's virtio-blk drive is a device node the host-agent makes itself under `/dev/engram/` (`mknod` from the kernel's major:minor, so no udev rule is in the @@ -251,7 +261,10 @@ thin-provisioning tools it calls at run time (`thin_dump`, /dev/engram/engram-pool.sb--root session root disk (snapshot of an image volume) /dev/engram/engram-pool.sb--swap session swap (a fresh volume at first; checkpointed like root) /dev/engram/engram-pool.sb--ckpt- a checkpoint snapshot, transient -//sandbox.json records the volume names +/dev/engram/engram-pool.sb---dest- the destination a hydrating live volume copies into +/dev/engram/engram-pool.sb---base- the base that destination is a snapshot of +/dev/ublkb the source a hydrating live volume reads cold regions from +//sandbox.json records the sandbox; the volume names are the pool's ``` The pool's metadata is the only record of which thins exist and the @@ -312,9 +325,34 @@ space falls under twice the placement floor. The eviction finalize (`eviction_finalize.rs`) keeps its shape: a host-owned job over durable artifacts, now the checkpoint snapshot instead of a frozen dirty file. -**Resume.** Decision 3. The dirty chunks are written before the VM -starts; nothing is faulted lazily from the chunk store on the disk path. -The memory path keeps its lazy base (ADR 0092). +**Resume.** Decision 3. The host snapshots the nearest base it holds +under the `dest` name, makes the live volume a dm-clone device over it +with hydration off and discard passdown off, discards every region the +base already holds in final form and every region the manifest leaves +out (a discard on the clone marks the region hydrated without a copy; +the guest never discards, so a discard always means "already correct"), +starts the source and turns hydration on with a tuned batch and +threshold, and boots the VM. The source answers a guest read of a cold +region with a ranged `Demand` read and the kernel's ascending copy +with whole-chunk `Hydrate` reads; the chunks the guest read first on +the session's last window are fetched at `Prefix` before the copy +reaches them. A settle task polls the clone's status and, when every +region is copied, suspends the device, reloads it as a thin snapshot of the +destination, resumes it and stops the source: the path Firecracker +opened never changes. A checkpoint in the window diffs the snapshot's +mapping against the base: a block whose mapping changed (copied or +written) is the snapshot's, any other is the manifest's, so nothing is +read through the source and no write is lost; the published manifest +takes the destination's name, and only a snapshot taken after the +settle becomes a checkpoint volume: one taken in the window holds base +bytes where the copy had not reached. A live volume that hydrates and +whose source this process does not serve is never deleted; the next +host-agent serves it again from the names alone, and retries until the +store answers. A store error that can pass stalls a read of the source +instead of failing it, so the guest never sees an I/O error for an +outage. A host without the ublk control device writes each volume in +full before its guest starts and logs an error at startup. The memory +path keeps its lazy base (ADR 0092). **Teleport (ADR 0123).** A live move carries the disk in three tiers. The target stages the checkpoint the source advertises at presetup. The @@ -510,11 +548,20 @@ that adds code without retiring the row it replaces is incomplete. repair use `thin_check`, `thin_dump` and `thin_repair`. `thin_check` on a 750 GiB pool takes about 4 s. Operators lose `ls` on a dirty file and gain `dmsetup status`. -- **Staging precedes the first boot on a node.** The prefetch +- **Staging an image precedes the first boot on a node.** The prefetch supervisor already downloads and pins the full base before marking an image ready, so this is the same cost moved from a cache directory to - a volume. It is listed here because it is now structural: there is no - lazy fallback. + a volume. A session's own difference from the image is not staged: + it hydrates behind the running guest. +- **A hydrating window has a userspace process on the cold-region + path.** The host-agent serves the clone's source; a guest read of a + region the copy has not reached waits on it, and on a host-agent + restart waits until the next process serves the volume again (the + kernel queues the request, under ublk user recovery). The window is + seconds long and bounded by the copy rate; after it the kernel alone + is in the path. Stopping a ublk device with a request queued and no + server blocks in the kernel until reboot, so a source is stopped only + after the device that mapped onto it is gone. ## Testing @@ -648,6 +695,56 @@ thin-pool host and the reverse. pick mirrors the Postgres in-transaction pick, which gates on RAM and CPU; the pool floor is a coordinator-side candidate filter, like the work-dir disk floor, and is covered by the placement unit test. +- **2026-11-09: the resume hydrates behind the running guest.** The + eager stage made a cold resume of an old session a function of its + difference from the staged image: 27 to 29 s of a 39 s wake for a + 31 GB root disk that shared nothing with the hourly image, and the + internal deployment's bakes move at least 40% of the disk's blocks + every hour, so a session older than an hour is the normal case. The + live volume is now a dm-clone device over a snapshot of the nearest + base, with a ublk source the host-agent serves from the fill engine + (Decision 3). Measured on a dev-cluster c3 host: dm-clone hydrates at + about 370 MiB/s with `hydration_batch_size 16` and + `hydration_threshold 128` against about 50 MiB/s with the kernel's + defaults of 1. With a 31.7 GiB difference beside a 13 GiB memory + fill, that setting and a source window of 4 chunks left the copies + waiting on the source or on their own limit; `hydration_threshold + 512` and a window of 12 chunks copy at 760 to 1017 MiB/s, where the + pool's backing disk is the limit, and settle in 53 s. Two limits + stay: dm-clone starts no background copy while guest I/O is in flight + on the device, and a checkpoint in the window reads every copied + block back from the same disk (a copied block and a written block + look the same in the pool's mapping), which nearly stops the copy + while it runs. So the periodic checkpoint waits for the settle, for + at most 10 minutes; an eviction or a snapshot does not wait. The two + values are host-fleet chart values (`thinPool.hydration`). A whole + chunk read for a volume is not hashed: a chunk is checked when it is + written to the store or to the cache. The copy starts about 1 s after + hydration is turned + on; a `dmsetup remove` right after a create meets udev's open for + about 0.2 s and is retried. Libublk's control commands run on a + thread-local ring, so every thread that stops, recovers or deletes a + device makes one first. The reattach generation goes to 3: the + pre-upgrade drain evicts every session, so no node carries a + hydrating volume into the roll. The drain acts on hosts below the + target's generation only, so a rollback past this change needs the + hosts drained by hand first, as for generations 1 and 2: the earlier + build serves no source, and a guest on a hydrating volume would wait + on its cold regions for ever. A source holds up to 14 chunks in + memory (224 MiB at 16 MiB chunks) for each hydrating volume. A source in + its window, or settled before its first checkpoint, moves by + snapshot: it has no checkpoint volume for a live move's block diff, + and the presetup says so. The live move's three block tiers stay; the + destination's volume hydrates while the tiers land, which the + checkpoint rule covers like a guest write. The heartbeat gains no + field and `sandbox.json` records no device: the pool's names are the + record, and a restart serves every hydrating volume again from them. + The base is chosen by content, not by name: the restore compares the + session's manifest with the kept checkpoint and every staged image + version and snapshots the one that holds the most chunks in place, + so an image whose tag moved since the session began still supplies + the blocks it shares. The snapshot's `base_disk_manifest` no longer + steers the disk. ## Facts this ADR establishes @@ -679,6 +776,27 @@ names the test that guards each one. As landed, the guards are: (`a_first_checkpoint_uploads_every_nonzero_chunk_and_stage_reproduces_it`, `a_checkpoint_over_the_pool_reproduces_through_stage`, `a_checkpoint_reproduces_the_guest_disk_on_another_disk`) +- A resume boots before its disk is copied in; the guest reads a cold + region through the source and the copy settles onto the destination + under the path Firecracker opened. + (`a_hydrating_device_copies_only_the_delta_and_settles_onto_its_destination`, + `a_lazy_resume_reads_cold_chunks_through_the_source_and_settles`, + `a_lazy_resume_starts_on_a_hydrating_volume_and_settles_onto_it`) +- A checkpoint in the window reads nothing through the source and + loses no write; no checkpoint volume exists before the settle. + (`a_checkpoint_of_a_hydrating_volume_lays_the_changed_blocks_over_the_wanted_disk`, + `a_checkpoint_during_hydration_publishes_the_disk_and_keeps_the_names_honest`) +- Only a snapshot taken after the settle becomes a checkpoint volume. + (`a_snapshot_taken_while_hydrating_never_becomes_the_checkpoint_volume`) +- A host-agent restart serves a hydrating volume again from the pool's + names; a volume nothing serves is never deleted. + (`a_killed_process_leaves_a_volume_the_next_one_serves_again`, + `a_restart_serves_a_hydrating_volume_again`) +- A role without a checkpoint volume moves by snapshot. + (`a_role_without_a_checkpoint_volume_refuses_a_live_move`) +- A resume starts from the volume on the host that shares the most + chunks with its manifest, whichever image version that is. + (`a_resume_starts_from_the_staged_volume_that_shares_the_most`) - The heartbeat carries observed facts only; no field asks the coordinator to take an action. diff --git a/flake.lock b/flake.lock index 115c23098..3d459bc4b 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1776877367, - "narHash": "sha256-EHq1/OX139R1RvBzOJ0aMRT3xnWyqtHBRUBuO1gFzjI=", + "lastModified": 1793865203, + "narHash": "sha256-/cpdPHaySDIyCgZI6EWngiM6jK3YLUNzv1b7lTzy6Bk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0726a0ecb6d4e08f6adced58726b95db924cef57", + "rev": "c56a87d8e41ce69bd2af57eb35dcc117a9dfca33", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a50d52b3c..eeaee1035 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "Engram — pinned dev toolchain (Rust + just + jq + sqlx-cli + cargo-watch + cargo-nextest)."; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; rust-overlay = { url = "github:oxalica/rust-overlay"; diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 0e4efc808..7350f29b4 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -183,47 +183,61 @@ winnow = { version = "1" } zeroize = { version = "1", features = ["derive"] } [target.x86_64-unknown-linux-gnu.dependencies] +concurrent-queue = { version = "2" } +event-listener-strategy = { version = "0.5" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } libc = { version = "0.2", default-features = false, features = ["use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nix = { version = "0.31", features = ["hostname", "ioctl", "mount", "socket"] } nom = { version = "7" } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } tower = { version = "0.5", default-features = false, features = ["timeout"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-unknown-linux-gnu.build-dependencies] +bindgen = { version = "0.69" } +concurrent-queue = { version = "2" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nom = { version = "7" } +prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-unknown-linux-musl.dependencies] +concurrent-queue = { version = "2" } +event-listener-strategy = { version = "0.5" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } libc = { version = "0.2", default-features = false, features = ["use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nix = { version = "0.31", features = ["hostname", "ioctl", "mount", "socket"] } nom = { version = "7" } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } tower = { version = "0.5", default-features = false, features = ["timeout"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-unknown-linux-musl.build-dependencies] +bindgen = { version = "0.69" } +concurrent-queue = { version = "2" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nom = { version = "7" } +prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.x86_64-apple-darwin.dependencies]