diff --git a/.config/nextest.toml b/.config/nextest.toml index d5ace72bc..9c6e10996 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -56,3 +56,10 @@ slow-timeout = { period = "60s", terminate-after = 10 } [[profile.default.overrides]] filter = "package(engram-tui) & binary(perf)" threads-required = "num-cpus" + +# The ublk root tests each finish in seconds. A device that no process +# serves holds its I/O, so a test that wedges is ended at 2 minutes +# instead of the default 3. +[[profile.default.overrides]] +filter = "package(engram-disk-source) & binary(ublk_device)" +slow-timeout = { period = "30s", terminate-after = 4 } diff --git a/.github/scripts/detect-rebake-lanes.py b/.github/scripts/detect-rebake-lanes.py index 5e9036236..43c5cd4f2 100755 --- a/.github/scripts/detect-rebake-lanes.py +++ b/.github/scripts/detect-rebake-lanes.py @@ -86,7 +86,8 @@ from pathlib import Path # Binaries that bake into each artifact. -FC_BINS = {"engram-host-agent", "engram-uffd-handler", "engram-egress-proxyd"} +FC_BINS = {"engram-host-agent", "engram-uffd-handler", "engram-egress-proxyd", + "engram-disk-source"} # ADR 0098 R-CoSim (rung 1): the coordinator↔host BOUNDARY simulator. # engram-dst-cosim deliberately spans BOTH closures — it pulls @@ -111,12 +112,16 @@ # copies it to /usr/local/bin) and is spawned by path, not a cargo # dependency, so it must be listed here or a daemon-only change # deploys nowhere. +# +# engram-disk-source (ADR 0127): the same shape again — the host-agent +# container carries it and spawns it by path. CONTAINER_BINS = { "engram-coordinator", "engram-host-agent", "engram-host-operator", "engram-uffd-handler", "engram-egress-proxyd", + "engram-disk-source", } # The built-in claude harness binary. ADR 0062: it is NOT baked into any # image — it rides the fleet `current_bundles` stamp (the @@ -158,6 +163,11 @@ "engram-egress-proxyd", } +# Crates whose root tests need the kernel's ublk driver (ADR 0127). They +# run on their own lane: the Firecracker runners' kernel has no ublk. +# engram-host-agent runs its disk sources as engram-disk-source processes. +UBLK_TEST_CRATES = {"engram-disk-source", "engram-host-agent"} + # Non-crate path prefixes per lane. `Cargo.lock`/root `Cargo.toml`/this # script/the bake workflow conservatively trip the binary lanes. BINARY_COMMON = ["Cargo.lock", "Cargo.toml", @@ -440,6 +450,7 @@ def main(): agentd_closure = release_closure(meta, AGENTD_BINS) e2e_closure = release_closure(meta, E2E_BINS) cosim_closure = release_closure(meta, COSIM_BINS) + ublk_closure = release_closure(meta, UBLK_TEST_CRATES) cli = release_closure(meta, CLI_BINS) # ADR 0045 Phase B: a Firecracker-fork bump (submodule pointer) restages the @@ -567,7 +578,12 @@ def main(): # fixtures and mount bundles via the sandbox crates already in the closure; # tripping FC on a bundle-payload edit was a needless ~KVM lane. The # e2e-STACK lane still gates on `e2e`, so bundle staging is validated there. - test_fc = ci_self or bool(cc & e2e_closure) or fc_fork or any_path(changed, FC_POOL_PATHS) + test_fc = ( + ci_self + or bool(cc & e2e_closure) + or fc_fork + or any_path(changed, FC_POOL_PATHS) + ) # web / orchestrator: their own sources or the protos they codegen from. # (The `engrams` CLI is a workspace crate: test_rust covers it.) test_web = ci_self or proto or any_path(changed, WEB_PATHS) @@ -584,6 +600,7 @@ def main(): # ADR 0098 R-CoSim: the coordinator↔host boundary sim — its own (spanning) # release closure. test_cosim = ci_self or bool(cc & cosim_closure) + test_ublk = ci_self or bool(cc & ublk_closure) # ── per-image bake selectivity ───────────────────────────────────── # Each image's lane evaluates against ITS OWN changed set: the registry @@ -602,8 +619,7 @@ def main(): if baselines else ({}, {}) coord_closure = release_closure(meta, {"engram-coordinator"}) - ha_closure = release_closure( - meta, {"engram-host-agent", "engram-uffd-handler", "engram-egress-proxyd"}) + ha_closure = release_closure(meta, FC_BINS) hop_closure = release_closure(meta, {"engram-host-operator"}) dirs = crate_dirs(meta, repo_root) @@ -660,7 +676,7 @@ def image_flag(name): print(f"-> test_rust={test_rust} test_cross={test_cross} test_fc={test_fc} " f"test_web={test_web} test_orchestrator={test_orchestrator} " f"test_buf={test_buf} ci_self={ci_self} proto={proto} " - f"test_cosim={test_cosim} " + f"test_cosim={test_cosim} test_ublk={test_ublk} " f"test_integrations_cli={test_integrations_cli} " f"test_deploy={test_deploy} test_site={test_site}", file=sys.stderr) @@ -695,6 +711,7 @@ def b(v): f.write(f"test_orchestrator={b(test_orchestrator)}\n") f.write(f"test_buf={b(test_buf)}\n") f.write(f"test_cosim={b(test_cosim)}\n") + f.write(f"test_ublk={b(test_ublk)}\n") f.write(f"test_integrations_cli={b(test_integrations_cli)}\n") f.write(f"test_deploy={b(test_deploy)}\n") f.write(f"test_site={b(test_site)}\n") diff --git a/.github/workflows/bake-images.yml b/.github/workflows/bake-images.yml index 09f66c10e..27193e82d 100644 --- a/.github/workflows/bake-images.yml +++ b/.github/workflows/bake-images.yml @@ -251,7 +251,7 @@ jobs: sudo cp -r /usr/include/linux /usr/include/asm-generic /opt/uapi/ sudo cp -r /usr/include/x86_64-linux-gnu/asm /opt/uapi/asm - - name: Build engram-host-agent + engram-uffd-handler + engram-egress-proxyd (musl-static) + - name: Build engram-host-agent + engram-uffd-handler + engram-egress-proxyd + engram-disk-source (musl-static) env: # Point the musl C compile (cc-rs) + bindgen/clang at the system # kernel uapi headers so userfaultfd-sys finds . @@ -269,12 +269,15 @@ jobs: ENGRAM_EGRESS_PROXYD_FINGERPRINT="$FP" cargo build \ --target x86_64-unknown-linux-musl \ --release \ - -p engram-host-agent -p engram-uffd-handler -p engram-egress-proxyd + -p engram-host-agent -p engram-uffd-handler -p engram-egress-proxyd \ + -p engram-disk-source mkdir -p dist cp target/x86_64-unknown-linux-musl/release/engram-host-agent dist/ cp target/x86_64-unknown-linux-musl/release/engram-uffd-handler dist/ cp target/x86_64-unknown-linux-musl/release/engram-egress-proxyd dist/ - file dist/engram-host-agent dist/engram-uffd-handler dist/engram-egress-proxyd + cp target/x86_64-unknown-linux-musl/release/engram-disk-source dist/ + file dist/engram-host-agent dist/engram-uffd-handler dist/engram-egress-proxyd \ + dist/engram-disk-source - name: Log in to GHCR uses: docker/login-action@v4 @@ -317,7 +320,8 @@ jobs: --artifact-type application/vnd.engrams.fc-host-binaries.v1 \ engram-host-agent:application/octet-stream \ engram-uffd-handler:application/octet-stream \ - engram-egress-proxyd:application/octet-stream + engram-egress-proxyd:application/octet-stream \ + engram-disk-source:application/octet-stream # Convenience tags. `oras tag` is a cheap registry-side re-point. tag_with_retry "$REPO:$SHA" "$SHORT" if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1fad25f2..8c552e017 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,6 +131,7 @@ jobs: test_web: ${{ steps.lanes.outputs.test_web }} test_orchestrator: ${{ steps.lanes.outputs.test_orchestrator }} test_cosim: ${{ steps.lanes.outputs.test_cosim }} + test_ublk: ${{ steps.lanes.outputs.test_ublk }} test_buf: ${{ steps.lanes.outputs.test_buf }} test_integrations_cli: ${{ steps.lanes.outputs.test_integrations_cli }} test_deploy: ${{ steps.lanes.outputs.test_deploy }} @@ -1166,15 +1167,38 @@ jobs: - name: Node-prep pool states (root) run: sudo sh scripts/thin-pool-states-test.sh - # The disk over the pool, then the session disk under a real guest - # over the same pool (`session_disk` reads ENGRAM_TEST_THIN_POOL). + - name: kernel block features (report) + run: | + uname -r + sudo dmsetup targets + ls /dev/ublk-control 2>&1 || true + for f in /proc/config.gz "/boot/config-$(uname -r)"; do + [ -r "$f" ] && { zcat -f "$f" | grep -E 'CONFIG_(DM_CLONE|DM_DELAY|BLK_DEV_UBLK)=' || true; } + done + grep -E 'dm-clone|dm-delay|ublk' "/lib/modules/$(uname -r)/modules.builtin" 2>/dev/null || true + + # The disk over the pool, its cloned drives over a loop-device + # source (dm-delay slows the copy where a test needs it), then the + # session disk under a real guest over the same pool (`session_disk` + # reads ENGRAM_TEST_THIN_POOL). A runner without the clone or delay + # target fails the lane, as a missing pool does. - name: dm-thin session-disk tests (root) env: ENGRAM_TEST_THIN_POOL: engram-pool run: | + # A target built into the kernel has no module, so the check is + # the kernel's target list, not modprobe. + for t in clone delay; do + sudo dmsetup targets | grep -q "^$t " || sudo modprobe "dm_$t" || true + done + for t in clone delay; do + sudo dmsetup targets | grep -q "^$t " \ + || { echo "FATAL: kernel $(uname -r) has no dm-$t target" >&2; exit 1; } + done sudo -E env "PATH=$PATH" cargo nextest run \ -p engram-disk-thin \ --test thin_pool \ + --test clone \ --run-ignored ignored-only \ --no-fail-fast \ --test-threads=1 @@ -1230,6 +1254,69 @@ jobs: run: | sudo chown -R "$USER":"$USER" target/ ~/.cargo ~/.docker 2>/dev/null || true + # ADR 0127: the disk source against the kernel's ublk driver. The + # Blacksmith kernel has no ublk, and the 6.17 kernel of the hosted + # 24.04 image fails on a device add, so this lane uses the hosted 22.04 + # image (kernel 6.8, the driver in its extra modules). A runner that + # cannot load the driver fails the lane. + test-ublk: + name: tests (ublk disk source, root) + needs: detect + if: needs.detect.outputs.test_ublk == 'true' + runs-on: ubuntu-22.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - name: Load the ublk driver + run: | + sudo apt-get update -qq + sudo modprobe ublk_drv || { + sudo apt-get install -y -qq --no-install-recommends "linux-modules-extra-$(uname -r)" + sudo modprobe ublk_drv + } + test -c /dev/ublk-control \ + || { echo "FATAL: kernel $(uname -r) has no ublk driver" >&2; exit 1; } + - uses: dtolnay/rust-toolchain@stable + # The 22.04 protoc is too old for proto3 optional fields; pin the + # release the bake uses. + - uses: arduino/setup-protoc@v3 + with: + version: "25.9" + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install libclang + run: sudo apt-get install -y -qq --no-install-recommends libclang-dev + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ublk + - name: Install nextest + uses: taiki-e/install-action@nextest + - name: ublk disk-source tests (root) + env: + ENGRAM_TEST_UBLK: "1" + run: | + cargo nextest run -p engram-disk-source --no-run + sudo -E env "PATH=$PATH" cargo nextest run \ + -p engram-disk-source \ + --test ublk_device \ + --run-ignored ignored-only \ + --no-fail-fast \ + --test-threads=1 + # The host-agent runs each disk source as a process of its own. + - name: host-agent disk-source process tests (root) + env: + ENGRAM_TEST_UBLK: "1" + run: | + cargo build -p engram-disk-source --bin engram-disk-source + cargo nextest run -p engram-host-agent --test disk_source_ublk --no-run + sudo -E env "PATH=$PATH" \ + "ENGRAM_DISK_SOURCE_BIN=$PWD/target/debug/engram-disk-source" \ + cargo nextest run \ + -p engram-host-agent \ + --test disk_source_ublk \ + --run-ignored ignored-only \ + --no-fail-fast \ + --test-threads=1 + test-fc-e2e: name: tests (firecracker true-e2e, Linux + KVM) needs: detect @@ -2417,6 +2504,7 @@ jobs: - test-fc-unprivileged - test-fc-pooled - test-fc-root + - test-ublk - test-fc-e2e - vz - test-macos diff --git a/Cargo.lock b/Cargo.lock index ae825eeef..a63a86f22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,6 +247,43 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + [[package]] name = "async-graphql-parser" version = "7.2.1" @@ -271,6 +308,82 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.6", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.6", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.6", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -293,6 +406,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.92" @@ -989,12 +1108,15 @@ dependencies = [ "itertools 0.12.1", "lazy_static", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex 1.3.0", "syn 2.0.117", + "which", ] [[package]] @@ -1055,6 +1177,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitmaps" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" + [[package]] name = "blake3" version = "1.8.7" @@ -1095,6 +1223,19 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "borsh" version = "1.8.1" @@ -1549,7 +1690,7 @@ dependencies = [ "futures-core", "mio", "parking_lot 0.12.5", - "rustix", + "rustix 1.1.6", "signal-hook", "signal-hook-mio", "winapi", @@ -1662,6 +1803,16 @@ dependencies = [ "darling_macro 0.20.11", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + [[package]] name = "darling" version = "0.24.1" @@ -1686,6 +1837,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + [[package]] name = "darling_core" version = "0.24.1" @@ -1710,6 +1875,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.24.1" @@ -1883,6 +2059,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_setters" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e6f6fa1f03c14ae082120b84b3c7fbd7b8588d924cf2d7c3daf9afd49df8b9" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -2352,12 +2540,40 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "engram-disk-source" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "clap", + "engram-blob-client", + "engram-chunk-store", + "engram-core", + "engram-storage-local", + "engram-testkit", + "futures", + "io-uring", + "libc", + "libublk", + "nix 0.31.3", + "parking_lot 0.13.0", + "proptest", + "smol", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "workspace-hack", +] + [[package]] name = "engram-disk-thin" version = "0.1.0" dependencies = [ "async-trait", "engram-chunk-store", + "engram-core", "engram-host-core", "engram-storage-local", "libc", @@ -3237,7 +3453,7 @@ dependencies = [ "proptest", "pulldown-cmark", "ratatui", - "rustix", + "rustix 1.1.6", "serde", "serde_json", "syntect", @@ -3337,7 +3553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3577,6 +3793,19 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.34" @@ -3600,6 +3829,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.34" @@ -4334,6 +4569,18 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bindgen", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.12.1" @@ -4807,6 +5054,41 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libublk" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212246283957aba80cbd6f04586d19eb57cf94aa49232c2a8a6b25b72c356239" +dependencies = [ + "async-lock", + "bitflags 2.13.2", + "bitmaps", + "derive_setters", + "futures-timer", + "io-uring", + "libc", + "libublk-rs-sys", + "log", + "serde", + "serde_json", + "slab", + "smol", + "thiserror 1.0.69", +] + +[[package]] +name = "libublk-rs-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd981fd7cf597c782453c2520f0d8f511dd6f66fc184b68577d7c7cf7108e44" +dependencies = [ + "anyhow", + "bindgen", + "libc", + "regex", + "serde", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -4816,6 +5098,12 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -5940,6 +6228,17 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -5973,6 +6272,20 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.6", + "windows-sys 0.61.2", +] + [[package]] name = "polyval" version = "0.7.3" @@ -6133,7 +6446,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.12.1", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -6154,7 +6467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -6701,6 +7014,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.6" @@ -6710,8 +7036,8 @@ dependencies = [ "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys", - "windows-sys 0.52.0", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", ] [[package]] @@ -6777,7 +7103,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7223,6 +7549,23 @@ dependencies = [ "serde", ] +[[package]] +name = "smol" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33bd3e260892199c3ccfc487c88b2da2265080acb316cd920da72fdfd7c599f" +dependencies = [ + "async-channel", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-net", + "async-process", + "blocking", + "futures-lite", +] + [[package]] name = "socket2" version = "0.6.5" @@ -7621,8 +7964,8 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", - "windows-sys 0.52.0", + "rustix 1.1.6", + "windows-sys 0.61.2", ] [[package]] @@ -7633,7 +7976,7 @@ checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" dependencies = [ "bitflags 2.13.2", "parking_lot 0.12.5", - "rustix", + "rustix 1.1.6", "signal-hook", "windows-sys 0.61.2", ] @@ -8862,6 +9205,18 @@ dependencies = [ "wezterm-dynamic", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "1.6.1" @@ -8894,7 +9249,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -9251,10 +9606,12 @@ dependencies = [ "aws-smithy-types", "axum", "axum-core", + "bindgen", "bitflags 2.13.2", "bytes", "cc", "chrono", + "concurrent-queue", "crc32fast", "crossbeam-utils", "crypto-common 0.1.7", @@ -9267,6 +9624,7 @@ dependencies = [ "displaydoc", "either", "errno", + "event-listener-strategy", "flate2", "form_urlencoded", "futures-channel", @@ -9287,8 +9645,10 @@ dependencies = [ "hyper-util", "idna", "indexmap", + "itertools 0.14.0", "lazy_static", "libc", + "linux-raw-sys 0.12.1", "log", "memchr", "mio", @@ -9303,6 +9663,7 @@ dependencies = [ "percent-encoding", "phf_shared", "ppv-lite86", + "prettyplease", "proc-macro2", "prost", "prost-build", @@ -9317,7 +9678,7 @@ dependencies = [ "regex-syntax", "reqwest", "rsa", - "rustix", + "rustix 1.1.6", "rustls", "serde", "serde_core", @@ -9385,7 +9746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08" dependencies = [ "libc", - "rustix", + "rustix 1.1.6", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5168c42d3..d790bb7c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/engram-host-agent", "crates/engram-host-core", "crates/engram-disk-thin", + "crates/engram-disk-source", "crates/engram-host-operator", "crates/engram-upgrade-drain", "crates/engram-chunk-store", diff --git a/crates/engram-disk-source/Cargo.toml b/crates/engram-disk-source/Cargo.toml new file mode 100644 index 000000000..170c0c9d7 --- /dev/null +++ b/crates/engram-disk-source/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "engram-disk-source" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "A read-only ublk block device whose bytes are one disk manifest, served from a local base volume, the chunk cache and the chunk store" + +[lints] +workspace = true + +[[bin]] +name = "engram-disk-source" +path = "src/main.rs" + +[dependencies] +bytes.workspace = true +clap.workspace = true +engram-blob-client.workspace = true +engram-chunk-store.workspace = true +engram-core.workspace = true +futures.workspace = true +nix = { version = "0.31", default-features = false, features = ["fs"] } +parking_lot.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +workspace-hack = { version = "0.1", path = "../../workspace-hack" } + +[target.'cfg(target_os = "linux")'.dependencies] +io-uring = "0.7" +libc = "0.2" +libublk = "0.4.8" +nix = { version = "0.31", default-features = false, features = ["event", "fs"] } +smol = "2" + +[dev-dependencies] +async-trait.workspace = true +engram-storage-local.workspace = true +engram-testkit.workspace = true +proptest.workspace = true +tempfile = "3" +tokio = { workspace = true, features = ["test-util"] } + +[target.'cfg(target_os = "linux")'.dev-dependencies] +libc = "0.2" +nix = { version = "0.31", default-features = false, features = ["signal"] } +tokio = { workspace = true, features = ["rt"] } diff --git a/crates/engram-disk-source/src/lib.rs b/crates/engram-disk-source/src/lib.rs new file mode 100644 index 000000000..f88ce8dca --- /dev/null +++ b/crates/engram-disk-source/src/lib.rs @@ -0,0 +1,13 @@ +//! A read-only block device whose bytes are one disk manifest (ADR 0127). +//! +//! [`DiskReader`] maps a read at any offset to the manifest's chunks and +//! runs on every platform, as does [`policy`], which decides when a +//! failed read is attempted again. [`ublk`] serves it as `/dev/ublkbN` +//! on Linux. + +pub mod policy; +pub mod reader; +#[cfg(target_os = "linux")] +pub mod ublk; + +pub use reader::{DiskReader, LocalBase}; diff --git a/crates/engram-disk-source/src/main.rs b/crates/engram-disk-source/src/main.rs new file mode 100644 index 000000000..723c293d4 --- /dev/null +++ b/crates/engram-disk-source/src/main.rs @@ -0,0 +1,192 @@ +//! `engram-disk-source serve`: serve one disk manifest as a read-only +//! ublk device until SIGTERM or SIGINT, then delete the device. Its state +//! is a function of its arguments and the blob-store environment +//! (`ENGRAM_BLOB_BACKEND` and the backend's settings), so a new process +//! with the same arguments and `--recover` takes over a device whose +//! process died. `engram-disk-source delete` removes a device that no +//! process serves. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::{Args, Parser, Subcommand}; +use engram_chunk_store::ManifestRef; + +#[derive(Parser)] +#[command( + name = "engram-disk-source", + about = "Serve a disk manifest as a read-only block device" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Create (or recover) the device, print " " + /// on one line when it serves I/O, and serve until SIGTERM or SIGINT. + Serve(ServeArgs), + /// Delete a device that no process serves: one its server stopped and + /// did not delete, or, with --quiesced, one whose server died. A device + /// that a process serves is never deleted. + Delete(DeleteArgs), +} + +#[derive(Args)] +struct DeleteArgs { + /// The device id. + #[arg(long)] + device: u32, + /// Also delete a device whose server died. Each read the device holds + /// fails, so use it only when nothing reads the device. + #[arg(long)] + quiesced: bool, +} + +#[derive(Args)] +struct ServeArgs { + /// The disk manifest to serve, as @v. + #[arg(long)] + manifest: ManifestRef, + /// A local volume to read chunks from when it holds a chunk with the + /// same hash. + #[arg(long, requires = "base_manifest")] + base_volume: Option, + /// The manifest the base volume holds, as @v. + #[arg(long, requires = "base_volume")] + base_manifest: Option, + /// The host chunk cache to read resident chunks from. It is never + /// written. + #[arg(long)] + cache_root: Option, + /// Attach to the existing device with this id instead of creating one. + #[arg(long)] + recover: Option, + /// Hardware queues. With --recover, the device's own count. + #[arg(long, default_value_t = 2)] + queues: u16, + /// Requests in flight per queue. With --recover, the device's own + /// depth. + #[arg(long, default_value_t = 64)] + depth: u16, +} + +fn main() -> ExitCode { + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + let result = match Cli::parse().command { + Command::Serve(args) => serve(args), + Command::Delete(args) => delete(args), + }; + match result { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("engram-disk-source: {e}"); + ExitCode::FAILURE + } + } +} + +#[cfg(not(target_os = "linux"))] +fn serve(_args: ServeArgs) -> std::io::Result<()> { + unsupported() +} + +#[cfg(not(target_os = "linux"))] +fn delete(_args: DeleteArgs) -> std::io::Result<()> { + unsupported() +} + +#[cfg(not(target_os = "linux"))] +fn unsupported() -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "ublk block devices need Linux", + )) +} + +#[cfg(target_os = "linux")] +fn delete(args: DeleteArgs) -> std::io::Result<()> { + engram_disk_source::ublk::delete_unserved(args.device, args.quiesced) +} + +#[cfg(target_os = "linux")] +fn serve(args: ServeArgs) -> std::io::Result<()> { + use std::io::Write; + use std::sync::Arc; + + use engram_disk_source::ublk; + use tokio::signal::unix::{signal, SignalKind}; + + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + // Before the device exists, so a stop that arrives during start-up + // still deletes it. + let (mut term, mut int) = { + let _guard = rt.enter(); + ( + signal(SignalKind::terminate())?, + signal(SignalKind::interrupt())?, + ) + }; + let stop = async move { + tokio::select! { + _ = term.recv() => {} + _ = int.recv() => {} + } + }; + let reader = rt.block_on(open_reader(&args))?; + let config = ublk::Config { + queues: args.queues, + depth: args.depth, + recover: args.recover, + }; + let on_ready = |device: &ublk::Device| { + tracing::info!(dev_id = device.dev_id, path = %device.path, "serving"); + let mut out = std::io::stdout().lock(); + let _ = writeln!(out, "{} {}", device.path, device.dev_id); + let _ = out.flush(); + }; + ublk::serve( + Arc::new(reader), + rt.handle().clone(), + &config, + on_ready, + stop, + ) +} + +#[cfg(target_os = "linux")] +async fn open_reader(args: &ServeArgs) -> std::io::Result { + use engram_chunk_store::reader::ChunkCacheReader; + use engram_chunk_store::ChunkStore; + use engram_disk_source::{DiskReader, LocalBase}; + + let blob = engram_blob_client::from_env() + .await + .map_err(std::io::Error::other)?; + let store = ChunkStore::new(blob); + let manifest = store + .get_manifest(args.manifest) + .await + .map_err(std::io::Error::other)?; + let mut reader = DiskReader::new(&manifest, store.clone())?; + if let (Some(path), Some(base_ref)) = (&args.base_volume, args.base_manifest) { + let base_manifest = store + .get_manifest(base_ref) + .await + .map_err(std::io::Error::other)?; + reader = reader.with_base(LocalBase::open(path, &base_manifest)?); + } + if let Some(root) = &args.cache_root { + reader = reader.with_cache(ChunkCacheReader::new(root)); + } + Ok(reader) +} diff --git a/crates/engram-disk-source/src/policy.rs b/crates/engram-disk-source/src/policy.rs new file mode 100644 index 000000000..9655713da --- /dev/null +++ b/crates/engram-disk-source/src/policy.rs @@ -0,0 +1,183 @@ +//! When a failed read is attempted again, and when it is refused. + +use std::io; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::reader::DiskReader; + +/// The first wait between attempts at a read that fails. +const FIRST_RETRY_DELAY: Duration = Duration::from_millis(50); +/// The longest wait between attempts at a read that fails. +const MAX_RETRY_DELAY: Duration = Duration::from_secs(5); + +/// A read that fails for a reason that may pass is attempted again until +/// it succeeds, so a slow or unreachable store stalls the guest's read +/// and never fails it. A chunk the store does not hold, or whose bytes +/// fail the hash check, is data loss: after one fresh fetch the read is +/// refused. A request outside the disk, or a read still waiting when +/// `stopping` turns true, is refused at once. +pub async fn read_until_served( + reader: &DiskReader, + offset: u64, + len: usize, + mut stopping: watch::Receiver, +) -> io::Result> { + let stopped = || io::Error::other("the server is stopping"); + let mut delay = FIRST_RETRY_DELAY; + let mut attempt = 1u32; + let mut lost_once = false; + loop { + let result = tokio::select! { + // A read that is ready is served even when a stop is too. + biased; + result = reader.read(offset, len) => result, + _ = stopping.wait_for(|s| *s) => return Err(stopped()), + }; + match result { + Ok(bytes) => return Ok(bytes), + Err(e) if e.kind() == io::ErrorKind::InvalidInput => return Err(e), + Err(e) if is_lost(&e) && lost_once => { + tracing::error!(offset, len, error = %e, "disk source data lost: the chunk store cannot serve this read"); + return Err(e); + } + Err(e) if is_lost(&e) => { + // The failed fetch left no cached copy, so this attempt + // fetches again. + tracing::warn!(offset, len, error = %e, "read failed for lost data; fetching once more"); + lost_once = true; + } + Err(e) => { + // An outage fails every attempt; log the 1st, 2nd, 4th, ... + if attempt.is_power_of_two() { + tracing::warn!(offset, len, attempt, error = %e, "read failed; trying again"); + } else { + tracing::debug!(offset, len, attempt, error = %e, "read failed; trying again"); + } + tokio::select! { + _ = tokio::time::sleep(delay) => {} + _ = stopping.wait_for(|s| *s) => return Err(stopped()), + } + delay = (delay * 2).min(MAX_RETRY_DELAY); + attempt = attempt.saturating_add(1); + } + } + } +} + +fn is_lost(e: &io::Error) -> bool { + matches!( + e.kind(), + io::ErrorKind::NotFound | io::ErrorKind::InvalidData + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use engram_chunk_store::{ChunkHash, ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind}; + use engram_core::traits::storage::BlobStorage; + use engram_storage_local::LocalBlobStorage; + use engram_testkit::storage::{ + Counters, FaultPlan, FaultyBlobStorage, GetFault, GetFaultKind, InjectedError, KeyMatch, + When, + }; + + const CHUNK: u64 = 64 * 1024; + + struct Fixture { + _dir: tempfile::TempDir, + reader: DiskReader, + counters: Arc, + expected: Vec, + } + + /// A one-chunk disk whose store applies `faults` to the chunk. + async fn fixture(faults: Vec<(When, GetFaultKind)>) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let expected: Vec = (0..CHUNK as usize).map(|i| (i % 251) as u8).collect(); + let hash = ChunkHash::of(&expected); + let local: Arc = Arc::new(LocalBlobStorage::new(dir.path())); + ChunkStore::new(Arc::clone(&local)) + .put_chunk(&expected) + .await + .unwrap(); + let plan = faults + .into_iter() + .fold(FaultPlan::new(), |plan, (when, kind)| { + plan.with_get(GetFault { + key: KeyMatch::Exact(hash.storage_key()), + when, + kind, + }) + }); + let (blob, counters) = FaultyBlobStorage::arc(local, plan); + let mut manifest = Manifest::empty(ManifestKind::Disk, CHUNK); + manifest.chunk_size = ChunkSize::bytes(CHUNK); + manifest.chunks = vec![ChunkRef { offset: 0, hash }]; + Fixture { + _dir: dir, + reader: DiskReader::new(&manifest, ChunkStore::new(blob)).unwrap(), + counters, + expected, + } + } + + fn unreachable() -> GetFaultKind { + GetFaultKind::NotFound(InjectedError::Sdk("the store is unreachable".into())) + } + + #[tokio::test(start_paused = true)] + async fn a_transient_store_error_is_tried_again_until_the_read_succeeds() { + let f = fixture((1..=3).map(|n| (When::Nth(n), unreachable())).collect()).await; + let (_stop, stopping) = watch::channel(false); + let got = read_until_served(&f.reader, 0, 4096, stopping) + .await + .unwrap(); + assert_eq!(got, &f.expected[..4096]); + assert_eq!(f.counters.gets_faulted(), 3); + assert_eq!(f.counters.gets_attempted(), 4); + } + + #[tokio::test(start_paused = true)] + async fn a_chunk_the_store_does_not_hold_is_fetched_twice_then_refused() { + let f = fixture(vec![( + When::Always, + GetFaultKind::NotFound(InjectedError::NotFound), + )]) + .await; + let (_stop, stopping) = watch::channel(false); + let err = read_until_served(&f.reader, 0, 4096, stopping) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::NotFound, "{err}"); + assert_eq!(f.counters.gets_attempted(), 2); + } + + #[tokio::test(start_paused = true)] + async fn a_stop_ends_a_read_that_is_trying_again() { + let f = Arc::new(fixture(vec![(When::Always, unreachable())]).await); + let (stop, stopping) = watch::channel(false); + let read = { + let f = Arc::clone(&f); + tokio::spawn(async move { read_until_served(&f.reader, 0, 4096, stopping).await }) + }; + while f.counters.gets_faulted() < 3 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!read.is_finished(), "the read waits while the store fails"); + stop.send(true).unwrap(); + let err = read.await.unwrap().unwrap_err(); + assert_eq!(err.to_string(), "the server is stopping"); + let attempts = f.counters.gets_attempted(); + tokio::time::sleep(MAX_RETRY_DELAY * 2).await; + assert_eq!( + f.counters.gets_attempted(), + attempts, + "a stopped read does not try again" + ); + } +} diff --git a/crates/engram-disk-source/src/reader.rs b/crates/engram-disk-source/src/reader.rs new file mode 100644 index 000000000..90a6e43f6 --- /dev/null +++ b/crates/engram-disk-source/src/reader.rs @@ -0,0 +1,758 @@ +//! [`DiskReader`]: the bytes of one disk manifest at any offset. +//! +//! A read takes each chunk it spans from the first tier that holds it: +//! a staged local volume with a chunk of the same hash, then the host's +//! chunk cache (read-only), then the chunk store. An offset the manifest +//! lists no chunk for reads as zeros and fetches nothing. Recently used +//! chunks stay in memory, and concurrent reads of one chunk share one +//! fetch. + +use std::collections::HashMap; +use std::fs::File; +use std::future::Future; +use std::io::{self, Seek, SeekFrom}; +use std::os::unix::fs::FileExt; +use std::path::Path; +use std::sync::Arc; + +use bytes::Bytes; +use engram_chunk_store::reader::ChunkCacheReader; +use engram_chunk_store::{ChunkHash, ChunkStore, Manifest, ManifestKind}; +use parking_lot::Mutex; +use tokio::sync::OnceCell; + +/// Chunks kept in memory when the caller sets no other bound. +pub const DEFAULT_MEMO_CHUNKS: usize = 8; + +/// A staged volume and the manifest it holds, indexed by chunk hash. +#[derive(Clone)] +pub struct LocalBase { + file: Arc, + size_bytes: u64, + offset_of: HashMap, +} + +impl LocalBase { + /// Open the volume at `path` read-only. `manifest` must describe + /// what the volume holds. + pub fn open(path: &Path, manifest: &Manifest) -> io::Result { + manifest.validate().map_err(io::Error::other)?; + let mut file = File::open(path)?; + // A block device reports no length in its metadata; its end does. + let size_bytes = file.seek(SeekFrom::End(0))?; + if size_bytes < manifest.total_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "base volume {} is {size_bytes} bytes; its manifest is {} bytes", + path.display(), + manifest.total_bytes + ), + )); + } + Ok(Self { + file: Arc::new(file), + size_bytes, + offset_of: manifest.chunks.iter().map(|c| (c.hash, c.offset)).collect(), + }) + } + + /// The chunk `hash` at `offset`; bytes that do not hash to `hash` + /// are refused. + async fn read(&self, hash: ChunkHash, offset: u64, len: usize) -> io::Result { + if offset.saturating_add(len as u64) > self.size_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("base read {offset}+{len} is past the end of the volume"), + )); + } + let file = Arc::clone(&self.file); + tokio::task::spawn_blocking(move || { + let mut buf = vec![0u8; len]; + file.read_exact_at(&mut buf, offset)?; + drop_cached_pages(&file, offset, len); + if ChunkHash::of(&buf) != hash { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("the {len} bytes at {offset} do not hash to {hash}"), + )); + } + Ok(Bytes::from(buf)) + }) + .await + .map_err(|e| io::Error::other(format!("base read task: {e}")))? + } +} + +/// The source keeps the chunk in its own memory, so the host page cache +/// gains nothing from a second copy. +#[cfg(target_os = "linux")] +fn drop_cached_pages(file: &File, offset: u64, len: usize) { + use nix::fcntl::{posix_fadvise, PosixFadviseAdvice}; + let _ = posix_fadvise( + file, + offset as i64, + len as i64, + PosixFadviseAdvice::POSIX_FADV_DONTNEED, + ); +} + +#[cfg(not(target_os = "linux"))] +fn drop_cached_pages(_file: &File, _offset: u64, _len: usize) {} + +/// The bytes of one disk manifest. +pub struct DiskReader { + chunk_size: u64, + total_bytes: u64, + /// Hash of the chunk at each index; `None` reads as zeros. + chunks: Vec>, + tiers: Arc, + memo: Memo, +} + +/// Where a chunk's bytes come from, first match wins. +#[derive(Clone)] +struct Tiers { + base: Option, + cache: Option, + store: ChunkStore, +} + +impl DiskReader { + /// A reader of `manifest` that fetches from `store`. + pub fn new(manifest: &Manifest, store: ChunkStore) -> io::Result { + manifest.validate().map_err(io::Error::other)?; + if manifest.kind != ManifestKind::Disk { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a disk source serves disk manifests only", + )); + } + let chunk_size = manifest.chunk_size.as_u64(); + let mut chunks = vec![None; manifest.expected_chunk_count() as usize]; + for c in &manifest.chunks { + chunks[(c.offset / chunk_size) as usize] = Some(c.hash); + } + Ok(Self { + chunk_size, + total_bytes: manifest.total_bytes, + chunks, + tiers: Arc::new(Tiers { + base: None, + cache: None, + store, + }), + memo: Memo::new(DEFAULT_MEMO_CHUNKS), + }) + } + + /// Serve chunks the staged volume `base` holds from it. + pub fn with_base(mut self, base: LocalBase) -> Self { + let mut tiers = Arc::unwrap_or_clone(self.tiers); + tiers.base = Some(base); + self.tiers = Arc::new(tiers); + self + } + + /// Serve chunks resident in the host chunk cache from it. The source + /// only reads the cache; the host-agent owns its contents. + pub fn with_cache(mut self, cache: ChunkCacheReader) -> Self { + let mut tiers = Arc::unwrap_or_clone(self.tiers); + tiers.cache = Some(cache); + self.tiers = Arc::new(tiers); + self + } + + /// Keep at most `chunks` (at least one) chunks in memory. + pub fn with_memo_chunks(mut self, chunks: usize) -> Self { + self.memo = Memo::new(chunks); + self + } + + pub fn size_bytes(&self) -> u64 { + self.total_bytes + } + + /// `len` bytes at `offset`; the range must lie inside the disk. + pub async fn read(&self, offset: u64, len: usize) -> io::Result> { + let end = offset + .checked_add(len as u64) + .filter(|&end| end <= self.total_bytes) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "read {offset}+{len} is past the end of the disk ({} bytes)", + self.total_bytes + ), + ) + })?; + let mut out = vec![0u8; len]; + if len == 0 { + return Ok(out); + } + let first = offset / self.chunk_size; + let last = (end - 1) / self.chunk_size; + let spans = (first..=last).filter_map(|index| { + let hash = self.chunks[index as usize]?; + Some(async move { Ok::<_, io::Error>((index, self.chunk(index, hash).await?)) }) + }); + for (index, bytes) in futures::future::try_join_all(spans).await? { + let chunk_start = index * self.chunk_size; + let from = offset.max(chunk_start); + let to = end.min(chunk_start + self.chunk_size); + out[(from - offset) as usize..(to - offset) as usize].copy_from_slice( + &bytes[(from - chunk_start) as usize..(to - chunk_start) as usize], + ); + } + Ok(out) + } + + /// The fetch runs in a task of its own, so the network, hashing and + /// copying of different chunks spread across the runtime's workers. + async fn chunk(&self, index: u64, hash: ChunkHash) -> io::Result { + let start = index * self.chunk_size; + let len = (self.total_bytes - start).min(self.chunk_size) as usize; + let tiers = Arc::clone(&self.tiers); + self.memo + .get_or_fetch(hash, || async move { + tokio::spawn(async move { tiers.fetch(hash, len).await }) + .await + .map_err(|e| io::Error::other(format!("chunk fetch task: {e}")))? + }) + .await + } +} + +impl Tiers { + async fn fetch(&self, hash: ChunkHash, len: usize) -> io::Result { + if let Some(base) = &self.base { + if let Some(&offset) = base.offset_of.get(&hash) { + match base.read(hash, offset, len).await { + Ok(bytes) => return Ok(bytes), + Err(e) => { + tracing::warn!(%hash, offset, error = %e, "base volume read failed; reading the chunk from the store") + } + } + } + } + if let Some(cache) = &self.cache { + let cache = cache.clone(); + let found = tokio::task::spawn_blocking(move || cache.read(hash)) + .await + .map_err(|e| io::Error::other(format!("cache read task: {e}")))?; + match found { + Ok(Some(bytes)) if bytes.len() == len => return Ok(Bytes::from(bytes)), + Ok(Some(bytes)) => { + tracing::warn!(%hash, len = bytes.len(), want = len, "cached chunk has the wrong length; reading it from the store") + } + Ok(None) => {} + Err(e) => { + tracing::warn!(%hash, error = %e, "chunk cache read failed; reading the chunk from the store") + } + } + } + let bytes = self.store.get_chunk(hash).await.map_err(store_error)?; + if bytes.len() != len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "chunk {hash} is {} bytes; the manifest needs {len}", + bytes.len() + ), + )); + } + Ok(bytes) + } +} + +/// A chunk the store does not hold reads as `NotFound`, and bytes that +/// fail the hash check as `InvalidData`: waiting does not repair either. +/// Every other store error may pass. +fn store_error(e: engram_chunk_store::ChunkStoreError) -> io::Error { + use engram_chunk_store::ChunkStoreError; + use engram_core::error::BlobError; + let kind = match &e { + ChunkStoreError::Blob(BlobError::NotFound) => io::ErrorKind::NotFound, + ChunkStoreError::HashMismatch { .. } => io::ErrorKind::InvalidData, + _ => io::ErrorKind::Other, + }; + io::Error::new(kind, e) +} + +/// Recently used chunks, least recently used first out. One cell per +/// chunk: the first reader fetches and the others wait on the cell. A +/// failed fetch leaves the cell empty, so the next reader fetches again. +/// A cell that a reader holds is never evicted, so concurrent reads of +/// one chunk share one fetch even when more chunks are in flight than +/// the memo keeps; the memo then holds more cells until those reads end. +struct Memo { + capacity: usize, + state: Mutex, +} + +#[derive(Default)] +struct MemoState { + clock: u64, + cells: HashMap>)>, +} + +impl Memo { + fn new(capacity: usize) -> Self { + Self { + capacity: capacity.max(1), + state: Mutex::default(), + } + } + + async fn get_or_fetch(&self, hash: ChunkHash, fetch: F) -> io::Result + where + F: FnOnce() -> Fut, + Fut: Future>, + { + self.cell(hash).get_or_try_init(fetch).await.cloned() + } + + fn cell(&self, hash: ChunkHash) -> Arc> { + let mut state = self.state.lock(); + state.clock += 1; + let now = state.clock; + let entry = state + .cells + .entry(hash) + .or_insert_with(|| (now, Arc::new(OnceCell::new()))); + entry.0 = now; + let cell = Arc::clone(&entry.1); + let excess = state.cells.len().saturating_sub(self.capacity); + if excess > 0 { + // Cells are cloned only under this lock, so a count of one + // means no reader holds the cell. The cell just returned is + // held. + let mut idle: Vec<(u64, ChunkHash)> = state + .cells + .iter() + .filter(|(_, (_, cell))| Arc::strong_count(cell) == 1) + .map(|(hash, (used, _))| (*used, *hash)) + .collect(); + idle.sort_unstable_by_key(|(used, _)| *used); + for (_, hash) in idle.into_iter().take(excess) { + state.cells.remove(&hash); + } + } + cell + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; + + use async_trait::async_trait; + use engram_chunk_store::{ChunkRef, ChunkSize}; + use engram_core::error::BlobError; + use engram_core::traits::storage::{BlobObjectMeta, BlobStorage, ByteStream}; + use engram_storage_local::LocalBlobStorage; + use engram_testkit::storage::{Counters, FaultPlan, FaultyBlobStorage}; + use proptest::prelude::*; + + const CHUNK: u64 = 64 * 1024; + + /// A disk of `CHUNK`-sized chunks with a short final chunk. `None` + /// slots are holes. + fn layout() -> Vec> { + vec![ + Some(1), + None, + Some(2), + Some(3), + None, + None, + Some(4), + Some(5), + ] + } + + const SHORT_TAIL: u64 = 12 * 1024; + + fn total_bytes() -> u64 { + (layout().len() as u64 - 1) * CHUNK + SHORT_TAIL + } + + /// Distinct, position-dependent bytes for chunk `seed`. + fn chunk_bytes(seed: u8, len: usize) -> Vec { + (0..len) + .map(|i| (i as u32).wrapping_mul(2_654_435_761).to_le_bytes()[seed as usize % 4] ^ seed) + .collect() + } + + struct Fixture { + _dir: tempfile::TempDir, + store: ChunkStore, + counters: Arc, + manifest: Manifest, + expected: Vec, + } + + async fn fixture() -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let local: Arc = Arc::new(LocalBlobStorage::new(dir.path())); + let (blob, counters) = FaultyBlobStorage::arc(local, FaultPlan::new()); + let store = ChunkStore::new(blob); + let total = total_bytes(); + let mut expected = vec![0u8; total as usize]; + let mut chunks = Vec::new(); + for (index, slot) in layout().into_iter().enumerate() { + let offset = index as u64 * CHUNK; + let len = (total - offset).min(CHUNK) as usize; + if let Some(seed) = slot { + let bytes = chunk_bytes(seed, len); + let hash = store.put_chunk(&bytes).await.unwrap(); + expected[offset as usize..offset as usize + len].copy_from_slice(&bytes); + chunks.push(ChunkRef { offset, hash }); + } + } + let mut manifest = Manifest::empty(ManifestKind::Disk, total); + manifest.chunk_size = ChunkSize::bytes(CHUNK); + manifest.chunks = chunks; + Fixture { + _dir: dir, + store, + counters, + manifest, + expected, + } + } + + /// One store and reader for every property case. A memo of two + /// chunks makes the cases exercise eviction and refetch as well. + static SHARED: std::sync::LazyLock<(tokio::runtime::Runtime, Fixture, DiskReader)> = + std::sync::LazyLock::new(|| { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .build() + .unwrap(); + let f = rt.block_on(fixture()); + let reader = DiskReader::new(&f.manifest, f.store.clone()) + .unwrap() + .with_memo_chunks(2); + (rt, f, reader) + }); + + proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn any_read_equals_the_manifest_bytes(offset in 0..total_bytes(), len in 0usize..(3 * CHUNK as usize)) { + let (rt, f, reader) = &*SHARED; + let len = len.min((total_bytes() - offset) as usize); + let got = rt.block_on(reader.read(offset, len)).unwrap(); + prop_assert_eq!(&got[..], &f.expected[offset as usize..offset as usize + len]); + } + } + + #[tokio::test] + async fn a_read_past_the_end_is_refused() { + let f = fixture().await; + let reader = DiskReader::new(&f.manifest, f.store.clone()).unwrap(); + let err = reader.read(total_bytes() - 1, 2).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert!(reader.read(total_bytes(), 0).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn holes_read_as_zeros_and_fetch_nothing() { + let f = fixture().await; + let reader = DiskReader::new(&f.manifest, f.store.clone()).unwrap(); + let before = f.counters.gets_attempted(); + assert_eq!( + reader.read(CHUNK, CHUNK as usize).await.unwrap(), + vec![0u8; CHUNK as usize] + ); + assert_eq!( + reader + .read(4 * CHUNK + 100, 2 * CHUNK as usize - 100) + .await + .unwrap(), + vec![0u8; 2 * CHUNK as usize - 100] + ); + assert_eq!(f.counters.gets_attempted(), before); + } + + #[tokio::test] + async fn a_chunk_the_store_does_not_hold_reads_as_not_found() { + let f = fixture().await; + let elsewhere = tempfile::tempdir().unwrap(); + let other = ChunkStore::new(Arc::new(LocalBlobStorage::new(elsewhere.path()))); + let absent = other + .put_chunk(&chunk_bytes(9, CHUNK as usize)) + .await + .unwrap(); + let mut manifest = f.manifest.clone(); + manifest.chunks[0].hash = absent; + let reader = DiskReader::new(&manifest, f.store.clone()).unwrap(); + let err = reader.read(0, 4096).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::NotFound, "{err}"); + assert_eq!(reader.read(2 * CHUNK, 4096).await.unwrap().len(), 4096); + } + + #[tokio::test] + async fn a_base_chunk_with_the_same_hash_at_any_offset_is_used_instead_of_the_store() { + let f = fixture().await; + // The base holds the target's chunks 0 and 2 at other offsets, and + // not chunk 3. + let base_dir = tempfile::tempdir().unwrap(); + let base_path = base_dir.path().join("base"); + let total = total_bytes(); + let mut base_bytes = vec![0u8; total as usize]; + let target_at = + |index: u64| &f.expected[(index * CHUNK) as usize..((index + 1) * CHUNK) as usize]; + base_bytes[(5 * CHUNK) as usize..(6 * CHUNK) as usize].copy_from_slice(target_at(0)); + base_bytes[CHUNK as usize..(2 * CHUNK) as usize].copy_from_slice(target_at(2)); + std::fs::write(&base_path, &base_bytes).unwrap(); + let hash_of = |index: u64| f.manifest.chunk_at(index * CHUNK).unwrap().hash; + let mut base_manifest = Manifest::empty(ManifestKind::Disk, total); + base_manifest.chunk_size = ChunkSize::bytes(CHUNK); + base_manifest.chunks = vec![ + ChunkRef { + offset: CHUNK, + hash: hash_of(2), + }, + ChunkRef { + offset: 5 * CHUNK, + hash: hash_of(0), + }, + ]; + let base = LocalBase::open(&base_path, &base_manifest).unwrap(); + let reader = DiskReader::new(&f.manifest, f.store.clone()) + .unwrap() + .with_base(base); + + let before = f.counters.gets_attempted(); + assert_eq!(reader.read(0, CHUNK as usize).await.unwrap(), target_at(0)); + assert_eq!( + reader.read(2 * CHUNK, CHUNK as usize).await.unwrap(), + target_at(2) + ); + assert_eq!( + f.counters.gets_attempted(), + before, + "base chunks must not reach the store" + ); + + assert_eq!( + reader.read(3 * CHUNK, CHUNK as usize).await.unwrap(), + target_at(3) + ); + assert_eq!(f.counters.gets_attempted(), before + 1); + } + + #[tokio::test] + async fn a_resident_cache_chunk_is_used_instead_of_the_store() { + let f = fixture().await; + let cache_dir = tempfile::tempdir().unwrap(); + let hash = f.manifest.chunk_at(0).unwrap().hash; + let path = engram_chunk_store::reader::chunk_path(cache_dir.path(), hash); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, &f.expected[..CHUNK as usize]).unwrap(); + let reader = DiskReader::new(&f.manifest, f.store.clone()) + .unwrap() + .with_cache(ChunkCacheReader::new(cache_dir.path())); + + let before = f.counters.gets_attempted(); + assert_eq!(reader.read(0, 4096).await.unwrap(), &f.expected[..4096]); + assert_eq!(f.counters.gets_attempted(), before); + assert_eq!( + reader.read(2 * CHUNK, 4096).await.unwrap(), + &f.expected[(2 * CHUNK) as usize..(2 * CHUNK) as usize + 4096] + ); + assert_eq!(f.counters.gets_attempted(), before + 1); + } + + #[tokio::test] + async fn the_memo_is_bounded_least_recently_used_first() { + let f = fixture().await; + let reader = DiskReader::new(&f.manifest, f.store.clone()) + .unwrap() + .with_memo_chunks(2); + let before = f.counters.gets_attempted(); + for index in [0, 2, 0, 3, 0] { + reader.read(index * CHUNK, 512).await.unwrap(); + } + assert_eq!( + f.counters.gets_attempted(), + before + 3, + "0, 2, 3 once each while 0 stays recent" + ); + reader.read(2 * CHUNK, 512).await.unwrap(); + assert_eq!( + f.counters.gets_attempted(), + before + 4, + "2 was evicted by 3" + ); + } + + /// Counts chunk GETs; each waits `delay`, then for a gate permit. + struct SlowBlob { + inner: Arc, + gets: AtomicU64, + delay: Duration, + gate: tokio::sync::Semaphore, + } + + impl SlowBlob { + fn new(inner: Arc, delay: Duration, open: bool) -> Arc { + let permits = if open { + tokio::sync::Semaphore::MAX_PERMITS + } else { + 0 + }; + Arc::new(Self { + inner, + gets: AtomicU64::new(0), + delay, + gate: tokio::sync::Semaphore::new(permits), + }) + } + } + + #[async_trait] + impl BlobStorage for SlowBlob { + async fn put_streaming(&self, key: &str, body: ByteStream) -> Result { + self.inner.put_streaming(key, body).await + } + async fn get_streaming(&self, key: &str) -> Result { + self.gets.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(self.delay).await; + let _permit = self.gate.acquire().await.expect("the gate never closes"); + self.inner.get_streaming(key).await + } + async fn head(&self, key: &str) -> Result { + self.inner.head(key).await + } + async fn delete(&self, key: &str) -> Result<(), BlobError> { + self.inner.delete(key).await + } + async fn list_prefix(&self, prefix: &str) -> Result, BlobError> { + self.inner.list_prefix(prefix).await + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_reads_of_one_chunk_share_one_fetch() { + let f = fixture().await; + let slow = SlowBlob::new(f.store.blob_storage(), Duration::ZERO, false); + let reader = Arc::new(DiskReader::new(&f.manifest, ChunkStore::new(slow.clone())).unwrap()); + let reads: Vec<_> = (0..16u64) + .map(|i| { + let reader = Arc::clone(&reader); + tokio::spawn(async move { reader.read(2 * CHUNK + i * 4096, 4096).await }) + }) + .collect(); + while slow.gets.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + // Every read has had time to reach the store if it were going to. + tokio::time::sleep(Duration::from_millis(100)).await; + slow.gate.add_permits(tokio::sync::Semaphore::MAX_PERMITS); + for (i, read) in reads.into_iter().enumerate() { + let at = (2 * CHUNK) as usize + i * 4096; + assert_eq!(read.await.unwrap().unwrap(), &f.expected[at..at + 4096]); + } + assert_eq!(slow.gets.load(Ordering::SeqCst), 1); + } + + /// Reads of different chunks fetch at the same time, even when one + /// task drives them all: the set takes less time than the fetches one + /// after another. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn reads_of_different_chunks_fetch_in_parallel() { + const DELAY: Duration = Duration::from_millis(300); + let f = fixture().await; + let slow = SlowBlob::new(f.store.blob_storage(), DELAY, true); + let reader = DiskReader::new(&f.manifest, ChunkStore::new(slow.clone())).unwrap(); + let offsets: Vec = f.manifest.chunks.iter().map(|c| c.offset).collect(); + assert!(offsets.len() >= 4); + + let began = std::time::Instant::now(); + let reads = offsets.iter().map(|&offset| reader.read(offset, 4096)); + let results = futures::future::try_join_all(reads).await.unwrap(); + let took = began.elapsed(); + + for (offset, got) in offsets.iter().zip(results) { + assert_eq!(got, &f.expected[*offset as usize..*offset as usize + 4096]); + } + assert_eq!(slow.gets.load(Ordering::SeqCst), offsets.len() as u64); + let serial = DELAY * offsets.len() as u32; + assert!( + took < serial - DELAY, + "{} fetches of {DELAY:?} each took {took:?}", + offsets.len() + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn more_chunks_in_flight_than_the_memo_keeps_each_fetch_once() { + let f = fixture().await; + let slow = SlowBlob::new(f.store.blob_storage(), Duration::ZERO, false); + let reader = Arc::new( + DiskReader::new(&f.manifest, ChunkStore::new(slow.clone())) + .unwrap() + .with_memo_chunks(2), + ); + let offsets: Vec = f.manifest.chunks.iter().map(|c| c.offset).collect(); + assert!(offsets.len() > 2); + let read_all = || -> Vec<_> { + offsets + .iter() + .map(|&offset| { + let reader = Arc::clone(&reader); + tokio::spawn(async move { (offset, reader.read(offset, 4096).await) }) + }) + .collect() + }; + let first = read_all(); + while slow.gets.load(Ordering::SeqCst) < offsets.len() as u64 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + // Each chunk's fetch is in flight; a second read of each must + // wait on it, not start another. + let second = read_all(); + tokio::time::sleep(Duration::from_millis(100)).await; + slow.gate.add_permits(tokio::sync::Semaphore::MAX_PERMITS); + for read in first.into_iter().chain(second) { + let (offset, got) = read.await.unwrap(); + let at = offset as usize; + assert_eq!(got.unwrap(), &f.expected[at..at + 4096]); + } + assert_eq!(slow.gets.load(Ordering::SeqCst), offsets.len() as u64); + } + + #[tokio::test] + async fn a_base_chunk_with_the_wrong_bytes_is_read_from_the_store() { + let f = fixture().await; + let base_dir = tempfile::tempdir().unwrap(); + let base_path = base_dir.path().join("base"); + let total = total_bytes(); + // The base manifest says chunk 0's hash is at offset 0, but the + // volume holds other bytes there. + std::fs::write(&base_path, vec![0xa5u8; total as usize]).unwrap(); + let mut base_manifest = Manifest::empty(ManifestKind::Disk, total); + base_manifest.chunk_size = ChunkSize::bytes(CHUNK); + base_manifest.chunks = vec![ChunkRef { + offset: 0, + hash: f.manifest.chunk_at(0).unwrap().hash, + }]; + let reader = DiskReader::new(&f.manifest, f.store.clone()) + .unwrap() + .with_base(LocalBase::open(&base_path, &base_manifest).unwrap()); + + let before = f.counters.gets_attempted(); + assert_eq!( + reader.read(0, CHUNK as usize).await.unwrap(), + &f.expected[..CHUNK as usize] + ); + assert_eq!(f.counters.gets_attempted(), before + 1); + } +} diff --git a/crates/engram-disk-source/src/ublk.rs b/crates/engram-disk-source/src/ublk.rs new file mode 100644 index 000000000..25077ca9d --- /dev/null +++ b/crates/engram-disk-source/src/ublk.rs @@ -0,0 +1,447 @@ +//! The ublk server: a read-only block device over a [`DiskReader`]. +//! +//! The device is created with user recovery and reissue. When the +//! process dies, the kernel keeps `/dev/ublkbN`, holds its I/O, and +//! gives the requests in flight again to the next process that recovers +//! it. Only a normal stop, or [`delete_unserved`] for a device no process +//! serves, deletes the device. +//! +//! Each queue thread runs one task per tag on a local executor driven by +//! the queue's io_uring. A read runs on the tokio runtime, and the tag's +//! task waits for it on an eventfd polled through the same ring, so a +//! slow read holds only its own tag. + +use std::future::Future; +use std::io; +use std::os::fd::AsRawFd; +use std::rc::Rc; +use std::sync::{Arc, Mutex}; + +use io_uring::{opcode, types}; +use libublk::ctrl::{UblkCtrl, UblkCtrlBuilder}; +use libublk::helpers::IoBuf; +use libublk::io::{UblkDev, UblkQueue}; +use libublk::uring_async::ublk_submit_sqe_async; +use libublk::{sys, BufDesc, UblkError, UblkFlags, UblkUringData}; +use nix::sys::eventfd::{EfdFlags, EventFd}; +use tokio::runtime::Handle; +use tokio::sync::{oneshot, watch}; + +use crate::policy::read_until_served; +use crate::reader::DiskReader; + +/// The device's logical and physical block size. +const BLOCK_SHIFT: u8 = 12; +const BLOCK_BYTES: u64 = 1 << BLOCK_SHIFT; +/// The largest single request the kernel sends. +const IO_BUF_BYTES: u32 = 512 * 1024; + +/// How the device is shaped and whether it is new. A recovered device +/// keeps its shape, so `queues` and `depth` must equal its own. +#[derive(Clone, Debug)] +pub struct Config { + pub queues: u16, + pub depth: u16, + /// Attach to this existing device instead of creating one. + pub recover: Option, +} + +/// The device a server serves. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Device { + pub dev_id: u32, + pub path: String, +} + +fn ublk_err(what: &str, e: UblkError) -> io::Error { + match e { + UblkError::IOError(e) => io::Error::new(e.kind(), format!("{what}: {e}")), + UblkError::UringIOError(errno) | UblkError::OtherError(errno) if errno < 0 => { + let os = io::Error::from_raw_os_error(-errno); + io::Error::new(os.kind(), format!("{what}: {os}")) + } + other => io::Error::other(format!("{what}: {other}")), + } +} + +/// The control commands report EBUSY as a success value. +fn check(what: &str, res: Result) -> io::Result { + match res { + Ok(r) if r < 0 => Err(ublk_err(what, UblkError::OtherError(r))), + Ok(r) => Ok(r), + Err(e) => Err(ublk_err(what, e)), + } +} + +/// Serve `reader` as a ublk device until `stop` completes, then delete +/// the device. `on_ready` runs once the device serves I/O. A process +/// that exits any other way leaves the device for [`Config::recover`]. +pub fn serve( + reader: Arc, + rt: Handle, + config: &Config, + on_ready: R, + stop: S, +) -> io::Result<()> +where + R: FnOnce(&Device) + Send + Sync + 'static, + S: Future + Send + 'static, +{ + let size = reader.size_bytes(); + if size == 0 || !size.is_multiple_of(BLOCK_BYTES) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("a disk of {size} bytes is not a whole number of {BLOCK_BYTES} byte blocks"), + )); + } + let ctrl = open_ctrl(config, size)?; + + let tgt_init = move |dev: &mut UblkDev| { + let depth = dev.dev_info.queue_depth; + // One request command and one eventfd poll per tag. + dev.tgt.sq_depth = depth.saturating_mul(2); + dev.tgt.cq_depth = depth.saturating_mul(2); + dev.tgt.dev_size = size; + dev.tgt.params = params(size, dev.dev_info.max_io_buf_bytes); + Ok(()) + }; + let stop_rt = rt.clone(); + // The kernel stops a device only once every request in flight has + // completed, so a stop first ends the reads still trying again. + let (stopping_tx, stopping) = watch::channel(false); + let serve_queue = move |qid: u16, dev: &UblkDev| { + serve_queue(qid, dev, &reader, &rt, &stopping); + }; + // `run_target` wants a `Sync` closure; the stop future is only `Send`. + let stop = Mutex::new(Some(stop)); + let on_started = move |ctrl: &UblkCtrl| { + on_ready(&Device { + dev_id: ctrl.dev_info().dev_id, + path: ctrl.get_bdev_path(), + }); + if let Some(stop) = stop.lock().ok().and_then(|mut s| s.take()) { + stop_rt.block_on(stop); + } + let _ = stopping_tx.send(true); + // Stops the queues; `run_target` then returns. + if let Err(e) = ctrl.kill_dev() { + tracing::warn!(error = %e, "stop the device"); + } + }; + check( + "serve the device", + ctrl.run_target(tgt_init, serve_queue, on_started), + )?; + check("delete the device", ctrl.del_dev())?; + Ok(()) +} + +fn params(size: u64, max_io_buf_bytes: u32) -> sys::ublk_params { + sys::ublk_params { + types: sys::UBLK_PARAM_TYPE_BASIC, + basic: sys::ublk_param_basic { + attrs: sys::UBLK_ATTR_READ_ONLY, + logical_bs_shift: BLOCK_SHIFT, + physical_bs_shift: BLOCK_SHIFT, + io_opt_shift: BLOCK_SHIFT, + io_min_shift: BLOCK_SHIFT, + max_sectors: max_io_buf_bytes >> 9, + dev_sectors: size >> 9, + ..Default::default() + }, + ..Default::default() + } +} + +fn open_ctrl(config: &Config, size: u64) -> io::Result { + let builder = UblkCtrlBuilder::default() + .name("engram-disk-source") + .nr_queues(config.queues) + .depth(config.depth) + .io_buf_bytes(IO_BUF_BYTES) + .ctrl_flags((sys::UBLK_F_USER_RECOVERY | sys::UBLK_F_USER_RECOVERY_REISSUE) as u64); + let Some(dev_id) = config.recover else { + return builder + .id(-1) + .dev_flags(UblkFlags::UBLK_DEV_F_ADD_DEV) + .build() + .map_err(|e| ublk_err("create the device", e)); + }; + let id = i32::try_from(dev_id).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("device id {dev_id} is out of range"), + ) + })?; + let ctrl = builder + .id(id) + .dev_flags(UblkFlags::UBLK_DEV_F_RECOVER_DEV) + .build() + .map_err(|e| ublk_err(&format!("open device {dev_id}"), e))?; + let info = ctrl.dev_info(); + if info.flags & sys::UBLK_F_USER_RECOVERY as u64 == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("device {dev_id} was not created for recovery"), + )); + } + // libublk sizes its per-queue state from the configured count but + // starts one thread per queue of the device. + if info.nr_hw_queues != config.queues || info.queue_depth != config.depth { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "device {dev_id} has {} queues of depth {}; this source was started with {} of depth {}", + info.nr_hw_queues, info.queue_depth, config.queues, config.depth + ), + )); + } + let mut kernel = sys::ublk_params::default(); + check("read the device parameters", ctrl.get_params(&mut kernel))?; + if kernel.basic.dev_sectors != size >> 9 + || kernel.basic.attrs & sys::UBLK_ATTR_READ_ONLY == 0 + || kernel.basic.logical_bs_shift != BLOCK_SHIFT + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "device {dev_id} is a {} byte device; this source is a read-only {size} byte disk", + kernel.basic.dev_sectors << 9 + ), + )); + } + check( + &format!("start recovery of device {dev_id}"), + ctrl.start_user_recover(), + )?; + Ok(ctrl) +} + +struct Tag<'a> { + reader: Arc, + rt: Handle, + stopping: watch::Receiver, + queue: Rc>, + tag: u16, +} + +fn serve_queue( + qid: u16, + dev: &UblkDev, + reader: &Arc, + rt: &Handle, + stopping: &watch::Receiver, +) { + let queue = match UblkQueue::new(qid, dev) { + Ok(queue) => Rc::new(queue), + Err(e) => { + tracing::error!(qid, error = %e, "open the queue"); + return; + } + }; + let exe = Rc::new(smol::LocalExecutor::new()); + let tasks: Vec<_> = queue + .tags() + .map(|tag| { + let tag = Tag { + reader: Arc::clone(reader), + rt: rt.clone(), + stopping: stopping.clone(), + queue: Rc::clone(&queue), + tag, + }; + exe.spawn(async move { + let id = tag.tag; + match serve_tag(tag).await { + Ok(()) | Err(UblkError::QueueIsDown) => {} + Err(e) => tracing::error!(qid, tag = id, error = %e, "tag stopped serving"), + } + }) + }) + .collect(); + let ticker = Rc::clone(&exe); + smol::block_on(exe.run(async { + let run_ops = || while ticker.try_tick() {}; + let is_done = || tasks.iter().all(|task| task.is_finished()); + match libublk::wait_and_handle_io_events(&queue, Some(20), run_ops, is_done).await { + Ok(()) | Err(UblkError::QueueIsDown) => {} + Err(e) => tracing::error!(qid, error = %e, "queue event loop"), + } + })); +} + +async fn serve_tag(t: Tag<'_>) -> Result<(), UblkError> { + let (queue, tag) = (&*t.queue, t.tag); + let done = Arc::new( + EventFd::from_value_and_flags(0, EfdFlags::EFD_NONBLOCK | EfdFlags::EFD_CLOEXEC) + .map_err(|e| UblkError::IOError(e.into()))?, + ); + let mut buf = IoBuf::::new(queue.dev.dev_info.max_io_buf_bytes as usize); + queue + .submit_io_prep_cmd(tag, BufDesc::Slice(buf.as_slice()), 0, Some(&buf)) + .await?; + loop { + let res = handle_io(&t, buf.as_mut_slice(), &done).await; + queue + .submit_io_commit_cmd(tag, BufDesc::Slice(buf.as_slice()), res) + .await?; + } +} + +/// The result the kernel gets for the request on `tag`: bytes read, or a +/// negative errno. +async fn handle_io(t: &Tag<'_>, buf: &mut [u8], done: &Arc) -> i32 { + let iod = t.queue.get_iod(t.tag); + match iod.op_flags & 0xff { + sys::UBLK_IO_OP_READ => { + let offset = iod.start_sector << 9; + let len = (iod.nr_sectors as usize) << 9; + if len > buf.len() { + return -libc::EINVAL; + } + match read_on_runtime(t, offset, len, done).await { + Ok(bytes) => { + buf[..len].copy_from_slice(&bytes); + len as i32 + } + Err(e) => { + tracing::error!(offset, len, error = %e, "read refused"); + if e.kind() == io::ErrorKind::InvalidInput { + -libc::EINVAL + } else { + -libc::EIO + } + } + } + } + // The device has no volatile cache, so the kernel sends no flush; + // one that arrives has nothing to make durable. + sys::UBLK_IO_OP_FLUSH => 0, + _ => -libc::EROFS, + } +} + +type ReadResult = io::Result>; + +/// The runtime side of a read: it sends the result, or drops the sender +/// when the task ends without one (a panic), and then signals `done`. +/// The tag's task therefore always wakes and finds the channel ready. +struct Finish { + tx: Option>, + done: Arc, +} + +impl Finish { + fn send(mut self, result: ReadResult) { + if let Some(tx) = self.tx.take() { + let _ = tx.send(result); + } + } +} + +impl Drop for Finish { + fn drop(&mut self) { + // The sender goes before the signal, so a woken waiter never + // finds the channel empty. + drop(self.tx.take()); + if let Err(e) = self.done.write(1) { + tracing::error!(error = %e, "signal a finished read"); + } + } +} + +/// Run the read on the tokio runtime and wait for it through the queue +/// ring: the runtime signals `done`, whose poll completes on the ring. +async fn read_on_runtime(t: &Tag<'_>, offset: u64, len: usize, done: &Arc) -> ReadResult { + let (tx, mut rx) = oneshot::channel(); + let reader = Arc::clone(&t.reader); + let stopping = t.stopping.clone(); + let finish = Finish { + tx: Some(tx), + done: Arc::clone(done), + }; + t.rt.spawn(async move { + let result = read_until_served(&reader, offset, len, stopping).await; + finish.send(result); + }); + loop { + let poll = opcode::PollAdd::new(types::Fd(done.as_raw_fd()), libc::POLLIN as u32).build(); + let res = ublk_submit_sqe_async(poll, UblkUringData::Target as u64) + .await + .map_err(|e| ublk_err("wait for a read", e))?; + // The read still runs, so a poll that ended early is armed again. + if matches!(-res, libc::EINTR | libc::EAGAIN | libc::ECANCELED) { + continue; + } + if res < 0 { + return Err(io::Error::from_raw_os_error(-res)); + } + // Reset the counter before the next request reuses it. + let _ = done.read(); + match rx.try_recv() { + Ok(result) => return result, + Err(oneshot::error::TryRecvError::Empty) => continue, + Err(oneshot::error::TryRecvError::Closed) => { + return Err(io::Error::other("the read task ended without a result")) + } + } + } +} + +/// Delete device `dev_id` when no process serves it. +/// +/// The kernel rules (drivers/block/ublk_drv.c) this relies on: +/// - A stop or delete of a LIVE device first quiesces its queue and waits +/// for the requests in flight. With no server to finish them, it waits +/// until the node reboots. So a LIVE device is refused. +/// - A DEAD device has no disk left: its server stopped it and did not +/// delete it. A delete only removes the control node. +/// - A QUIESCED device is one whose server died (user recovery). A delete +/// marks its queues to abort, so each request it holds fails with an +/// I/O error, and the delete does not wait. It is deleted only with +/// `quiesced`, for a caller that knows nothing reads the device. +/// +/// A device whose recorded server process still runs is refused: that +/// server can be in the middle of a recovery. +pub fn delete_unserved(dev_id: u32, quiesced: bool) -> io::Result<()> { + let id = i32::try_from(dev_id).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("device id {dev_id} is out of range"), + ) + })?; + let ctrl = + UblkCtrl::new_simple(id).map_err(|e| ublk_err(&format!("open device {dev_id}"), e))?; + let info = ctrl.dev_info(); + let state = u32::from(info.state); + let allowed = state == sys::UBLK_S_DEV_DEAD || (quiesced && state == sys::UBLK_S_DEV_QUIESCED); + if !allowed { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + format!("device {dev_id} is in state {state}; it is not deleted"), + )); + } + if info.ublksrv_pid > 0 && server_runs(info.ublksrv_pid) { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + format!( + "device {dev_id}: its server process {} still runs; it is not deleted", + info.ublksrv_pid + ), + )); + } + check(&format!("delete device {dev_id}"), ctrl.del_dev())?; + Ok(()) +} + +/// Whether `pid` is a live (not zombie) engram-disk-source process. +fn server_runs(pid: i32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let Some((comm, rest)) = stat.split_once(" (").and_then(|(_, r)| r.rsplit_once(") ")) else { + return false; + }; + // The kernel keeps the first 15 bytes of the name. + comm == &"engram-disk-source"[..15] && !rest.starts_with('Z') +} diff --git a/crates/engram-disk-source/tests/ublk_device.rs b/crates/engram-disk-source/tests/ublk_device.rs new file mode 100644 index 000000000..9e2a45c08 --- /dev/null +++ b/crates/engram-disk-source/tests/ublk_device.rs @@ -0,0 +1,471 @@ +#![cfg(target_os = "linux")] +//! The `engram-disk-source` binary against the kernel's ublk driver, as +//! root. `ENGRAM_TEST_UBLK=1` runs them (the driver must be loaded: +//! `modprobe ublk_drv`); unset, every test skips. + +use std::fs::{File, OpenOptions}; +use std::io::{BufRead, BufReader}; +use std::os::unix::fs::{FileExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; + +use engram_chunk_store::{ + ChunkHash, ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind, ManifestRef, +}; +use engram_storage_local::LocalBlobStorage; +use nix::sys::signal::{kill, Signal}; +use nix::unistd::Pid; + +const CHUNK: u64 = 1 << 20; +/// Chunk seeds; `None` is a hole. +const LAYOUT: [Option; 6] = [Some(1), None, Some(2), Some(3), None, Some(4)]; +const READ: usize = 64 * 1024; +const QUEUES: u16 = 2; +const READY_TIMEOUT: Duration = Duration::from_secs(60); + +macro_rules! ublk_or_skip { + () => { + if std::env::var_os("ENGRAM_TEST_UBLK").is_none() { + eprintln!("SKIP: ENGRAM_TEST_UBLK is not set"); + return; + } + }; +} + +struct Disk { + _dir: tempfile::TempDir, + store_root: PathBuf, + manifest: ManifestRef, + expected: Vec, +} + +fn chunk_bytes(seed: u8) -> Vec { + (0..CHUNK as usize) + .map(|i| ((i as u32).wrapping_mul(2_654_435_761) >> (seed % 24)) as u8 ^ seed) + .collect() +} + +/// A manifest of `LAYOUT` published to a local store. +fn publish() -> Disk { + let dir = tempfile::tempdir().unwrap(); + let store_root = dir.path().to_path_buf(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let total = LAYOUT.len() as u64 * CHUNK; + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(store_root.join("blobs")))); + let mut expected = vec![0u8; total as usize]; + let mut chunks = Vec::new(); + for (index, seed) in LAYOUT.iter().enumerate() { + let Some(seed) = seed else { continue }; + let offset = index as u64 * CHUNK; + let bytes = chunk_bytes(*seed); + let hash = rt.block_on(store.put_chunk(&bytes)).unwrap(); + expected[offset as usize..(offset + CHUNK) as usize].copy_from_slice(&bytes); + chunks.push(ChunkRef { offset, hash }); + } + let mut manifest = Manifest::empty(ManifestKind::Disk, total); + manifest.chunk_size = ChunkSize::bytes(CHUNK); + manifest.chunks = chunks; + let manifest_ref = ManifestRef::new(); + rt.block_on(store.put_manifest(manifest_ref, &manifest)) + .unwrap(); + Disk { + _dir: dir, + store_root, + manifest: manifest_ref, + expected, + } +} + +/// A running binary. Dropped while it runs (a failed assertion), it is +/// stopped so the device does not outlive the test. +struct Source { + child: Option, + path: PathBuf, + dev_id: u32, +} + +/// Start the binary and wait for its ready line. +fn start(disk: &Disk, recover: Option) -> Source { + try_start(disk, recover, QUEUES).unwrap_or_else(|e| panic!("{e}")) +} + +/// [`start`] with `queues` queues, or why it failed. A binary that +/// printed no ready line is reaped, and killed first if it still runs. +fn try_start(disk: &Disk, recover: Option, queues: u16) -> Result { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_engram-disk-source")); + cmd.args([ + "serve", + "--manifest", + &disk.manifest.to_string(), + "--queues", + &queues.to_string(), + "--depth", + "16", + ]) + .env("ENGRAM_BLOB_BACKEND", "local") + .env("ENGRAM_LOCAL_PATH", &disk.store_root) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + if let Some(id) = recover { + cmd.args(["--recover", &id.to_string()]); + } + let mut child = cmd + .spawn() + .map_err(|e| format!("spawn engram-disk-source: {e}"))?; + let stdout = child.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let mut line = String::new(); + let _ = BufReader::new(stdout).read_line(&mut line); + let _ = tx.send(line); + }); + let line = rx.recv_timeout(READY_TIMEOUT); + let ready = line.as_deref().ok().and_then(|line| { + let (path, id) = line.trim().split_once(' ')?; + Some((PathBuf::from(path), id.parse().ok()?)) + }); + let Some((path, dev_id)) = ready else { + // An empty line is the end of its output: the binary exits. + if !matches!(line.as_deref(), Ok("")) { + let _ = child.kill(); + } + let status = child.wait().map(|s| s.to_string()); + return Err(format!( + "engram-disk-source gave {line:?}, not a ready line \" \" ({status:?})" + )); + }; + Ok(Source { + child: Some(child), + path, + dev_id, + }) +} + +impl Source { + /// SIGTERM: a normal stop, which deletes the device. + fn stop(mut self) { + let mut child = self.child.take().unwrap(); + kill(Pid::from_raw(child.id() as i32), Signal::SIGTERM).unwrap(); + let status = child.wait().unwrap(); + assert!(status.success(), "a stopped source exits cleanly: {status}"); + let sysfs = PathBuf::from(format!("/sys/block/ublkb{}", self.dev_id)); + assert!(!sysfs.exists(), "a normal stop deletes the device"); + } + + /// SIGKILL: the device stays for recovery. + fn crash(mut self, disk: &Disk) -> Crashed<'_> { + let mut child = self.child.take().unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + Crashed { + disk, + dev_id: Some(self.dev_id), + } + } +} + +/// A device whose process died. Dropped before [`Crashed::recover`] +/// succeeds, it recovers the device and stops it: a device left without +/// a process holds its I/O, and a read blocked on it keeps the test +/// process from exiting. +struct Crashed<'a> { + disk: &'a Disk, + dev_id: Option, +} + +impl Crashed<'_> { + fn dev_id(&self) -> u32 { + self.dev_id.unwrap() + } + + fn recover(mut self) -> Source { + let source = start(self.disk, self.dev_id); + self.dev_id = None; + source + } +} + +impl Drop for Crashed<'_> { + fn drop(&mut self) { + let Some(dev_id) = self.dev_id else { return }; + match try_start(self.disk, Some(dev_id), QUEUES) { + // Its drop stops it, which deletes the device. + Ok(source) => drop(source), + Err(e) => eprintln!("device {dev_id} stays without a process: {e}"), + } + } +} + +impl Drop for Source { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = kill(Pid::from_raw(child.id() as i32), Signal::SIGTERM); + let _ = child.wait(); + } + } +} + +/// A buffer at an address aligned for direct I/O. +struct Aligned { + buf: Vec, + start: usize, + len: usize, +} + +impl Aligned { + fn new(len: usize) -> Self { + let buf = vec![0u8; len + 4096]; + let start = buf.as_ptr().align_offset(4096); + Self { buf, start, len } + } + fn as_mut(&mut self) -> &mut [u8] { + &mut self.buf[self.start..self.start + self.len] + } +} + +fn open_direct(path: &Path) -> File { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECT) + .open(path) + .unwrap() +} + +#[test] +#[ignore] +fn the_device_reads_as_the_manifest_bytes() { + ublk_or_skip!(); + let disk = publish(); + let source = start(&disk, None); + let size = std::fs::read_to_string(format!("/sys/block/ublkb{}/size", source.dev_id)).unwrap(); + assert_eq!( + size.trim().parse::().unwrap() * 512, + disk.expected.len() as u64 + ); + + let device = open_direct(&source.path); + let mut buf = Aligned::new(READ); + for offset in (0..disk.expected.len()).step_by(READ) { + device.read_exact_at(buf.as_mut(), offset as u64).unwrap(); + assert!( + buf.as_mut() == &disk.expected[offset..offset + READ], + "bytes at {offset} differ from the manifest" + ); + } + drop(device); + source.stop(); +} + +#[test] +#[ignore] +fn writes_are_refused() { + ublk_or_skip!(); + let disk = publish(); + let source = start(&disk, None); + let ro = std::fs::read_to_string(format!("/sys/block/ublkb{}/ro", source.dev_id)).unwrap(); + assert_eq!(ro.trim(), "1", "the kernel marks the device read-only"); + // Root may open a read-only device for writing; the write itself fails. + let refused = match OpenOptions::new().write(true).open(&source.path) { + Err(e) => e, + Ok(device) => device + .write_at(&[0u8; 4096], 0) + .expect_err("a write to a read-only device fails"), + }; + assert!( + matches!( + refused.raw_os_error(), + Some(libc::EROFS) | Some(libc::EACCES) | Some(libc::EPERM) + ), + "unexpected error {refused}" + ); + source.stop(); +} + +#[test] +#[ignore] +fn reads_stall_across_a_crash_and_complete_after_recovery() { + ublk_or_skip!(); + let disk = Arc::new(publish()); + let source = start(&disk, None); + let path = source.path.clone(); + + let reading = Arc::new(AtomicBool::new(true)); + let reader = { + let disk = Arc::clone(&disk); + let reading = Arc::clone(&reading); + std::thread::spawn(move || { + let device = open_direct(&path); + let mut buf = Aligned::new(READ); + let mut spans = Vec::new(); + let mut offset = 0usize; + while reading.load(Ordering::SeqCst) { + let began = Instant::now(); + // A crash must stall this read, never fail it. + device.read_exact_at(buf.as_mut(), offset as u64).unwrap(); + assert!( + buf.as_mut() == &disk.expected[offset..offset + READ], + "bytes at {offset} differ from the manifest" + ); + spans.push((began, Instant::now())); + offset = (offset + READ) % disk.expected.len(); + } + spans + }) + }; + + std::thread::sleep(Duration::from_millis(200)); + let crashed_at = Instant::now(); + let crashed = source.crash(&disk); + let dev_id = crashed.dev_id(); + // The kernel holds the device's I/O while no process serves it. + std::thread::sleep(Duration::from_millis(500)); + let recovered = crashed.recover(); + let recovered_at = Instant::now(); + assert_eq!(recovered.dev_id, dev_id); + assert_eq!(recovered.path, source_path(dev_id)); + std::thread::sleep(Duration::from_millis(200)); + reading.store(false, Ordering::SeqCst); + let spans = reader.join().expect("no read failed"); + + assert!( + spans.iter().any( + |(began, ended)| *began < crashed_at + Duration::from_millis(100) + && *ended > recovered_at - Duration::from_millis(100) + ), + "a read issued around the crash waits for the recovered process" + ); + assert!( + spans.iter().any(|(began, _)| *began > recovered_at), + "reads continue after recovery" + ); + recovered.stop(); +} + +#[test] +#[ignore] +fn a_recovery_with_another_queue_count_is_refused_and_leaves_the_device() { + ublk_or_skip!(); + let disk = publish(); + let crashed = start(&disk, None).crash(&disk); + let err = match try_start(&disk, Some(crashed.dev_id()), QUEUES - 1) { + Ok(_) => panic!("a recovery with another queue count is refused"), + Err(e) => e, + }; + // A refusal exits 1; a panic exits 101. + assert!(err.contains("exit status: 1\""), "{err}"); + let recovered = crashed.recover(); + let mut buf = Aligned::new(READ); + open_direct(&recovered.path) + .read_exact_at(buf.as_mut(), 0) + .unwrap(); + assert!(buf.as_mut() == &disk.expected[..READ]); + recovered.stop(); +} + +#[test] +#[ignore] +fn a_stop_ends_a_read_that_cannot_be_served() { + ublk_or_skip!(); + let disk = publish(); + // A directory where chunk 3's blob should be: the store fails to read + // it without reporting the chunk absent, so a read of it tries again + // until the server stops. + let blob = disk + .store_root + .join("blobs") + .join(ChunkHash::of(&chunk_bytes(3)).storage_key()); + std::fs::remove_file(&blob).unwrap(); + std::fs::create_dir(&blob).unwrap(); + let source = start(&disk, None); + let path = source.path.clone(); + let reader = std::thread::spawn(move || { + let mut buf = Aligned::new(READ); + open_direct(&path).read_exact_at(buf.as_mut(), 3 * CHUNK) + }); + std::thread::sleep(Duration::from_millis(300)); + assert!( + !reader.is_finished(), + "the read waits while the store cannot serve it" + ); + // The kernel stops a device only once its requests complete; the stop + // must end the waiting read, or it never finishes. + source.stop(); + let err = reader + .join() + .unwrap() + .expect_err("a read ended by a stop fails"); + assert_eq!( + err.raw_os_error(), + Some(libc::EIO), + "unexpected error {err}" + ); +} + +#[test] +#[ignore] +fn a_chunk_the_store_does_not_hold_fails_its_read() { + ublk_or_skip!(); + let disk = publish(); + let lost = ChunkHash::of(&chunk_bytes(3)); + std::fs::remove_file(disk.store_root.join("blobs").join(lost.storage_key())).unwrap(); + let source = start(&disk, None); + let mut buf = Aligned::new(READ); + let err = open_direct(&source.path) + .read_exact_at(buf.as_mut(), 3 * CHUNK) + .expect_err("a read of lost data fails"); + assert_eq!( + err.raw_os_error(), + Some(libc::EIO), + "unexpected error {err}" + ); + open_direct(&source.path) + .read_exact_at(buf.as_mut(), 0) + .expect("a read of data the store holds still succeeds"); + source.stop(); +} + +/// Run `engram-disk-source delete`; whether it succeeded. +fn delete(dev_id: u32, quiesced: bool) -> bool { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_engram-disk-source")); + cmd.args(["delete", "--device", &dev_id.to_string()]); + if quiesced { + cmd.arg("--quiesced"); + } + cmd.status().unwrap().success() +} + +#[test] +#[ignore] +fn delete_refuses_a_served_device_and_removes_one_whose_server_died() { + ublk_or_skip!(); + let disk = publish(); + let source = start(&disk, None); + let dev_id = source.dev_id; + assert!(!delete(dev_id, true), "a served device is refused"); + let mut crashed = source.crash(&disk); + assert!( + !delete(dev_id, false), + "a device whose server died needs --quiesced" + ); + // The kernel quiesces the device once it sees the server's queues go. + let deleted = (0..100).any(|_| { + delete(dev_id, true) || { + std::thread::sleep(Duration::from_millis(20)); + false + } + }); + assert!(deleted, "a device whose server died is deleted"); + assert!(!PathBuf::from(format!("/sys/class/ublk-char/ublkc{dev_id}")).exists()); + // Nothing is left to recover. + crashed.dev_id = None; +} + +fn source_path(dev_id: u32) -> PathBuf { + PathBuf::from(format!("/dev/ublkb{dev_id}")) +} diff --git a/crates/engram-disk-thin/Cargo.toml b/crates/engram-disk-thin/Cargo.toml index e4ea1c7b6..387e40595 100644 --- a/crates/engram-disk-thin/Cargo.toml +++ b/crates/engram-disk-thin/Cargo.toml @@ -26,5 +26,6 @@ workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] engram-chunk-store.workspace = true +engram-core.workspace = true engram-storage-local.workspace = true tempfile = "3" diff --git a/crates/engram-disk-thin/src/disk.rs b/crates/engram-disk-thin/src/disk.rs index 906241e9d..7ec10b15c 100644 --- a/crates/engram-disk-thin/src/disk.rs +++ b/crates/engram-disk-thin/src/disk.rs @@ -1,24 +1,46 @@ //! The disk over the ioctl layer; Linux only. +use std::collections::HashMap; +use std::fs::File; +use std::future::Future; use std::io; +use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; use crate::dm; -use crate::parse::{self, DevNo, ThinParams, METADATA_BLOCK_BYTES}; +use crate::parse::{self, CloneTable, DevNo, ThinParams, METADATA_BLOCK_BYTES}; use async_trait::async_trait; -use engram_host_core::disk::{Capacity, SessionDisk, Volume, VolumeName}; +use engram_host_core::disk::{ + check_fresh_bitmap, discard, skip_ranges, Capacity, ChunkIndex, CloneState, Hydration, + SessionDisk, Volume, VolumeName, CLONE_HEADER_BYTES, CLONE_REGION_BYTES, +}; const SECTOR_BYTES: u64 = 512; +const REGION_SECTORS: u64 = CLONE_REGION_BYTES / SECTOR_BYTES; +/// A view is loaded with hydration off and never turned on, so its copy +/// tuning has no effect. +const VIEW: Hydration = Hydration { + enabled: false, + threshold_regions: 1, + batch_regions: 1, +}; /// dm-thin device ids are 24 bits wide. const ID_MASK: u32 = 0x00FF_FFFF; /// A `delete` waits this long in all for the last holder of a device /// to close it. const REMOVE_ATTEMPTS: u32 = 10; const REMOVE_RETRY: Duration = Duration::from_millis(100); +/// A resume that fails is retried this many times in all before the +/// device is reported left suspended. +const RESUME_ATTEMPTS: u32 = 5; +const RESUME_RETRY: Duration = Duration::from_millis(50); +/// Copies share the pool's id counter, metadata-snapshot lock and drive +/// locks, so an operation can run in a task of its own. +#[derive(Clone)] pub struct ThinPoolDisk { dm: dm::Dm, pool: String, @@ -32,14 +54,17 @@ pub struct ThinPoolDisk { /// is safe. next_id: Arc, /// The pool holds one reserved metadata snapshot at a time. - metadata_snap: tokio::sync::Mutex<()>, + metadata_snap: Arc>, + /// One lock per drive or view name, held across each operation on it. + drive_locks: Arc>>>>, } impl ThinPoolDisk { /// Attach to an active pool whose block size divides `chunk_size`, - /// delete thins that have no device (a crash between `delete` and - /// the pool message), and seed the id counter. Device nodes are made - /// under `nodes_dir`. + /// resume its devices that a process death left suspended, delete + /// thins that have no device (a crash between `delete` and the pool + /// message), and seed the id counter. Device nodes are made under + /// `nodes_dir`. No other process may operate on the pool meanwhile. pub async fn open(pool: &str, chunk_size: u64, nodes_dir: &Path) -> io::Result { let dm = dm::Dm::open()?; let params = parse::pool_params(&single_target(&dm.table(pool).await?, "thin-pool")?)?; @@ -62,8 +87,10 @@ impl ThinPoolDisk { chunk_size, block_bytes, next_id: Arc::new(AtomicU32::new(1)), - metadata_snap: tokio::sync::Mutex::new(()), + metadata_snap: Arc::new(tokio::sync::Mutex::new(())), + drive_locks: Arc::new(Mutex::new(HashMap::new())), }; + disk.resume_suspended().await?; let in_pool = disk.thin_ids().await?; let active = disk.active_ids().await?; for id in in_pool.iter().filter(|id| !active.contains(id)) { @@ -79,6 +106,45 @@ impl ThinPoolDisk { &self.pool } + /// Resume every device of this pool that is suspended: thins first, + /// then the drives and views stacked on them. + async fn resume_suspended(&self) -> io::Result<()> { + let prefix = format!("{}.", self.pool); + let mut suspended = Vec::new(); + for (name, _) in self.dm.list().await? { + if !name.starts_with(&prefix) { + continue; + } + match self.dm.info(&name).await { + Ok(info) if info.suspended => {} + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + _ => continue, + } + let thin = match self.dm.table(&name).await { + Ok(targets) => targets.iter().any(|t| t.kind == "thin"), + Err(e) if e.kind() == io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + }; + suspended.push((!thin, name)); + } + suspended.sort(); + for (_, name) in suspended { + tracing::warn!(device = %name, "resuming a device a process death left suspended"); + self.dm.resume(&name).await?; + } + Ok(()) + } + + /// The lock of drive or view `name`. + async fn lock_drive(&self, name: &VolumeName) -> tokio::sync::OwnedMutexGuard<()> { + let lock = { + let mut locks = self.drive_locks.lock().unwrap_or_else(|e| e.into_inner()); + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + Arc::clone(locks.entry(name.as_str().to_string()).or_default()) + }; + lock.lock_owned().await + } + /// Every thin id the pool holds, with or without a device. pub async fn thin_ids(&self) -> io::Result> { self.with_metadata_snap(dm::thin_dump_ids(&self.metadata_path)) @@ -155,14 +221,10 @@ impl ThinPoolDisk { self.dm .create(&self.device_name(name), sectors, "thin", ¶ms) .await?; - match self.volume(name, sectors).await { - Ok(volume) => Ok(volume), - Err(e) => { - let _ = tokio::fs::remove_file(self.node_of(name)).await; - let _ = self.dm.remove(&self.device_name(name)).await; - Err(e) - } - } + self.open_created(name, |node| { + Volume::open(name.clone(), node, sectors * SECTOR_BYTES, self.chunk_size) + }) + .await } .await; if activated.is_err() { @@ -173,19 +235,258 @@ impl ThinPoolDisk { activated } - async fn volume(&self, name: &VolumeName, sectors: u64) -> io::Result { + /// The device node of `name`, made when it is missing or stale. + async fn node(&self, name: &VolumeName) -> io::Result { let node = self.node_of(name); dm::make_node(&node, self.dm.devno(&self.device_name(name)).await?)?; + Ok(node) + } + + async fn volume(&self, name: &VolumeName, sectors: u64) -> io::Result { + let node = self.node(name).await?; Volume::open(name.clone(), &node, sectors * SECTOR_BYTES, self.chunk_size) } + /// Open a device just created through its node. On failure the node + /// and the device are removed again. + async fn open_created( + &self, + name: &VolumeName, + open: impl FnOnce(&Path) -> io::Result, + ) -> io::Result { + let opened = match self.node(name).await { + Ok(node) => open(&node), + Err(e) => Err(e), + }; + if opened.is_err() { + let _ = tokio::fs::remove_file(self.node_of(name)).await; + let _ = self.dm.remove(&self.device_name(name)).await; + } + opened + } + + /// Remove `name`'s device, waiting a bounded time for its last holder + /// to close it, then its node. + async fn remove_device(&self, name: &VolumeName) -> io::Result<()> { + let mut attempt = 1; + loop { + match self.dm.remove(&self.device_name(name)).await { + Err(e) if e.kind() == io::ErrorKind::ResourceBusy && attempt < REMOVE_ATTEMPTS => { + attempt += 1; + tokio::time::sleep(REMOVE_RETRY).await; + } + removed => break removed?, + } + } + match tokio::fs::remove_file(self.node_of(name)).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } + } + + /// What `name` is to the drive operations; `None` when it exists and + /// is not a drive or a view. + async fn drive_kind(&self, name: &VolumeName) -> io::Result> { + let device = self.device_name(name); + let targets = self.dm.table(&device).await?; + Ok(match targets.as_slice() { + [] => Some(Drive::Incomplete), + [target] if target.kind == "linear" => Some(Drive::CutOver { + sectors: target.length_sectors, + }), + [target] if target.kind == "clone" => { + let table = parse::clone_table(&target.params)?; + let sectors = target.length_sectors; + Some(if self.dm.info(&device).await?.read_only { + Drive::View { sectors } + } else { + Drive::Copying { table, sectors } + }) + } + _ => None, + }) + } + + async fn drive(&self, name: &VolumeName) -> io::Result { + self.drive_kind(name).await?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is not a cloned drive or a view"), + ) + }) + } + + /// The clone table of a drive that copies; `InvalidInput` for any + /// other drive or view. + async fn copying_drive(&self, drive: &VolumeName) -> io::Result<(CloneTable, u64)> { + match self.drive(drive).await? { + Drive::Copying { table, sectors } => Ok((table, sectors)), + other => Err(other.refused(drive)), + } + } + + /// A new empty thin with no device, by id. + async fn create_thin(&self) -> io::Result { + loop { + let id = self.next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; + match self + .dm + .message(&self.pool, &format!("create_thin {id}")) + .await + { + Ok(()) => return Ok(id), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + } + } + + /// The `block_bytes` blocks of `volume` in which thins `left` and + /// `right` map different pool blocks; `block_bytes` is a multiple of + /// the pool's block size. + async fn delta( + &self, + left: u32, + right: u32, + volume: &Volume, + block_bytes: u64, + ) -> io::Result> { + if !block_bytes.is_multiple_of(self.block_bytes) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "block size {block_bytes} is not a multiple of the pool's {} bytes", + self.block_bytes + ), + )); + } + let pool_blocks = self + .with_metadata_snap(dm::thin_delta(&self.metadata_path, left, right)) + .await?; + let per_block = block_bytes / self.block_bytes; + let count = engram_host_core::disk::block_count(volume, block_bytes); + let mut blocks: Vec = pool_blocks + .into_iter() + .map(|block| block / per_block) + .filter(|block| *block < count) + .collect(); + blocks.dedup(); + Ok(blocks) + } + + /// A clone table over two thins of this pool and `source`. + async fn clone_table( + &self, + dest: &Volume, + bitmap: &Volume, + source: &Path, + region_sectors: u64, + ) -> io::Result<(CloneTable, u64)> { + let (_, sectors) = self.thin_of(&dest.name).await?; + self.thin_of(&bitmap.name).await?; + let table = CloneTable { + metadata_dev: self.dm.devno(&self.device_name(&bitmap.name)).await?, + dest_dev: self.dm.devno(&self.device_name(&dest.name)).await?, + source_dev: dm::block_devno(source)?, + region_sectors, + }; + Ok((table, sectors)) + } + + /// The region counts of a `clone` target. + async fn clone_status(&self, drive: &VolumeName) -> io::Result { + let status = self.dm.status(&self.device_name(drive)).await?; + parse::clone_status(&single_target(&status, "clone")?) + } + + /// `ResourceBusy` while a device is stacked on `volume`. + async fn check_unused(&self, volume: &VolumeName) -> io::Result<()> { + let devno = self.dm.devno(&self.device_name(volume)).await?; + if dm::holders(devno)? > 0 { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!("{volume} is used by a drive or a view"), + )); + } + Ok(()) + } + + async fn cut_over_locked(&self, drive: &VolumeName) -> io::Result<()> { + let (table, sectors) = match self.drive(drive).await? { + Drive::CutOver { .. } => return Ok(()), + Drive::Copying { table, sectors } => (table, sectors), + other => return Err(other.refused(drive)), + }; + if !self.clone_status(drive).await?.is_hydrated() { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!("{drive} is still hydrating"), + )); + } + let device = self.device_name(drive); + let suspended = Suspended::suspend(&self.dm, &device).await?; + let loaded = self + .dm + .load(&device, sectors, "linear", &format!("{} 0", table.dest_dev)) + .await; + let resumed = suspended.resume().await; + loaded.and(resumed) + } + + async fn snapshot_clone_locked( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> io::Result<(Volume, Volume)> { + let (table, _) = self.copying_drive(drive).await?; + let dest_dev = self.dm.devno(&self.device_name(&dest.name)).await?; + let bitmap_dev = self.dm.devno(&self.device_name(&bitmap.name)).await?; + if table.dest_dev != dest_dev || table.metadata_dev != bitmap_dev { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{drive} copies into {} with bitmap {}, not {} and {}", + table.dest_dev, table.metadata_dev, dest.name, bitmap.name + ), + )); + } + let suspended = Suspended::suspend(&self.dm, &self.device_name(drive)).await?; + let mut made = Vec::with_capacity(2); + let taken = async { + for (origin, name) in [(bitmap, bitmap_snapshot), (dest, dest_snapshot)] { + made.push(self.snapshot(origin, name).await?); + } + Ok(()) + } + .await; + let resumed = suspended.resume().await; + if let Err(e) = taken.and(resumed) { + // The destination snapshot goes first: one left without its + // bitmap snapshot is refused, never read as a whole disk. + for snap in made.into_iter().rev() { + let name = snap.name.clone(); + drop(snap); + if let Err(error) = self.delete(&name).await { + tracing::warn!(volume = %name, %error, "snapshot left behind by a failed snapshot_clone"); + } + } + return Err(e); + } + let dest_snap = made.pop().expect("two snapshots taken"); + let bitmap_snap = made.pop().expect("two snapshots taken"); + Ok((dest_snap, bitmap_snap)) + } + /// Run `read` over a reserved metadata snapshot. The reserve is /// released when the guard drops, so a cancelled read releases it /// too. A reserve another holder left behind (a process that died, /// a read cancelled before this guard existed) is released first. async fn with_metadata_snap( &self, - read: impl std::future::Future>, + read: impl Future>, ) -> io::Result { let _held = self.metadata_snap.lock().await; if let Err(error) = self.dm.message(&self.pool, "reserve_metadata_snap").await { @@ -216,6 +517,83 @@ struct Thin { params: ThinParams, } +/// What a name is to the drive operations. +enum Drive { + /// A read-write `clone` target: copying into its destination. + Copying { table: CloneTable, sectors: u64 }, + /// A `linear` target onto the destination. + CutOver { sectors: u64 }, + /// A read-only `clone` target: a checkpoint's view. + View { sectors: u64 }, + /// No live table: a `clone_drive` that did not finish. + Incomplete, +} + +impl Drive { + fn refused(&self, name: &VolumeName) -> io::Error { + let what = match self { + Self::Copying { .. } => "is copying", + Self::CutOver { .. } => "is cut over", + Self::View { .. } => "is a view", + Self::Incomplete => "has no table", + }; + io::Error::new(io::ErrorKind::InvalidInput, format!("{name} {what}")) + } +} + +/// A suspended device. Until a resume succeeds it stays armed: a drop +/// resumes it, so a task aborted between the two still resumes it. +struct Suspended { + device: Option<(dm::Dm, String)>, +} + +impl Suspended { + async fn suspend(dm: &dm::Dm, name: &str) -> io::Result { + dm.suspend(name).await?; + Ok(Self { + device: Some((dm.clone(), name.to_string())), + }) + } + + /// Resume, retrying a bounded number of times. On failure the drop + /// tries once more and logs the device as left suspended. + async fn resume(mut self) -> io::Result<()> { + let (dm, name) = self.device.as_ref().expect("armed until resumed"); + let mut attempt = 1; + loop { + match dm.resume(name).await { + Ok(()) => break, + Err(error) if attempt < RESUME_ATTEMPTS => { + tracing::warn!(device = %name, attempt, %error, "resume failed; retrying"); + attempt += 1; + tokio::time::sleep(RESUME_RETRY).await; + } + Err(error) => return Err(error), + } + } + self.device = None; + Ok(()) + } +} + +impl Drop for Suspended { + fn drop(&mut self) { + if let Some((dm, name)) = self.device.take() { + if let Err(error) = dm.resume_blocking(&name) { + tracing::error!(device = %name, %error, "device left suspended"); + } + } + } +} + +/// Run `op` in a task of its own, so a caller that is dropped cannot stop +/// it between a suspend and the resume. +async fn detached( + op: impl Future> + Send + 'static, +) -> io::Result { + tokio::spawn(op).await.map_err(io::Error::other)? +} + /// A reserved metadata snapshot; released on drop. struct MetadataSnap<'a> { dm: &'a dm::Dm, @@ -244,18 +622,8 @@ fn single_target(targets: &[dm::Target], kind: &str) -> io::Result { impl SessionDisk for ThinPoolDisk { async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { let sectors = Self::sectors_of(size_bytes)?; - loop { - let id = self.next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; - match self - .dm - .message(&self.pool, &format!("create_thin {id}")) - .await - { - Ok(()) => return self.activate(name, id, sectors).await, - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, - Err(e) => return Err(e), - } - } + let id = self.create_thin().await?; + self.activate(name, id, sectors).await } async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { @@ -264,11 +632,8 @@ impl SessionDisk for ThinPoolDisk { let pool = self.pool.clone(); let next_id = self.next_id.clone(); let origin_name = self.device_name(&origin.name); - // The origin holds all I/O while it is suspended, so the resume - // must run whatever happens to the caller: the calls are one owned - // task. - let id = tokio::spawn(async move { - dm.suspend(&origin_name).await?; + let id = detached(async move { + let suspended = Suspended::suspend(&dm, &origin_name).await?; let created = loop { let id = next_id.fetch_add(1, Ordering::Relaxed) & ID_MASK; match dm @@ -279,7 +644,7 @@ impl SessionDisk for ThinPoolDisk { created => break created.map(|()| id), } }; - let resumed = dm.resume(&origin_name).await; + let resumed = suspended.resume().await; match (created, resumed) { (Ok(id), Ok(())) => Ok(id), (Ok(id), Err(e)) => { @@ -292,8 +657,7 @@ impl SessionDisk for ThinPoolDisk { (Err(e), _) => Err(e), } }) - .await - .map_err(io::Error::other)??; + .await?; self.activate(name, id, sectors).await } @@ -305,20 +669,7 @@ impl SessionDisk for ThinPoolDisk { /// Waits a bounded time for the last holder to close the device. async fn delete(&self, name: &VolumeName) -> io::Result<()> { let (thin, _) = self.thin_of(name).await?; - let mut attempt = 1; - loop { - match self.dm.remove(&self.device_name(name)).await { - Err(e) if e.kind() == io::ErrorKind::ResourceBusy && attempt < REMOVE_ATTEMPTS => { - attempt += 1; - tokio::time::sleep(REMOVE_RETRY).await; - } - removed => break removed?, - } - } - match tokio::fs::remove_file(self.node_of(name)).await { - Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), - _ => {} - } + self.remove_device(name).await?; self.dm .message(&self.pool, &format!("delete {}", thin.id)) .await @@ -337,6 +688,12 @@ impl SessionDisk for ThinPoolDisk { } async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { + if self.drive_kind(from).await?.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{from} is a drive or a view, which is not renamed"), + )); + } self.dm .rename(&self.device_name(from), &self.device_name(to)) .await?; @@ -359,6 +716,182 @@ impl SessionDisk for ThinPoolDisk { }) } + /// A `clone` target over the two thins and `source`, with region + /// [`CLONE_REGION_BYTES`]. + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + hydration: Hydration, + ) -> io::Result { + let _held = self.lock_drive(drive).await; + let device = self.device_name(drive); + match self.dm.info(&device).await { + Ok(_) => { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("{drive} already exists"), + )) + } + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + Err(_) => {} + } + let (table, sectors) = self + .clone_table(dest, bitmap, source, REGION_SECTORS) + .await?; + self.check_unused(&dest.name).await?; + self.check_unused(&bitmap.name).await?; + check_fresh_bitmap(bitmap).await?; + self.dm + .create(&device, sectors, "clone", &table.params(hydration)) + .await?; + self.open_created(drive, |node| { + Volume::open(drive.clone(), node, sectors * SECTOR_BYTES, self.chunk_size) + }) + .await + } + + async fn open_clone(&self, drive: &VolumeName) -> io::Result { + let (sectors, open): (u64, fn(_, &Path, _, _) -> _) = match self.drive(drive).await? { + Drive::Copying { sectors, .. } | Drive::CutOver { sectors } => (sectors, Volume::open), + Drive::View { sectors } => (sectors, Volume::open_read_only), + Drive::Incomplete => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{drive} has no table; remove it and make it again"), + )) + } + }; + let node = self.node(drive).await?; + open( + drive.clone(), + &node, + sectors * SECTOR_BYTES, + self.chunk_size, + ) + } + + async fn clone_state(&self, drive: &VolumeName) -> io::Result { + match self.drive(drive).await? { + Drive::Incomplete => Ok(CloneState::Incomplete), + Drive::CutOver { .. } => Ok(CloneState::CutOver), + Drive::Copying { .. } | Drive::View { .. } => self.clone_status(drive).await, + } + } + + /// One `BLKDISCARD` per joined range on the drive, then a flush. What + /// this relies on in `drivers/md/dm-clone-target.c` (6.8 and 6.12): + /// - A discard marks every region it covers whole as copied and + /// copies nothing (`process_deferred_discards`). With + /// `no_discard_passdown` the bio then completes without reaching + /// the destination (`complete_discard_bio`). Regions it covers in + /// part are left as they are (`bio_region_range` rounds inward). + /// - Neither path checks whether hydration is on. With the metadata + /// read-only, or a failed metadata update, the discard completes + /// without an error and marks nothing; those regions are copied. + /// - The marks are in memory until a commit: a flush or a suspend + /// commits them, as it commits copied regions. A region unmarked + /// by a crash is copied, which writes the same bytes again. + /// - Without passdown the drive takes discards up to its own limit, + /// region-aligned, and the block layer splits a larger range at + /// region boundaries, so a range needs no split here. + async fn skip_hydration(&self, drive: &VolumeName, ranges: &[Range]) -> io::Result<()> { + let _held = self.lock_drive(drive).await; + let (_, sectors) = self.copying_drive(drive).await?; + let ranges = skip_ranges(ranges, sectors * SECTOR_BYTES)?; + let node = self.node(drive).await?; + tokio::task::spawn_blocking(move || { + let file = File::options().write(true).open(node)?; + for range in ranges { + discard(&file, range.start, range.end - range.start)?; + } + file.sync_all() + }) + .await + .map_err(io::Error::other)? + } + + async fn enable_hydration(&self, drive: &VolumeName) -> io::Result<()> { + let _held = self.lock_drive(drive).await; + self.copying_drive(drive).await?; + self.dm + .message(&self.device_name(drive), "enable_hydration") + .await + } + + /// Suspend, load `linear` onto the destination, resume. A region the + /// clone has copied stays copied, so the check before the suspend + /// still holds at the swap. + async fn cut_over(&self, drive: &VolumeName) -> io::Result<()> { + let (disk, drive) = (self.clone(), drive.clone()); + detached(async move { + let _held = disk.lock_drive(&drive).await; + disk.cut_over_locked(&drive).await + }) + .await + } + + /// Both snapshots, the bitmap first, are taken while the drive is + /// suspended: the suspend stops the background copy and commits the + /// bitmap. + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> io::Result<(Volume, Volume)> { + let disk = self.clone(); + let (drive, dest, bitmap) = (drive.clone(), dest.clone(), bitmap.clone()); + let (dest_snapshot, bitmap_snapshot) = (dest_snapshot.clone(), bitmap_snapshot.clone()); + detached(async move { + let _held = disk.lock_drive(&drive).await; + disk.snapshot_clone_locked(&drive, &dest, &bitmap, &dest_snapshot, &bitmap_snapshot) + .await + }) + .await + } + + /// A read-only `clone` target with hydration off, with the region size + /// the bitmap snapshot's superblock records. + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + source: &Path, + ) -> io::Result { + let _held = self.lock_drive(view).await; + let superblock = bitmap_snapshot.read_direct(0, CLONE_HEADER_BYTES).await?; + let region_sectors = parse::clone_superblock_region_sectors(&superblock)?; + let (table, sectors) = self + .clone_table(dest_snapshot, bitmap_snapshot, source, region_sectors) + .await?; + self.dm + .create_read_only( + &self.device_name(view), + sectors, + "clone", + &table.params(VIEW), + ) + .await?; + self.open_created(view, |node| { + Volume::open_read_only(view.clone(), node, sectors * SECTOR_BYTES, self.chunk_size) + }) + .await + } + + /// Also removes a drive left with no table by a crash inside + /// `clone_drive`. + async fn remove_clone(&self, drive: &VolumeName) -> io::Result<()> { + let _held = self.lock_drive(drive).await; + self.drive(drive).await?; + self.remove_device(drive).await + } + /// Reads the pool's mapping tree instead of the volumes. A block is /// dirty when any pool block inside it differs; `block_bytes` is a /// multiple of the pool's block size. @@ -370,28 +903,34 @@ impl SessionDisk for ThinPoolDisk { ) -> io::Result> { engram_host_core::disk::same_origin(before, after)?; engram_host_core::disk::blocks_per_chunk(before, block_bytes)?; - if !block_bytes.is_multiple_of(self.block_bytes) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "block size {block_bytes} is not a multiple of the pool's {} bytes", - self.block_bytes - ), - )); - } let (left, _) = self.thin_of(&before.name).await?; let (right, _) = self.thin_of(&after.name).await?; - let pool_blocks = self - .with_metadata_snap(dm::thin_delta(&self.metadata_path, left.id, right.id)) - .await?; - let per_block = block_bytes / self.block_bytes; - let count = engram_host_core::disk::block_count(before, block_bytes); - let mut blocks: Vec = pool_blocks - .into_iter() - .map(|block| block / per_block) - .filter(|block| *block < count) - .collect(); - blocks.dedup(); - Ok(blocks) + self.delta(left.id, right.id, before, block_bytes).await + } + + /// The pool blocks the snapshot maps differently from `origin`. With + /// no origin, from an empty thin made for the read and deleted after + /// it; one a crash leaves is deleted by the next [`Self::open`]. + async fn written_chunks( + &self, + origin: Option<&Volume>, + snapshot: &Volume, + ) -> io::Result> { + if let Some(origin) = origin { + return self.dirty_chunks(origin, snapshot).await; + } + let (right, _) = self.thin_of(&snapshot.name).await?; + let empty = self.create_thin().await?; + let blocks = self + .delta(empty, right.id, snapshot, snapshot.chunk_size) + .await; + if let Err(error) = self + .dm + .message(&self.pool, &format!("delete {empty}")) + .await + { + tracing::warn!(pool = %self.pool, id = empty, %error, "empty thin left behind"); + } + Ok(blocks?.into_iter().map(ChunkIndex).collect()) } } diff --git a/crates/engram-disk-thin/src/dm.rs b/crates/engram-disk-thin/src/dm.rs index 073caaa5b..d4594b7b4 100644 --- a/crates/engram-disk-thin/src/dm.rs +++ b/crates/engram-disk-thin/src/dm.rs @@ -25,6 +25,7 @@ const DM_TABLE_LOAD_CMD: u32 = 9; const DM_TABLE_STATUS_CMD: u32 = 12; const DM_TARGET_MSG_CMD: u32 = 14; +const DM_READONLY_FLAG: u32 = 1 << 0; const DM_SUSPEND_FLAG: u32 = 1 << 1; const DM_STATUS_TABLE_FLAG: u32 = 1 << 4; const DM_BUFFER_FULL_FLAG: u32 = 1 << 8; @@ -55,6 +56,15 @@ const fn request(cmd: u32) -> libc::Ioctl { (3 << 30 | (HEADER_LEN as u32) << 16 | DM_IOCTL << 8 | cmd) as libc::Ioctl } +/// A device's number and state, from its status reply. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Info { + pub dev: DevNo, + pub suspended: bool, + /// The live table was loaded read-only. + pub read_only: bool, +} + /// One target line of a table or status. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Target { @@ -322,6 +332,30 @@ impl Dm { length_sectors: u64, kind: &str, params: &str, + ) -> io::Result<()> { + self.create_with(name, length_sectors, kind, params, 0) + .await + } + + /// [`Self::create`] for a device the kernel refuses writes to. + pub async fn create_read_only( + &self, + name: &str, + length_sectors: u64, + kind: &str, + params: &str, + ) -> io::Result<()> { + self.create_with(name, length_sectors, kind, params, DM_READONLY_FLAG) + .await + } + + async fn create_with( + &self, + name: &str, + length_sectors: u64, + kind: &str, + params: &str, + load_flags: u32, ) -> io::Result<()> { self.call( "create", @@ -331,19 +365,11 @@ impl Dm { io::ErrorKind::AlreadyExists, ) .await?; - let mut load = Request::new(name, 0)?; - load.push_target(length_sectors, kind, params); - let loaded = self - .call( - "load", - name, - load, - DM_TABLE_LOAD_CMD, - io::ErrorKind::ResourceBusy, - ) - .await; - let ready = match loaded { - Ok(_) => self.resume(name).await, + let ready = match self + .load_with(name, length_sectors, kind, params, load_flags) + .await + { + Ok(()) => self.resume(name).await, Err(e) => Err(e), }; if ready.is_err() { @@ -352,6 +378,39 @@ impl Dm { ready } + /// Load a single-target table into the device's inactive slot; the + /// next `resume` makes it live. + pub async fn load( + &self, + name: &str, + length_sectors: u64, + kind: &str, + params: &str, + ) -> io::Result<()> { + self.load_with(name, length_sectors, kind, params, 0).await + } + + async fn load_with( + &self, + name: &str, + length_sectors: u64, + kind: &str, + params: &str, + flags: u32, + ) -> io::Result<()> { + let mut load = Request::new(name, flags)?; + load.push_target(length_sectors, kind, params); + self.call( + "load", + name, + load, + DM_TABLE_LOAD_CMD, + io::ErrorKind::ResourceBusy, + ) + .await + .map(drop) + } + pub async fn remove(&self, name: &str) -> io::Result<()> { self.call( "remove", @@ -410,6 +469,14 @@ impl Dm { .map(drop) } + /// [`Self::resume`] on the calling thread, for a `Drop`. + pub fn resume_blocking(&self, name: &str) -> io::Result<()> { + Request::new(name, 0)? + .ioctl_blocking(&self.control, DM_DEV_SUSPEND_CMD) + .map(drop) + .map_err(|e| describe("resume", name, io::ErrorKind::ResourceBusy, e)) + } + pub async fn message(&self, name: &str, text: &str) -> io::Result<()> { let mut req = Request::new(name, 0)?; req.push_message(text); @@ -434,6 +501,10 @@ impl Dm { } pub async fn devno(&self, name: &str) -> io::Result { + Ok(self.info(name).await?.dev) + } + + pub async fn info(&self, name: &str) -> io::Result { let reply = self .call( "status", @@ -443,7 +514,12 @@ impl Dm { io::ErrorKind::ResourceBusy, ) .await?; - Ok(reply.dev()) + let flags = reply.flags(); + Ok(Info { + dev: reply.dev(), + suspended: flags & DM_SUSPEND_FLAG != 0, + read_only: flags & DM_READONLY_FLAG != 0, + }) } /// The active table's target lines. @@ -486,6 +562,29 @@ pub async fn device_path(devno: DevNo) -> io::Result { Ok(PathBuf::from("/dev").join(parse::devname(&uevent)?)) } +/// The device number of the block device node at `path`. +pub fn block_devno(path: &Path) -> io::Result { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + let meta = std::fs::metadata(path)?; + if !meta.file_type().is_block_device() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a block device", path.display()), + )); + } + let rdev = meta.rdev(); + Ok(DevNo { + major: libc::major(rdev), + minor: libc::minor(rdev), + }) +} + +/// The number of devices stacked on `devno`: a device-mapper table that +/// uses it, a partition, a RAID member. +pub fn holders(devno: DevNo) -> io::Result { + Ok(std::fs::read_dir(format!("{}/holders", devno.sysfs_path()))?.count()) +} + /// Create a block device node, replacing one that points elsewhere. #[allow(unsafe_code)] pub fn make_node(path: &Path, devno: DevNo) -> io::Result<()> { diff --git a/crates/engram-disk-thin/src/parse.rs b/crates/engram-disk-thin/src/parse.rs index 1fa11c43c..396f5e7cc 100644 --- a/crates/engram-disk-thin/src/parse.rs +++ b/crates/engram-disk-thin/src/parse.rs @@ -4,6 +4,8 @@ use std::io; +use engram_host_core::disk::{CloneState, Hydration}; + /// A block device number: `major:minor`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DevNo { @@ -110,6 +112,106 @@ pub fn pool_status(status: &str) -> io::Result { } } +/// `clone` target params: ` `, +/// then the feature and core arguments. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CloneTable { + pub metadata_dev: DevNo, + pub dest_dev: DevNo, + pub source_dev: DevNo, + pub region_sectors: u64, +} + +impl CloneTable { + /// The params a table load takes. The kernel reads `no_hydration` + /// only at load; afterwards the `enable_hydration` message turns it + /// on. `no_discard_passdown` keeps a discard, which marks regions + /// copied, from reaching the destination: its blocks stay as they + /// are. + pub fn params(&self, hydration: Hydration) -> String { + let features = if hydration.enabled { + "1 no_discard_passdown" + } else { + "2 no_hydration no_discard_passdown" + }; + format!( + "{} {} {} {} {features} 4 hydration_threshold {} hydration_batch_size {}", + self.metadata_dev, + self.dest_dev, + self.source_dev, + self.region_sectors, + hydration.threshold_regions, + hydration.batch_regions + ) + } +} + +pub fn clone_table(params: &str) -> io::Result { + let fields: Vec<&str> = params.split_whitespace().collect(); + match fields.as_slice() { + [metadata, dest, source, region_sectors, ..] => Ok(CloneTable { + metadata_dev: DevNo::parse(metadata)?, + dest_dev: DevNo::parse(dest)?, + source_dev: DevNo::parse(source)?, + region_sectors: number(region_sectors)?, + }), + _ => Err(invalid(format!("not clone params: {params:?}"))), + } +} + +/// `clone` status: ` / / +/// ... ... `. +/// +/// A clone whose metadata is not read-write (`ro`, or a single-word +/// `Fail`) is an error: it cannot record a copy, so its counts no longer +/// move. +pub fn clone_status(status: &str) -> io::Result { + let fields: Vec<&str> = status.split_whitespace().collect(); + let [_, _, _, regions, _, features, rest @ ..] = fields.as_slice() else { + return match fields.as_slice() { + [word] => Err(invalid(format!("clone is {word}"))), + _ => Err(invalid(format!("not clone status: {status:?}"))), + }; + }; + let n_features: usize = number(features)?; + let feature_args = rest + .get(..n_features) + .ok_or_else(|| invalid(format!("clone status is short: {status:?}")))?; + match rest.last() { + Some(&"rw") => {} + Some(mode) => return Err(invalid(format!("clone is {mode}"))), + None => return Err(invalid(format!("not clone status: {status:?}"))), + } + let (hydrated_regions, total_regions) = fraction(regions)?; + Ok(CloneState::Copying { + hydrated_regions, + total_regions, + hydration_enabled: !feature_args.contains(&"no_hydration"), + }) +} + +const CLONE_MAGIC: u64 = 0x8af2_7f64; +const CLONE_MAGIC_AT: usize = 32; +const CLONE_REGION_AT: usize = 172; + +/// The region size, in sectors, that a dm-clone superblock records +/// (`struct superblock_disk` in `drivers/md/dm-clone-metadata.c`: the +/// magic at byte 32 and the region size at byte 172, little-endian). +/// The kernel refuses a table whose region size differs. +pub fn clone_superblock_region_sectors(superblock: &[u8]) -> io::Result { + let le64 = |at: usize| { + superblock + .get(at..at + 8) + .map(|b| u64::from_le_bytes(b.try_into().unwrap())) + .ok_or_else(|| invalid("dm-clone superblock is short")) + }; + if le64(CLONE_MAGIC_AT)? != CLONE_MAGIC { + return Err(invalid("no dm-clone superblock")); + } + le64(CLONE_REGION_AT) +} + /// The `DEVNAME=` line of a sysfs `uevent` file. pub fn devname(uevent: &str) -> io::Result { uevent @@ -278,4 +380,124 @@ mod tests { .unwrap() .is_empty()); } + + fn dev(major: u32, minor: u32) -> DevNo { + DevNo { major, minor } + } + + /// `dmsetup table` of a clone made with hydration off, kernel 6.8. + const CLONE_TABLE: &str = "252:2 252:1 7:4 128 2 no_hydration no_discard_passdown 4 hydration_threshold 2 hydration_batch_size 3"; + + #[test] + fn clone_params_format_what_the_kernel_reads_back() { + let table = CloneTable { + metadata_dev: dev(252, 2), + dest_dev: dev(252, 1), + source_dev: dev(7, 4), + region_sectors: 128, + }; + let off = Hydration { + enabled: false, + threshold_regions: 2, + batch_regions: 3, + }; + assert_eq!(table.params(off), CLONE_TABLE); + assert_eq!( + table.params(Hydration { + enabled: true, + ..off + }), + "252:2 252:1 7:4 128 1 no_discard_passdown 4 hydration_threshold 2 hydration_batch_size 3" + ); + assert_eq!(clone_table(CLONE_TABLE).unwrap(), table); + assert!(clone_table("252:2 252:1 7:4").is_err()); + } + + #[test] + fn clone_status_reads_the_region_counts_and_the_hydration_flag() { + // `dmsetup status` of one clone, kernel 6.8: new, after one guest + // write, hydrated with hydration on. + let fresh = clone_status( + "8 6/1024 128 0/64 0 1 no_hydration 4 hydration_threshold 2 hydration_batch_size 3 rw", + ) + .unwrap(); + assert_eq!( + fresh, + CloneState::Copying { + hydrated_regions: 0, + total_regions: 64, + hydration_enabled: false + } + ); + let written = clone_status( + "8 6/1024 128 1/64 0 1 no_hydration 4 hydration_threshold 2 hydration_batch_size 3 rw", + ) + .unwrap(); + assert!(matches!( + written, + CloneState::Copying { + hydrated_regions: 1, + .. + } + )); + let done = clone_status( + "8 6/1024 128 64/64 0 0 4 hydration_threshold 2 hydration_batch_size 3 rw", + ) + .unwrap(); + assert_eq!( + done, + CloneState::Copying { + hydrated_regions: 64, + total_regions: 64, + hydration_enabled: true + } + ); + // The kernel emits `no_hydration` before `no_discard_passdown`. + for (features, enabled) in [ + ("2 no_hydration no_discard_passdown", false), + ("1 no_discard_passdown", true), + ] { + let status = format!( + "8 6/1024 128 5/64 0 {features} 4 hydration_threshold 2 hydration_batch_size 3 rw" + ); + assert_eq!( + clone_status(&status).unwrap(), + CloneState::Copying { + hydrated_regions: 5, + total_regions: 64, + hydration_enabled: enabled + } + ); + } + } + + #[test] + fn the_region_size_comes_from_a_clone_superblock() { + // The first bytes of a bitmap after one `clone` table load with + // region 128 and 16384 sectors, kernel 6.8. + let mut superblock = vec![0u8; 4096]; + superblock[..4].copy_from_slice(&[0x4a, 0x96, 0x31, 0xd2]); + superblock[32..36].copy_from_slice(&[0x64, 0x7f, 0xf2, 0x8a]); + superblock[40] = 1; + superblock[172] = 0x80; + superblock[181] = 0x40; + assert_eq!(clone_superblock_region_sectors(&superblock).unwrap(), 128); + assert!(clone_superblock_region_sectors(&vec![0u8; 4096]).is_err()); + assert!(clone_superblock_region_sectors(&superblock[..100]).is_err()); + } + + #[test] + fn clone_status_rejects_a_clone_that_cannot_record_a_copy() { + for (status, want) in [ + ("Fail", "clone is Fail"), + ( + "8 6/1024 128 3/64 0 0 4 hydration_threshold 2 hydration_batch_size 3 ro", + "clone is ro", + ), + ] { + assert_eq!(clone_status(status).unwrap_err().to_string(), want); + } + assert!(clone_status("8 6/1024 128 3/64").is_err()); + assert!(clone_status("8 6/1024 128 3/64 0 2 no_hydration").is_err()); + } } diff --git a/crates/engram-disk-thin/tests/clone.rs b/crates/engram-disk-thin/tests/clone.rs new file mode 100644 index 000000000..f90f86094 --- /dev/null +++ b/crates/engram-disk-thin/tests/clone.rs @@ -0,0 +1,1430 @@ +#![cfg(target_os = "linux")] +//! Cloned drives against a real dm-thin pool, as root. +//! `ENGRAM_TEST_THIN_POOL` names the pool; unset, every test skips. The +//! source is a read-only loop device over a file, behind dm-delay where +//! a test needs the background copy to take time. + +use std::collections::HashMap; +use std::io::ErrorKind; +use std::os::unix::fs::{FileExt, MetadataExt}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use engram_chunk_store::{ChunkStore, ManifestRef}; +use engram_core::types::ids::SandboxId; +use engram_core::types::snapshot::DiskRole; +use engram_disk_thin::ThinPoolDisk; +use engram_host_core::disk::{ + clone_bitmap_bytes, ChunkIndex, CloneState, Hydration, SessionDisk, Volume, VolumeName, + CLONE_REGION_BYTES, +}; +use engram_host_core::disk_sources::Served; +use engram_host_core::image_volumes; +use engram_host_core::session_volumes::{ + checkpoint_name, dest_name, live_name, Cutover, Source, Waiting, +}; +use engram_host_core::{checkpoint, stage, DiskSources, FileDisk, FileSources, SessionVolumes}; +use engram_storage_local::LocalBlobStorage; + +const CHUNK: u64 = 65536; +const REGION: u64 = CLONE_REGION_BYTES; +const REGIONS: u64 = 64; +const SIZE: u64 = REGION * REGIONS; +const HYDRATION_DEADLINE: Duration = Duration::from_secs(60); +const NODES: &str = "/dev/engram-test"; + +/// A pool, a source device of `SIZE` bytes, and the names one test uses. +/// Dropping it removes every device and node the test left, so a failed +/// test leaves none behind; a thin it leaves is deleted by the next +/// `ThinPoolDisk::open`. +struct Rig { + disk: ThinPoolDisk, + test: &'static str, + source: PathBuf, + bytes: Vec, + loop_dev: String, + _dir: tempfile::TempDir, +} + +impl Rig { + /// `None`, with a `SKIP:` line, when no pool is named. + async fn new(test: &'static str, read_delay_ms: Option) -> Option { + let Ok(pool) = std::env::var("ENGRAM_TEST_THIN_POOL") else { + eprintln!("SKIP: ENGRAM_TEST_THIN_POOL is not set"); + return None; + }; + let disk = ThinPoolDisk::open(&pool, CHUNK, Path::new(NODES)) + .await + .expect("open pool"); + let dir = tempfile::tempdir().unwrap(); + let bytes = pattern(SIZE as usize, 0x9E37_79B9); + let file = dir.path().join("source.img"); + std::fs::write(&file, &bytes).unwrap(); + let loop_dev = run( + "losetup", + &["--read-only", "--show", "-f", file.to_str().unwrap()], + ); + let mut rig = Self { + source: PathBuf::from(&loop_dev), + disk, + test, + bytes, + loop_dev, + _dir: dir, + }; + if let Some(ms) = read_delay_ms { + let delay = rig.device(&rig.name("delay")); + let table = format!("0 {} delay {} 0 {ms}", SIZE / 512, rig.loop_dev); + run( + "dmsetup", + &["create", "--noudevsync", &delay, "--table", &table], + ); + run("dmsetup", &["mknodes", &delay]); + rig.source = PathBuf::from("/dev/mapper").join(&delay); + } + Some(rig) + } + + fn name(&self, role: &str) -> VolumeName { + VolumeName::new(format!("t-{}-{role}-{}", self.test, std::process::id())).unwrap() + } + + fn device(&self, name: &VolumeName) -> String { + self.disk.device_name(name) + } + + /// Every device-mapper device this test made that still exists. + fn leftovers(&self) -> Vec { + let suffix = format!("-{}", std::process::id()); + let prefix = format!("{}.t-{}-", self.disk.pool(), self.test); + run("dmsetup", &["ls"]) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| name.starts_with(&prefix) && name.ends_with(&suffix)) + .map(str::to_string) + .collect() + } + + /// [`Self::leftovers`] the disk made: all but the source's delay. + fn disk_leftovers(&self) -> Vec { + let delay = self.device(&self.name("delay")); + self.leftovers() + .into_iter() + .filter(|device| *device != delay) + .collect() + } + + async fn volume(&self, role: &str, size: u64) -> Volume { + self.disk.create(&self.name(role), size).await.unwrap() + } + + async fn clone_drive(&self, dest: &Volume, bitmap: &Volume, hydration: Hydration) -> Volume { + self.disk + .clone_drive(&self.name("drive"), dest, bitmap, &self.source, hydration) + .await + .unwrap() + } + + async fn state(&self, role: &str) -> CloneState { + self.disk.clone_state(&self.name(role)).await.unwrap() + } + + /// Poll until the drive has copied `at_least` regions. + async fn hydrated(&self, at_least: u64) -> u64 { + let deadline = tokio::time::Instant::now() + HYDRATION_DEADLINE; + loop { + match self.state("drive").await { + CloneState::Copying { + hydrated_regions, .. + } if hydrated_regions >= at_least => return hydrated_regions, + CloneState::Copying { .. } => {} + state => panic!("the drive is {state:?}"), + } + assert!( + tokio::time::Instant::now() < deadline, + "fewer than {at_least} regions copied in {HYDRATION_DEADLINE:?}" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } +} + +impl Drop for Rig { + fn drop(&mut self) { + // A drive or view holds its thins, and every device holds the + // source, so remove in passes until nothing is left. A device a + // failed test left suspended is resumed first, so no I/O waits on + // it. + for _ in 0..4 { + for device in self.leftovers() { + for args in [vec!["resume"], vec!["remove", "--noudevsync"]] { + let _ = Command::new("dmsetup").args(args).arg(&device).output(); + } + let _ = std::fs::remove_file(Path::new(NODES).join(&device)); + } + } + let _ = Command::new("losetup") + .args(["-d", &self.loop_dev]) + .output(); + } +} + +fn run(program: &str, args: &[&str]) -> String { + let output = Command::new(program).args(args).output().unwrap(); + assert!( + output.status.success(), + "{program} {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_string() +} + +/// `len` bytes of xorshift output: no region repeats another or is zero. +fn pattern(len: usize, mut state: u32) -> Vec { + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect() +} + +fn off(threshold_regions: u32, batch_regions: u32) -> Hydration { + Hydration { + enabled: false, + threshold_regions, + batch_regions, + } +} + +async fn write(volume: &Volume, expected: &mut [u8], offset: u64, bytes: Vec) { + expected[offset as usize..offset as usize + bytes.len()].copy_from_slice(&bytes); + volume.write_range(offset, bytes).await.unwrap(); +} + +async fn whole(volume: &Volume) -> Vec { + volume.read_range(0, volume.size_bytes).await.unwrap() +} + +/// `major:minor` of a volume's device node. +fn devno(volume: &Volume) -> String { + let rdev = std::fs::metadata(&volume.path).unwrap().rdev(); + format!("{}:{}", libc::major(rdev), libc::minor(rdev)) +} + +/// The first region at which two images differ, for a short message. +fn first_difference(got: &[u8], want: &[u8]) -> Option { + got.iter() + .zip(want) + .position(|(g, w)| g != w) + .map(|byte| byte as u64 / REGION) +} + +#[tokio::test] +#[ignore] +async fn a_clone_reads_the_source_and_cuts_over_with_a_handle_open() { + let Some(rig) = Rig::new("cutover", None).await else { + return; + }; + let dest = rig.volume("dest", SIZE).await; + let bitmap = rig.volume("bitmap", clone_bitmap_bytes(SIZE)).await; + let drive = rig.clone_drive(&dest, &bitmap, off(4, 4)).await; + assert_eq!( + drive.path, + Path::new(NODES).join(rig.device(&rig.name("drive"))) + ); + assert!(!rig.disk.list().await.unwrap().contains(&rig.name("drive"))); + + // Reads of uncopied regions come from the source and copy nothing. + let mut expected = rig.bytes.clone(); + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + assert_eq!( + rig.state("drive").await, + CloneState::Copying { + hydrated_regions: 0, + total_regions: REGIONS, + hydration_enabled: false + } + ); + + // A partial write copies its region first; a whole-region write does + // not need to. Both land. + write(&drive, &mut expected, 3 * REGION + 4096, vec![0xA5; 4096]).await; + write( + &drive, + &mut expected, + 5 * REGION, + pattern(REGION as usize, 7), + ) + .await; + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + assert_eq!(rig.hydrated(2).await, 2); + assert_eq!( + rig.disk + .cut_over(&rig.name("drive")) + .await + .unwrap_err() + .kind(), + ErrorKind::ResourceBusy + ); + + rig.disk.enable_hydration(&rig.name("drive")).await.unwrap(); + rig.hydrated(REGIONS).await; + assert!(matches!( + rig.state("drive").await, + CloneState::Copying { + hydration_enabled: true, + .. + } + )); + + let held = std::fs::File::open(&drive.path).unwrap(); + let rdev = held.metadata().unwrap().rdev(); + rig.disk.cut_over(&rig.name("drive")).await.unwrap(); + rig.disk.cut_over(&rig.name("drive")).await.unwrap(); + assert_eq!(rig.state("drive").await, CloneState::CutOver); + let table = run("dmsetup", &["table", &rig.device(&rig.name("drive"))]); + assert_eq!(table.split_whitespace().nth(2), Some("linear"), "{table}"); + assert_eq!(std::fs::metadata(&drive.path).unwrap().rdev(), rdev); + + // The bitmap is free, and both handles opened before the cutover work. + drop(bitmap); + rig.disk.delete(&rig.name("bitmap")).await.unwrap(); + write(&drive, &mut expected, 9 * REGION, vec![0x5A; 8192]).await; + drive.sync().await.unwrap(); + let mut through_held = vec![0u8; 8192]; + held.read_exact_at(&mut through_held, 9 * REGION).unwrap(); + assert_eq!(through_held, vec![0x5A; 8192]); + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + assert_eq!(first_difference(&whole(&dest).await, &expected), None); + let reopened = rig.disk.open_clone(&rig.name("drive")).await.unwrap(); + assert_eq!(reopened.path, drive.path); + + drop((held, drive, reopened)); + rig.disk.remove_clone(&rig.name("drive")).await.unwrap(); + assert_eq!(first_difference(&whole(&dest).await, &expected), None); + drop(dest); + rig.disk.delete(&rig.name("dest")).await.unwrap(); + assert_eq!(rig.disk_leftovers(), Vec::::new()); +} + +/// The checkpoint view: snapshots of the destination and the bitmap, +/// taken while the copy runs and after guest writes, composed again with +/// the source read exactly what the guest saw. The drive keeps copying +/// and cuts over unharmed. +#[tokio::test] +#[ignore] +async fn a_view_over_snapshots_reads_what_the_guest_saw_mid_hydration() { + let Some(rig) = Rig::new("view", Some(30)).await else { + return; + }; + let dest = rig.volume("dest", SIZE).await; + let bitmap = rig.volume("bitmap", clone_bitmap_bytes(SIZE)).await; + let drive = rig.clone_drive(&dest, &bitmap, off(1, 1)).await; + let mut expected = rig.bytes.clone(); + write(&drive, &mut expected, 3 * REGION + 512, vec![0x11; 4096]).await; + write( + &drive, + &mut expected, + 5 * REGION, + pattern(REGION as usize, 5), + ) + .await; + drive.sync().await.unwrap(); + rig.disk.enable_hydration(&rig.name("drive")).await.unwrap(); + rig.hydrated(8).await; + + let (dest_snap, bitmap_snap) = rig + .disk + .snapshot_clone( + &rig.name("drive"), + &dest, + &bitmap, + &rig.name("dest-snap"), + &rig.name("bitmap-snap"), + ) + .await + .unwrap(); + let seen = expected.clone(); + // Writes after the snapshot, to a copied and an uncopied region. + write(&drive, &mut expected, REGION, vec![0x22; REGION as usize]).await; + write(&drive, &mut expected, 60 * REGION + 4096, vec![0x33; 4096]).await; + drive.sync().await.unwrap(); + + let view = rig + .disk + .clone_view(&rig.name("view"), &dest_snap, &bitmap_snap, &rig.source) + .await + .unwrap(); + let CloneState::Copying { + hydrated_regions: at_snapshot, + hydration_enabled: false, + .. + } = rig.state("view").await + else { + panic!("a view is a clone with hydration off"); + }; + eprintln!("{at_snapshot} of {REGIONS} regions copied at the snapshot"); + assert!( + (8..REGIONS).contains(&at_snapshot), + "{at_snapshot} regions copied at the snapshot" + ); + // The destination snapshot alone holds zeros where nothing was + // copied; the view reads the source there. + assert_eq!( + dest_snap.read_range(63 * REGION, REGION).await.unwrap(), + vec![0; REGION as usize] + ); + assert_eq!(first_difference(&whole(&view).await, &seen), None); + assert!(matches!( + rig.state("view").await, + CloneState::Copying { hydrated_regions, .. } if hydrated_regions == at_snapshot + )); + assert!(view.write_range(0, vec![0xFF; 4096]).await.is_err()); + let reopened = rig.disk.open_clone(&rig.name("view")).await.unwrap(); + assert!(reopened.write_range(0, vec![0xFF; 4096]).await.is_err()); + let refused = [ + rig.disk.cut_over(&rig.name("view")).await, + rig.disk.enable_hydration(&rig.name("view")).await, + rig.disk.rename(&rig.name("view"), &rig.name("moved")).await, + rig.disk + .snapshot_clone( + &rig.name("view"), + &dest_snap, + &bitmap_snap, + &rig.name("s1"), + &rig.name("s2"), + ) + .await + .map(drop), + ]; + for result in refused { + assert_eq!(result.unwrap_err().kind(), ErrorKind::InvalidInput); + } + drop(reopened); + + // The drive copies on and cuts over with the guest's every write. + rig.hydrated(REGIONS).await; + rig.disk.cut_over(&rig.name("drive")).await.unwrap(); + drop(bitmap); + rig.disk.delete(&rig.name("bitmap")).await.unwrap(); + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + assert_eq!(first_difference(&whole(&dest).await, &expected), None); + // The view depends on its snapshots only. + assert_eq!(first_difference(&whole(&view).await, &seen), None); + + drop((view, drive, dest, dest_snap, bitmap_snap)); + rig.disk.remove_clone(&rig.name("view")).await.unwrap(); + rig.disk.remove_clone(&rig.name("drive")).await.unwrap(); + for role in ["dest-snap", "bitmap-snap", "dest"] { + rig.disk.delete(&rig.name(role)).await.unwrap(); + } + assert_eq!(rig.disk_leftovers(), Vec::::new()); +} + +#[tokio::test] +#[ignore] +async fn a_drive_refuses_what_its_state_does_not_allow() { + let Some(rig) = Rig::new("refuse", None).await else { + return; + }; + let dest = rig.volume("dest", SIZE).await; + let bitmap = rig.volume("bitmap", clone_bitmap_bytes(SIZE)).await; + let big = rig.volume("big", 2 * SIZE).await; + let other = rig.volume("other", clone_bitmap_bytes(SIZE)).await; + let dest2 = rig.volume("dest2", SIZE).await; + let small = rig.volume("small", 512).await; + let taken = rig.volume("taken", clone_bitmap_bytes(SIZE)).await; + + // The source is smaller than the destination: the kernel refuses the + // table and no device is left. + let err = rig + .disk + .clone_drive(&rig.name("drive"), &big, &bitmap, &rig.source, off(1, 1)) + .await + .unwrap_err(); + assert!( + !rig.leftovers().iter().any(|d| d.contains("-drive-")), + "{err}" + ); + + let err = rig + .disk + .clone_drive(&rig.name("drive"), &dest, &small, &rig.source, off(1, 1)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput, "{err}"); + + let drive = rig.clone_drive(&dest, &bitmap, off(1, 1)).await; + let err = rig + .disk + .clone_drive(&rig.name("drive"), &dest, &bitmap, &rig.source, off(1, 1)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::AlreadyExists); + // A second drive over the first one's destination or bitmap. + for (d, b) in [(&dest, &other), (&dest2, &bitmap)] { + let err = rig + .disk + .clone_drive(&rig.name("second"), d, b, &rig.source, off(1, 1)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ResourceBusy, "{err}"); + } + assert_eq!( + rig.disk.delete(&rig.name("dest")).await.unwrap_err().kind(), + ErrorKind::ResourceBusy + ); + assert_eq!( + rig.disk + .rename(&rig.name("drive"), &rig.name("moved")) + .await + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + + // A snapshot name that is taken: the call fails, deletes nothing it + // did not make, and leaves the drive running. + let err = rig + .disk + .snapshot_clone( + &rig.name("drive"), + &dest, + &bitmap, + &rig.name("taken"), + &rig.name("bitmap-snap"), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::AlreadyExists, "{err}"); + rig.disk.open(&rig.name("taken")).await.unwrap(); + assert_eq!( + rig.disk + .open(&rig.name("bitmap-snap")) + .await + .unwrap_err() + .kind(), + ErrorKind::NotFound + ); + let mut expected = rig.bytes.clone(); + write(&drive, &mut expected, 4096, vec![0x77; 4096]).await; + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + let err = rig + .disk + .snapshot_clone( + &rig.name("drive"), + &dest, + &other, + &rig.name("dest-snap"), + &rig.name("bitmap-snap"), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + assert_eq!( + rig.disk + .remove_clone(&rig.name("dest")) + .await + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + assert_eq!( + rig.disk + .remove_clone(&rig.name("drive")) + .await + .unwrap_err() + .kind(), + ErrorKind::ResourceBusy + ); + + drop(drive); + rig.disk.enable_hydration(&rig.name("drive")).await.unwrap(); + rig.hydrated(REGIONS).await; + rig.disk.cut_over(&rig.name("drive")).await.unwrap(); + assert_eq!( + rig.disk + .enable_hydration(&rig.name("drive")) + .await + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + // The cutover frees the bitmap, which still holds the drive's header. + let err = rig + .disk + .clone_drive(&rig.name("second"), &dest2, &bitmap, &rig.source, off(1, 1)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput, "{err}"); + rig.disk.remove_clone(&rig.name("drive")).await.unwrap(); + assert_eq!( + rig.disk + .clone_state(&rig.name("drive")) + .await + .unwrap_err() + .kind(), + ErrorKind::NotFound + ); + + drop((dest, bitmap, big, other, dest2, small, taken)); + for role in ["dest", "bitmap", "big", "other", "dest2", "small", "taken"] { + rig.disk.delete(&rig.name(role)).await.unwrap(); + } + assert_eq!(rig.disk_leftovers(), Vec::::new()); +} + +/// A process death between a suspend and its resume leaves the drive and +/// its destination suspended. Opening the pool again resumes both: the +/// guest's write that waited completes, and the copy runs to the cutover. +#[tokio::test] +#[ignore] +async fn a_drive_left_suspended_resumes_when_the_pool_is_opened_again() { + let Some(rig) = Rig::new("resume", Some(30)).await else { + return; + }; + let dest = rig.volume("dest", SIZE).await; + let bitmap = rig.volume("bitmap", clone_bitmap_bytes(SIZE)).await; + let drive = rig.clone_drive(&dest, &bitmap, off(1, 1)).await; + rig.disk.enable_hydration(&rig.name("drive")).await.unwrap(); + rig.hydrated(2).await; + + for role in ["drive", "dest"] { + run("dmsetup", &["suspend", &rig.device(&rig.name(role))]); + } + let mut expected = rig.bytes.clone(); + let write = pattern(8192, 3); + expected[(7 * REGION) as usize..(7 * REGION) as usize + 8192].copy_from_slice(&write); + let path = drive.path.clone(); + let writer = std::thread::spawn(move || { + let file = std::fs::File::options().write(true).open(path).unwrap(); + file.write_all_at(&write, 7 * REGION).unwrap(); + file.sync_all().unwrap(); + }); + std::thread::sleep(Duration::from_millis(200)); + assert!(!writer.is_finished(), "a write to a suspended drive waits"); + + let pool = std::env::var("ENGRAM_TEST_THIN_POOL").unwrap(); + let disk = ThinPoolDisk::open(&pool, CHUNK, Path::new(NODES)) + .await + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !writer.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "the write still waits after the pool was opened again" + ); + std::thread::sleep(Duration::from_millis(20)); + } + writer.join().unwrap(); + rig.hydrated(REGIONS).await; + disk.cut_over(&rig.name("drive")).await.unwrap(); + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + + drop((drive, bitmap)); + rig.disk.delete(&rig.name("bitmap")).await.unwrap(); + rig.disk.remove_clone(&rig.name("drive")).await.unwrap(); + assert_eq!(first_difference(&whole(&dest).await, &expected), None); + drop(dest); + rig.disk.delete(&rig.name("dest")).await.unwrap(); + assert_eq!(rig.disk_leftovers(), Vec::::new()); +} + +/// A `clone_drive` cut after the device was made leaves it with no live +/// table, and with its table loaded inactive the kernel has already +/// written the bitmap's header. The restore removes the drive, makes the +/// bitmap again, and converges. +#[tokio::test] +#[ignore] +async fn a_half_made_drive_is_removed_and_made_again() { + let Some(rig) = Rig::new("half", None).await else { + return; + }; + for loaded in [false, true] { + let dest = rig.volume("dest", SIZE).await; + let bitmap = rig.volume("bitmap", clone_bitmap_bytes(SIZE)).await; + let device = rig.device(&rig.name("drive")); + run("dmsetup", &["create", "--noudevsync", "--notable", &device]); + if loaded { + let table = format!( + "0 {} clone {} {} {} 128 1 no_hydration", + SIZE / 512, + devno(&bitmap), + devno(&dest), + rig.source.display() + ); + run("dmsetup", &["load", &device, "--table", &table]); + } + assert_eq!(rig.state("drive").await, CloneState::Incomplete); + assert_eq!( + rig.disk + .open_clone(&rig.name("drive")) + .await + .unwrap_err() + .kind(), + ErrorKind::InvalidData + ); + + rig.disk.remove_clone(&rig.name("drive")).await.unwrap(); + if loaded { + let err = rig + .disk + .clone_drive(&rig.name("drive"), &dest, &bitmap, &rig.source, off(1, 1)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput, "{err}"); + } + drop(bitmap); + rig.disk.delete(&rig.name("bitmap")).await.unwrap(); + let bitmap = rig.volume("bitmap", clone_bitmap_bytes(SIZE)).await; + let drive = rig.clone_drive(&dest, &bitmap, off(4, 4)).await; + let mut expected = rig.bytes.clone(); + write(&drive, &mut expected, REGION + 512, vec![0xC3; 4096]).await; + rig.disk.enable_hydration(&rig.name("drive")).await.unwrap(); + rig.hydrated(REGIONS).await; + rig.disk.cut_over(&rig.name("drive")).await.unwrap(); + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + + drop((drive, bitmap)); + rig.disk.remove_clone(&rig.name("drive")).await.unwrap(); + rig.disk.delete(&rig.name("bitmap")).await.unwrap(); + assert_eq!(first_difference(&whole(&dest).await, &expected), None); + drop(dest); + rig.disk.delete(&rig.name("dest")).await.unwrap(); + } + assert_eq!(rig.disk_leftovers(), Vec::::new()); +} + +/// [`FileSources`] behind a read-only loop device, as the block device a +/// drive needs, and behind dm-delay when reads must be slow. +struct LoopSources { + files: FileSources, + delay_ms: Option, + devices: Mutex>, +} + +struct LoopSource { + loop_dev: String, + /// The dm-delay device over the loop device. + delay: Option, +} + +impl LoopSource { + fn path(&self) -> PathBuf { + match &self.delay { + Some(delay) => Path::new("/dev/mapper").join(delay), + None => PathBuf::from(&self.loop_dev), + } + } +} + +impl LoopSources { + /// The devices, also after a test panicked while it held them. + fn devices(&self) -> std::sync::MutexGuard<'_, HashMap<(SandboxId, DiskRole), LoopSource>> { + self.devices.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn detach_all(&self) { + let sources: Vec = self.devices().drain().map(|(_, s)| s).collect(); + for source in sources { + if let Some(delay) = &source.delay { + let _ = remove_dm(delay); + } + let _ = Command::new("losetup") + .args(["-d", &source.loop_dev]) + .output(); + } + } +} + +/// Remove a device-mapper device, waiting a bounded time for a holder +/// that goes by itself, such as a udev probe of a new device. +fn remove_dm(name: &str) -> std::io::Result<()> { + let mut stderr = String::new(); + for _ in 0..10 { + let output = Command::new("dmsetup") + .args(["remove", "--retry", "--noudevsync", name]) + .output()?; + if output.status.success() { + return Ok(()); + } + stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + std::thread::sleep(Duration::from_millis(200)); + } + Err(std::io::Error::new( + ErrorKind::ResourceBusy, + format!("dmsetup remove {name}: {stderr}"), + )) +} + +#[async_trait] +impl DiskSources for LoopSources { + fn can_start(&self) -> bool { + true + } + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option<(&Volume, ManifestRef)>, + ) -> std::io::Result { + let file = self.files.start(sandbox, role, manifest, base).await?; + let loop_dev = run( + "losetup", + &["--read-only", "--show", "-f", file.to_str().unwrap()], + ); + let delay = self.delay_ms.map(|ms| { + let name = format!("engram-test-delay-{}-{role}", sandbox.as_uuid().simple()); + let sectors = std::fs::metadata(&file).unwrap().len() / 512; + let table = format!("0 {sectors} delay {loop_dev} 0 {ms}"); + run( + "dmsetup", + &["create", "--noudevsync", &name, "--table", &table], + ); + run("dmsetup", &["mknodes", &name]); + name + }); + let source = LoopSource { loop_dev, delay }; + let path = source.path(); + self.devices().insert((sandbox, role), source); + Ok(path) + } + + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result<()> { + self.files.ensure_alive(sandbox, role).await + } + + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result { + let served = self.files.served(sandbox, role).await?; + let device = self.devices().get(&(sandbox, role)).map(LoopSource::path); + Ok(Served { + device: device.ok_or(ErrorKind::NotFound)?, + ..served + }) + } + + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result<()> { + let source = self.devices().remove(&(sandbox, role)); + if let Some(source) = source { + if let Some(delay) = &source.delay { + if let Err(e) = remove_dm(delay) { + self.devices().insert((sandbox, role), source); + return Err(e); + } + } + let detached = Command::new("losetup") + .args(["-d", &source.loop_dev]) + .output()?; + if !detached.status.success() { + return Err(std::io::Error::other(format!( + "losetup -d {}: {}", + source.loop_dev, + String::from_utf8_lossy(&detached.stderr) + ))); + } + } + self.files.stop(sandbox, role).await + } + + async fn list(&self) -> std::io::Result> { + self.files.list().await + } +} + +/// Every device-mapper device whose name holds `id`: a sandbox's with +/// its source's delay, or an image's. +fn devices_naming(id: impl std::fmt::Display) -> Vec { + let id = id.to_string(); + run("dmsetup", &["ls"]) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|name| name.contains(&id)) + .map(str::to_string) + .collect() +} + +/// Session volumes over the pool with loop-device sources, an image +/// manifest of `SIZE` pattern bytes, and one sandbox. Dropping it removes +/// what a failed test left of the sandbox and the image: their devices, +/// in passes because a drive holds its thins, then the loop devices. +struct SessionRig { + disk: ThinPoolDisk, + store: ChunkStore, + files: FileDisk, + sources: Arc, + volumes: SessionVolumes, + bytes: Vec, + image: ManifestRef, + sandbox: SandboxId, + _dir: tempfile::TempDir, +} + +impl SessionRig { + /// `None`, with a `SKIP:` line, when no pool is named. + async fn new(read_delay_ms: Option) -> Option { + let Ok(pool) = std::env::var("ENGRAM_TEST_THIN_POOL") else { + eprintln!("SKIP: ENGRAM_TEST_THIN_POOL is not set"); + return None; + }; + let disk = ThinPoolDisk::open(&pool, CHUNK, Path::new(NODES)) + .await + .expect("open pool"); + let dir = tempfile::tempdir().unwrap(); + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.path().join("blobs")))); + let files = FileDisk::new(dir.path().join("files"), CHUNK).unwrap(); + let bytes = pattern(SIZE as usize, 0x2545_F491); + let image = publish(&files, &store, "image", &bytes).await; + let sources = Arc::new(LoopSources { + files: FileSources::new(dir.path().join("sources"), store.clone()).unwrap(), + delay_ms: read_delay_ms, + devices: Mutex::new(HashMap::new()), + }); + let volumes = SessionVolumes::new(Arc::new(disk.clone()), store.clone()) + .with_sources(sources.clone()); + Some(Self { + disk, + store, + files, + sources, + volumes, + bytes, + image, + sandbox: SandboxId::new(), + _dir: dir, + }) + } + + /// The sandbox's root, provisioned lazily from the image. + async fn lazy_root(&self) -> Volume { + self.root(Source::LazyFork { + manifest: self.image, + base: None, + }) + .await + } + + /// The sandbox's root, provisioned from `source`. + async fn root(&self, source: Source) -> Volume { + let provisioned = self + .volumes + .provision(self.sandbox, source, None) + .await + .unwrap(); + Volume::open( + VolumeName::new("probe").unwrap(), + &provisioned.root, + SIZE, + CHUNK, + ) + .unwrap() + } + + /// The pool's used data bytes. + async fn pool_used(&self) -> u64 { + let capacity = self.disk.capacity().await.unwrap(); + capacity.data_total_bytes - capacity.data_free_bytes + } + + /// The regions the root drive has copied. + async fn root_hydrated(&self) -> u64 { + match self.root_state().await { + CloneState::Copying { + hydrated_regions, .. + } => hydrated_regions, + state => panic!("the drive is {state:?}"), + } + } + + /// Poll until the root drive has copied every region. + async fn root_hydrated_in_full(&self) { + let deadline = tokio::time::Instant::now() + HYDRATION_DEADLINE; + while !self.root_state().await.is_hydrated() { + assert!( + tokio::time::Instant::now() < deadline, + "not hydrated in {HYDRATION_DEADLINE:?}" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + async fn root_state(&self) -> CloneState { + self.disk + .clone_state(&live_name(self.sandbox, DiskRole::Root)) + .await + .unwrap() + } + + /// Release the sandbox and check that nothing of it is left. + async fn release_clean(&self) { + self.volumes.release(self.sandbox).await.unwrap(); + assert_eq!(self.sources.list().await.unwrap(), Vec::new()); + assert_eq!(self.sources.devices().len(), 0); + assert_eq!( + devices_naming(self.sandbox.as_uuid().simple()), + Vec::::new() + ); + } +} + +impl Drop for SessionRig { + fn drop(&mut self) { + let ids = [ + self.sandbox.as_uuid().simple().to_string(), + self.image.manifest_id.simple().to_string(), + ]; + for _ in 0..4 { + for device in ids.iter().flat_map(devices_naming) { + let _ = Command::new("dmsetup") + .args(["remove", "--retry", "--noudevsync", &device]) + .output(); + let _ = std::fs::remove_file(Path::new(NODES).join(&device)); + } + } + self.sources.detach_all(); + } +} + +/// Publish `bytes` as a disk manifest, through a file volume `name`. +async fn publish(files: &FileDisk, store: &ChunkStore, name: &str, bytes: &[u8]) -> ManifestRef { + let volume = files + .create(&VolumeName::new(name).unwrap(), bytes.len() as u64) + .await + .unwrap(); + volume.write_range(0, bytes.to_vec()).await.unwrap(); + let snap = VolumeName::new(format!("{name}-snap")).unwrap(); + let (_, manifest) = checkpoint(files, store, &volume, &snap, None) + .await + .unwrap(); + let published = ManifestRef::new(); + store.put_manifest(published, &manifest).await.unwrap(); + published +} + +/// The manifest's bytes, staged into a file volume `name`. +async fn staged_bytes(files: &FileDisk, store: &ChunkStore, name: &str, m: ManifestRef) -> Vec { + let manifest = store.get_manifest(m).await.unwrap(); + let staged = stage( + files, + store, + &VolumeName::new(name).unwrap(), + &manifest, + None, + ) + .await + .unwrap(); + whole(&staged).await +} + +/// A lazy provision boots the role over a drive that reads the manifest +/// from its source; a release removes the drive, stops the source and +/// deletes every volume of the sandbox. +#[tokio::test] +#[ignore] +async fn a_lazy_session_role_reads_its_manifest_and_releases_clean() { + let Some(rig) = SessionRig::new(None).await else { + return; + }; + let drive = rig.lazy_root().await; + assert_eq!(first_difference(&whole(&drive).await, &rig.bytes), None); + let state = rig.root_state().await; + assert!( + matches!( + state, + CloneState::Copying { + hydration_enabled: true, + .. + } + ), + "{state:?}" + ); + drop(drive); + rig.release_clean().await; +} + +/// A checkpoint while the drive copies reads through a view over the +/// snapshot pair and the source: staged again, the published manifest is +/// exactly what the drive read at the take, guest writes included. +#[tokio::test] +#[ignore] +async fn a_checkpoint_mid_hydration_stages_into_what_the_drive_read() { + let Some(rig) = SessionRig::new(Some(30)).await else { + return; + }; + let drive = rig.lazy_root().await; + let mut expected = rig.bytes.clone(); + write(&drive, &mut expected, 60 * REGION + 4096, vec![0x44; 4096]).await; + write( + &drive, + &mut expected, + 5 * REGION, + pattern(REGION as usize, 9), + ) + .await; + let CloneState::Copying { + hydrated_regions, .. + } = rig.root_state().await + else { + panic!("the drive is not copying"); + }; + eprintln!("{hydrated_regions} of {REGIONS} regions copied before the take"); + assert!(hydrated_regions < REGIONS); + + let published = rig + .volumes + .checkpoint(rig.sandbox, DiskRole::Root) + .await + .unwrap(); + + let staged = staged_bytes(&rig.files, &rig.store, "staged", published).await; + assert_eq!(first_difference(&staged, &expected), None); + assert!(matches!(rig.root_state().await, CloneState::Copying { .. })); + drop(drive); + rig.release_clean().await; +} + +/// Hydrated to the end, the role cuts over through the session volumes: +/// a handle opened before reads the manifest and the guest's write, and +/// the bitmap, the source and its loop device are gone. The first +/// checkpoint after the cutover stages into what the drive reads. +#[tokio::test] +#[ignore] +async fn a_hydrated_session_role_cuts_over_and_stops_its_source() { + let Some(rig) = SessionRig::new(None).await else { + return; + }; + let drive = rig.lazy_root().await; + let mut expected = rig.bytes.clone(); + write(&drive, &mut expected, 3 * REGION + 4096, vec![0x5A; 4096]).await; + let deadline = tokio::time::Instant::now() + HYDRATION_DEADLINE; + loop { + match rig + .volumes + .cut_over(rig.sandbox, DiskRole::Root) + .await + .unwrap() + { + Cutover::Done => break, + Cutover::Waiting(Waiting::Copying(_)) => {} + waiting => panic!("{waiting:?}"), + } + assert!( + tokio::time::Instant::now() < deadline, + "not hydrated in {HYDRATION_DEADLINE:?}" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + // The host-agent stops the source after it releases the capture lock. + rig.volumes + .stop_source(rig.sandbox, DiskRole::Root) + .await + .unwrap(); + + assert_eq!(rig.root_state().await, CloneState::CutOver); + assert_eq!(first_difference(&whole(&drive).await, &expected), None); + assert_eq!(rig.sources.list().await.unwrap(), Vec::new()); + assert_eq!(rig.sources.devices().len(), 0); + let bitmap = format!("{}-root-bitmap", rig.sandbox.as_uuid().simple()); + let devices = devices_naming(rig.sandbox.as_uuid().simple()); + assert!(!devices.iter().any(|d| d.ends_with(&bitmap)), "{devices:?}"); + assert_eq!( + rig.volumes + .hydration(rig.sandbox, DiskRole::Root) + .await + .unwrap(), + None + ); + + let published = rig + .volumes + .checkpoint(rig.sandbox, DiskRole::Root) + .await + .unwrap(); + let staged = staged_bytes(&rig.files, &rig.store, "staged", published).await; + assert_eq!(first_difference(&staged, &expected), None); + drop(drive); + let report = rig.volumes.purge(rig.sandbox).await.unwrap(); + assert!(report.failed.is_empty(), "{:?}", report.failed); + assert_eq!( + devices_naming(rig.sandbox.as_uuid().simple()), + Vec::::new() + ); +} + +/// Over a new destination, hydration copies only the chunks the manifest +/// lists: the zero regions are marked copied before the drive starts, the +/// destination maps the listed chunks alone, and the drive reads the +/// manifest's bytes. +#[tokio::test] +#[ignore] +async fn a_lazy_role_over_a_new_volume_copies_only_the_listed_chunks() { + let Some(rig) = SessionRig::new(None).await else { + return; + }; + let listed = [3u64, 20, 21, 50]; + let mut bytes = vec![0u8; SIZE as usize]; + for (i, chunk) in listed.iter().enumerate() { + let at = (chunk * CHUNK) as usize; + bytes[at..at + CHUNK as usize].copy_from_slice(&pattern(CHUNK as usize, i as u32 + 1)); + } + let manifest = publish(&rig.files, &rig.store, "sparse", &bytes).await; + let used_before = rig.pool_used().await; + + let provisioned = rig + .volumes + .provision( + rig.sandbox, + Source::LazyContinue { + manifest, + base: None, + }, + None, + ) + .await + .unwrap(); + let skipped = (SIZE - listed.len() as u64 * CHUNK) / REGION; + assert!(rig.root_hydrated().await >= skipped); + rig.root_hydrated_in_full().await; + let drive = Volume::open( + VolumeName::new("probe").unwrap(), + &provisioned.root, + SIZE, + CHUNK, + ) + .unwrap(); + drive.sync().await.unwrap(); + assert_eq!(first_difference(&whole(&drive).await, &bytes), None); + + let dest = rig + .disk + .open(&dest_name(rig.sandbox, DiskRole::Root, manifest)) + .await + .unwrap(); + assert_eq!( + rig.disk.written_chunks(None, &dest).await.unwrap(), + listed.map(ChunkIndex).to_vec() + ); + let grown = rig.pool_used().await - used_before; + eprintln!("the pool grew by {grown} bytes"); + assert!(grown < SIZE / 4, "the pool grew by {grown} bytes"); + drop((drive, dest)); + rig.release_clean().await; +} + +/// The bytes the kernel has read from a volume's device. +fn bytes_read(volume: &Volume) -> u64 { + let stat = std::fs::read_to_string(format!("/sys/dev/block/{}/stat", devno(volume))).unwrap(); + let sectors: u64 = stat.split_whitespace().nth(2).unwrap().parse().unwrap(); + sectors * 512 +} + +/// After the cutover of a mostly-zero disk over a new destination, the +/// first checkpoint reads only the chunks the destination maps: the ones +/// hydration copied and the one the guest wrote. Staged again, the +/// published manifest is what the drive reads. +#[tokio::test] +#[ignore] +async fn the_first_checkpoint_after_a_cutover_reads_only_the_mapped_chunks() { + let Some(rig) = SessionRig::new(None).await else { + return; + }; + let size = 16 * SIZE; + let listed = [3u64, 20, 21, 50]; + let mut bytes = vec![0u8; size as usize]; + for (i, chunk) in listed.iter().enumerate() { + let at = (chunk * CHUNK) as usize; + bytes[at..at + CHUNK as usize].copy_from_slice(&pattern(CHUNK as usize, i as u32 + 1)); + } + let manifest = publish(&rig.files, &rig.store, "sparse", &bytes).await; + let provisioned = rig + .volumes + .provision( + rig.sandbox, + Source::LazyContinue { + manifest, + base: None, + }, + None, + ) + .await + .unwrap(); + let drive = Volume::open( + VolumeName::new("probe").unwrap(), + &provisioned.root, + size, + CHUNK, + ) + .unwrap(); + let deadline = tokio::time::Instant::now() + HYDRATION_DEADLINE; + while rig + .volumes + .cut_over(rig.sandbox, DiskRole::Root) + .await + .unwrap() + != Cutover::Done + { + assert!( + tokio::time::Instant::now() < deadline, + "not cut over in {HYDRATION_DEADLINE:?}" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + rig.volumes + .stop_source(rig.sandbox, DiskRole::Root) + .await + .unwrap(); + let mut expected = bytes; + write( + &drive, + &mut expected, + 40 * CHUNK, + pattern(CHUNK as usize, 0x40), + ) + .await; + + let published = rig + .volumes + .checkpoint(rig.sandbox, DiskRole::Root) + .await + .unwrap(); + + let volume = rig + .disk + .open(&checkpoint_name(rig.sandbox, DiskRole::Root, published)) + .await + .unwrap(); + assert_eq!( + rig.disk.written_chunks(None, &volume).await.unwrap(), + [3, 20, 21, 40, 50].map(ChunkIndex).to_vec() + ); + let read = bytes_read(&volume); + eprintln!("the checkpoint read {read} of {size} bytes"); + assert!(read < size / 8, "the checkpoint read {read} bytes"); + let staged = staged_bytes(&rig.files, &rig.store, "staged", published).await; + assert_eq!(first_difference(&staged, &expected), None); + drop((drive, volume)); + let report = rig.volumes.purge(rig.sandbox).await.unwrap(); + assert!(report.failed.is_empty(), "{:?}", report.failed); +} + +/// Over a staged base, hydration copies only the chunks that differ from +/// the base: the destination keeps sharing every other block with the +/// image volume, and the drive reads the wanted bytes. A discard that +/// reached the destination would unmap those blocks and read zeros. +#[tokio::test] +#[ignore] +async fn a_lazy_role_over_a_staged_base_copies_only_what_differs() { + let Some(rig) = SessionRig::new(None).await else { + return; + }; + let image_volume = image_volumes::volume_name(rig.image); + let base = rig.store.get_manifest(rig.image).await.unwrap(); + let staged = stage(&rig.disk, &rig.store, &image_volume, &base, None) + .await + .unwrap(); + let changed = [7u64, 8, 33]; + let mut wanted = rig.bytes.clone(); + for (i, chunk) in changed.iter().enumerate() { + let at = (chunk * CHUNK) as usize; + wanted[at..at + CHUNK as usize].copy_from_slice(&pattern(CHUNK as usize, 0x77 + i as u32)); + } + let manifest = publish(&rig.files, &rig.store, "continued", &wanted).await; + let used_before = rig.pool_used().await; + + let drive = rig + .root(Source::LazyContinue { + manifest, + base: Some(rig.image), + }) + .await; + let skipped = (SIZE - changed.len() as u64 * CHUNK) / REGION; + assert!(rig.root_hydrated().await >= skipped); + rig.root_hydrated_in_full().await; + drive.sync().await.unwrap(); + assert_eq!(first_difference(&whole(&drive).await, &wanted), None); + + let dest = rig + .disk + .open(&dest_name(rig.sandbox, DiskRole::Root, manifest)) + .await + .unwrap(); + assert_eq!( + rig.disk.written_chunks(Some(&staged), &dest).await.unwrap(), + changed.map(ChunkIndex).to_vec() + ); + let grown = rig.pool_used().await - used_before; + eprintln!("the pool grew by {grown} bytes"); + assert!(grown < SIZE / 4, "the pool grew by {grown} bytes"); + drop((drive, dest, staged)); + rig.release_clean().await; + rig.disk.delete(&image_volume).await.unwrap(); +} + +/// Over a staged base the destination is a snapshot of the image +/// volume. A chunk the destination never wrote is the source's, also +/// where the base differs; a chunk the guest wrote with the base's own +/// bytes is the guest's, although its content equals the origin's. +#[tokio::test] +#[ignore] +async fn a_checkpoint_over_a_staged_base_stages_into_what_the_drive_read() { + let Some(rig) = SessionRig::new(Some(50)).await else { + return; + }; + let image_volume = image_volumes::volume_name(rig.image); + let base = rig.store.get_manifest(rig.image).await.unwrap(); + drop( + stage(&rig.disk, &rig.store, &image_volume, &base, None) + .await + .unwrap(), + ); + let mut wanted = rig.bytes.clone(); + let late = (40 * REGION) as usize; + wanted[late..].copy_from_slice(&pattern(SIZE as usize - late, 0x1357_9BDF)); + let manifest = publish(&rig.files, &rig.store, "continued", &wanted).await; + + let drive = rig + .root(Source::LazyContinue { + manifest, + base: Some(rig.image), + }) + .await; + let mut expected = wanted.clone(); + let region = (50 * REGION) as usize..(51 * REGION) as usize; + write( + &drive, + &mut expected, + 50 * REGION, + rig.bytes[region].to_vec(), + ) + .await; + let CloneState::Copying { + hydrated_regions, .. + } = rig.root_state().await + else { + panic!("the drive is not copying"); + }; + eprintln!("{hydrated_regions} of {REGIONS} regions copied before the take"); + assert!((40..REGIONS).contains(&hydrated_regions)); + + let published = rig + .volumes + .checkpoint(rig.sandbox, DiskRole::Root) + .await + .unwrap(); + + let staged = staged_bytes(&rig.files, &rig.store, "staged", published).await; + assert_eq!(first_difference(&staged, &expected), None); + drop(drive); + rig.release_clean().await; + rig.disk.delete(&image_volume).await.unwrap(); + assert_eq!( + devices_naming(rig.image.manifest_id.simple()), + Vec::::new() + ); +} diff --git a/crates/engram-disk-thin/tests/thin_pool.rs b/crates/engram-disk-thin/tests/thin_pool.rs index 5e15190bc..8830e7cc5 100644 --- a/crates/engram-disk-thin/tests/thin_pool.rs +++ b/crates/engram-disk-thin/tests/thin_pool.rs @@ -9,7 +9,9 @@ use std::sync::Arc; use async_trait::async_trait; use engram_chunk_store::ChunkStore; use engram_disk_thin::ThinPoolDisk; -use engram_host_core::disk::{Capacity, ChunkIndex, SessionDisk, Volume, VolumeName}; +use engram_host_core::disk::{ + Capacity, ChunkIndex, CloneState, Hydration, SessionDisk, Volume, VolumeName, +}; use engram_host_core::{checkpoint, stage}; use engram_storage_local::LocalBlobStorage; @@ -77,6 +79,63 @@ impl SessionDisk for Oracle<'_> { async fn rename(&self, from: &VolumeName, to: &VolumeName) -> std::io::Result<()> { self.0.rename(from, to).await } + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + hydration: Hydration, + ) -> std::io::Result { + self.0 + .clone_drive(drive, dest, bitmap, source, hydration) + .await + } + async fn open_clone(&self, drive: &VolumeName) -> std::io::Result { + self.0.open_clone(drive).await + } + async fn clone_state(&self, drive: &VolumeName) -> std::io::Result { + self.0.clone_state(drive).await + } + async fn skip_hydration( + &self, + drive: &VolumeName, + ranges: &[std::ops::Range], + ) -> std::io::Result<()> { + self.0.skip_hydration(drive, ranges).await + } + async fn enable_hydration(&self, drive: &VolumeName) -> std::io::Result<()> { + self.0.enable_hydration(drive).await + } + async fn cut_over(&self, drive: &VolumeName) -> std::io::Result<()> { + self.0.cut_over(drive).await + } + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> std::io::Result<(Volume, Volume)> { + self.0 + .snapshot_clone(drive, dest, bitmap, dest_snapshot, bitmap_snapshot) + .await + } + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + source: &Path, + ) -> std::io::Result { + self.0 + .clone_view(view, dest_snapshot, bitmap_snapshot, source) + .await + } + async fn remove_clone(&self, drive: &VolumeName) -> std::io::Result<()> { + self.0.remove_clone(drive).await + } } #[tokio::test] diff --git a/crates/engram-host-agent/src/config.rs b/crates/engram-host-agent/src/config.rs index 3c58ee986..aa723f3c1 100644 --- a/crates/engram-host-agent/src/config.rs +++ b/crates/engram-host-agent/src/config.rs @@ -50,6 +50,62 @@ pub struct HostAgentConfig { /// the mode=all / in-process test-harness backend, which never /// constructs a `HostAgentConfig` through `main.rs`'s match anyway. pub backend_name: String, + /// When a restore boots before its disk is complete (ADR 0127). + pub lazy_restore: LazyRestore, + /// What `auto` allows an eager restore to write. + pub eager_budget: EagerBudget, + /// How often a sandbox's hydration task checks whether its disks + /// are copied and can be cut over. + pub hydration_interval: Duration, +} + +/// The longest an eager restore of a disk may write before `auto` boots +/// it over a source, and the write rates its estimate uses. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EagerBudget { + pub seconds: f64, + pub rates: engram_host_core::eager_estimate::Rates, +} + +impl Default for EagerBudget { + fn default() -> Self { + Self { + seconds: 2.0, + rates: engram_host_core::eager_estimate::Rates { + local: 800e6, + remote: 200e6, + }, + } + } +} + +/// When a restore boots over a source before its disk is complete. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum LazyRestore { + /// Every restore writes the whole disk before it boots. The host + /// starts no disk source. + #[default] + Off, + /// Every restore boots over a source. + Force, + /// A restore boots over a source when it cannot finish its writes + /// within the eager budget. + Auto, +} + +impl std::str::FromStr for LazyRestore { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "off" => Ok(Self::Off), + "force" => Ok(Self::Force), + "auto" => Ok(Self::Auto), + other => Err(format!( + "invalid lazy restore mode `{other}`: expected `off`, `force` or `auto`" + )), + } + } } impl Default for HostAgentConfig { @@ -67,6 +123,9 @@ impl Default for HostAgentConfig { grpc_advertise_addr: None, migrate_peer_listen_addr: None, backend_name: "process".to_string(), + lazy_restore: LazyRestore::Off, + eager_budget: EagerBudget::default(), + hydration_interval: crate::hydration::DEFAULT_INTERVAL, } } } diff --git a/crates/engram-host-agent/src/coord_client.rs b/crates/engram-host-agent/src/coord_client.rs index 0eeb387f1..a0a436a46 100644 --- a/crates/engram-host-agent/src/coord_client.rs +++ b/crates/engram-host-agent/src/coord_client.rs @@ -552,6 +552,10 @@ pub struct HeartbeatRequest { /// Active → Unreachable. #[serde(default)] pub unreachable_guests: Vec<(SandboxId, SessionId)>, + /// Sandboxes whose disks still copy from a source, or whose cutover + /// has steps left (ADR 0127 Decision 9). + #[serde(default)] + pub hydrating: Vec, } #[derive(Deserialize)] @@ -740,11 +744,13 @@ mod tests { harness_attached: Vec::new(), capture_job_reports: Vec::new(), unreachable_guests: Vec::new(), + hydrating: vec![engram_core::SandboxId::new()], }; let v = serde_json::to_value(&req).unwrap(); assert_eq!(v["current_bundles"][0]["sha256"], "ff00"); // ADR 0035 amendment D2: the per-sandbox attachment leg rides the same wire. assert_eq!(v["sandbox_bundles"][0]["bundles"][0]["sha256"], "ab12"); + assert_eq!(v["hydrating"].as_array().map(Vec::len), Some(1)); } /// ADR 0084 P1b: the new capture-job HTTP-mirror fields must decode diff --git a/crates/engram-host-agent/src/disk_source.rs b/crates/engram-host-agent/src/disk_source.rs new file mode 100644 index 000000000..89b34c6e8 --- /dev/null +++ b/crates/engram-host-agent/src/disk_source.rs @@ -0,0 +1,1409 @@ +//! Lazy disk sources as processes of their own (ADR 0127 Decision 4). +//! +//! Each source is an `engram-disk-source serve` process. It outlives the +//! host-agent: it is not killed when the host-agent exits, it runs in the +//! node cgroup leaf `/-disk-`, and its output goes +//! to `/disk-sources/-.log`. A durable record per +//! role, `/disk-sources/-.json`, names the +//! process, its device and the serve arguments. A source whose process +//! died is started again on the same device with `--recover`; the kernel +//! holds the device's I/O until then. +//! +//! A ublk device stopped or deleted while it is LIVE, with a request that +//! no server will finish, blocks in the kernel until the node reboots. So +//! a SIGTERM to a serving process deletes a device (the server fails the +//! reads it cannot serve, then deletes the device), a recovery that fails +//! ends with SIGKILL, which keeps the device for the next recovery, and a +//! stop recovers a dead source first. Only a device that no process +//! serves is deleted from outside, by `engram-disk-source delete`, which +//! refuses a LIVE device. +//! +//! Each start, recovery and stop runs in a task of its own, so a caller +//! that is dropped cannot leave a process without a record. + +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use engram_core::types::ids::SandboxId; +use engram_core::types::manifest::ManifestRef; +use engram_core::types::snapshot::DiskRole; +use engram_host_core::disk::Volume; +use engram_host_core::disk_sources::{DiskSources, Served}; +use engram_host_core::{HostFs, TokioFs}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::ChildStdout; + +use crate::durable_record; + +const WHAT: &str = "disk source record"; +const POLL: Duration = Duration::from_millis(20); +/// How many sources the startup recovery starts at once. +const RECOVER_AT_ONCE: usize = 4; + +/// How [`ProcessSources`] runs its processes. +#[derive(Clone, Debug)] +pub struct ProcessSourcesConfig { + /// The `engram-disk-source` binary. + pub bin: PathBuf, + /// Records and logs go in `/disk-sources`. + pub work_dir: PathBuf, + /// The node cgroup that holds each source's leaf. `None` keeps the + /// sources in the host-agent's cgroup. + pub cgroup_parent: Option, + /// The host chunk cache a source reads resident chunks from. + pub cache_root: Option, + pub queues: u16, + pub depth: u16, + /// How long a process may take to report its device. + pub ready_timeout: Duration, + /// How long a process may take to exit after a signal, and a delete + /// may take. + pub stop_timeout: Duration, +} + +impl ProcessSourcesConfig { + pub fn new(bin: impl Into, work_dir: impl Into) -> Self { + Self { + bin: bin.into(), + work_dir: work_dir.into(), + cgroup_parent: None, + cache_root: None, + queues: 2, + depth: 64, + ready_timeout: Duration::from_secs(60), + stop_timeout: Duration::from_secs(10), + } + } +} + +/// One source process, as its durable record holds it. Two builds read +/// it during a roll: a field added later is `#[serde(default)]`, and an +/// older build ignores a field it does not know. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceRecord { + pub sandbox: SandboxId, + pub role: DiskRole, + pub pid: u32, + /// The process start time in clock ticks after boot (field 22 of + /// `/proc//stat`). With the pid it identifies the process, so a + /// reused pid is never taken for the source. + pub start_time: u64, + /// `None` until the process reports its device. + #[serde(default)] + pub device: Option, + /// A recovery must give the device its own queue count and depth. + pub queues: u16, + pub depth: u16, + pub args: ServeArgs, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Device { + pub id: u32, + pub path: PathBuf, +} + +/// The arguments of `engram-disk-source serve`; a recovery repeats them. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServeArgs { + pub manifest: ManifestRef, + #[serde(default)] + pub base: Option, + #[serde(default)] + pub cache_root: Option, +} + +/// A staged volume and the manifest it holds. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Base { + pub volume: PathBuf, + pub manifest: ManifestRef, +} + +/// What the host-agent reads about processes and ublk devices. +pub trait Probe: Send + Sync { + /// The start time of `pid` while it runs. `None` when no such process + /// runs; a zombie has exited. + fn start_time(&self, pid: u32) -> Option; + /// Whether the kernel has ublk device `id`, served or not. + fn device_exists(&self, id: u32) -> bool; + /// Whether the ublk driver is loaded. + fn driver_loaded(&self) -> bool; + /// Every ublk device id the kernel has. + fn devices(&self) -> Vec; + /// Every running `engram-disk-source` process, with its start time. + fn source_processes(&self) -> Vec<(u32, u64)>; +} + +/// [`Probe`] over `/proc` and sysfs. +pub struct KernelProbe; + +/// The control nodes of the ublk devices; one lives from a device's add +/// to its delete. +const UBLK_CHAR_CLASS: &str = "/sys/class/ublk-char"; + +impl Probe for KernelProbe { + fn start_time(&self, pid: u32) -> Option { + use engram_sandbox_firecracker::sandbox_manifest::{ + is_zombie, read_proc_start_time_jiffies, + }; + if is_zombie(pid) { + return None; + } + read_proc_start_time_jiffies(pid) + } + + fn device_exists(&self, id: u32) -> bool { + Path::new(UBLK_CHAR_CLASS) + .join(format!("ublkc{id}")) + .exists() + } + + fn driver_loaded(&self) -> bool { + Path::new("/dev/ublk-control").exists() + } + + fn devices(&self) -> Vec { + let Ok(entries) = std::fs::read_dir(UBLK_CHAR_CLASS) else { + return Vec::new(); + }; + entries + .flatten() + .filter_map(|e| e.file_name().to_str()?.strip_prefix("ublkc")?.parse().ok()) + .collect() + } + + fn source_processes(&self) -> Vec<(u32, u64)> { + use engram_sandbox_firecracker::sandbox_manifest::read_proc_comm; + let Ok(entries) = std::fs::read_dir("/proc") else { + return Vec::new(); + }; + entries + .flatten() + .filter_map(|e| e.file_name().to_str()?.parse::().ok()) + .filter(|&pid| { + // The kernel keeps the first 15 bytes of the name. + read_proc_comm(pid).is_some_and(|comm| comm == "engram-disk-source"[..15]) + }) + .filter_map(|pid| Some((pid, self.start_time(pid)?))) + .collect() + } +} + +/// The state of a recorded source. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Health { + /// The recorded process runs and has reported its device. + Serving(Device), + /// The process died. The kernel holds the device for a recovery. + Dead(Device), + /// A start that did not report its device; its process still runs. + Unreported, + /// No device is left to serve: the process died before it reported + /// one (`None`), or a stop deleted the device and did not finish. + Gone(Option), +} + +/// `device_lost`: the kernel has no device of that id, or another record +/// names it (a device made after this one's was deleted). +fn health(device: Option, alive: bool, device_lost: bool) -> Health { + match (device, alive) { + (Some(device), true) => Health::Serving(device), + (None, true) => Health::Unreported, + (Some(device), false) if !device_lost => Health::Dead(device), + (device, false) => Health::Gone(device.map(|d| d.id)), + } +} + +/// Run `op` in a task of its own, so a caller that is dropped cannot stop +/// it between a spawn and the record of the process. +async fn detached( + op: impl Future> + Send + 'static, +) -> io::Result { + tokio::spawn(op).await.map_err(io::Error::other)? +} + +/// [`DiskSources`] as `engram-disk-source` processes, one per role. +pub struct ProcessSources(Arc); + +/// The state and steps of [`ProcessSources`], shared with the tasks that +/// run each step. +struct Sources { + cfg: ProcessSourcesConfig, + dir: PathBuf, + fs: Arc, + probe: Arc, + /// One lock per role: a start, a recovery and a stop of one role never + /// overlap. + locks: std::sync::Mutex>>>, +} + +type RoleKey = (SandboxId, DiskRole); + +impl ProcessSources { + pub fn new(cfg: ProcessSourcesConfig) -> Self { + Self::with_probe(cfg, Arc::new(TokioFs), Arc::new(KernelProbe)) + } + + pub fn with_probe( + cfg: ProcessSourcesConfig, + fs: Arc, + probe: Arc, + ) -> Self { + let dir = cfg.work_dir.join("disk-sources"); + Self(Arc::new(Sources { + cfg, + dir, + fs, + probe, + locks: Default::default(), + })) + } + + /// Whether this host can start a source; the reason when it cannot. + pub fn check_driver(&self) -> io::Result<()> { + self.0.check_driver() + } + + /// Every record on this host. + pub async fn records(&self) -> Vec { + self.0.records().await + } + + /// Log every source process and ublk device that no record names, and + /// delete each such device that no process serves. Run once at + /// startup, before anything starts a source. A device a running + /// process serves is never deleted. A device whose server died is + /// deleted only when every record file could be read: no drive is + /// made over a device before its record names it, so such a device + /// has no reader. + pub async fn sweep_unrecorded(&self) { + let sources = self.0.clone(); + if let Err(error) = detached(async move { sources.sweep_unrecorded().await }).await { + tracing::warn!(%error, "disk source sweep failed"); + } + } +} + +impl Sources { + fn check_driver(&self) -> io::Result<()> { + if self.probe.driver_loaded() { + return Ok(()); + } + Err(io::Error::new( + io::ErrorKind::Unsupported, + "the ublk driver is not loaded (/dev/ublk-control is absent), so this host cannot \ + start a disk source; node prep loads ublk_drv", + )) + } + + async fn lock(&self, key: RoleKey) -> tokio::sync::OwnedMutexGuard<()> { + let lock = { + let mut locks = self.locks.lock().unwrap(); + // An entry only the map holds has no holder and no waiter. + locks.retain(|k, l| *k == key || Arc::strong_count(l) > 1); + locks.entry(key).or_default().clone() + }; + lock.lock_owned().await + } + + fn id(sandbox: SandboxId, role: DiskRole) -> String { + format!("{sandbox}-{role}") + } + + fn log_path(&self, sandbox: SandboxId, role: DiskRole) -> PathBuf { + self.dir.join(format!("{}.log", Self::id(sandbox, role))) + } + + fn leaf(sandbox: SandboxId, role: DiskRole) -> String { + format!("{sandbox}-disk-{role}") + } + + async fn load(&self, sandbox: SandboxId, role: DiskRole) -> io::Result> { + durable_record::load(self.fs.as_ref(), &self.dir, Self::id(sandbox, role), WHAT).await + } + + async fn persist(&self, record: &SourceRecord) -> io::Result<()> { + let id = Self::id(record.sandbox, record.role); + durable_record::persist(self.fs.as_ref(), &self.dir, id, record, WHAT).await + } + + async fn records(&self) -> Vec { + durable_record::load_all(self.fs.as_ref(), &self.dir, WHAT).await + } + + /// How many record files there are, readable or not. + async fn record_files(&self) -> usize { + match self.fs.read_dir(&self.dir).await { + Ok(paths) => paths + .iter() + .filter(|p| p.extension().and_then(|e| e.to_str()) == Some("json")) + .count(), + Err(_) => 0, + } + } + + async fn health(&self, record: &SourceRecord) -> Health { + let alive = self.probe.start_time(record.pid) == Some(record.start_time); + let lost = match (&record.device, alive) { + (Some(device), false) => { + !self.probe.device_exists(device.id) + || self.records().await.iter().any(|other| { + (other.sandbox, other.role) != (record.sandbox, record.role) + && other.device.as_ref().map(|d| d.id) == Some(device.id) + }) + } + _ => false, + }; + health(record.device.clone(), alive, lost) + } + + /// Spawn `serve` with `queues` and `depth`, on device `recover` when it + /// is set, and move it to the role's cgroup leaf. Returns the pid, its + /// start time and the stdout that carries the ready line. + async fn spawn( + &self, + sandbox: SandboxId, + role: DiskRole, + args: &ServeArgs, + (queues, depth): (u16, u16), + recover: Option, + ) -> io::Result<(u32, u64, ChildStdout)> { + self.fs.create_dir(&self.dir).await?; + let log_path = self.log_path(sandbox, role); + // A file, never a pipe: a pipe breaks when the host-agent exits. + let log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path)?; + let mut cmd = tokio::process::Command::new(&self.cfg.bin); + cmd.arg("serve") + .arg("--manifest") + .arg(args.manifest.to_string()) + .arg("--queues") + .arg(queues.to_string()) + .arg("--depth") + .arg(depth.to_string()); + if let Some(base) = &args.base { + cmd.arg("--base-volume") + .arg(&base.volume) + .arg("--base-manifest") + .arg(base.manifest.to_string()); + } + if let Some(root) = &args.cache_root { + cmd.arg("--cache-root").arg(root); + } + if let Some(id) = recover { + cmd.arg("--recover").arg(id.to_string()); + } + // No kill_on_drop: the source outlives the host-agent. + let mut child = cmd + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(log)) + .spawn() + .map_err(|e| { + io::Error::new(e.kind(), format!("spawn {}: {e}", self.cfg.bin.display())) + })?; + let stdout = child.stdout.take().expect("stdout is piped"); + let pid = child + .id() + .expect("a child that was not waited for has a pid"); + let start_time = self.probe.start_time(pid); + // Reaps the process when it exits while this host-agent runs. + tokio::spawn(async move { + let _ = child.wait().await; + }); + let Some(start_time) = start_time else { + return Err(io::Error::other(format!( + "disk source {} exited at once; see {}", + Self::id(sandbox, role), + log_path.display() + ))); + }; + if let Some(parent) = &self.cfg.cgroup_parent { + let leaf = Self::leaf(sandbox, role); + if let Err(e) = engram_sandbox_firecracker::escape_to_node_cgroup(parent, &leaf, &[pid]) + { + tracing::warn!( + %sandbox, %role, pid, parent = %parent.display(), error = %e, + "could not move the disk source to its node cgroup; a host-agent pod \ + restart kills it, and the next host-agent recovers it", + ); + } + } + Ok((pid, start_time, stdout)) + } + + /// Wait for the ready line ` `. + async fn ready( + &self, + stdout: ChildStdout, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result { + let id = Self::id(sandbox, role); + let log = self.log_path(sandbox, role); + let mut line = String::new(); + let mut stdout = BufReader::new(stdout); + let read = stdout.read_line(&mut line); + match tokio::time::timeout(self.cfg.ready_timeout, read).await { + Err(_) => Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "disk source {id} did not serve within {:?}; see {}", + self.cfg.ready_timeout, + log.display() + ), + )), + Ok(Err(e)) => Err(e), + Ok(Ok(0)) => Err(io::Error::other(format!( + "disk source {id} exited before it served; see {}", + log.display() + ))), + Ok(Ok(_)) => parse_ready(&line), + } + } + + /// Delete device `id`, which no process serves, with + /// `engram-disk-source delete`. `quiesced` also deletes a device whose + /// server died; the binary refuses a device that a process serves. + async fn delete_device(&self, id: u32, quiesced: bool) -> io::Result<()> { + let mut cmd = tokio::process::Command::new(&self.cfg.bin); + cmd.arg("delete").arg("--device").arg(id.to_string()); + if quiesced { + cmd.arg("--quiesced"); + } + cmd.stdin(Stdio::null()).kill_on_drop(true); + let output = tokio::time::timeout(self.cfg.stop_timeout, cmd.output()) + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::TimedOut, + format!("the delete of device {id} did not finish"), + ) + })??; + if output.status.success() { + return Ok(()); + } + Err(io::Error::other(format!( + "delete device {id}: {} ({})", + String::from_utf8_lossy(&output.stderr).trim(), + output.status + ))) + } + + /// Send `signal` to the process if it is still the recorded one. + fn signal(&self, pid: u32, start_time: u64, signal: nix::sys::signal::Signal) { + if self.probe.start_time(pid) != Some(start_time) { + return; + } + let Ok(raw) = i32::try_from(pid) else { return }; + if let Err(e) = nix::sys::signal::kill(nix::unistd::Pid::from_raw(raw), signal) { + tracing::debug!(pid, %signal, error = %e, "signal the disk source"); + } + } + + /// Whether the process exits within the stop timeout. + async fn exits(&self, pid: u32, start_time: u64) -> bool { + let polls = self.cfg.stop_timeout.as_millis() / POLL.as_millis() + 1; + for _ in 0..polls { + if self.probe.start_time(pid) != Some(start_time) { + return true; + } + tokio::time::sleep(POLL).await; + } + self.probe.start_time(pid) != Some(start_time) + } + + /// End a process: SIGTERM, then SIGKILL if it does not exit in time. + /// SIGTERM deletes the device of a process that serves, so this is for + /// a source that nothing reads. Whether the process exited. + async fn terminate(&self, pid: u32, start_time: u64) -> bool { + use nix::sys::signal::Signal; + self.signal(pid, start_time, Signal::SIGTERM); + if self.exits(pid, start_time).await { + return true; + } + tracing::warn!(pid, timeout = ?self.cfg.stop_timeout, "disk source ignored SIGTERM; SIGKILL"); + self.kill(pid, start_time).await + } + + /// SIGKILL: the device stays for a recovery. Whether the process exited. + async fn kill(&self, pid: u32, start_time: u64) -> bool { + self.signal(pid, start_time, nix::sys::signal::Signal::SIGKILL); + self.exits(pid, start_time).await + } + + /// Remove what a role's source leaves on the host; the record last. + async fn clean_up(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + engram_sandbox_firecracker::remove_node_cgroup( + self.cfg.cgroup_parent.as_deref(), + &Self::leaf(sandbox, role), + ); + match self.fs.remove_file(&self.log_path(sandbox, role)).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + _ => {} + } + durable_record::delete(self.fs.as_ref(), &self.dir, Self::id(sandbox, role)).await + } + + /// Start a recovery process on the record's device, with the device's + /// own queues and depth. + async fn recover(&self, record: &SourceRecord, device: &Device) -> io::Result { + self.check_driver()?; + let (sandbox, role) = (record.sandbox, record.role); + let shape = (record.queues, record.depth); + let (pid, start_time, stdout) = self + .spawn(sandbox, role, &record.args, shape, Some(device.id)) + .await?; + let next = SourceRecord { + pid, + start_time, + ..record.clone() + }; + // Before the ready line: a host-agent that dies now leaves a record + // that names the process that serves the device. + if let Err(e) = self.persist(&next).await { + self.kill(pid, start_time).await; + return Err(e); + } + let served = self.ready(stdout, sandbox, role).await; + match served { + Ok(served) if served == *device => { + tracing::info!(%sandbox, %role, pid, device = device.id, "disk source recovered"); + Ok(next) + } + other => { + self.kill(pid, start_time).await; + Err(match other { + Ok(served) => io::Error::other(format!( + "the recovery of device {} serves {served:?}", + device.id + )), + Err(e) => e, + }) + } + } + } + + /// Make `record`'s source serve, and return its record and device. + /// `NotFound`, after its residue is removed, when no device is left. + async fn revive(&self, record: SourceRecord) -> io::Result<(SourceRecord, Device)> { + let (sandbox, role) = (record.sandbox, record.role); + let id = Self::id(sandbox, role); + let gone = match self.health(&record).await { + Health::Serving(device) => return Ok((record, device)), + Health::Dead(device) => { + tracing::warn!(%sandbox, %role, pid = record.pid, device = device.id, + "disk source process is gone; recovering its device"); + return Ok((self.recover(&record, &device).await?, device)); + } + Health::Unreported => { + if !self.terminate(record.pid, record.start_time).await { + return Err(io::Error::other(format!( + "disk source {id} (pid {}) did not report a device and does not exit", + record.pid + ))); + } + "it did not report a device; it is stopped".to_string() + } + Health::Gone(Some(device)) => format!("its device {device} is gone"), + Health::Gone(None) => "it exited before it reported a device".to_string(), + }; + tracing::warn!(%sandbox, %role, reason = %gone, "disk source record removed"); + self.clean_up(sandbox, role).await?; + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("disk source {id}: {gone}"), + )) + } + + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option, + ) -> io::Result { + self.check_driver()?; + let _guard = self.lock((sandbox, role)).await; + if self.load(sandbox, role).await?.is_some() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("disk source {} already exists", Self::id(sandbox, role)), + )); + } + let args = ServeArgs { + manifest, + base, + cache_root: self.cfg.cache_root.clone(), + }; + let shape = (self.cfg.queues, self.cfg.depth); + let (pid, start_time, stdout) = self.spawn(sandbox, role, &args, shape, None).await?; + let record = SourceRecord { + sandbox, + role, + pid, + start_time, + device: None, + queues: self.cfg.queues, + depth: self.cfg.depth, + args, + }; + let served = async { + // Before the ready line, so a host-agent that dies now leaves a + // record of the process. + self.persist(&record).await?; + let device = self.ready(stdout, sandbox, role).await?; + let serving = SourceRecord { + device: Some(device.clone()), + ..record + }; + self.persist(&serving).await?; + Ok::<_, io::Error>(device.path) + }; + match served.await { + Ok(path) => Ok(path), + Err(e) => { + // Nothing reads the device yet; SIGTERM deletes it. + if self.terminate(pid, start_time).await { + self.clean_up(sandbox, role).await?; + } + Err(e) + } + } + } + + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + let _guard = self.lock((sandbox, role)).await; + let record = self + .load(sandbox, role) + .await? + .ok_or_else(|| no_source(sandbox, role))?; + self.revive(record).await.map(drop) + } + + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + let _guard = self.lock((sandbox, role)).await; + let Some(record) = self.load(sandbox, role).await? else { + return Ok(()); + }; + let device = match self.revive(record).await { + Ok((record, device)) => { + if !self.terminate(record.pid, record.start_time).await { + return Err(io::Error::other(format!( + "disk source {} (pid {}) does not exit", + Self::id(sandbox, role), + record.pid + ))); + } + device + } + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + // The kernel refuses a recovery of a device that its server + // stopped and did not delete. With no process left, a delete + // is the one way to remove it. + Err(e) => match self.load(sandbox, role).await? { + Some(SourceRecord { + pid, + start_time, + device: Some(device), + .. + }) if self.probe.start_time(pid) != Some(start_time) => { + tracing::warn!(%sandbox, %role, device = device.id, error = %e, + "disk source cannot be recovered; deleting its device"); + device + } + _ => return Err(e), + }, + }; + if self.probe.device_exists(device.id) { + // The caller removed every drive over the device, so nothing + // reads it: a device whose server died may go too. + if let Err(e) = self.delete_device(device.id, true).await { + tracing::warn!(%sandbox, %role, device = device.id, error = %e, + "device of a stopped disk source not deleted"); + } + } + if self.probe.device_exists(device.id) { + // The record stays, so the next stop tries again. + return Err(io::Error::other(format!( + "device {} of disk source {} stays after its process exited", + device.id, + Self::id(sandbox, role) + ))); + } + self.clean_up(sandbox, role).await + } + + async fn sweep_unrecorded(&self) -> io::Result<()> { + let records = self.records().await; + if !self.probe.driver_loaded() { + if !records.is_empty() { + tracing::error!( + records = records.len(), + "disk source records exist, but the ublk driver is not loaded; their \ + sources cannot serve" + ); + } + return Ok(()); + } + let readable = self.record_files().await == records.len(); + let processes: HashSet<(u32, u64)> = + records.iter().map(|r| (r.pid, r.start_time)).collect(); + for (pid, start_time) in self.probe.source_processes() { + if !processes.contains(&(pid, start_time)) { + tracing::warn!(pid, "a disk source process runs that no record names"); + } + } + let devices: HashSet = records + .iter() + .filter_map(|r| r.device.as_ref().map(|d| d.id)) + .collect(); + for id in self.probe.devices() { + if devices.contains(&id) { + continue; + } + match self.delete_device(id, readable).await { + Ok(()) => tracing::warn!(device = id, "deleted a ublk device that no record names"), + Err(error) => tracing::warn!( + device = id, + %error, + "a ublk device that no record names stays" + ), + } + } + Ok(()) + } +} + +fn parse_ready(line: &str) -> io::Result { + let parsed = line.trim().split_once(' ').and_then(|(path, id)| { + Some(Device { + id: id.parse().ok()?, + path: PathBuf::from(path), + }) + }); + parsed.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("disk source ready line {line:?} is not \" \""), + ) + }) +} + +fn no_source(sandbox: SandboxId, role: DiskRole) -> io::Error { + io::Error::new( + io::ErrorKind::NotFound, + format!("the {role} disk of sandbox {sandbox} has no source"), + ) +} + +#[async_trait] +impl DiskSources for ProcessSources { + fn can_start(&self) -> bool { + self.0.probe.driver_loaded() + } + + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option<(&Volume, ManifestRef)>, + ) -> io::Result { + let base = base.map(|(volume, manifest)| Base { + volume: volume.path.clone(), + manifest, + }); + let sources = self.0.clone(); + detached(async move { sources.start(sandbox, role, manifest, base).await }).await + } + + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + let sources = self.0.clone(); + detached(async move { sources.ensure_alive(sandbox, role).await }).await + } + + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + let record = self + .0 + .load(sandbox, role) + .await? + .ok_or_else(|| no_source(sandbox, role))?; + let device = record.device.ok_or_else(|| no_source(sandbox, role))?; + Ok(Served { + device: device.path, + manifest: record.args.manifest, + base: record.args.base.map(|b| b.manifest), + }) + } + + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + let sources = self.0.clone(); + detached(async move { sources.stop(sandbox, role).await }).await + } + + async fn list(&self) -> io::Result> { + Ok(self + .0 + .records() + .await + .into_iter() + .map(|r| (r.sandbox, r.role)) + .collect()) + } +} + +/// Make every recorded source serve, after the pool opens and before the +/// volume reconcile, which removes drives: a drive over a dead source +/// cannot be removed. A few recoveries run at once, so sources that each +/// take the full ready timeout do not add up. +pub async fn recover_all(sources: &dyn DiskSources) { + use futures::StreamExt; + let roles = match sources.list().await { + Ok(roles) => roles, + Err(error) => { + tracing::warn!(%error, "disk source recovery: cannot list the sources"); + return; + } + }; + futures::stream::iter(roles) + .for_each_concurrent(RECOVER_AT_ONCE, |(sandbox, role)| async move { + match sources.ensure_alive(sandbox, role).await { + Ok(()) => tracing::debug!(%sandbox, %role, "disk source serves"), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + tracing::info!(%sandbox, %role, %error, "disk source removed"); + } + Err(error) => { + tracing::error!(%sandbox, %role, %error, "disk source recovery failed"); + } + } + }) + .await; +} + +#[cfg(test)] +mod tests { + //! Records and the restart decision, with a fake probe and a shell + //! script in place of the binary; the kernel side is the root lane's + //! (`tests/disk_source_ublk.rs`). Crash states follow ADR 0099 H5: each + //! post-crash file state is built directly. + + use std::os::unix::fs::PermissionsExt; + use std::sync::Mutex; + + use super::*; + + /// Processes run unless the test says otherwise. A device exists while + /// a file of its id is in `devices`, which the fake binary's `delete` + /// removes. + struct FakeProbe { + start_times: Mutex>>, + devices: PathBuf, + processes: Mutex>, + no_driver: bool, + } + + impl Probe for FakeProbe { + fn start_time(&self, pid: u32) -> Option { + if let Some(time) = self.start_times.lock().unwrap().get(&pid) { + return *time; + } + let pid = nix::unistd::Pid::from_raw(i32::try_from(pid).ok()?); + nix::sys::signal::kill(pid, None).is_ok().then_some(1) + } + fn device_exists(&self, id: u32) -> bool { + self.devices.join(id.to_string()).exists() + } + fn driver_loaded(&self) -> bool { + !self.no_driver + } + fn devices(&self) -> Vec { + std::fs::read_dir(&self.devices) + .unwrap() + .filter_map(|e| e.ok()?.file_name().to_str()?.parse().ok()) + .collect() + } + fn source_processes(&self) -> Vec<(u32, u64)> { + self.processes.lock().unwrap().clone() + } + } + + struct Rig { + dir: tempfile::TempDir, + probe: Arc, + sources: ProcessSources, + } + + /// `script` is the body of the fake binary. `$ARGS` names a file that + /// `serve` writes its arguments to, `$DELETES` one that `delete` + /// appends its arguments to, and `$DEVS` the fake's device folder. + fn rig(script: &str, no_driver: bool) -> Rig { + let dir = tempfile::tempdir().unwrap(); + let devices = dir.path().join("devices"); + std::fs::create_dir(&devices).unwrap(); + let bin = dir.path().join("fake-disk-source"); + let body = script + .replace("$ARGS", &dir.path().join("args").display().to_string()) + .replace( + "$DELETES", + &dir.path().join("deletes").display().to_string(), + ) + .replace("$DEVS", &devices.display().to_string()); + std::fs::write(&bin, format!("#!/bin/sh\n{body}\n")).unwrap(); + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut cfg = ProcessSourcesConfig::new(bin, dir.path()); + cfg.ready_timeout = Duration::from_secs(10); + cfg.stop_timeout = Duration::from_secs(5); + let probe = Arc::new(FakeProbe { + start_times: Default::default(), + devices, + processes: Default::default(), + no_driver, + }); + let sources = ProcessSources::with_probe(cfg, Arc::new(TokioFs), probe.clone()); + Rig { + dir, + probe, + sources, + } + } + + const DELETES: &str = + "if [ \"$1\" = delete ]; then echo \"$@\" >> $DELETES; rm -f $DEVS/$3; exit 0; fi"; + const DELETE_REFUSED: &str = + "if [ \"$1\" = delete ]; then echo \"$@\" >> $DELETES; echo refused >&2; exit 1; fi"; + const RECOVERY_REFUSED: &str = "case \"$*\" in *--recover*) exit 1;; esac"; + const SERVE: &str = "echo \"$@\" > $ARGS\necho /dev/ublkb7 7\nexec sleep 60"; + + fn serves() -> String { + format!("{DELETES}\n{SERVE}") + } + + impl Rig { + fn args(&self) -> Option { + std::fs::read_to_string(self.dir.path().join("args")).ok() + } + fn deletes(&self) -> Option { + std::fs::read_to_string(self.dir.path().join("deletes")).ok() + } + fn record_path(&self, sandbox: SandboxId, role: DiskRole) -> PathBuf { + durable_record::record_path(&self.sources.0.dir, Sources::id(sandbox, role)) + } + fn mark_dead(&self, pid: u32) { + self.probe.start_times.lock().unwrap().insert(pid, None); + } + fn mark_alive(&self, pid: u32) { + self.probe.start_times.lock().unwrap().insert(pid, Some(1)); + } + fn add_device(&self, id: u32) { + std::fs::write(self.probe.devices.join(id.to_string()), b"").unwrap(); + } + async fn persist(&self, record: &SourceRecord) { + self.sources.0.persist(record).await.unwrap(); + } + } + + fn record(sandbox: SandboxId, role: DiskRole, pid: u32, device: Option) -> SourceRecord { + SourceRecord { + sandbox, + role, + pid, + start_time: 1, + device: device.map(|id| Device { + id, + path: PathBuf::from(format!("/dev/ublkb{id}")), + }), + queues: 3, + depth: 9, + args: ServeArgs { + manifest: ManifestRef::new(), + base: Some(Base { + volume: PathBuf::from("/dev/engram/base"), + manifest: ManifestRef::new(), + }), + cache_root: Some(PathBuf::from("/cache")), + }, + } + } + + fn kill_now(pid: u32) { + let pid = nix::unistd::Pid::from_raw(pid as i32); + let _ = nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGKILL); + } + + #[test] + fn health_restarts_only_a_dead_process_whose_device_is_its_own() { + let device = Device { + id: 3, + path: PathBuf::from("/dev/ublkb3"), + }; + let some = || Some(device.clone()); + assert_eq!(health(some(), true, false), Health::Serving(device.clone())); + assert_eq!(health(some(), false, false), Health::Dead(device.clone())); + assert_eq!(health(some(), false, true), Health::Gone(Some(3))); + assert_eq!(health(None, true, false), Health::Unreported); + assert_eq!(health(None, false, false), Health::Gone(None)); + } + + /// The build before a field was added wrote no such field, and the + /// build after it writes one this build does not know. + #[test] + fn a_record_reads_across_one_release_in_both_directions() { + let rec = record(SandboxId::new(), DiskRole::Root, 9, None); + let mut json = serde_json::to_value(&rec).unwrap(); + let fields = json.as_object_mut().unwrap(); + fields.remove("device"); + fields["args"].as_object_mut().unwrap().remove("cache_root"); + fields.insert("a_later_field".into(), serde_json::json!(1)); + let read: SourceRecord = serde_json::from_value(json).unwrap(); + let expected = SourceRecord { + args: ServeArgs { + cache_root: None, + ..rec.args.clone() + }, + ..rec + }; + assert_eq!(read, expected); + } + + #[tokio::test] + async fn a_record_round_trips_and_names_what_the_source_serves() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Swap); + let rec = record(sandbox, role, 4242, Some(5)); + r.persist(&rec).await; + + assert_eq!(r.sources.records().await, vec![rec.clone()]); + assert_eq!(r.sources.list().await.unwrap(), vec![(sandbox, role)]); + assert_eq!( + r.sources.served(sandbox, role).await.unwrap(), + Served { + device: PathBuf::from("/dev/ublkb5"), + manifest: rec.args.manifest, + base: rec.args.base.map(|b| b.manifest), + } + ); + } + + #[tokio::test] + async fn a_torn_record_is_an_error_for_its_role_and_skipped_in_the_list() { + let r = rig(&serves(), false); + let (torn, sibling) = (SandboxId::new(), SandboxId::new()); + let role = DiskRole::Root; + r.persist(&record(sibling, role, 1, Some(1))).await; + r.persist(&record(torn, role, 2, Some(2))).await; + let path = r.record_path(torn, role); + let full = std::fs::read(&path).unwrap(); + for offset in 0..full.len() { + std::fs::write(&path, &full[..offset]).unwrap(); + assert_eq!( + r.sources.list().await.unwrap(), + vec![(sibling, role)], + "offset {offset}" + ); + let err = r.sources.served(torn, role).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData, "offset {offset}"); + } + // A record that cannot be read is never taken as no source: a stop + // could otherwise leave a device without a server. + for err in [ + r.sources.ensure_alive(torn, role).await.unwrap_err(), + r.sources.stop(torn, role).await.unwrap_err(), + ] { + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + } + } + + #[tokio::test] + async fn a_missing_record_is_no_source() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + assert!(r.sources.list().await.unwrap().is_empty()); + for err in [ + r.sources.served(sandbox, role).await.unwrap_err(), + r.sources.ensure_alive(sandbox, role).await.unwrap_err(), + ] { + assert_eq!(err.kind(), io::ErrorKind::NotFound); + } + r.sources.stop(sandbox, role).await.unwrap(); + } + + #[tokio::test] + async fn garbage_siblings_are_not_records() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + let rec = record(sandbox, role, 1, Some(1)); + r.persist(&rec).await; + let dir = &r.sources.0.dir; + // A crash before the rename, a log, and junk. + let other = record(SandboxId::new(), role, 2, Some(2)); + let other_id = Sources::id(other.sandbox, role); + let sealed = + crate::durable_envelope::seal(&other_id, &serde_json::to_string(&other).unwrap()); + std::fs::write(dir.join(format!("{other_id}.json.partial")), sealed).unwrap(); + std::fs::write(r.sources.0.log_path(sandbox, role), b"x").unwrap(); + std::fs::write(dir.join("junk.json"), b"\x00\xff not json").unwrap(); + assert_eq!(r.sources.records().await, vec![rec]); + } + + #[tokio::test] + async fn a_live_source_is_left_alone() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + let rec = record(sandbox, role, 4242, Some(5)); + r.mark_alive(4242); + r.persist(&rec).await; + r.sources.ensure_alive(sandbox, role).await.unwrap(); + assert_eq!(r.args(), None, "nothing was started"); + assert_eq!(r.sources.records().await, vec![rec]); + } + + /// A dead process, or a live one that is not the recorded process + /// (its pid was reused), is started again on the same device with the + /// same arguments, queues and depth. + #[tokio::test] + async fn a_gone_or_reused_pid_is_recovered_on_the_same_device() { + for reused in [false, true] { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + let rec = record(sandbox, role, 4242, Some(7)); + let time = if reused { Some(99) } else { None }; + r.probe.start_times.lock().unwrap().insert(4242, time); + r.add_device(7); + r.persist(&rec).await; + + r.sources.ensure_alive(sandbox, role).await.unwrap(); + + let base = rec.args.base.as_ref().unwrap(); + assert_eq!( + r.args().unwrap().trim(), + format!( + "serve --manifest {} --queues 3 --depth 9 --base-volume /dev/engram/base \ + --base-manifest {} --cache-root /cache --recover 7", + rec.args.manifest, base.manifest + ), + "reused={reused}" + ); + let now = r.sources.records().await.pop().unwrap(); + assert_ne!(now.pid, 4242); + let before = SourceRecord { + pid: 4242, + start_time: 1, + ..now.clone() + }; + assert_eq!(before, rec, "only the process changes"); + kill_now(now.pid); + } + } + + #[tokio::test] + async fn a_source_whose_device_is_gone_is_removed() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.mark_dead(4242); + r.persist(&record(sandbox, role, 4242, Some(7))).await; + let err = r.sources.ensure_alive(sandbox, role).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::NotFound); + assert!(r.sources.records().await.is_empty()); + assert_eq!(r.args(), None); + } + + /// The kernel reuses a deleted device's id. A dead source whose device + /// id another record names lost its device to a stop that did not + /// finish; a recovery would attach to the other source's device. + #[tokio::test] + async fn a_device_id_another_source_holds_is_not_recovered() { + let r = rig(&serves(), false); + let role = DiskRole::Root; + let (stale, owner) = (SandboxId::new(), SandboxId::new()); + r.mark_dead(4242); + r.mark_alive(4343); + r.add_device(7); + r.persist(&record(stale, role, 4242, Some(7))).await; + let held = record(owner, role, 4343, Some(7)); + r.persist(&held).await; + + let err = r.sources.ensure_alive(stale, role).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::NotFound); + assert_eq!(r.sources.records().await, vec![held]); + assert_eq!(r.args(), None); + } + + #[tokio::test] + async fn a_start_that_never_reported_a_device_is_removed() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.mark_dead(4242); + r.persist(&record(sandbox, role, 4242, None)).await; + let err = r.sources.ensure_alive(sandbox, role).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::NotFound); + assert!(r.sources.records().await.is_empty()); + } + + #[tokio::test] + async fn start_records_the_device_and_stop_removes_every_trace() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + let manifest = ManifestRef::new(); + let path = r + .sources + .start(sandbox, role, manifest, None) + .await + .unwrap(); + assert_eq!(path, PathBuf::from("/dev/ublkb7")); + assert_eq!( + r.args().unwrap().trim(), + format!("serve --manifest {manifest} --queues 2 --depth 64") + ); + let rec = r.sources.records().await.pop().unwrap(); + assert_eq!(rec.device.as_ref().map(|d| d.id), Some(7)); + assert_eq!((rec.queues, rec.depth), (2, 64)); + let again = r.sources.start(sandbox, role, manifest, None).await; + assert_eq!(again.unwrap_err().kind(), io::ErrorKind::AlreadyExists); + + // The fake has no device, as after a normal stop. + r.sources.stop(sandbox, role).await.unwrap(); + assert!(r.probe.start_time(rec.pid).is_none(), "the process exited"); + assert!(r.sources.records().await.is_empty()); + assert!(!r.sources.0.log_path(sandbox, role).exists()); + assert_eq!(r.deletes(), None, "no delete was needed"); + r.sources.stop(sandbox, role).await.unwrap(); + } + + /// The server's own delete failed after it stopped the device: the + /// stop deletes the device without a server. + #[tokio::test] + async fn a_device_left_by_its_server_is_deleted_by_the_stop() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.sources + .start(sandbox, role, ManifestRef::new(), None) + .await + .unwrap(); + r.add_device(7); + r.sources.stop(sandbox, role).await.unwrap(); + assert_eq!(r.deletes().unwrap().trim(), "delete --device 7 --quiesced"); + assert!(!r.probe.device_exists(7)); + assert!(r.sources.records().await.is_empty()); + } + + /// The kernel refuses a recovery of a device its server stopped and did + /// not delete; the stop deletes it instead of failing forever. + #[tokio::test] + async fn a_device_that_cannot_be_recovered_is_deleted_by_the_stop() { + let r = rig(&format!("{DELETES}\n{RECOVERY_REFUSED}\n{SERVE}"), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.mark_dead(4242); + r.add_device(7); + r.persist(&record(sandbox, role, 4242, Some(7))).await; + r.sources.ensure_alive(sandbox, role).await.unwrap_err(); + r.sources.stop(sandbox, role).await.unwrap(); + assert_eq!(r.deletes().unwrap().trim(), "delete --device 7 --quiesced"); + assert!(r.sources.records().await.is_empty()); + } + + #[tokio::test] + async fn a_stop_whose_delete_is_refused_keeps_the_record() { + let r = rig(&format!("{DELETE_REFUSED}\n{SERVE}"), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.sources + .start(sandbox, role, ManifestRef::new(), None) + .await + .unwrap(); + r.add_device(7); + let err = r.sources.stop(sandbox, role).await.unwrap_err(); + assert!(err.to_string().contains("stays"), "{err}"); + assert_eq!(r.sources.list().await.unwrap(), vec![(sandbox, role)]); + // The next stop recovers the device first, then stops it. + std::fs::remove_file(r.dir.path().join("args")).unwrap(); + r.sources.stop(sandbox, role).await.unwrap_err(); + assert!(r.args().unwrap().contains("--recover 7")); + std::fs::remove_file(r.probe.devices.join("7")).unwrap(); + r.sources.stop(sandbox, role).await.unwrap(); + assert!(r.sources.records().await.is_empty()); + } + + #[tokio::test] + async fn a_start_that_does_not_serve_leaves_nothing() { + let r = rig("exit 3", false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.sources + .start(sandbox, role, ManifestRef::new(), None) + .await + .unwrap_err(); + assert!(r.sources.records().await.is_empty()); + } + + /// A caller dropped while the start waits for the ready line leaves a + /// source that its record names, never a process without one. + #[tokio::test] + async fn a_dropped_start_finishes_in_its_own_task() { + let r = rig(&format!("{DELETES}\nsleep 1\n{SERVE}"), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + let start = r.sources.start(sandbox, role, ManifestRef::new(), None); + let cut = tokio::time::timeout(Duration::from_millis(200), start).await; + assert!(cut.is_err(), "the caller gave up"); + for _ in 0..200 { + if r.sources.records().await.iter().any(|r| r.device.is_some()) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let rec = r.sources.records().await.pop().unwrap(); + assert_eq!(rec.device.map(|d| d.id), Some(7)); + r.sources.stop(sandbox, role).await.unwrap(); + } + + #[tokio::test] + async fn no_driver_no_start() { + let r = rig(&serves(), true); + assert!(!r.sources.can_start()); + let err = r + .sources + .start(SandboxId::new(), DiskRole::Root, ManifestRef::new(), None) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::Unsupported); + assert!(err.to_string().contains("/dev/ublk-control"), "{err}"); + assert_eq!(r.args(), None); + } + + /// The sweep deletes devices that no record names and that no process + /// serves; the binary refuses a device a process serves. + #[tokio::test] + async fn the_sweep_deletes_only_unrecorded_devices() { + let r = rig(&serves(), false); + let (sandbox, role) = (SandboxId::new(), DiskRole::Root); + r.mark_alive(4242); + r.persist(&record(sandbox, role, 4242, Some(3))).await; + r.add_device(3); + r.add_device(8); + r.probe.processes.lock().unwrap().push((5151, 1)); + r.sources.sweep_unrecorded().await; + assert_eq!(r.deletes().unwrap().trim(), "delete --device 8 --quiesced"); + assert!(r.probe.device_exists(3), "a recorded device stays"); + assert!(!r.probe.device_exists(8)); + } + + /// A record file that cannot be read may name a device that a drive + /// reads; the sweep then deletes only devices whose server stopped them. + #[tokio::test] + async fn an_unreadable_record_limits_the_sweep_to_stopped_devices() { + let r = rig(&serves(), false); + std::fs::create_dir_all(&r.sources.0.dir).unwrap(); + std::fs::write(r.sources.0.dir.join("torn.json"), b"{").unwrap(); + r.add_device(8); + r.sources.sweep_unrecorded().await; + assert_eq!(r.deletes().unwrap().trim(), "delete --device 8"); + } +} diff --git a/crates/engram-host-agent/src/durable_record.rs b/crates/engram-host-agent/src/durable_record.rs index 4175cc7dd..1d96cc4a0 100644 --- a/crates/engram-host-agent/src/durable_record.rs +++ b/crates/engram-host-agent/src/durable_record.rs @@ -1,14 +1,17 @@ //! One-file-per-record durable JSON persistence — the shared engine -//! under `checkpoint::CheckpointRecord` (ADR 0028 Fix A) and -//! `capture_job::CaptureJobRecord` (ADR 0084 §A). +//! under `checkpoint::CheckpointRecord` (ADR 0028 Fix A), +//! `capture_job::CaptureJobRecord` (ADR 0084 §A) and +//! `disk_source::SourceRecord` (ADR 0127). //! -//! The contract both rely on: +//! The contract they rely on: //! - **persist** = write to a `.json.partial` temp, fsync, rename, then //! fsync the parent dir — the rename publishes complete bytes or //! nothing, and the dir fsync makes the rename itself crash-durable; //! - **load_all** = torn-write-tolerant: unreadable/unparseable files //! are skipped with a warn, never an error — a torn write must not //! wedge the heartbeat loop that re-advertises these records; +//! - **load** = one record by id; a torn or corrupt file is an +//! `InvalidData` error, because the caller asked for that record; //! - **delete** = idempotent (NotFound is fine): the coordinator acked, //! the PG row owns the reference now. //! @@ -32,6 +35,8 @@ use std::path::{Path, PathBuf}; use engram_host_core::HostFs; use serde::de::DeserializeOwned; + +use crate::durable_envelope::OpenError; use serde::Serialize; pub fn record_path(dir: &Path, id: impl Display) -> PathBuf { @@ -83,39 +88,85 @@ pub async fn load_all(fs: &dyn HostFs, dir: &Path, what: &s // The record's expected identity is its filename stem — bind the // envelope's sealed id to it so a misdirected read is rejected. let expected_id = p.file_stem().and_then(|s| s.to_str()).unwrap_or_default(); - match fs.read(&p).await { - Ok(bytes) => match crate::durable_envelope::open(&bytes, expected_id) { - Ok(body) => match serde_json::from_slice::(&body) { - Ok(r) => out.push(r), - Err(e) => { - tracing::warn!(path = %p.display(), error = %e, - "unparseable {what} body; skipping"); - } - }, - // R5: DISTINCT logs — a checksum/identity failure is - // corruption (a real finding), not a torn write. - Err(e @ crate::durable_envelope::OpenError::ChecksumMismatch { .. }) => { - tracing::warn!(path = %p.display(), error = %e, - "corrupt {what} (checksum); skipping"); - } - Err(e @ crate::durable_envelope::OpenError::IdMismatch { .. }) => { - tracing::warn!(path = %p.display(), error = %e, - "misdirected {what} (identity); skipping"); - } - Err(e @ crate::durable_envelope::OpenError::Malformed(_)) => { - tracing::warn!(path = %p.display(), error = %e, - "malformed {what} envelope; skipping"); - } - }, + let bytes = match fs.read(&p).await { + Ok(bytes) => bytes, Err(e) => { tracing::warn!(path = %p.display(), error = %e, "unreadable {what}; skipping"); + continue; + } + }; + match decode(&bytes, expected_id) { + Ok(r) => out.push(r), + Err(Unusable::Body(e)) => { + tracing::warn!(path = %p.display(), error = %e, + "unparseable {what} body; skipping"); + } + // R5: DISTINCT logs — a checksum/identity failure is + // corruption (a real finding), not a torn write. + Err(Unusable::Envelope(e @ OpenError::ChecksumMismatch { .. })) => { + tracing::warn!(path = %p.display(), error = %e, + "corrupt {what} (checksum); skipping"); + } + Err(Unusable::Envelope(e @ OpenError::IdMismatch { .. })) => { + tracing::warn!(path = %p.display(), error = %e, + "misdirected {what} (identity); skipping"); + } + Err(Unusable::Envelope(e @ OpenError::Malformed(_))) => { + tracing::warn!(path = %p.display(), error = %e, + "malformed {what} envelope; skipping"); } } } out } +/// The record `/.json`: `None` when there is none, `InvalidData` +/// when its file is torn or corrupt. +pub async fn load( + fs: &dyn HostFs, + dir: &Path, + id: impl Display, + what: &str, +) -> std::io::Result> { + let id = id.to_string(); + let path = record_path(dir, &id); + let bytes = match fs.read(&path).await { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + decode(&bytes, &id).map(Some).map_err(|e| { + let e = match e { + Unusable::Envelope(e) => e.to_string(), + Unusable::Body(e) => e.to_string(), + }; + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{what} {}: {e}", path.display()), + ) + }) +} + +enum Unusable { + Envelope(OpenError), + Body(serde_json::Error), +} + +/// Open the envelope of `bytes` for the record id `id` and parse its body. +fn decode(bytes: &[u8], id: &str) -> Result { + let body = crate::durable_envelope::open(bytes, id).map_err(Unusable::Envelope)?; + serde_json::from_slice(&body).map_err(Unusable::Body) +} + +/// Delete the record `/.json`; NotFound is fine (already gone). +pub async fn delete(fs: &dyn HostFs, dir: &Path, id: impl Display) -> std::io::Result<()> { + match fs.remove_file(&record_path(dir, id)).await { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } +} + /// Delete the acked records' files; NotFound is fine (already gone). pub async fn delete_acked( fs: &dyn HostFs, @@ -124,12 +175,10 @@ pub async fn delete_acked( what: &str, ) { for id in acked { - let p = record_path(dir, id); - if let Err(e) = fs.remove_file(&p).await { - if e.kind() != std::io::ErrorKind::NotFound { - tracing::warn!(path = %p.display(), error = %e, - "failed to delete acked {what}"); - } + let id = id.to_string(); + if let Err(e) = delete(fs, dir, &id).await { + tracing::warn!(path = %record_path(dir, &id).display(), error = %e, + "failed to delete acked {what}"); } } } diff --git a/crates/engram-host-agent/src/eviction_finalize.rs b/crates/engram-host-agent/src/eviction_finalize.rs index 2cbddfbed..8b088a39f 100644 --- a/crates/engram-host-agent/src/eviction_finalize.rs +++ b/crates/engram-host-agent/src/eviction_finalize.rs @@ -841,4 +841,49 @@ mod tests { }; assert!(message.contains("never published"), "{message}"); } + + /// A cloned role's publish renames its destination to the new + /// version. The redrive answers with that version while nothing is + /// pending. A pending pair publishes through the role's source, so a + /// host without disk sources fails the leg. + #[tokio::test] + async fn a_redrive_reads_a_cloned_role_lineage_from_its_destination() { + use engram_host_core::session_volumes::{bitmap_snapshot_name, dest_name}; + let tmp = tempfile::tempdir().unwrap(); + let (finalizer, volumes) = finalizer_with_a_root(tmp.path(), SandboxId::new()).await; + let sandbox = SandboxId::new(); + let previous = ManifestRef::new(); + let published = ManifestRef { + version: previous.version + 1, + ..previous + }; + let disk = volumes.disk(); + disk.create(&dest_name(sandbox, DiskRole::Root, published), 128 * 1024) + .await + .unwrap(); + let record = record_for( + sandbox, + vec![PendingDisk { + role: DiskRole::Root, + previous, + }], + ); + + let (root, _) = disk_leg_work(&finalizer, &record).await.unwrap(); + assert_eq!(root, Some(published)); + + let pair = engram_host_core::VolumeName::new(format!( + "tmp-sb-{}-root", + sandbox.as_uuid().simple() + )) + .unwrap(); + disk.create(&pair, 128 * 1024).await.unwrap(); + disk.create(&bitmap_snapshot_name(sandbox, DiskRole::Root), 4096) + .await + .unwrap(); + let Err(SandboxError::Snapshot(message)) = disk_leg_work(&finalizer, &record).await else { + panic!("a cloned role's pair must not publish without its source"); + }; + assert!(message.contains("disk sources"), "{message}"); + } } diff --git a/crates/engram-host-agent/src/heartbeat.rs b/crates/engram-host-agent/src/heartbeat.rs index 2d671081a..584e7956d 100644 --- a/crates/engram-host-agent/src/heartbeat.rs +++ b/crates/engram-host-agent/src/heartbeat.rs @@ -44,6 +44,7 @@ pub fn build_heartbeat( // (dormant P1a) — always empty until a later commit wires the // durable-record source. capture_job_reports: Vec::new(), + hydrating: Vec::new(), } } diff --git a/crates/engram-host-agent/src/hydration.rs b/crates/engram-host-agent/src/hydration.rs new file mode 100644 index 000000000..3472e9130 --- /dev/null +++ b/crates/engram-host-agent/src/hydration.rs @@ -0,0 +1,261 @@ +//! The hydration of a sandbox's cloned disks (ADR 0127 Decisions 5-7): +//! one task per sandbox keeps each source serving and cuts each disk +//! over to its destination once it is copied. The task belongs to the +//! sandbox: `destroy` stops it, and after a host-agent restart it starts +//! again for a live sandbox with a disk left, from the bitmap the kernel +//! kept. A source failure never ends the sandbox: the task logs and tries +//! again, later each time the same disk fails again. + +use std::collections::HashMap; +use std::io; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use dashmap::DashMap; +use engram_core::types::snapshot::DiskRole; +use engram_core::SandboxId; +use engram_host_core::session_volumes::{Cutover, Hydrating, Waiting}; +use engram_host_core::SessionVolumes; + +use crate::pooled_backend::{capture_lock_in, AbortTaskOnDrop, CaptureLocks}; + +/// How often a task looks at its disks unless the host sets it. +pub(crate) const DEFAULT_INTERVAL: Duration = Duration::from_secs(1); + +/// The longest wait before a disk whose step failed is tried again. +const MAX_BACKOFF: Duration = Duration::from_secs(60); + +/// The hydration task of each sandbox that has one, with the generation +/// that armed it. +pub(crate) struct Hydrations { + tasks: DashMap, + generation: AtomicU64, + interval: Duration, +} + +impl Hydrations { + pub(crate) fn new(interval: Duration) -> Arc { + Arc::new(Self { + tasks: DashMap::new(), + generation: AtomicU64::new(0), + interval, + }) + } + + /// Start the sandbox's task, unless it has one. The task ends when no + /// disk is left to cut over. It takes the sandbox's capture lock for + /// each cutover, which serializes the cutover with the checkpoints. + pub(crate) fn arm( + self: &Arc, + id: SandboxId, + volumes: Arc, + capture_locks: Arc, + ) { + let dashmap::Entry::Vacant(slot) = self.tasks.entry(id) else { + return; + }; + let generation = self.generation.fetch_add(1, Ordering::Relaxed); + let owner: Weak = Arc::downgrade(self); + let interval = self.interval; + let task = tokio::spawn(async move { + hydrate(&volumes, &capture_locks, id, interval).await; + // An abort takes effect only at an await, so a disarmed task can + // get here; it then removes nothing that a later `arm` put in. + if let Some(owner) = owner.upgrade() { + if owner + .tasks + .remove_if(&id, |_, (armed, _)| *armed == generation) + .is_some() + { + owner.publish_gauge(); + } + } + }); + slot.insert((generation, AbortTaskOnDrop(task))); + self.publish_gauge(); + } + + /// Stop the sandbox's task. + pub(crate) fn disarm(&self, id: SandboxId) { + if self.tasks.remove(&id).is_some() { + self.publish_gauge(); + } + } + + /// The sandboxes with a task. + pub(crate) fn sandboxes(&self) -> Vec { + let mut ids: Vec = self.tasks.iter().map(|e| *e.key()).collect(); + ids.sort(); + ids + } + + fn publish_gauge(&self) { + metrics::gauge!(crate::metrics::DISK_HYDRATING_SANDBOXES).set(self.tasks.len() as f64); + } +} + +/// When each disk whose step failed is tried again, in ticks. +#[derive(Default)] +struct Backoff { + roles: HashMap, +} + +impl Backoff { + /// Whether `role` is due at `tick`. + fn due(&self, role: DiskRole, tick: u64) -> bool { + self.roles.get(&role).is_none_or(|&(_, next)| tick >= next) + } + + /// Record the outcome of `role`'s step at `tick`. Each failure in a row + /// doubles the wait, up to `max_ticks`; a success clears it. + fn record(&mut self, role: DiskRole, tick: u64, failed: bool, max_ticks: u64) { + if !failed { + self.roles.remove(&role); + return; + } + let failures = self.roles.get(&role).map_or(0, |&(f, _)| f) + 1; + let wait = 1u64 + .checked_shl(failures - 1) + .unwrap_or(u64::MAX) + .min(max_ticks.max(1)); + self.roles.insert(role, (failures, tick + wait)); + } +} + +/// Tick until no disk of the sandbox is left to cut over. A failure is +/// logged when it differs from the failures of the tick before. +async fn hydrate( + volumes: &SessionVolumes, + capture_locks: &CaptureLocks, + id: SandboxId, + interval: Duration, +) { + let armed = crate::time_source::metrics_now(); + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let max_ticks = (MAX_BACKOFF.as_millis() / interval.as_millis().max(1)) as u64; + let mut backoff = Backoff::default(); + let mut reported: Vec = Vec::new(); + for tick in 0u64.. { + ticker.tick().await; + let pass = pass(volumes, capture_locks, id, tick, max_ticks, &mut backoff).await; + if pass.errors != reported { + for error in &pass.errors { + tracing::warn!(sandbox_id = %id, %error, "disk hydration failed; retrying"); + } + reported = pass.errors; + } + if !pass.left { + break; + } + } + metrics::histogram!(crate::metrics::DISK_HYDRATION_SECONDS) + .record(armed.elapsed().as_secs_f64()); + tracing::info!(sandbox_id = %id, "disk hydration finished"); +} + +struct Pass { + /// A disk is still to cut over, or its step failed. + left: bool, + errors: Vec, +} + +async fn pass( + volumes: &SessionVolumes, + capture_locks: &CaptureLocks, + id: SandboxId, + tick: u64, + max_ticks: u64, + backoff: &mut Backoff, +) -> Pass { + let roles = match volumes.roles(id).await { + Ok(roles) => roles, + Err(e) => { + return Pass { + left: true, + errors: vec![format!("list the disks: {e}")], + } + } + }; + let mut pass = Pass { + left: false, + errors: Vec::new(), + }; + for role in roles { + if !backoff.due(role, tick) { + pass.left = true; + continue; + } + let result = step(volumes, capture_locks, id, role).await; + backoff.record(role, tick, result.is_err(), max_ticks); + match result { + Ok(done) => pass.left |= !done, + Err(e) => { + pass.left = true; + pass.errors.push(format!("{role} disk: {e}")); + } + } + } + pass +} + +/// Move one disk on; `true` when nothing of it is left to cut over. +/// `hydration` recovers a dead source before the capture lock is taken, +/// and the source stops after the lock is released: neither can hold up +/// a checkpoint for a process start or stop. +async fn step( + volumes: &SessionVolumes, + capture_locks: &CaptureLocks, + id: SandboxId, + role: DiskRole, +) -> io::Result { + match volumes.hydration(id, role).await? { + None => Ok(true), + Some(Hydrating::Copying(_)) => Ok(false), + Some(Hydrating::Ready) => { + let cut = { + let lock = capture_lock_in(capture_locks, id); + let _held = lock.lock().await; + volumes.cut_over(id, role).await + }; + let outcome = match &cut { + Ok(Cutover::Done) => "done", + Ok(Cutover::Waiting(Waiting::Checkpoint)) => "checkpoint_in_flight", + Ok(Cutover::Waiting(Waiting::Copying(_))) => "copying", + Err(_) => "error", + }; + metrics::counter!(crate::metrics::DISK_CUTOVERS_TOTAL, "outcome" => outcome) + .increment(1); + if cut? != Cutover::Done { + return Ok(false); + } + volumes.stop_source(id, role).await?; + tracing::info!(sandbox_id = %id, %role, "disk cut over to its destination"); + Ok(true) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_failing_disk_waits_twice_as_long_each_time_up_to_the_cap() { + let mut backoff = Backoff::default(); + let role = DiskRole::Root; + let mut tick = 0; + let mut waits = Vec::new(); + for _ in 0..8 { + backoff.record(role, tick, true, 60); + let next = (tick..).find(|t| backoff.due(role, *t)).unwrap(); + waits.push(next - tick); + tick = next; + } + assert_eq!(waits, vec![1, 2, 4, 8, 16, 32, 60, 60]); + assert!(backoff.due(DiskRole::Swap, tick), "each disk waits alone"); + backoff.record(role, tick, false, 60); + assert!(backoff.due(role, tick), "a success clears the wait"); + } +} diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index c4cae1bfe..9ecc05586 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -70,6 +70,7 @@ pub mod checkpoint; pub mod config; pub mod coord_client; pub mod dirty_map; +pub mod disk_source; pub mod durable_envelope; pub mod durable_record; pub mod egress; @@ -87,6 +88,7 @@ pub use host_client::LocalHostClient; #[cfg(target_os = "linux")] pub mod heap_profile; pub mod heartbeat; +mod hydration; pub mod idle_evictor; pub mod image_cache; pub mod image_prefetch; @@ -129,6 +131,9 @@ pub struct HostAgent { /// Where image and session volumes live. `None` keeps readiness on /// the chunk cache alone and leaves the disks to the backend. pub session_disk: Option>, + /// The sources lazy restores read from (ADR 0127), whatever the lazy + /// restore mode. `None` only in tests. + pub disk_sources: Option>, /// This host's stable id, stamped on snapshot trace hints and passed /// to the UFFD handler. `None` generates one at run time. pub host_id: Option, @@ -153,6 +158,7 @@ impl HostAgent { chunk_cache: None, fc_for_reattach: None, session_disk: None, + disk_sources: None, host_id: None, base_shm_protected: base_shm_gc::ProtectedPaths::new(), } @@ -191,6 +197,11 @@ impl HostAgent { self } + pub fn with_disk_sources(mut self, sources: Arc) -> Self { + self.disk_sources = Some(sources); + self + } + /// Must match the id on `FirecrackerConfig.host_id`: both key the /// trace file `traces//.json`. pub fn with_host_id(mut self, id: engram_core::HostId) -> Self { @@ -294,9 +305,11 @@ impl HostAgent { } if let Some(cs) = self.chunk_store.clone() { if let Some(disk) = self.session_disk.clone() { - p = p.with_session_volumes(Arc::new( - engram_host_core::SessionVolumes::new(disk, cs.clone()), - )); + let mut volumes = engram_host_core::SessionVolumes::new(disk, cs.clone()); + if let Some(sources) = self.disk_sources.clone() { + volumes = volumes.with_sources(sources); + } + p = p.with_session_volumes(Arc::new(volumes)); } p = p.with_chunk_store(cs); // ADR 0080 §C: the MaterializeImage scratch root — @@ -331,6 +344,9 @@ impl HostAgent { // from ever engaging the chain or the periodic driver // below — VZ has no guest-memory snapshot, so a // checkpoint there is just a wasteful VM pause. + p = p + .with_hydration_interval(self.cfg.hydration_interval) + .with_lazy_restore(self.cfg.lazy_restore, self.cfg.eager_budget); let checkpoints_supported = self.sandbox.supports_diff_checkpoints(); if self.chunk_store.is_some() && checkpoints_supported { p = p.with_checkpoint_dir(self.cfg.work_dir.join("checkpoints")); @@ -351,7 +367,11 @@ impl HostAgent { // (2026-07-13 incident: a survivor's evict capture ran // 40+ minutes). arc.rehydrate_chain_heads().await; + if let Some(sources) = &self.disk_sources { + disk_source::recover_all(sources.as_ref()).await; + } arc.reconcile_volumes().await; + arc.rearm_hydrations().await; arc }; // ADR 0084 P1b: the capture-job executor + durable-record @@ -1443,6 +1463,7 @@ impl HostAgent { // ADR 0091: control-plane-dead guests; the coord // flips their sessions Active → Unreachable. unreachable_guests: pooled_for_heartbeat.unreachable_guests(), + hydrating: pooled_for_heartbeat.hydrating(), }; match coord_for_heartbeat.heartbeat(host_id, &req).await { Ok(resp) => { diff --git a/crates/engram-host-agent/src/live_attach.rs b/crates/engram-host-agent/src/live_attach.rs index 712324708..c72ec7986 100644 --- a/crates/engram-host-agent/src/live_attach.rs +++ b/crates/engram-host-agent/src/live_attach.rs @@ -751,6 +751,12 @@ mod tests { std::fs::create_dir_all(tmp.path().join("chunk-cache")).unwrap(); std::fs::create_dir_all(tmp.path().join("bindings")).unwrap(); std::fs::write(tmp.path().join("bindings/keep.json"), b"{}").unwrap(); + // A disk source outlives its sandbox's VM (ADR 0127): its record + // and log stay, even for a dead sandbox id. + let source = tmp.path().join("disk-sources"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join(format!("{dead_dir_id}-root.json")), b"{}").unwrap(); + std::fs::write(source.join(format!("{dead_dir_id}-root.log")), b"").unwrap(); // ADR 0112: leaked swap backings. Design is unlink-after-attach, // so ANY .img at startup is a leak — a dead sandbox's (aborted // create) and a LIVE one's (failed unlink; FC holds the fd, so @@ -795,6 +801,11 @@ mod tests { && tmp.path().join("bindings/keep.json").exists(), "non-uuid residents untouched" ); + assert!( + source.join(format!("{dead_dir_id}-root.json")).exists() + && source.join(format!("{dead_dir_id}-root.log")).exists(), + "disk source records and logs untouched" + ); assert!( kept_dir.exists() && tmp.path().join(format!("{kept_id}.vsock")).exists(), "a manifest-bearing dir and its sockets are the reattach \ diff --git a/crates/engram-host-agent/src/main.rs b/crates/engram-host-agent/src/main.rs index a5ec861c9..b0a7fad0c 100644 --- a/crates/engram-host-agent/src/main.rs +++ b/crates/engram-host-agent/src/main.rs @@ -62,6 +62,29 @@ struct Cli { #[arg(long, env = "ENGRAM_THIN_POOL")] thin_pool: Option, + /// When a restore boots before its disk is complete: `off`, `force` + /// or `auto`. Any mode but `off` runs a disk source process per + /// restoring disk, which needs the ublk driver on a host with a thin + /// pool; without the driver every restore is eager. Sources that an + /// earlier run started are served to their end in every mode. + #[arg(long, env = "ENGRAM_LAZY_RESTORE", default_value = "off")] + lazy_restore: engram_host_agent::config::LazyRestore, + + /// In `auto` mode, the longest an eager restore of a disk may write, + /// in seconds, before the restore boots over a source instead. + #[arg(long, env = "ENGRAM_LAZY_RESTORE_BUDGET_SECS", default_value_t = engram_host_agent::config::EagerBudget::default().seconds, value_parser = non_negative)] + lazy_restore_budget_secs: f64, + + /// In `auto` mode, the rate in MB/s (10^6 bytes) at which an eager + /// restore writes chunks it fetches from the chunk store. + #[arg(long, env = "ENGRAM_LAZY_RESTORE_REMOTE_MBPS", default_value_t = engram_host_agent::config::EagerBudget::default().rates.remote / 1e6, value_parser = positive)] + lazy_restore_remote_mbps: f64, + + /// In `auto` mode, the rate in MB/s (10^6 bytes) at which an eager + /// restore writes chunks the host's chunk cache holds. + #[arg(long, env = "ENGRAM_LAZY_RESTORE_LOCAL_MBPS", default_value_t = engram_host_agent::config::EagerBudget::default().rates.local / 1e6, value_parser = positive)] + lazy_restore_local_mbps: f64, + /// Coordinator endpoint to dial via WebSocket. #[arg(long, env = "ENGRAM_COORDINATOR_ENDPOINT")] coordinator: Option, @@ -321,6 +344,14 @@ async fn main() -> Result<(), HostAgentError> { BackendChoice::Firecracker => "firecracker".to_string(), BackendChoice::Vz => "vz".to_string(), }, + lazy_restore: cli.lazy_restore, + eager_budget: engram_host_agent::config::EagerBudget { + seconds: cli.lazy_restore_budget_secs, + rates: engram_host_core::eager_estimate::Rates { + local: cli.lazy_restore_local_mbps * 1e6, + remote: cli.lazy_restore_remote_mbps * 1e6, + }, + }, ..HostAgentConfig::default() }; @@ -686,6 +717,7 @@ async fn main() -> Result<(), HostAgentError> { .unwrap_or_else(|| "http://127.0.0.1:8080".to_string()); let observe_token = cfg.coordinator_token.clone(); + let disk_sources = disk_sources(&cli, &chunk_store).await?; let mut agent = HostAgent::new(cfg, sandbox, cloud) .with_chunk_store(chunk_store) .with_chunk_cache(chunk_cache) @@ -696,6 +728,7 @@ async fn main() -> Result<(), HostAgentError> { if let Some(fc) = fc_for_reattach { agent = agent.with_fc_reattach(fc); } + agent = agent.with_disk_sources(disk_sources); // The egress proxy is mandatory (issue #240) and, since ADR 0121, // lives in `engram-egress-proxyd` — a node-local daemon that // outlives this pod so in-flight guest streams survive a roll. @@ -755,6 +788,66 @@ async fn main() -> Result<(), HostAgentError> { agent.run().await } +/// A number above zero, for a rate. +fn positive(text: &str) -> Result { + match text.parse::() { + Ok(n) if n > 0.0 && n.is_finite() => Ok(n), + _ => Err(format!("`{text}` is not a number above zero")), + } +} + +/// A number of zero or more, for a budget. +fn non_negative(text: &str) -> Result { + match text.parse::() { + Ok(n) if n >= 0.0 && n.is_finite() => Ok(n), + _ => Err(format!("`{text}` is not a number of zero or more")), + } +} + +/// The sources lazy restores read from (ADR 0127): a process per source +/// over a thin pool, a staged file per source over sparse files. Built +/// whatever the lazy restore mode: a host that ran with it on keeps +/// sources that drives still read, and only this host can recover, cut +/// over and stop them. The mode gates only new lazy restores. +async fn disk_sources( + cli: &Cli, + chunk_store: &engram_chunk_store::ChunkStore, +) -> Result, HostAgentError> { + use engram_host_agent::config::LazyRestore; + use engram_host_agent::disk_source::{ProcessSources, ProcessSourcesConfig}; + if cli.thin_pool.is_none() { + let files = engram_host_core::FileSources::new( + cli.work_dir.join("file-sources"), + chunk_store.clone(), + ) + .map_err(HostAgentError::Io)?; + return Ok(Arc::new(files)); + } + let bin = std::env::var_os("ENGRAM_DISK_SOURCE_BIN") + .map_or_else(|| PathBuf::from("engram-disk-source"), PathBuf::from); + let mut cfg = ProcessSourcesConfig::new(bin, cli.work_dir.clone()); + // The VMs' node cgroup (ADR 0044 K2): a source outlives the pod too. + cfg.cgroup_parent = std::env::var("ENGRAM_FC_VM_CGROUP_PARENT") + .ok() + .filter(|p| !p.is_empty()) + .map(PathBuf::from); + cfg.cache_root = Some(cli.work_dir.join("chunk-cache")); + let sources = ProcessSources::new(cfg); + if let Err(error) = sources.check_driver() { + if cli.lazy_restore != LazyRestore::Off { + tracing::error!( + %error, + mode = ?cli.lazy_restore, + "lazy restore is on, but this host cannot start a disk source; every restore is eager", + ); + } + } + // Before anything starts a source. Without the driver it reports the + // records whose sources cannot serve. + sources.sweep_unrecorded().await; + Ok(Arc::new(sources)) +} + /// Load the egress CA material for the daemon (ADR 0121): the daemon /// itself only ever reads PEMs from its spawn env, so every CA source /// (env / local-disk / GCP or AWS Secrets Manager) resolves HERE, in the one diff --git a/crates/engram-host-agent/src/metrics.rs b/crates/engram-host-agent/src/metrics.rs index e4355d890..7e5d888ef 100644 --- a/crates/engram-host-agent/src/metrics.rs +++ b/crates/engram-host-agent/src/metrics.rs @@ -29,6 +29,12 @@ pub fn init(addr: SocketAddr) { 1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1200.0, 2400.0, 3600.0, 7200.0, ]; + // Restore estimates and provision times sit around the eager budget + // (seconds) and reach minutes for a full disk. + let restore_buckets = &[ + 0.1, 0.25, 0.5, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0, 300.0, + ]; + let mut builder = PrometheusBuilder::new() .with_http_listener(addr) .set_buckets_for_metric( @@ -41,6 +47,7 @@ pub fn init(addr: SocketAddr) { SNAPSHOT_CREATE_SECONDS, SNAPSHOT_FINISH_SECONDS, EVICTION_FINALIZE_STAGE_SECONDS, + DISK_HYDRATION_SECONDS, ] { builder = builder .set_buckets_for_metric( @@ -58,6 +65,17 @@ pub fn init(addr: SocketAddr) { let probe_buckets = &[ 0.0002, 0.0005, 0.001, 0.0025, 0.005, 0.010, 0.025, 0.050, 0.100, 0.250, 1.0, 5.0, ]; + for name in [ + DISK_RESTORE_ESTIMATE_SECONDS, + DISK_RESTORE_PROVISION_SECONDS, + ] { + builder = builder + .set_buckets_for_metric( + metrics_exporter_prometheus::Matcher::Full(name.to_string()), + restore_buckets, + ) + .expect("install restore histogram buckets"); + } builder = builder .set_buckets_for_metric( metrics_exporter_prometheus::Matcher::Full(TOKIO_SCHED_DELAY_SECONDS.to_string()), @@ -465,6 +483,29 @@ pub const SUBSTRATE_POPULATE_REQUESTS_TOTAL: &str = "engram_substrate_populate_r pub const BUNDLE_STARTUP_PUBLISH_FAILURES_TOTAL: &str = "engram_bundle_startup_publish_failures_total"; +/// Gauge (ADR 0127). Sandboxes with a hydration task: their disks still +/// copy from a source, or their cutover has steps left. +pub const DISK_HYDRATING_SANDBOXES: &str = "engram_disk_hydrating_sandboxes"; +/// Histogram (ADR 0127). Seconds from the start of a sandbox's hydration +/// task to the cutover of its last disk. A task that a host-agent restart +/// starts again counts from the restart. +pub const DISK_HYDRATION_SECONDS: &str = "engram_disk_hydration_seconds"; +/// Counter (ADR 0127). Cutover attempts of a hydrated disk, by `outcome`: +/// `done`, `checkpoint_in_flight` (waits for the publish), `copying` +/// (regions left), `error` (tried again on the next tick). +pub const DISK_CUTOVERS_TOTAL: &str = "engram_disk_cutovers_total"; + +/// Counter (ADR 0127 Decision 1). Disk restores by `path` (`eager`, +/// `lazy`) and `role` (`root`, `swap`), while the lazy-restore mode is +/// `force` or `auto`. +pub const DISK_RESTORE_TOTAL: &str = "engram_disk_restore_total"; +/// Histogram (ADR 0127 Decision 1). The estimate, in seconds, of an eager +/// restore's writes for one disk, by `role`. +pub const DISK_RESTORE_ESTIMATE_SECONDS: &str = "engram_disk_restore_estimate_seconds"; +/// Histogram. Seconds a restore spends provisioning its disks, by `path`: +/// `lazy` when any disk boots over a source, else `eager`. +pub const DISK_RESTORE_PROVISION_SECONDS: &str = "engram_disk_restore_provision_seconds"; + /// Histogram (ADR 0116 C1). Observed-minus-expected delay of a 100 ms /// heartbeat tick on the host-agent's main tokio runtime — the /// executor-starvation probe. Steady-state ≈ 0; sustained excursions diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index 85c107e06..2ce26f585 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -349,8 +349,9 @@ pub struct PooledBackend { /// checkpoint completed; every later capture is a diff against it. checkpoint_chains: Arc>, /// Serializes captures per sandbox: a periodic checkpoint never - /// interleaves with an eviction snapshot. - capture_locks: Arc>>>, + /// interleaves with an eviction snapshot, and a disk cutover never + /// runs between a checkpoint's take and its publish. + capture_locks: Arc, /// `sandbox_id → snapshot_id` of an in-flight, durably persisted /// eviction finalize job (issue #529). Makes `snapshot_begin` /// idempotent under an eviction-scanner retry storm, and keeps @@ -377,6 +378,15 @@ pub struct PooledBackend { drained_dests: Arc>, /// Optional hint polling stops as soon as the memory drain terminates. hint_pollers: Arc>, + /// The task of each sandbox whose disks still copy from a source. + hydrations: Arc, + /// The background memory prefetch of a restore, which goes with its + /// sandbox. + memory_prefetches: Arc>, + /// When a restore boots over a source before its disks are written. + lazy_restore: crate::config::LazyRestore, + /// What `auto` allows an eager restore to write. + eager_budget: crate::config::EagerBudget, /// The SIGTERM capture quiesce: raised once, never lowered. /// `capture_phase` refuses under the capture lock before anything is /// consumed, so a capture never walks into the runtime teardown. @@ -520,12 +530,9 @@ fn record_warm_hook_failure_metric(kind: engram_core::types::CaptureFailureKind) .increment(1); } -/// Abort the wrapped task on drop, including hint polling after DrainDone. -/// A leg wrapped by -/// [`spawn_leg_keepalive`] stops resending stale progress the moment -/// the guard goes out of scope — on every path, including an early -/// `?`-return, since `Drop` runs during unwind too. -struct AbortTaskOnDrop(tokio::task::JoinHandle<()>); +/// Aborts the wrapped task when dropped, on every path: an early +/// `?`-return and an unwind drop it too. +pub(crate) struct AbortTaskOnDrop(pub(crate) tokio::task::JoinHandle<()>); impl Drop for AbortTaskOnDrop { fn drop(&mut self) { @@ -533,6 +540,17 @@ impl Drop for AbortTaskOnDrop { } } +/// The capture lock of each sandbox. +pub(crate) type CaptureLocks = DashMap>>; + +/// The sandbox's capture lock, made on first use. +pub(crate) fn capture_lock_in(locks: &CaptureLocks, id: SandboxId) -> Arc> { + locks + .entry(id) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() +} + /// Observe the local handler independently of the coordinator's drain-wait RPC. /// ControlTx replays terminal reports to each subscriber, including late ones. fn monitor_hint_poller(poller: AbortTaskOnDrop, sock: Option) -> AbortTaskOnDrop { @@ -1235,12 +1253,8 @@ impl PooledBackend { selected_mounts: Vec, ) -> Result<(), SandboxError> { let new_id = id; - let post_copy_dest = self.postcopy_dests.contains_key(&metadata.id) - || self - .inner - .snapshot_path_for(metadata.id) - .join(engram_sandbox_firecracker::MIGRATION_PEER_FILE) - .exists(); + let post_copy_dest = + self.is_post_copy_dest(metadata.id, &self.inner.snapshot_path_for(metadata.id)); // Memory is served lazily (UFFD faults / peer) for a substrate // base-create, a UFFD resume, and every staged migration restore; // a File-mode restore needs a contiguous `memory.bin`. One @@ -1270,7 +1284,7 @@ impl PooledBackend { } } else if let Some(cs) = self.chunk_store.clone() { let md = metadata.clone(); - tokio::spawn(tracing::Instrument::instrument( + let prefetch = tokio::spawn(tracing::Instrument::instrument( async move { if let Err(e) = Self::prefetch_memory_chunks_inner(&cs, &md).await { tracing::warn!( @@ -1282,6 +1296,7 @@ impl PooledBackend { }, tracing::info_span!("restore.prefetch_memory_bg"), )); + self.memory_prefetches.insert(id, AbortTaskOnDrop(prefetch)); } let src = self.inner.snapshot_path_for(metadata.id); @@ -1309,15 +1324,27 @@ impl PooledBackend { } } - let disks = match self.volumes.as_ref() { - Some(volumes) => Some( - tracing::Instrument::instrument( + let (disks, lazy_disks) = match self.volumes.as_ref() { + Some(volumes) => { + let provisioned = tracing::Instrument::instrument( self.provision_for_restore(volumes, id, &metadata, &src, fresh, fork_disk), - tracing::info_span!("restore.provision_disks"), + tracing::info_span!( + "restore.provision_disks", + root_path = tracing::field::Empty, + root_estimate_seconds = tracing::field::Empty, + swap_path = tracing::field::Empty, + swap_estimate_seconds = tracing::field::Empty, + provision_seconds = tracing::field::Empty, + ), ) - .await?, - ), - None => None, + .await; + if provisioned.is_err() { + self.memory_prefetches.remove(&id); + } + let (disks, lazy) = provisioned?; + (Some(disks), lazy) + } + None => (None, false), }; let restore = async { // A staged post-copy destination arms its fetch poller as soon @@ -1383,6 +1410,14 @@ impl PooledBackend { let inner = self.inner.clone(); let migration_roles = self.migration_roles.clone(); let hint_pollers = self.hint_pollers.clone(); + let hydration = match (lazy_disks, &self.volumes) { + (true, Some(volumes)) => Some(( + self.hydrations.clone(), + volumes.clone(), + self.capture_locks.clone(), + )), + _ => None, + }; let disks = disks.clone(); tokio::spawn(async move { if fresh { @@ -1401,6 +1436,9 @@ impl PooledBackend { if post_copy_dest { migration_roles.insert(new_id, crate::migration::MigrationRole::PostCopyDest); } + if let Some((hydrations, volumes, capture_locks)) = hydration { + hydrations.arm(new_id, volumes, capture_locks); + } Ok(()) }) .await @@ -1408,6 +1446,7 @@ impl PooledBackend { } .await; if restore.is_err() { + self.memory_prefetches.remove(&id); if let Some(volumes) = self.volumes.as_ref() { if let Err(error) = volumes.release(id).await { tracing::warn!(sandbox_id = %id, %error, "release after a failed restore failed"); @@ -1417,7 +1456,18 @@ impl PooledBackend { restore } - /// Provision a restored sandbox's disks from the snapshot's manifests. + /// The restore of `snapshot` from its folder `dir` is the destination + /// of a live move. + fn is_post_copy_dest(&self, snapshot: engram_core::SnapshotId, dir: &Path) -> bool { + self.postcopy_dests.contains_key(&snapshot) + || dir + .join(engram_sandbox_firecracker::MIGRATION_PEER_FILE) + .exists() + } + + /// Provision a restored sandbox's disks from the snapshot's manifests, + /// each eagerly or over a source as [`Self::restore_paths`] picks. + /// Returns the disks and whether any boots over a source. async fn provision_for_restore( &self, volumes: &SessionVolumes, @@ -1426,7 +1476,7 @@ impl PooledBackend { src: &Path, fresh: bool, fork_disk: bool, - ) -> Result { + ) -> Result<(SessionDisks, bool), SandboxError> { let manifest = metadata.disk_manifest.ok_or_else(|| { SandboxError::MemoryImageUnusable("snapshot has no disk manifest".into()) })?; @@ -1462,14 +1512,158 @@ impl PooledBackend { size_bytes: mib << 20, }), }; + let mut disks = vec![(DiskRole::Root, root)]; + disks.extend(swap.map(|source| (DiskRole::Swap, source))); + let move_dest = self.is_post_copy_dest(metadata.id, src); + let mut chosen = self + .restore_paths(volumes, id, &disks, move_dest) + .await + .into_iter(); + let root = chosen.next().expect("a source for each disk"); + let swap = chosen.next(); + let lazy = [Some(root), swap] + .into_iter() + .flatten() + .any(|s| matches!(s, Source::LazyFork { .. } | Source::LazyContinue { .. })); + let started = crate::time_source::metrics_now(); let provisioned = volumes .provision(id, root, swap) .await .map_err(|e| SandboxError::Vm(format!("provision session disks: {e}").into()))?; - Ok(SessionDisks { + let seconds = started.elapsed().as_secs_f64(); + tracing::Span::current().record("provision_seconds", seconds); + metrics::histogram!( + crate::metrics::DISK_RESTORE_PROVISION_SECONDS, + "path" => if lazy { "lazy" } else { "eager" }, + ) + .record(seconds); + let disks = SessionDisks { root: provisioned.root, swap: provisioned.swap, - }) + }; + Ok((disks, lazy)) + } + + /// The source each disk restores from: its eager source, or the lazy + /// counterpart (ADR 0127 Decision 1). `off` is always eager. `force` is + /// lazy for each disk whose eager provision would write. `auto` gives + /// the disks one budget ([`lazy_disks`]); a disk whose estimate cannot + /// be read is eager. A host that cannot start a source, and a disk of zeros, + /// is eager. `force` and `auto` record each estimate. + /// + /// A live move's destination (`move_dest`) is eager in every mode: the + /// move writes the source's later blocks into the live volumes before + /// the guest boots, and a drive that still copies cannot open as a + /// volume. Every other restore writes its disks only through the guest, + /// so it can be lazy. + /// + /// [`lazy_disks`]: engram_host_core::eager_estimate::lazy_disks + async fn restore_paths( + &self, + volumes: &SessionVolumes, + id: SandboxId, + disks: &[(DiskRole, Source)], + move_dest: bool, + ) -> Vec { + use crate::config::LazyRestore; + let considered = |source: &Source| { + self.lazy_restore != LazyRestore::Off + && !move_dest + && volumes.can_start_sources() + && !matches!(source, Source::Empty { .. }) + }; + let mut estimates = Vec::with_capacity(disks.len()); + for (role, source) in disks { + if !considered(source) { + estimates.push(None); + continue; + } + estimates.push(match self.eager_estimate(volumes, id, *role, *source).await { + Ok(estimate) => Some(estimate), + Err(error) => { + tracing::warn!(sandbox_id = %id, %role, %error, "eager restore estimate failed"); + None + } + }); + } + let lazy: Vec = match self.lazy_restore { + LazyRestore::Off => vec![false; disks.len()], + LazyRestore::Force => disks + .iter() + .zip(&estimates) + .map(|((_, source), e)| considered(source) && e.is_none_or(|e| e.writes())) + .collect(), + LazyRestore::Auto => { + let seconds: Vec> = + estimates.iter().map(|e| e.map(|e| e.seconds)).collect(); + engram_host_core::eager_estimate::lazy_disks(&seconds, self.eager_budget.seconds) + } + }; + let span = tracing::Span::current(); + disks + .iter() + .zip(estimates) + .zip(lazy) + .map(|((&(role, source), estimate), lazy)| { + if !considered(&source) { + return source; + } + let path = if lazy { "lazy" } else { "eager" }; + let (path_field, estimate_field) = match role { + DiskRole::Root => ("root_path", "root_estimate_seconds"), + DiskRole::Swap => ("swap_path", "swap_estimate_seconds"), + }; + span.record(path_field, path); + if let Some(e) = estimate { + span.record(estimate_field, e.seconds); + metrics::histogram!(crate::metrics::DISK_RESTORE_ESTIMATE_SECONDS, "role" => role.to_string()) + .record(e.seconds); + tracing::info!( + sandbox_id = %id, + %role, + path, + estimate_seconds = e.seconds, + local_bytes = e.local_bytes, + remote_bytes = e.remote_bytes, + "disk restore path", + ); + } + metrics::counter!(crate::metrics::DISK_RESTORE_TOTAL, "path" => path, "role" => role.to_string()) + .increment(1); + if lazy { + source.into_lazy() + } else { + source + } + }) + .collect() + } + + /// The time an eager provision of `source` writes before the guest + /// boots. A chunk in the chunk cache counts as local. + async fn eager_estimate( + &self, + volumes: &SessionVolumes, + id: SandboxId, + role: DiskRole, + source: Source, + ) -> std::io::Result { + use engram_host_core::eager_estimate::{estimate, Estimate}; + let Some(plan) = volumes.eager_plan(id, role, source).await? else { + return Ok(Estimate { + local_bytes: 0, + remote_bytes: 0, + seconds: 0.0, + }); + }; + let cache = self.chunk_cache.as_ref(); + estimate( + &plan.wanted, + plan.base.as_ref(), + |hash| cache.is_some_and(|c| c.contains_on_disk(hash)), + self.eager_budget.rates, + ) + .map_err(std::io::Error::other) } pub fn new(inner: Arc) -> Self { @@ -1507,6 +1701,10 @@ impl PooledBackend { migration_roles: Arc::new(DashMap::new()), drained_dests: Arc::new(DashMap::new()), hint_pollers: Arc::new(DashMap::new()), + hydrations: crate::hydration::Hydrations::new(crate::hydration::DEFAULT_INTERVAL), + memory_prefetches: Arc::new(DashMap::new()), + lazy_restore: crate::config::LazyRestore::Off, + eager_budget: crate::config::EagerBudget::default(), captures_quiesced: Arc::new(std::sync::atomic::AtomicBool::new(false)), clock: Arc::new(engram_core::traits::SystemClock::new()), host_fs: Arc::new(engram_host_core::TokioFs), @@ -1523,6 +1721,86 @@ impl PooledBackend { self.volumes.as_ref() } + /// When a restore boots over a source, and what `auto` allows an + /// eager restore to write. + pub fn with_lazy_restore( + mut self, + mode: crate::config::LazyRestore, + budget: crate::config::EagerBudget, + ) -> Self { + self.lazy_restore = mode; + self.eager_budget = budget; + self + } + + /// How often a sandbox's hydration task looks at its disks. + pub fn with_hydration_interval(mut self, interval: std::time::Duration) -> Self { + self.hydrations = crate::hydration::Hydrations::new(interval); + self + } + + /// Start the task that cuts the sandbox's cloned disks over to their + /// destinations once they are copied, unless it has one. A restore + /// that boots over a source arms it once the VM runs. + pub fn arm_hydration(&self, id: SandboxId) { + if let Some(volumes) = &self.volumes { + self.hydrations + .arm(id, volumes.clone(), self.capture_locks.clone()); + } + } + + /// Arm the hydration of every live sandbox with a disk that still + /// copies or whose cutover has steps left. Run once at startup, after + /// the sources are recovered and the volumes reconciled. + pub async fn rearm_hydrations(&self) { + if self.volumes.is_none() { + return; + } + let live = match self.inner.list().await { + Ok(ids) => ids, + Err(error) => { + tracing::warn!(%error, "hydration re-arm skipped: cannot list sandboxes"); + return; + } + }; + for id in live { + if self.hydrates(id).await { + self.arm_hydration(id); + } + } + } + + /// The sandbox has a disk left to cut over. A disk whose state cannot + /// be read counts: the task tries again and logs why. + async fn hydrates(&self, id: SandboxId) -> bool { + let Some(volumes) = &self.volumes else { + return false; + }; + let roles = match volumes.roles(id).await { + Ok(roles) => roles, + Err(error) => { + tracing::warn!(sandbox_id = %id, %error, "hydration re-arm: cannot list the disks"); + return true; + } + }; + for role in roles { + match volumes.hydration(id, role).await { + Ok(None) => {} + Ok(Some(_)) => return true, + Err(error) => { + tracing::warn!(sandbox_id = %id, %role, %error, "hydration re-arm: cannot read the disk"); + return true; + } + } + } + false + } + + /// The sandboxes with a hydration task, for the heartbeat. + pub fn hydrating(&self) -> Vec { + self.hydrations.sandboxes() + } + fn volumes(&self) -> Result<&Arc, SandboxError> { self.volumes .as_ref() @@ -1673,10 +1951,7 @@ impl PooledBackend { /// The per-sandbox capture serialization lock (periodic /// checkpoint vs eviction/drain/SIGTERM snapshot). fn capture_lock(&self, id: SandboxId) -> Arc> { - self.capture_locks - .entry(id) - .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) - .clone() + capture_lock_in(&self.capture_locks, id) } /// ADR 0038 B1: is a capture (eviction / evac / drain / a prior @@ -5743,6 +6018,8 @@ impl SandboxBackend for PooledBackend { // A held source forgets its swap policy and metadata with the VM. self.held_snapshots.remove(&id); self.hint_pollers.remove(&id); + self.hydrations.disarm(id); + self.memory_prefetches.remove(&id); self.teardowns .run_or_join(id, || { let session_bindings = self.session_bindings.clone(); @@ -8981,6 +9258,335 @@ mod tests { ); } + /// Session volumes over a file-backed disk with file sources, and a + /// published manifest of a disk whose block 0 holds `root_block`. + async fn volumes_with_sources( + tmp: &std::path::Path, + ) -> ( + Arc, + engram_core::types::manifest::ManifestRef, + ) { + volumes_with_these_sources(tmp, true).await + } + + /// [`volumes_with_sources`] whose sources start only when `can_start`. + async fn volumes_with_these_sources( + tmp: &std::path::Path, + can_start: bool, + ) -> ( + Arc, + engram_core::types::manifest::ManifestRef, + ) { + use engram_chunk_store::ChunkStore; + use engram_storage_local::LocalBlobStorage; + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(tmp.join("blob")))); + let disk: Arc = Arc::new( + engram_host_core::FileDisk::new(tmp.join("volumes"), MIGRATION_DISK_BLOCK_BYTES) + .unwrap(), + ); + let files = engram_host_core::FileSources::new(tmp.join("sources"), store.clone()).unwrap(); + let sources: Arc = if can_start { + Arc::new(files) + } else { + Arc::new(NoStart(files)) + }; + let volumes = Arc::new(SessionVolumes::new(disk, store).with_sources(sources)); + let published = SandboxId::new(); + volumes + .provision( + published, + Source::Empty { + size_bytes: 2 * MIGRATION_DISK_BLOCK_BYTES, + }, + None, + ) + .await + .unwrap(); + let live = volumes.live(published, DiskRole::Root).await.unwrap(); + live.write_chunk(ChunkIndex(0), root_block()).await.unwrap(); + drop(live); + let manifest = volumes.checkpoint(published, DiskRole::Root).await.unwrap(); + volumes.purge(published).await.unwrap(); + (volumes, manifest) + } + + /// [`volumes_with_sources`] with a root `id` restored lazily from the + /// manifest. `FileDisk` copies the whole source at the restore, so the + /// root is hydrated. + async fn volumes_with_a_lazy_root(tmp: &std::path::Path, id: SandboxId) -> Arc { + let (volumes, manifest) = volumes_with_sources(tmp).await; + volumes + .provision( + id, + Source::LazyFork { + manifest, + base: None, + }, + None, + ) + .await + .unwrap(); + volumes + } + + /// `off` restores eagerly, `force` over a source when the eager path + /// would write, and `auto` over a source only when the estimate passes + /// the budget. + #[tokio::test] + async fn the_lazy_restore_mode_picks_the_path_of_each_disk() { + use crate::config::{EagerBudget, LazyRestore}; + let tmp = tempfile::tempdir().unwrap(); + let (volumes, manifest) = volumes_with_sources(tmp.path()).await; + let (src, metadata) = root_only_snapshot(tmp.path(), manifest); + let over = |seconds| EagerBudget { + seconds, + ..EagerBudget::default() + }; + for (mode, budget, lazy) in [ + (LazyRestore::Off, over(0.0), false), + (LazyRestore::Force, over(2.0), true), + (LazyRestore::Auto, over(2.0), false), + (LazyRestore::Auto, over(0.0), true), + ] { + let pooled = hydrating_backend(&volumes).with_lazy_restore(mode, budget); + let id = SandboxId::new(); + + let (disks, got) = pooled + .provision_for_restore(&volumes, id, &metadata, &src, false, false) + .await + .unwrap(); + + assert_eq!(got, lazy, "{mode:?} with a budget of {}", budget.seconds); + let drive = engram_host_core::session_volumes::live_name(id, DiskRole::Root); + assert_eq!(volumes.disk().clone_state(&drive).await.is_ok(), lazy); + let root = std::fs::read(&disks.root).unwrap(); + assert_eq!(&root[..root_block().len()], &root_block()[..]); + volumes.purge(id).await.unwrap(); + } + + // A checkpoint this host kept at exactly the manifest is adopted, + // which writes nothing, so `force` restores it eagerly. + let released = SandboxId::new(); + let continued = Source::Continue { + manifest, + base: None, + }; + volumes.provision(released, continued, None).await.unwrap(); + volumes.release(released).await.unwrap(); + let pooled = hydrating_backend(&volumes).with_lazy_restore(LazyRestore::Force, over(2.0)); + let id = SandboxId::new(); + let (_, lazy) = pooled + .provision_for_restore(&volumes, id, &metadata, &src, false, false) + .await + .unwrap(); + assert!(!lazy, "an adopted checkpoint restores eagerly under force"); + } + + /// A live move's destination restores eagerly under `force`: the move + /// writes into its live volumes before the guest boots. + #[tokio::test] + async fn a_move_destination_restores_eagerly_under_force() { + use crate::config::{EagerBudget, LazyRestore}; + let tmp = tempfile::tempdir().unwrap(); + let (volumes, manifest) = volumes_with_sources(tmp.path()).await; + let (src, metadata) = root_only_snapshot(tmp.path(), manifest); + std::fs::write( + src.join(engram_sandbox_firecracker::MIGRATION_PEER_FILE), + b"{}", + ) + .unwrap(); + let pooled = hydrating_backend(&volumes).with_lazy_restore( + LazyRestore::Force, + EagerBudget { + seconds: 0.0, + ..EagerBudget::default() + }, + ); + let id = SandboxId::new(); + + let (_, lazy) = pooled + .provision_for_restore(&volumes, id, &metadata, &src, false, false) + .await + .unwrap(); + + assert!(!lazy); + assert!(volumes.live(id, DiskRole::Root).await.is_ok()); + } + + /// A snapshot folder and metadata for a restore of a root disk from + /// `manifest`, with no swap and no memory. + fn root_only_snapshot( + tmp: &std::path::Path, + manifest: engram_core::types::manifest::ManifestRef, + ) -> (std::path::PathBuf, SnapshotMetadata) { + let src = tmp.join("snapshot"); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("manifest.json"), br#"{"spec":{}}"#).unwrap(); + let metadata = SnapshotMetadata { + id: engram_core::SnapshotId::new(), + size_bytes: 4096, + created_at: chrono::Utc::now(), + image_version: "image:1".into(), + swap_manifest: None, + disk_manifest: Some(manifest), + memory_manifest: None, + base_memory_manifest: None, + base_disk_manifest: None, + migration_source: None, + source_sandbox_id: None, + state_blob_key: None, + sidecar_blob_key: None, + rootfs_blob_key: None, + working_set_blob_key: None, + aux_bundles: vec![], + paused_at: None, + peer_hints: Vec::new(), + }; + (src, metadata) + } + + /// File sources on a host that cannot start one, as a host without + /// the ublk driver. + struct NoStart(engram_host_core::FileSources); + + #[async_trait::async_trait] + impl engram_host_core::DiskSources for NoStart { + fn can_start(&self) -> bool { + false + } + async fn start( + &self, + _: SandboxId, + _: DiskRole, + _: engram_core::types::manifest::ManifestRef, + _: Option<( + &engram_host_core::Volume, + engram_core::types::manifest::ManifestRef, + )>, + ) -> std::io::Result { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) + } + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result<()> { + self.0.ensure_alive(sandbox, role).await + } + async fn served( + &self, + sandbox: SandboxId, + role: DiskRole, + ) -> std::io::Result { + self.0.served(sandbox, role).await + } + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result<()> { + self.0.stop(sandbox, role).await + } + async fn list(&self) -> std::io::Result> { + self.0.list().await + } + } + + /// A host that cannot start a source restores eagerly in every mode, + /// where a lazy restore would fail. + #[tokio::test] + async fn a_host_that_cannot_start_a_source_restores_eagerly() { + use crate::config::{EagerBudget, LazyRestore}; + let tmp = tempfile::tempdir().unwrap(); + let (volumes, manifest) = volumes_with_these_sources(tmp.path(), false).await; + let (src, metadata) = root_only_snapshot(tmp.path(), manifest); + let budget = EagerBudget { + seconds: 0.0, + ..EagerBudget::default() + }; + for mode in [LazyRestore::Force, LazyRestore::Auto] { + let pooled = hydrating_backend(&volumes).with_lazy_restore(mode, budget); + let id = SandboxId::new(); + let (_, lazy) = pooled + .provision_for_restore(&volumes, id, &metadata, &src, false, false) + .await + .unwrap(); + assert!(!lazy, "{mode:?}"); + volumes.purge(id).await.unwrap(); + } + } + + fn hydrating_backend(volumes: &Arc) -> PooledBackend { + PooledBackend::new(Arc::new(ResumeSpy { + capture_root: None, + resumes: Default::default(), + })) + .with_session_volumes(volumes.clone()) + .with_hydration_interval(std::time::Duration::from_millis(1)) + } + + /// A drive cut over before a crash, with its bitmap and source left, + /// needs the task on re-arm: it finishes the cutover and ends. A + /// sandbox without a cloned disk needs none. + #[tokio::test] + async fn a_rearmed_hydration_finishes_a_cutover_cut_short_and_ends() { + let tmp = tempfile::tempdir().unwrap(); + let id = SandboxId::new(); + let volumes = volumes_with_a_lazy_root(tmp.path(), id).await; + volumes + .disk() + .cut_over(&engram_host_core::session_volumes::live_name( + id, + DiskRole::Root, + )) + .await + .unwrap(); + let plain = SandboxId::new(); + volumes + .provision( + plain, + Source::Empty { + size_bytes: MIGRATION_DISK_BLOCK_BYTES, + }, + None, + ) + .await + .unwrap(); + let pooled = hydrating_backend(&volumes); + assert!(pooled.hydrates(id).await); + assert!(!pooled.hydrates(plain).await); + + pooled.arm_hydration(id); + pooled.arm_hydration(id); + assert_eq!(pooled.hydrating(), vec![id]); + for _ in 0..500 { + if pooled.hydrating().is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + + assert!(pooled.hydrating().is_empty()); + assert_eq!(volumes.hydration(id, DiskRole::Root).await.unwrap(), None); + assert!(!pooled.hydrates(id).await); + } + + /// A cutover waits while a checkpoint pair exists, so the task keeps + /// running; destroy stops it with the sandbox. + #[tokio::test] + async fn destroy_stops_the_hydration_task() { + let tmp = tempfile::tempdir().unwrap(); + let id = SandboxId::new(); + let volumes = volumes_with_a_lazy_root(tmp.path(), id).await; + drop(volumes.take(id, DiskRole::Root).await.unwrap()); + let pooled = hydrating_backend(&volumes); + + pooled.arm_hydration(id); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert_eq!(pooled.hydrating(), vec![id]); + pooled.destroy(id).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + + assert!(pooled.hydrating().is_empty()); + assert!(volumes.roles(id).await.unwrap().is_empty()); + assert!( + !pooled.capture_locks.contains_key(&id), + "a stopped task takes no capture lock" + ); + } + /// Aborting an export discards its seal, removes the export dir and /// resumes the guest in place. #[tokio::test] diff --git a/crates/engram-host-agent/tests/disk_source_ublk.rs b/crates/engram-host-agent/tests/disk_source_ublk.rs new file mode 100644 index 000000000..a2c0172ea --- /dev/null +++ b/crates/engram-host-agent/tests/disk_source_ublk.rs @@ -0,0 +1,227 @@ +//! `ProcessSources` against the kernel's ublk driver, as root: a source +//! killed while a read waits on its device is recovered by `ensure_alive`, +//! the read completes with the manifest's bytes, and a stop leaves no +//! process and no device; the startup sweep deletes a device that no +//! record names and no process serves. +//! +//! `ENGRAM_TEST_UBLK=1` and `ENGRAM_DISK_SOURCE_BIN` (a built +//! `engram-disk-source`) run it; unset, it skips. The driver must be loaded. +#![cfg(target_os = "linux")] + +use std::fs::OpenOptions; +use std::os::unix::fs::{FileExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestKind, ManifestRef}; +use engram_core::types::ids::SandboxId; +use engram_core::types::snapshot::DiskRole; +use engram_host_agent::disk_source::{KernelProbe, Probe, ProcessSources, ProcessSourcesConfig}; +use engram_host_core::DiskSources; +use engram_storage_local::LocalBlobStorage; +use nix::sys::signal::{kill, Signal}; +use nix::unistd::Pid; + +const CHUNK: u64 = 1 << 20; +const CHUNKS: u64 = 4; +const READ: usize = 64 * 1024; + +fn chunk_bytes(seed: u8) -> Vec { + (0..CHUNK as usize) + .map(|i| ((i as u32).wrapping_mul(2_654_435_761) >> (seed % 24)) as u8 ^ seed) + .collect() +} + +/// Publish a disk of `CHUNKS` distinct chunks to a local store. +async fn publish(store_root: &Path) -> (ManifestRef, Vec) { + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(store_root.join("blobs")))); + let mut expected = Vec::new(); + let mut chunks = Vec::new(); + for index in 0..CHUNKS { + let bytes = chunk_bytes(index as u8 + 1); + let hash = store.put_chunk(&bytes).await.unwrap(); + chunks.push(ChunkRef { + offset: index * CHUNK, + hash, + }); + expected.extend_from_slice(&bytes); + } + let mut manifest = Manifest::empty(ManifestKind::Disk, CHUNKS * CHUNK); + manifest.chunk_size = ChunkSize::bytes(CHUNK); + manifest.chunks = chunks; + let manifest_ref = ManifestRef::new(); + store.put_manifest(manifest_ref, &manifest).await.unwrap(); + (manifest_ref, expected) +} + +/// One direct read, so the page cache does not serve it. +fn read_direct(path: &Path, offset: u64) -> std::io::Result> { + let device = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECT) + .open(path)?; + let mut buf = vec![0u8; READ + 4096]; + let start = buf.as_ptr().align_offset(4096); + device.read_exact_at(&mut buf[start..start + READ], offset)?; + Ok(buf[start..start + READ].to_vec()) +} + +/// Stops the source if the test fails first: a device left without a +/// process holds the blocked read, and the test process cannot exit. +struct StopOnDrop<'a> { + rt: &'a tokio::runtime::Runtime, + sources: &'a ProcessSources, + sandbox: SandboxId, +} + +impl Drop for StopOnDrop<'_> { + fn drop(&mut self) { + if let Err(e) = self + .rt + .block_on(self.sources.stop(self.sandbox, DiskRole::Root)) + { + eprintln!("stop the source: {e}"); + } + } +} + +struct Rig { + work: tempfile::TempDir, + rt: tokio::runtime::Runtime, + cfg: ProcessSourcesConfig, + manifest: ManifestRef, + expected: Vec, +} + +/// A published disk and a source config, or `None` when the lane's +/// variables are not set. +fn rig() -> Option { + if std::env::var_os("ENGRAM_TEST_UBLK").is_none() { + eprintln!("SKIP: ENGRAM_TEST_UBLK is not set"); + return None; + } + let Some(bin) = std::env::var_os("ENGRAM_DISK_SOURCE_BIN").map(PathBuf::from) else { + eprintln!("SKIP: ENGRAM_DISK_SOURCE_BIN is not set"); + return None; + }; + let work = tempfile::tempdir().unwrap(); + // The source reads the blob store from the environment it inherits. + // nextest runs each test in a process of its own, before any other + // thread starts. + std::env::set_var("ENGRAM_BLOB_BACKEND", "local"); + std::env::set_var("ENGRAM_LOCAL_PATH", work.path()); + let rt = tokio::runtime::Runtime::new().unwrap(); + let (manifest, expected) = rt.block_on(publish(work.path())); + let mut cfg = ProcessSourcesConfig::new(bin, work.path()); + cfg.queues = 1; + cfg.depth = 8; + Some(Rig { + work, + rt, + cfg, + manifest, + expected, + }) +} + +/// A host-agent that died before it recorded a source leaves a device +/// that no record names. When its server has died too, the startup sweep +/// deletes it. +#[test] +#[ignore = "requires root and the ublk driver"] +fn the_sweep_deletes_a_device_no_record_names() { + let Some(r) = rig() else { return }; + let sources = ProcessSources::new(r.cfg.clone()); + let sandbox = SandboxId::new(); + r.rt.block_on(sources.start(sandbox, DiskRole::Root, r.manifest, None)) + .unwrap(); + let rec = r.rt.block_on(sources.records()).pop().unwrap(); + let device = rec.device.unwrap(); + kill(Pid::from_raw(rec.pid as i32), Signal::SIGKILL).unwrap(); + let probe = KernelProbe; + while probe.start_time(rec.pid) == Some(rec.start_time) { + std::thread::sleep(Duration::from_millis(10)); + } + std::fs::remove_file( + r.work + .path() + .join("disk-sources") + .join(format!("{sandbox}-root.json")), + ) + .unwrap(); + + // The kernel quiesces the device once it sees the server's queues go. + for _ in 0..100 { + r.rt.block_on(sources.sweep_unrecorded()); + if !probe.device_exists(device.id) { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + assert!( + !probe.device_exists(device.id), + "the sweep deleted the device" + ); +} + +#[test] +#[ignore = "requires root and the ublk driver"] +fn a_killed_source_is_recovered_under_a_waiting_read_and_a_stop_leaves_nothing() { + let Some(Rig { + work: _work, + rt, + cfg, + manifest, + expected, + }) = rig() + else { + return; + }; + let sources = ProcessSources::new(cfg); + let sandbox = SandboxId::new(); + let path = rt + .block_on(sources.start(sandbox, DiskRole::Root, manifest, None)) + .unwrap(); + let _stop = StopOnDrop { + rt: &rt, + sources: &sources, + sandbox, + }; + assert_eq!(read_direct(&path, 0).unwrap(), expected[..READ]); + + let first = rt.block_on(sources.records()).pop().unwrap(); + let device = first.device.clone().unwrap(); + kill(Pid::from_raw(first.pid as i32), Signal::SIGKILL).unwrap(); + let probe = KernelProbe; + while probe.start_time(first.pid) == Some(first.start_time) { + std::thread::sleep(Duration::from_millis(10)); + } + let offset = 2 * CHUNK + 4096; + let reader = { + let path = path.clone(); + std::thread::spawn(move || read_direct(&path, offset)) + }; + std::thread::sleep(Duration::from_millis(300)); + assert!( + !reader.is_finished(), + "the read waits while no process serves" + ); + + rt.block_on(sources.ensure_alive(sandbox, DiskRole::Root)) + .unwrap(); + let bytes = reader + .join() + .unwrap() + .expect("the read completes after the recovery"); + assert_eq!(bytes, expected[offset as usize..offset as usize + READ]); + let second = rt.block_on(sources.records()).pop().unwrap(); + assert_ne!(second.pid, first.pid); + assert_eq!(second.device, Some(device.clone())); + + rt.block_on(sources.stop(sandbox, DiskRole::Root)).unwrap(); + assert_eq!(probe.start_time(second.pid), None, "the process exited"); + assert!(!probe.device_exists(device.id), "the device is deleted"); + assert!(!PathBuf::from(format!("/sys/block/ublkb{}", device.id)).exists()); + assert!(rt.block_on(sources.records()).is_empty()); +} diff --git a/crates/engram-host-agent/tests/session_disk.rs b/crates/engram-host-agent/tests/session_disk.rs index c004a72cc..eb5ef0cbd 100644 --- a/crates/engram-host-agent/tests/session_disk.rs +++ b/crates/engram-host-agent/tests/session_disk.rs @@ -24,8 +24,9 @@ use engram_core::types::ids::SandboxId; use engram_core::types::sandbox::{CpuLimit, DiskLimit, ExecRequest, MemoryLimit, SandboxSpec}; use engram_core::types::snapshot::DiskRole; use engram_host_agent::pooled_backend::PooledBackend; -use engram_host_core::disk::{Capacity, SessionDisk, Volume, VolumeName}; -use engram_host_core::SessionVolumes; +use engram_host_core::disk::{Capacity, CloneState, Hydration, SessionDisk, Volume, VolumeName}; +use engram_host_core::disk_sources::Served; +use engram_host_core::{DiskSources, SessionVolumes}; use engram_rootfs_materializer::{InitInjection, Transport}; use engram_sandbox_firecracker::{ FirecrackerBackend, FirecrackerConfig, RestoreMode, ENGRAM_AGENTD_PORT, @@ -240,6 +241,159 @@ async fn a_host_agent_restart_keeps_the_guest_and_every_write() { generation_b.destroy(sandbox).await.expect("destroy"); } +/// With lazy restore forced, a restore boots over a source: the guest +/// reads the checkpoint's bytes at once, the disk cuts over in the +/// background, and the source goes with the cutover. +#[tokio::test] +#[ignore = "requires Linux + KVM + firecracker; bakes a rootfs and boots microVMs"] +async fn a_forced_lazy_restore_boots_over_a_source_and_cuts_over() { + let Some(rig) = Rig::gate().await else { return }; + let disk = common::session_disk(rig.work.path()).await; + let (eager, _) = rig.pooled(disk.clone()); + let sandbox = SandboxId::new(); + eager + .create(sandbox, rig.spec().await) + .await + .expect("create"); + eager.bind_session(engram_core::SessionId::new(), sandbox); + let sum = exec( + &eager, + sandbox, + "head -c 1048576 /dev/urandom > /root/data && sync \ + && sha256sum /root/data | cut -d' ' -f1", + ) + .await; + let checkpoint = eager.checkpoint_sandbox(sandbox).await.expect("checkpoint"); + eager.destroy(sandbox).await.expect("destroy"); + + let sources = Arc::new(LoopSources { + files: engram_host_core::FileSources::new( + rig.work.path().join("sources"), + rig.chunk_store.clone(), + ) + .expect("file sources"), + loops: std::sync::Mutex::new(HashMap::new()), + }); + let (lazy, volumes) = rig.pooled_with( + disk, + Some(sources.clone()), + engram_host_agent::config::LazyRestore::Force, + ); + let resumed = SandboxId::new(); + lazy.restore(resumed, checkpoint, None) + .await + .expect("lazy restore"); + let drive = engram_host_core::session_volumes::live_name(resumed, DiskRole::Root); + assert!( + volumes.disk().clone_state(&drive).await.is_ok(), + "the root is a cloned drive" + ); + let again = exec(&lazy, resumed, "sha256sum /root/data | cut -d' ' -f1").await; + assert_eq!(again.trim(), sum.trim(), "the guest reads the checkpoint"); + + let cut = common::poll_until_async(Duration::from_secs(60), Duration::from_millis(100), || { + let lazy = lazy.clone(); + async move { lazy.hydrating().is_empty() } + }) + .await; + assert!(cut, "the disk cuts over"); + assert_eq!( + volumes.disk().clone_state(&drive).await.expect("drive"), + CloneState::CutOver + ); + assert!(sources.list().await.expect("sources").is_empty()); + assert!( + sources.loops.lock().unwrap().is_empty(), + "no loop device is left" + ); + let after = exec(&lazy, resumed, "sha256sum /root/data | cut -d' ' -f1").await; + assert_eq!(after.trim(), sum.trim()); + lazy.destroy(resumed).await.expect("destroy resumed"); +} + +/// [`engram_host_core::FileSources`] behind a read-only loop device, the +/// block device a drive over the pool needs. Over sparse files the file +/// itself is the source. +struct LoopSources { + files: engram_host_core::FileSources, + loops: std::sync::Mutex>, +} + +fn over_pool() -> bool { + std::env::var_os("ENGRAM_TEST_THIN_POOL").is_some() +} + +#[async_trait] +impl DiskSources for LoopSources { + fn can_start(&self) -> bool { + true + } + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: engram_chunk_store::ManifestRef, + base: Option<(&Volume, engram_chunk_store::ManifestRef)>, + ) -> std::io::Result { + let file = self.files.start(sandbox, role, manifest, base).await?; + if !over_pool() { + return Ok(file); + } + let out = std::process::Command::new("losetup") + .args(["--read-only", "--show", "-f"]) + .arg(&file) + .output()?; + if !out.status.success() { + return Err(std::io::Error::other(format!( + "losetup: {}", + String::from_utf8_lossy(&out.stderr) + ))); + } + let device = String::from_utf8_lossy(&out.stdout).trim().to_string(); + self.loops + .lock() + .unwrap() + .insert((sandbox, role), device.clone()); + Ok(PathBuf::from(device)) + } + + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result<()> { + self.files.ensure_alive(sandbox, role).await + } + + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result { + let served = self.files.served(sandbox, role).await?; + match self.loops.lock().unwrap().get(&(sandbox, role)) { + Some(device) => Ok(Served { + device: PathBuf::from(device), + ..served + }), + None => Ok(served), + } + } + + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> std::io::Result<()> { + let device = self.loops.lock().unwrap().remove(&(sandbox, role)); + if let Some(device) = device { + let out = std::process::Command::new("losetup") + .args(["-d", &device]) + .output()?; + if !out.status.success() { + self.loops.lock().unwrap().insert((sandbox, role), device); + return Err(std::io::Error::other(format!( + "losetup -d: {}", + String::from_utf8_lossy(&out.stderr) + ))); + } + } + self.files.stop(sandbox, role).await + } + + async fn list(&self) -> std::io::Result> { + self.files.list().await + } +} + /// A guest command that runs `script` detached from the exec. The guest's /// busybox has no `nohup`; agentd's exec has no controlling tty, so none is /// needed. The shell's own stdio goes to /dev/null first: busybox ash leaks @@ -314,6 +468,70 @@ impl SessionDisk for CountingDisk { ) -> std::io::Result> { self.inner.dirty_blocks(before, after, block_bytes).await } + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + hydration: Hydration, + ) -> std::io::Result { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner + .clone_drive(drive, dest, bitmap, source, hydration) + .await + } + async fn open_clone(&self, drive: &VolumeName) -> std::io::Result { + self.inner.open_clone(drive).await + } + async fn clone_state(&self, drive: &VolumeName) -> std::io::Result { + self.inner.clone_state(drive).await + } + async fn skip_hydration( + &self, + drive: &VolumeName, + ranges: &[std::ops::Range], + ) -> std::io::Result<()> { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner.skip_hydration(drive, ranges).await + } + async fn enable_hydration(&self, drive: &VolumeName) -> std::io::Result<()> { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner.enable_hydration(drive).await + } + async fn cut_over(&self, drive: &VolumeName) -> std::io::Result<()> { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner.cut_over(drive).await + } + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> std::io::Result<(Volume, Volume)> { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner + .snapshot_clone(drive, dest, bitmap, dest_snapshot, bitmap_snapshot) + .await + } + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + source: &Path, + ) -> std::io::Result { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner + .clone_view(view, dest_snapshot, bitmap_snapshot, source) + .await + } + async fn remove_clone(&self, drive: &VolumeName) -> std::io::Result<()> { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner.remove_clone(drive).await + } } struct Rig { @@ -400,13 +618,29 @@ impl Rig { /// A backend over `disk`, with the volumes it owns. fn pooled(&self, disk: Arc) -> (Arc, Arc) { - let volumes = Arc::new(SessionVolumes::new(disk, self.chunk_store.clone())); + self.pooled_with(disk, None, engram_host_agent::config::LazyRestore::Off) + } + + /// [`Self::pooled`] with disk sources and a lazy-restore mode. + fn pooled_with( + &self, + disk: Arc, + sources: Option>, + lazy_restore: engram_host_agent::config::LazyRestore, + ) -> (Arc, Arc) { + let mut volumes = SessionVolumes::new(disk, self.chunk_store.clone()); + if let Some(sources) = sources { + volumes = volumes.with_sources(sources); + } + let volumes = Arc::new(volumes); let inner = Arc::new(FirecrackerBackend::new(self.work.path(), self.cfg.clone())); let pooled = Arc::new( PooledBackend::new(inner) .with_chunk_store(self.chunk_store.clone()) .with_session_volumes(volumes.clone()) - .with_checkpoint_dir(self.work.path().join("checkpoints")), + .with_checkpoint_dir(self.work.path().join("checkpoints")) + .with_lazy_restore(lazy_restore, Default::default()) + .with_hydration_interval(Duration::from_millis(100)), ); pooled.set_self_ref(&pooled); (pooled, volumes) diff --git a/crates/engram-host-core/src/disk.rs b/crates/engram-host-core/src/disk.rs index f18a87af3..81e51d72d 100644 --- a/crates/engram-host-core/src/disk.rs +++ b/crates/engram-host-core/src/disk.rs @@ -6,9 +6,15 @@ //! per-platform lifecycle around it. Everything is addressed in chunks //! of `chunk_size` bytes; an all-zero chunk is a hole and is never //! listed in a manifest. +//! +//! A cloned drive (ADR 0127) is a volume the guest uses before its +//! bytes are local: reads of regions not yet copied go to a read-only +//! source, writes land in a destination volume, and a bitmap volume +//! records the copied regions until the cutover. use std::fs::File; use std::io; +use std::ops::Range; use std::os::fd::AsRawFd; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; @@ -72,7 +78,77 @@ impl Capacity { } } -/// An open volume. Clones share the file handle. +/// The unit a cloned drive copies in (ADR 0127), equal to the default +/// block size of a production pool. It is part of the on-disk format: a +/// bitmap records the region size it was made with, and the kernel +/// refuses a table over that bitmap with any other size. +pub const CLONE_REGION_BYTES: u64 = 64 * 1024; + +/// The start of a bitmap that holds its header (a dm-clone superblock is +/// one 4 KiB metadata block). A bitmap no drive has used reads as zeros +/// here. +pub const CLONE_HEADER_BYTES: u64 = 4096; + +/// The size of the bitmap volume for a drive of `volume_bytes`. The +/// kernel stores one bit per region in 4 KiB metadata blocks and writes a +/// commit to new blocks before it frees the old ones, so the bits count +/// twice; 1 MiB holds the superblock, the space map and the tree nodes. +/// It is at least 4 MiB and a whole number of MiB. A thin volume takes +/// pool space only where the kernel writes, so the margin is free. +pub fn clone_bitmap_bytes(volume_bytes: u64) -> u64 { + const MIB: u64 = 1 << 20; + let bits = volume_bytes.div_ceil(CLONE_REGION_BYTES); + let bytes = 2 * bits.div_ceil(8) + MIB; + bytes.max(4 * MIB).div_ceil(MIB) * MIB +} + +/// How a cloned drive copies the regions the guest has not written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Hydration { + /// Copy in the background from the start. Off, only guest writes + /// copy regions until [`SessionDisk::enable_hydration`]. + pub enabled: bool, + /// Regions the background copy holds in flight at most. + pub threshold_regions: u32, + /// Adjacent regions the background copy joins into one copy. + pub batch_regions: u32, +} + +/// Where a cloned drive is in its life. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloneState { + /// Reads of uncopied regions still go to the source. + Copying { + hydrated_regions: u64, + total_regions: u64, + hydration_enabled: bool, + }, + /// The drive maps the destination alone; the source and the bitmap + /// are no longer read. + CutOver, + /// A [`SessionDisk::clone_drive`] that did not finish left the name + /// with no live table. The bitmap can hold a header already: remove + /// the drive, and make it again on a new bitmap. + Incomplete, +} + +impl CloneState { + /// Every region is in the destination, so [`SessionDisk::cut_over`] + /// is allowed. + pub fn is_hydrated(&self) -> bool { + match *self { + Self::Copying { + hydrated_regions, + total_regions, + .. + } => hydrated_regions == total_regions, + Self::CutOver => true, + Self::Incomplete => false, + } + } +} + +/// An open volume. Its copies share the file handle. #[derive(Clone, Debug)] pub struct Volume { pub name: VolumeName, @@ -112,6 +188,24 @@ impl Volume { }) } + /// Open the device or file at `path` read-only. Writes fail. + pub fn open_read_only( + name: VolumeName, + path: &Path, + size_bytes: u64, + chunk_size: u64, + ) -> io::Result { + assert!(chunk_size > 0, "chunk_size must be non-zero"); + Ok(Self { + name, + path: path.to_path_buf(), + size_bytes, + chunk_size, + file: Arc::new(File::open(path)?), + direct: None, + }) + } + /// Number of chunks, including a short final one. pub fn chunk_count(&self) -> u64 { self.size_bytes.div_ceil(self.chunk_size) @@ -192,6 +286,33 @@ impl Volume { .map_err(|e| io::Error::other(format!("read_range task: {e}")))? } + /// [`Self::read_range`] past the host page cache, where the platform + /// allows and the range is aligned for it: for bytes the kernel writes + /// beneath the cache, such as a dm-clone superblock. + pub async fn read_direct(&self, offset: u64, len: u64) -> io::Result> { + self.check_range(offset, len)?; + let aligned = + offset.is_multiple_of(DIRECT_ALIGN as u64) && len.is_multiple_of(DIRECT_ALIGN as u64); + let direct = self.direct.as_ref().filter(|_| aligned).map(Arc::clone); + let file = Arc::clone(&self.file); + tokio::task::spawn_blocking(move || { + let zeros = vec![0u8; len as usize]; + match direct.zip(AlignedBuf::copy_of(&zeros)) { + Some((direct, mut buf)) => { + direct.read_exact_at(buf.as_mut_slice(), offset)?; + Ok(buf.as_slice().to_vec()) + } + None => { + let mut buf = zeros; + file.read_exact_at(&mut buf, offset)?; + Ok(buf) + } + } + }) + .await + .map_err(|e| io::Error::other(format!("read_direct task: {e}")))? + } + /// Write `bytes` at `offset`; the range must lie inside the volume. pub async fn write_range(&self, offset: u64, bytes: Vec) -> io::Result<()> { self.check_range(offset, bytes.len() as u64)?; @@ -278,6 +399,56 @@ impl AlignedBuf { fn as_slice(&self) -> &[u8] { &self.buf[self.start..self.start + self.len] } + + fn as_mut_slice(&mut self) -> &mut [u8] { + &mut self.buf[self.start..self.start + self.len] + } +} + +/// `bitmap` can take a new drive: it holds a header and no drive has +/// written one. `InvalidInput` otherwise. +pub async fn check_fresh_bitmap(bitmap: &Volume) -> io::Result<()> { + if bitmap.size_bytes < CLONE_HEADER_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "bitmap {} has {} bytes, under the {CLONE_HEADER_BYTES}-byte header", + bitmap.name, bitmap.size_bytes + ), + )); + } + if !is_zero_chunk(&bitmap.read_direct(0, CLONE_HEADER_BYTES).await?) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("bitmap {} was used by a drive before", bitmap.name), + )); + } + Ok(()) +} + +/// The ranges of a [`SessionDisk::skip_hydration`] on a drive of +/// `drive_bytes`, checked and with adjacent ranges joined. Empty ranges +/// are dropped. +pub fn skip_ranges(ranges: &[Range], drive_bytes: u64) -> io::Result>> { + let mut joined: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges.iter().filter(|r| !r.is_empty()) { + let aligned = range.start.is_multiple_of(CLONE_REGION_BYTES) + && range.end.is_multiple_of(CLONE_REGION_BYTES); + let after_last = joined.last().is_none_or(|last| last.end <= range.start); + if !aligned || range.end > drive_bytes || !after_last { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "skip range {range:?} is not region-aligned, ascending and inside {drive_bytes} bytes" + ), + )); + } + match joined.last_mut() { + Some(last) if last.end == range.start => last.end = range.end, + _ => joined.push(range.clone()), + } + } + Ok(joined) } /// Two volumes are snapshots of one origin when their shapes match. @@ -326,17 +497,7 @@ pub fn is_zero_chunk(bytes: &[u8]) -> bool { pub fn punch_hole(file: &File, offset: u64, length: u64) -> io::Result<()> { use std::os::unix::fs::FileTypeExt; if file.metadata()?.file_type().is_block_device() { - // _IO(0x12, 119); argument is [start, length] in bytes. - const BLKDISCARD: libc::Ioctl = 0x1277; - let range: [u64; 2] = [offset, length]; - // SAFETY: the kernel copies the two-element range in and writes - // nothing back. The fd is owned by `file` for the whole call. - let result = unsafe { libc::ioctl(file.as_raw_fd(), BLKDISCARD, range.as_ptr()) }; - return if result == 0 { - Ok(()) - } else { - Err(io::Error::last_os_error()) - }; + return discard(file, offset, length); } // SAFETY: fallocate takes its arguments by value and writes nothing // back. The fd is owned by `file` for the whole call; the range was @@ -356,6 +517,25 @@ pub fn punch_hole(file: &File, offset: u64, length: u64) -> io::Result<()> { } } +/// `BLKDISCARD` of `length` bytes at `offset` on a block device opened +/// for writing. What a discard means is the device's: a dm-thin volume +/// unmaps the range, a dm-clone drive marks it copied. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +pub fn discard(file: &File, offset: u64, length: u64) -> io::Result<()> { + // _IO(0x12, 119); argument is [start, length] in bytes. + const BLKDISCARD: libc::Ioctl = 0x1277; + let range: [u64; 2] = [offset, length]; + // SAFETY: the kernel copies the two-element range in and writes + // nothing back. The fd is owned by `file` for the whole call. + let result = unsafe { libc::ioctl(file.as_raw_fd(), BLKDISCARD, range.as_ptr()) }; + if result == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + /// Release `length` bytes at `offset` so they read as zeros. The range /// must be block-aligned. #[cfg(target_os = "macos")] @@ -379,6 +559,15 @@ pub fn punch_hole(file: &File, offset: u64, length: u64) -> io::Result<()> { } /// The platform's volume store. +/// +/// Volumes, drives and views share one name space. A drive (ADR 0127) +/// is made by [`Self::clone_drive`]; a view by [`Self::clone_view`]. +/// While a drive or view uses a volume, `delete` of that volume fails +/// with `ResourceBusy`. Operations on one drive are serialized. An +/// operation that suspends a drive or a volume resumes it before it +/// returns, even when its caller is dropped; a process death between +/// the two leaves the device suspended, and the platform's disk resumes +/// it when it is opened again (`ThinPoolDisk::open`). #[async_trait] pub trait SessionDisk: Send + Sync { /// A new volume of `size_bytes` that reads as zeros. @@ -391,18 +580,99 @@ pub trait SessionDisk: Send + Sync { /// A volume that already exists. `NotFound` when it does not. async fn open(&self, name: &VolumeName) -> io::Result; - /// Remove a volume. Fails while a process holds it open. + /// Remove a volume. Fails while a process, a drive or a view holds + /// it. async fn delete(&self, name: &VolumeName) -> io::Result<()>; - /// Every volume that exists, in no particular order. + /// Every volume that exists, in no particular order. Drives and views + /// are not volumes. async fn list(&self) -> io::Result>; /// Give a volume a new name atomically. `AlreadyExists` when `to` - /// is taken. Open handles keep working under the new name. + /// is taken. Open handles keep working under the new name. A drive + /// or a view is not renamed: `InvalidInput`. async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()>; async fn capacity(&self) -> io::Result; + /// A new drive `drive` that reads as `source` and writes into `dest`, + /// with the copy bitmap on `bitmap`. `source` is a read-only block + /// device (a file on [`FileDisk`]) at least as large as `dest`, and + /// the drive is as large as `dest`. `AlreadyExists` when the name is + /// taken; `ResourceBusy` when another drive or view uses `dest` or + /// `bitmap`; `InvalidInput` when `bitmap` is smaller than + /// [`CLONE_HEADER_BYTES`] or a drive wrote its header before. + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + hydration: Hydration, + ) -> io::Result; + + /// A drive [`Self::clone_drive`] made, before or after its cutover, + /// or a view, which opens read-only. `NotFound` when it does not + /// exist; `InvalidData` for [`CloneState::Incomplete`]. + async fn open_clone(&self, drive: &VolumeName) -> io::Result; + + /// A view reports the regions copied when its snapshots were taken, + /// with hydration off. + async fn clone_state(&self, drive: &VolumeName) -> io::Result; + + /// Record the byte `ranges` of a drive as copied without copying + /// them, so hydration never writes them. The caller states that the + /// destination already holds the source's bytes there; the guest + /// must not run yet. A range the disk cannot mark is copied as + /// usual, so a skip never changes what the drive reads. `ranges` are ascending, do not overlap, and start and + /// end on [`CLONE_REGION_BYTES`] inside the drive (`InvalidInput` + /// otherwise). `InvalidInput` after the cutover and on a view. + async fn skip_hydration(&self, drive: &VolumeName, ranges: &[Range]) -> io::Result<()>; + + /// Start the background copy of a drive made with hydration off. + /// `InvalidInput` after the cutover and on a view. + async fn enable_hydration(&self, drive: &VolumeName) -> io::Result<()>; + + /// Make a fully hydrated drive map its destination alone. The drive's + /// path and every open handle stay valid; afterwards the caller can + /// delete the bitmap and stop the source. `ResourceBusy` while a + /// region is not yet copied; `InvalidInput` on a view. A drive + /// already cut over is left as it is. + async fn cut_over(&self, drive: &VolumeName) -> io::Result<()>; + + /// Point-in-time copies of a drive's destination and bitmap that + /// agree with each other: no region is copied between the two. The + /// guest must be paused and the drive synced first, so the bitmap + /// records every region the guest wrote. Returns `(dest, bitmap)`. + /// On failure neither copy is left. `InvalidInput` after the cutover, + /// on a view, and when `dest` and `bitmap` are not the drive's. + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> io::Result<(Volume, Volume)>; + + /// A read-only drive `view` over a [`Self::snapshot_clone`] pair and + /// the drive's source. It reads exactly what the guest saw when the + /// snapshots were taken, and copies nothing. + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + source: &Path, + ) -> io::Result; + + /// Remove a drive or a view; its volumes stay. `InvalidInput` for a + /// name that is neither. A `ThinPoolDisk` refuses while a process + /// holds the drive open; a [`FileDisk`] cannot see open handles and + /// removes it, and the handles keep reading and writing the + /// destination. + async fn remove_clone(&self, drive: &VolumeName) -> io::Result<()>; + /// Ascending indices of the `block_bytes`-sized blocks that differ /// between two snapshots of one origin; `block_bytes` divides the /// chunk size. The default reads both in full and is the oracle a @@ -432,6 +702,19 @@ pub trait SessionDisk: Send + Sync { Ok(dirty) } + /// Ascending indices of the chunks of `snapshot` that a write can + /// have changed since it was made from `origin`, or, with no origin, + /// since it was a new volume. A chunk written with the bytes it held + /// is listed, so the list does not depend on content. The default + /// lists every chunk; a disk that records writes lists fewer. + async fn written_chunks( + &self, + _origin: Option<&Volume>, + snapshot: &Volume, + ) -> io::Result> { + Ok((0..snapshot.chunk_count()).map(ChunkIndex).collect()) + } + /// Ascending indices of the chunks that differ between two snapshots /// of one origin. async fn dirty_chunks(&self, before: &Volume, after: &Volume) -> io::Result> { @@ -449,12 +732,35 @@ pub trait SessionDisk: Send + Sync { /// Volumes as sparse files in one directory: the macOS implementation /// (`fs::copy` is a clone on APFS) and the Linux one without a thin pool. +/// +/// A drive or view `d` is files under `.drives/`, which +/// [`SessionDisk::list`] does not read: `d`, a hard link to its +/// destination (the destination snapshot for a view), which exists only +/// once the rest is made; `d.bitmap`, a hard link to its bitmap while +/// its table would read it; and a `d.cut-over` or `d.view` marker. A +/// volume with more than one link is in use. `FileDisk` hydrates +/// eagerly: `clone_drive` copies the source into the destination before +/// it returns, so every drive is hydrated from the start and `cut_over` +/// only records the state. A view reads its destination snapshot, which +/// already holds every byte the guest saw. #[derive(Clone, Debug)] pub struct FileDisk { root: PathBuf, chunk_size: u64, } +/// What a name under `.drives/` is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FileDrive { + Copying, + CutOver, + View, +} + +/// The header `FileDisk` writes into a bitmap, as the kernel writes a +/// superblock: a later `clone_drive` sees the bitmap as used. +const FILE_BITMAP_HEADER: &[u8] = b"engram FileDisk clone bitmap\n"; + impl FileDisk { pub fn new(root: impl Into, chunk_size: u64) -> io::Result { assert!(chunk_size > 0, "chunk_size must be non-zero"); @@ -470,24 +776,174 @@ impl FileDisk { fn path_of(&self, name: &VolumeName) -> PathBuf { self.root.join(name.as_str()) } + + /// `.` is not a volume-name character, so no volume shadows these. + fn drives_dir(&self) -> PathBuf { + self.root.join(".drives") + } + + fn drive_path(&self, drive: &VolumeName) -> PathBuf { + self.drives_dir().join(drive.as_str()) + } + + fn drive_file(&self, drive: &VolumeName, suffix: &str) -> PathBuf { + self.drives_dir().join(format!("{drive}.{suffix}")) + } + + /// `name` is free for a new volume, drive or view. + fn check_free(&self, name: &VolumeName) -> io::Result<()> { + if self.path_of(name).exists() || self.drive_path(name).exists() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("{name} already exists"), + )); + } + Ok(()) + } + + /// The drive or view `name`. `NotFound` when the name is free; + /// `InvalidInput` when it is a volume. + fn drive(&self, name: &VolumeName) -> io::Result { + match std::fs::metadata(self.drive_path(name)) { + Err(e) if e.kind() == io::ErrorKind::NotFound && self.path_of(name).exists() => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} is not a cloned drive or a view"), + )) + } + found => found?, + }; + Ok(if self.drive_file(name, "view").exists() { + FileDrive::View + } else if self.drive_file(name, "cut-over").exists() { + FileDrive::CutOver + } else { + FileDrive::Copying + }) + } + + /// The drive `name` before its cutover; `InvalidInput` for a view or a + /// drive cut over. + fn copying(&self, name: &VolumeName) -> io::Result<()> { + match self.drive(name)? { + FileDrive::Copying => Ok(()), + other => Err(refused(name, other)), + } + } + + fn remove_drive_files(&self, name: &VolumeName) -> io::Result<()> { + for suffix in ["bitmap", "cut-over", "view"] { + remove_if_present(&self.drive_file(name, suffix))?; + } + Ok(()) + } + + /// Remove `name`'s files; the link to the destination, which makes + /// `name` exist, goes last. + fn unlink_drive(&self, name: &VolumeName) -> io::Result<()> { + self.remove_drive_files(name)?; + std::fs::remove_file(self.drive_path(name)) + } + + /// Make `name`'s files: the bitmap link and `marker`, then the link to + /// `dest`. Leftovers of a make that did not finish are replaced. + fn link_drive( + &self, + name: &VolumeName, + dest: &Volume, + bitmap: &Volume, + marker: Option<&str>, + ) -> io::Result { + std::fs::create_dir_all(self.drives_dir())?; + self.remove_drive_files(name)?; + std::fs::hard_link(&bitmap.path, self.drive_file(name, "bitmap"))?; + if let Some(marker) = marker { + File::create(self.drive_file(name, marker))?; + } + let path = self.drive_path(name); + std::fs::hard_link(&dest.path, &path)?; + Ok(path) + } +} + +fn refused(name: &VolumeName, drive: FileDrive) -> io::Error { + let what = match drive { + FileDrive::CutOver => "is cut over", + FileDrive::View => "is a view", + FileDrive::Copying => "is copying", + }; + io::Error::new(io::ErrorKind::InvalidInput, format!("{name} {what}")) +} + +fn remove_if_present(path: &Path) -> io::Result<()> { + match std::fs::remove_file(path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } +} + +/// `ResourceBusy` when a drive or view links `volume`'s file. +fn check_unused(volume: &Path, name: &VolumeName) -> io::Result<()> { + use std::os::unix::fs::MetadataExt; + if std::fs::metadata(volume)?.nlink() > 1 { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!("{name} is used by a drive or a view"), + )); + } + Ok(()) +} + +/// `a` and `b` are one file. +fn same_file(a: &Path, b: &Path) -> io::Result { + use std::os::unix::fs::MetadataExt; + let (a, b) = (std::fs::metadata(a)?, std::fs::metadata(b)?); + Ok(a.dev() == b.dev() && a.ino() == b.ino()) +} + +/// Make the first `len` bytes of `dest` equal those of `source`, writing +/// only the ranges that differ, then sync `dest`. +fn hydrate_file(source: &Path, dest: &File, len: u64) -> io::Result<()> { + use std::io::{Seek, SeekFrom}; + const STEP: u64 = 1 << 20; + let mut source_file = File::open(source)?; + let source_len = source_file.seek(SeekFrom::End(0))?; + if source_len < len { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "source {} has {source_len} bytes; the drive needs {len}", + source.display() + ), + )); + } + let mut want = vec![0u8; STEP as usize]; + let mut have = vec![0u8; STEP as usize]; + let mut offset = 0; + while offset < len { + let n = (len - offset).min(STEP) as usize; + source_file.read_exact_at(&mut want[..n], offset)?; + dest.read_exact_at(&mut have[..n], offset)?; + if want[..n] != have[..n] { + dest.write_all_at(&want[..n], offset)?; + } + offset += n as u64; + } + dest.sync_all() } #[async_trait] impl SessionDisk for FileDisk { async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { + self.check_free(name)?; let path = self.path_of(name); File::create_new(&path)?.set_len(size_bytes)?; Volume::open(name.clone(), &path, size_bytes, self.chunk_size) } async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { + self.check_free(name)?; let path = self.path_of(name); - if path.exists() { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - format!("volume {name} already exists"), - )); - } let (from, to) = (origin.path.clone(), path.clone()); tokio::task::spawn_blocking(move || std::fs::copy(from, to)) .await @@ -502,7 +958,9 @@ impl SessionDisk for FileDisk { } async fn delete(&self, name: &VolumeName) -> io::Result<()> { - std::fs::remove_file(self.path_of(name)) + let path = self.path_of(name); + check_unused(&path, name)?; + std::fs::remove_file(path) } async fn list(&self) -> io::Result> { @@ -517,14 +975,14 @@ impl SessionDisk for FileDisk { } async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { - let target = self.path_of(to); - if target.exists() { + if self.drive_path(from).exists() { return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - format!("volume {to} already exists"), + io::ErrorKind::InvalidInput, + format!("{from} is a drive or a view, which is not renamed"), )); } - std::fs::rename(self.path_of(from), target) + self.check_free(to)?; + std::fs::rename(self.path_of(from), self.path_of(to)) } async fn capacity(&self) -> io::Result { @@ -535,6 +993,142 @@ impl SessionDisk for FileDisk { metadata_total_bytes: 0, }) } + + /// Copies `source` into `dest` before it returns; `hydration` has + /// nothing to tune. + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + _hydration: Hydration, + ) -> io::Result { + self.check_free(drive)?; + check_unused(&dest.path, &dest.name)?; + check_unused(&bitmap.path, &bitmap.name)?; + check_fresh_bitmap(bitmap).await?; + let (file, len, from) = ( + Arc::clone(&dest.file), + dest.size_bytes, + source.to_path_buf(), + ); + tokio::task::spawn_blocking(move || hydrate_file(&from, &file, len)) + .await + .map_err(|e| io::Error::other(format!("clone task: {e}")))??; + bitmap.write_range(0, FILE_BITMAP_HEADER.to_vec()).await?; + bitmap.sync().await?; + let path = self.link_drive(drive, dest, bitmap, None)?; + Volume::open(drive.clone(), &path, dest.size_bytes, self.chunk_size) + } + + async fn open_clone(&self, drive: &VolumeName) -> io::Result { + let kind = self.drive(drive)?; + let path = self.drive_path(drive); + let size_bytes = std::fs::metadata(&path)?.len(); + match kind { + FileDrive::View => { + Volume::open_read_only(drive.clone(), &path, size_bytes, self.chunk_size) + } + FileDrive::Copying | FileDrive::CutOver => { + Volume::open(drive.clone(), &path, size_bytes, self.chunk_size) + } + } + } + + async fn clone_state(&self, drive: &VolumeName) -> io::Result { + let kind = self.drive(drive)?; + let regions = std::fs::metadata(self.drive_path(drive))? + .len() + .div_ceil(CLONE_REGION_BYTES); + Ok(match kind { + FileDrive::CutOver => CloneState::CutOver, + FileDrive::Copying | FileDrive::View => CloneState::Copying { + hydrated_regions: regions, + total_regions: regions, + hydration_enabled: kind == FileDrive::Copying, + }, + }) + } + + /// The drive is copied in full at `clone_drive`, so there is nothing + /// to skip; the ranges are only checked. + async fn skip_hydration(&self, drive: &VolumeName, ranges: &[Range]) -> io::Result<()> { + self.copying(drive)?; + skip_ranges(ranges, std::fs::metadata(self.drive_path(drive))?.len()).map(drop) + } + + async fn enable_hydration(&self, drive: &VolumeName) -> io::Result<()> { + self.copying(drive) + } + + /// Also removes a bitmap link that a cutover cut short left behind. + async fn cut_over(&self, drive: &VolumeName) -> io::Result<()> { + match self.drive(drive)? { + FileDrive::View => return Err(refused(drive, FileDrive::View)), + FileDrive::Copying => File::create(self.drive_file(drive, "cut-over")).map(drop)?, + FileDrive::CutOver => {} + } + remove_if_present(&self.drive_file(drive, "bitmap")) + } + + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> io::Result<(Volume, Volume)> { + self.copying(drive)?; + if !same_file(&dest.path, &self.drive_path(drive))? + || !same_file(&bitmap.path, &self.drive_file(drive, "bitmap"))? + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{drive} does not copy into {} with bitmap {}", + dest.name, bitmap.name + ), + )); + } + let bitmap_snap = self.snapshot(bitmap, bitmap_snapshot).await?; + match self.snapshot(dest, dest_snapshot).await { + Ok(dest_snap) => Ok((dest_snap, bitmap_snap)), + Err(e) => { + drop(bitmap_snap); + match self.delete(bitmap_snapshot).await { + Ok(()) => Err(e), + Err(left) => Err(io::Error::new( + e.kind(), + format!("{e}; snapshot {bitmap_snapshot} is left behind: {left}"), + )), + } + } + } + } + + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + _source: &Path, + ) -> io::Result { + self.check_free(view)?; + let path = self.link_drive(view, dest_snapshot, bitmap_snapshot, Some("view"))?; + Volume::open_read_only( + view.clone(), + &path, + dest_snapshot.size_bytes, + self.chunk_size, + ) + } + + async fn remove_clone(&self, drive: &VolumeName) -> io::Result<()> { + self.drive(drive)?; + self.unlink_drive(drive) + } } #[cfg(test)] @@ -553,6 +1147,15 @@ mod tests { VolumeName::new(s).expect("valid name") } + #[test] + fn a_bitmap_holds_two_bits_per_region_and_at_least_four_mib() { + const MIB: u64 = 1 << 20; + assert_eq!(clone_bitmap_bytes(4 * MIB), 4 * MIB); + assert_eq!(clone_bitmap_bytes(26 << 30), 4 * MIB); + // 2^24 regions: 2 MiB of bits, counted twice, and 1 MiB. + assert_eq!(clone_bitmap_bytes(1 << 40), 5 * MIB); + } + #[test] fn volume_name_accepts_device_safe_names_only() { assert!(VolumeName::new("img-abc-v3").is_ok()); @@ -753,4 +1356,682 @@ mod tests { .unwrap() .is_empty()); } + + const REGION: u64 = CLONE_REGION_BYTES; + const DRIVE: u64 = 4 * REGION; + + fn hydration() -> Hydration { + Hydration { + enabled: false, + threshold_regions: 1, + batch_regions: 1, + } + } + + /// A source file of `DRIVE` bytes where byte `i` is `i % 251 + 1`. + fn source_file(dir: &Path) -> (PathBuf, Vec) { + let bytes: Vec = (0..DRIVE).map(|i| (i % 251) as u8 + 1).collect(); + let path = dir.join("source.img"); + std::fs::write(&path, &bytes).unwrap(); + (path, bytes) + } + + async fn write(volume: &Volume, expected: &mut [u8], offset: u64, bytes: Vec) { + expected[offset as usize..offset as usize + bytes.len()].copy_from_slice(&bytes); + volume.write_range(offset, bytes).await.unwrap(); + } + + async fn whole(volume: &Volume) -> Vec { + volume.read_range(0, volume.size_bytes).await.unwrap() + } + + #[tokio::test] + async fn a_file_clone_reads_the_source_and_keeps_writes_through_the_cutover() { + let (dir, disk) = disk(); + let (source, mut expected) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + // Bytes the destination held before the clone do not show. + dest.write_range(REGION, vec![0xEE; 4096]).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + + let drive = disk + .clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + assert_eq!(whole(&drive).await, expected); + assert_eq!( + disk.clone_state(&name("drive")).await.unwrap(), + CloneState::Copying { + hydrated_regions: 4, + total_regions: 4, + hydration_enabled: true + } + ); + let mut names = disk.list().await.unwrap(); + names.sort_by(|x, y| x.as_str().cmp(y.as_str())); + assert_eq!(names, vec![name("bitmap"), name("dest")]); + + write(&drive, &mut expected, REGION + 512, vec![0xAA; 4096]).await; + write( + &drive, + &mut expected, + 2 * REGION, + vec![0xBB; REGION as usize], + ) + .await; + disk.enable_hydration(&name("drive")).await.unwrap(); + disk.cut_over(&name("drive")).await.unwrap(); + disk.cut_over(&name("drive")).await.unwrap(); + assert_eq!( + disk.clone_state(&name("drive")).await.unwrap(), + CloneState::CutOver + ); + assert_eq!( + disk.enable_hydration(&name("drive")) + .await + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + disk.delete(&name("bitmap")).await.unwrap(); + + write(&drive, &mut expected, 3 * REGION, vec![0xCC; 100]).await; + assert_eq!(whole(&drive).await, expected); + assert_eq!(whole(&dest).await, expected); + let reopened = disk.open_clone(&name("drive")).await.unwrap(); + assert_eq!(reopened.path, drive.path); + assert_eq!(whole(&reopened).await, expected); + + disk.remove_clone(&name("drive")).await.unwrap(); + assert_eq!( + disk.clone_state(&name("drive")).await.unwrap_err().kind(), + io::ErrorKind::NotFound + ); + assert_eq!( + whole(&disk.open(&name("dest")).await.unwrap()).await, + expected + ); + } + + #[tokio::test] + async fn a_file_view_reads_what_the_guest_saw_at_the_snapshot() { + let (dir, disk) = disk(); + let (source, mut expected) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + let drive = disk + .clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + write(&drive, &mut expected, 4096, vec![1; 8192]).await; + drive.sync().await.unwrap(); + + let (dest_snap, bitmap_snap) = disk + .snapshot_clone( + &name("drive"), + &dest, + &bitmap, + &name("dest-snap"), + &name("bitmap-snap"), + ) + .await + .unwrap(); + drive.write_range(0, vec![2; 8192]).await.unwrap(); + + let view = disk + .clone_view(&name("view"), &dest_snap, &bitmap_snap, &source) + .await + .unwrap(); + assert_eq!(whole(&view).await, expected); + assert!(view.write_range(0, vec![3; 512]).await.is_err()); + disk.remove_clone(&name("view")).await.unwrap(); + assert_eq!(whole(&dest_snap).await, expected); + } + + #[test] + fn skip_ranges_join_adjacent_ranges_and_refuse_bad_ones() { + let r = REGION; + assert_eq!( + skip_ranges(&[0..r, r..2 * r, 2 * r..2 * r, 3 * r..4 * r], DRIVE).unwrap(), + vec![0..2 * r, 3 * r..4 * r] + ); + assert_eq!(skip_ranges(&[], DRIVE).unwrap(), vec![]); + for bad in [ + vec![0..r, r..r + 512], + vec![0..r, r + 512..2 * r], + vec![0..r, r..DRIVE + r], + vec![r..2 * r, 0..r], + vec![0..2 * r, r..3 * r], + ] { + assert_eq!( + skip_ranges(&bad, DRIVE).unwrap_err().kind(), + io::ErrorKind::InvalidInput, + "{bad:?}" + ); + } + } + + #[tokio::test] + async fn a_file_skip_changes_no_byte_and_is_refused_after_the_cutover() { + let (dir, disk) = disk(); + let (source, expected) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + let drive = disk + .clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + + disk.skip_hydration(&name("drive"), &[0..REGION, 2 * REGION..DRIVE]) + .await + .unwrap(); + assert_eq!(whole(&drive).await, expected); + assert_eq!( + disk.skip_hydration(&name("drive"), &[0..REGION, REGION..REGION + 512]) + .await + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + + disk.enable_hydration(&name("drive")).await.unwrap(); + disk.cut_over(&name("drive")).await.unwrap(); + assert_eq!( + disk.skip_hydration(&name("drive"), &[0..REGION, REGION..2 * REGION]) + .await + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + } + + #[tokio::test] + async fn a_file_clone_refuses_a_short_source_and_a_taken_name() { + let (dir, disk) = disk(); + let (source, _) = source_file(dir.path()); + let big = disk.create(&name("big"), 2 * DRIVE).await.unwrap(); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + let err = disk + .clone_drive(&name("drive"), &big, &bitmap, &source, hydration()) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + disk.open_clone(&name("drive")).await.unwrap_err().kind(), + io::ErrorKind::NotFound + ); + + disk.clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + let err = disk + .clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::AlreadyExists); + } + + /// A [`FileDisk`] whose every [`SessionDisk`] call is traced and, from + /// an armed index on, refused without effect: the volumes left behind + /// are what a process death at that op leaves. Volume reads and + /// writes are not cut; they go to the kernel in production too. + struct CrashDisk { + inner: FileDisk, + trace: std::sync::Mutex>, + crash_at: Option, + } + + impl CrashDisk { + fn new(root: &Path, crash_at: Option) -> Self { + Self { + inner: FileDisk::new(root, CHUNK).unwrap(), + trace: std::sync::Mutex::new(Vec::new()), + crash_at, + } + } + + fn trace(&self) -> Vec<&'static str> { + self.trace.lock().unwrap().clone() + } + + fn gate(&self, op: &'static str) -> io::Result<()> { + let mut trace = self.trace.lock().unwrap(); + let index = trace.len(); + trace.push(op); + if self.crash_at.is_some_and(|k| index >= k) { + return Err(io::Error::other(format!( + "injected crash at op {index} ({op})" + ))); + } + Ok(()) + } + } + + #[async_trait] + impl SessionDisk for CrashDisk { + async fn create(&self, name: &VolumeName, size_bytes: u64) -> io::Result { + self.gate("create")?; + self.inner.create(name, size_bytes).await + } + async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { + self.gate("snapshot")?; + self.inner.snapshot(origin, name).await + } + async fn open(&self, name: &VolumeName) -> io::Result { + self.gate("open")?; + self.inner.open(name).await + } + async fn delete(&self, name: &VolumeName) -> io::Result<()> { + self.gate("delete")?; + self.inner.delete(name).await + } + async fn list(&self) -> io::Result> { + self.gate("list")?; + self.inner.list().await + } + async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { + self.gate("rename")?; + self.inner.rename(from, to).await + } + async fn capacity(&self) -> io::Result { + self.gate("capacity")?; + self.inner.capacity().await + } + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + hydration: Hydration, + ) -> io::Result { + self.gate("clone_drive")?; + self.inner + .clone_drive(drive, dest, bitmap, source, hydration) + .await + } + async fn open_clone(&self, drive: &VolumeName) -> io::Result { + self.gate("open_clone")?; + self.inner.open_clone(drive).await + } + async fn clone_state(&self, drive: &VolumeName) -> io::Result { + self.gate("clone_state")?; + self.inner.clone_state(drive).await + } + async fn skip_hydration( + &self, + drive: &VolumeName, + ranges: &[Range], + ) -> io::Result<()> { + self.gate("skip_hydration")?; + self.inner.skip_hydration(drive, ranges).await + } + async fn enable_hydration(&self, drive: &VolumeName) -> io::Result<()> { + self.gate("enable_hydration")?; + self.inner.enable_hydration(drive).await + } + async fn cut_over(&self, drive: &VolumeName) -> io::Result<()> { + self.gate("cut_over")?; + self.inner.cut_over(drive).await + } + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> io::Result<(Volume, Volume)> { + self.gate("snapshot_clone")?; + self.inner + .snapshot_clone(drive, dest, bitmap, dest_snapshot, bitmap_snapshot) + .await + } + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + source: &Path, + ) -> io::Result { + self.gate("clone_view")?; + self.inner + .clone_view(view, dest_snapshot, bitmap_snapshot, source) + .await + } + async fn remove_clone(&self, drive: &VolumeName) -> io::Result<()> { + self.gate("remove_clone")?; + self.inner.remove_clone(drive).await + } + } + + async fn create_or_open(disk: &dyn SessionDisk, n: &str, size: u64) -> io::Result { + match disk.create(&name(n), size).await { + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => disk.open(&name(n)).await, + created => created, + } + } + + /// The guest's writes on pass `pass`: a partial-region write on the + /// first pass, a whole-region write on the second. + fn guest_writes(pass: u8) -> (u64, Vec) { + match pass { + 1 => (REGION + 512, vec![0xA1; 4096]), + _ => (2 * REGION, vec![0xB2; REGION as usize]), + } + } + + /// A lazy restore and its cutover, written to converge from any state + /// an earlier pass left: the drive's state decides what is left to + /// do. Each guest write that lands is applied to `expected`. + async fn lazy_restore( + disk: &dyn SessionDisk, + source: &Path, + pass: u8, + expected: &mut [u8], + ) -> io::Result { + let drive = match disk.clone_state(&name("drive")).await { + Ok(CloneState::Incomplete) => { + disk.remove_clone(&name("drive")).await?; + match disk.delete(&name("bitmap")).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + _ => {} + } + new_drive(disk, source).await? + } + Ok(_) => disk.open_clone(&name("drive")).await?, + Err(e) if e.kind() == io::ErrorKind::NotFound => new_drive(disk, source).await?, + Err(e) => return Err(e), + }; + let (offset, bytes) = guest_writes(pass); + write(&drive, expected, offset, bytes).await; + if !disk.clone_state(&name("drive")).await?.is_hydrated() { + return Err(io::Error::other("the drive is not hydrated")); + } + disk.cut_over(&name("drive")).await?; + match disk.delete(&name("bitmap")).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + _ => {} + } + Ok(drive) + } + + async fn new_drive(disk: &dyn SessionDisk, source: &Path) -> io::Result { + let dest = create_or_open(disk, "dest", DRIVE).await?; + let bitmap = create_or_open(disk, "bitmap", REGION).await?; + disk.clone_drive(&name("drive"), &dest, &bitmap, source, hydration()) + .await + } + + /// A second pass keeps every write the first pass made: a pass that + /// copied the source over a live destination fails here. + #[tokio::test] + async fn a_lazy_restore_cut_at_any_op_converges_on_the_next_pass() { + let dir = tempfile::tempdir().unwrap(); + let (source, bytes) = source_file(dir.path()); + + let recording = CrashDisk::new(&dir.path().join("recorded"), None); + lazy_restore(&recording, &source, 1, &mut bytes.clone()) + .await + .unwrap(); + assert_eq!( + recording.trace(), + vec![ + "clone_state", + "create", + "create", + "clone_drive", + "clone_state", + "cut_over", + "delete" + ] + ); + + for k in 0..recording.trace().len() { + let root = dir.path().join(format!("cut-{k}")); + let mut expected = bytes.clone(); + let cut = CrashDisk::new(&root, Some(k)); + let err = lazy_restore(&cut, &source, 1, &mut expected) + .await + .unwrap_err(); + assert!(err.to_string().contains("injected crash"), "k={k}: {err}"); + let first_pass_wrote = expected != bytes; + assert_eq!(first_pass_wrote, k > 3, "k={k}"); + + let disk = FileDisk::new(&root, CHUNK).unwrap(); + let drive = lazy_restore(&disk, &source, 2, &mut expected) + .await + .unwrap(); + assert_eq!(whole(&drive).await, expected, "k={k}"); + assert_eq!( + whole(&disk.open(&name("dest")).await.unwrap()).await, + expected, + "k={k}" + ); + assert_eq!( + disk.clone_state(&name("drive")).await.unwrap(), + CloneState::CutOver, + "k={k}" + ); + assert_eq!(disk.list().await.unwrap(), vec![name("dest")], "k={k}"); + } + } + + #[tokio::test] + async fn a_file_clone_takes_only_a_fresh_bitmap_and_unused_volumes() { + let (dir, disk) = disk(); + let (source, _) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + let small = disk.create(&name("small"), 512).await.unwrap(); + let used = disk.create(&name("used"), REGION).await.unwrap(); + used.write_range(100, vec![1]).await.unwrap(); + let clone = |bitmap: &Volume| { + let (dest, bitmap) = (dest.clone(), bitmap.clone()); + let (disk, source) = (disk.clone(), source.clone()); + async move { + disk.clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .map(drop) + .unwrap_err() + .kind() + } + }; + assert_eq!(clone(&small).await, io::ErrorKind::InvalidInput); + assert_eq!(clone(&used).await, io::ErrorKind::InvalidInput); + disk.delete(&name("small")).await.unwrap(); + assert_eq!(clone(&small).await, io::ErrorKind::NotFound); + + disk.clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + let other = disk.create(&name("other"), DRIVE).await.unwrap(); + let fresh = disk.create(&name("fresh"), REGION).await.unwrap(); + for (d, b) in [(&dest, &fresh), (&other, &bitmap)] { + let err = disk + .clone_drive(&name("second"), d, b, &source, hydration()) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::ResourceBusy, "{err}"); + } + for volume in ["dest", "bitmap"] { + assert_eq!( + disk.delete(&name(volume)).await.unwrap_err().kind(), + io::ErrorKind::ResourceBusy + ); + } + + // After the cutover the bitmap is free but holds a header. + disk.cut_over(&name("drive")).await.unwrap(); + let err = disk + .clone_drive(&name("second"), &other, &bitmap, &source, hydration()) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + disk.delete(&name("bitmap")).await.unwrap(); + assert_eq!( + disk.delete(&name("dest")).await.unwrap_err().kind(), + io::ErrorKind::ResourceBusy + ); + disk.remove_clone(&name("drive")).await.unwrap(); + disk.delete(&name("dest")).await.unwrap(); + } + + #[tokio::test] + async fn file_drives_share_the_volume_name_space_and_are_not_renamed() { + let (dir, disk) = disk(); + let (source, _) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + let err = disk + .clone_drive(&name("dest"), &dest, &bitmap, &source, hydration()) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::AlreadyExists); + disk.clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + assert_eq!( + disk.create(&name("drive"), REGION) + .await + .unwrap_err() + .kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!( + disk.snapshot(&dest, &name("drive")) + .await + .unwrap_err() + .kind(), + io::ErrorKind::AlreadyExists + ); + let other = disk.create(&name("other"), REGION).await.unwrap(); + assert_eq!( + disk.rename(&other.name, &name("drive")) + .await + .unwrap_err() + .kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!( + disk.rename(&name("drive"), &name("moved")) + .await + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + disk.remove_clone(&name("other")).await.unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + disk.remove_clone(&name("free")).await.unwrap_err().kind(), + io::ErrorKind::NotFound + ); + } + + #[tokio::test] + async fn a_file_view_opens_read_only_and_refuses_drive_operations() { + let (dir, disk) = disk(); + let (source, bytes) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + disk.clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + let (dest_snap, bitmap_snap) = disk + .snapshot_clone( + &name("drive"), + &dest, + &bitmap, + &name("dest-snap"), + &name("bitmap-snap"), + ) + .await + .unwrap(); + disk.clone_view(&name("view"), &dest_snap, &bitmap_snap, &source) + .await + .unwrap(); + + let view = disk.open_clone(&name("view")).await.unwrap(); + assert_eq!(whole(&view).await, bytes); + assert!(view.write_range(0, vec![3; 512]).await.is_err()); + assert!(matches!( + disk.clone_state(&name("view")).await.unwrap(), + CloneState::Copying { + hydration_enabled: false, + .. + } + )); + let refused = [ + disk.cut_over(&name("view")).await, + disk.enable_hydration(&name("view")).await, + disk.snapshot_clone( + &name("view"), + &dest_snap, + &bitmap_snap, + &name("s1"), + &name("s2"), + ) + .await + .map(drop), + ]; + for result in refused { + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput); + } + for volume in ["dest-snap", "bitmap-snap"] { + assert_eq!( + disk.delete(&name(volume)).await.unwrap_err().kind(), + io::ErrorKind::ResourceBusy + ); + } + disk.remove_clone(&name("view")).await.unwrap(); + disk.delete(&name("dest-snap")).await.unwrap(); + disk.delete(&name("bitmap-snap")).await.unwrap(); + } + + /// A failed `snapshot_clone` leaves no snapshot of its own and deletes + /// nothing it did not make, so a retry under a free name succeeds. + #[tokio::test] + async fn a_failed_file_snapshot_clone_removes_only_what_it_made() { + let (dir, disk) = disk(); + let (source, _) = source_file(dir.path()); + let dest = disk.create(&name("dest"), DRIVE).await.unwrap(); + let bitmap = disk.create(&name("bitmap"), REGION).await.unwrap(); + disk.clone_drive(&name("drive"), &dest, &bitmap, &source, hydration()) + .await + .unwrap(); + disk.create(&name("taken"), REGION).await.unwrap(); + let other = disk.create(&name("other"), REGION).await.unwrap(); + + let snap = |dest_snap: &'static str, bitmap: Volume| { + let (disk, dest) = (disk.clone(), dest.clone()); + async move { + disk.snapshot_clone( + &name("drive"), + &dest, + &bitmap, + &name(dest_snap), + &name("bitmap-snap"), + ) + .await + } + }; + assert_eq!( + snap("taken", other).await.unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + snap("taken", bitmap.clone()).await.unwrap_err().kind(), + io::ErrorKind::AlreadyExists + ); + let mut names = disk.list().await.unwrap(); + names.sort_by(|x, y| x.as_str().cmp(y.as_str())); + assert_eq!( + names, + vec![name("bitmap"), name("dest"), name("other"), name("taken")] + ); + snap("dest-snap", bitmap).await.unwrap(); + } } diff --git a/crates/engram-host-core/src/disk_flows.rs b/crates/engram-host-core/src/disk_flows.rs index f7c7b4306..d1d23a57b 100644 --- a/crates/engram-host-core/src/disk_flows.rs +++ b/crates/engram-host-core/src/disk_flows.rs @@ -3,7 +3,9 @@ use std::io; -use engram_chunk_store::{ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestDelta, ManifestKind}; +use engram_chunk_store::{ + ChunkHash, ChunkRef, ChunkSize, ChunkStore, Manifest, ManifestDelta, ManifestKind, +}; use futures::stream::{self, TryStreamExt}; use crate::disk::{is_zero_chunk, ChunkIndex, SessionDisk, Volume, VolumeName}; @@ -15,7 +17,7 @@ fn store_err(e: engram_chunk_store::ChunkStoreError) -> io::Error { io::Error::other(e) } -fn shape_matches(volume: &Volume, manifest: &Manifest) -> io::Result<()> { +pub(crate) fn shape_matches(volume: &Volume, manifest: &Manifest) -> io::Result<()> { if volume.chunk_size != manifest.chunk_size.as_u64() || volume.size_bytes != manifest.total_bytes { @@ -137,15 +139,31 @@ pub mod checkpoint_split { } }; - let chunk_size = snapshot.chunk_size; + upload_over(store, snapshot, indices, &base_manifest).await + } + + /// `base` with the chunks at `indices` replaced by what `from` holds + /// there. A chunk equal to the one `base` holds at its offset is not + /// uploaded; an all-zero chunk becomes a hole. + pub async fn upload_over( + store: &ChunkStore, + from: &Volume, + indices: Vec, + base: &Manifest, + ) -> io::Result { + shape_matches(from, base)?; + let chunk_size = from.chunk_size; let results: Vec<(u64, Option)> = stream::iter(indices.into_iter().map(Ok)) .map_ok(|index| async move { let offset = index.0 * chunk_size; - let bytes = snapshot.read_chunk(index).await?; + let bytes = from.read_chunk(index).await?; if is_zero_chunk(&bytes) { return Ok::<_, io::Error>((offset, None)); } - let hash = store.put_chunk(&bytes).await.map_err(store_err)?; + let hash = ChunkHash::of(&bytes); + if base.chunk_at(offset).is_none_or(|c| c.hash != hash) { + store.put_chunk(&bytes).await.map_err(store_err)?; + } Ok((offset, Some(ChunkRef { offset, hash }))) }) .try_buffer_unordered(CONCURRENCY) @@ -159,7 +177,31 @@ pub mod checkpoint_split { None => delta.zero.push(offset), } } - Ok(base_manifest.apply(&delta)) + Ok(base.apply(&delta)) + } + + /// The manifest of `from` when only the chunks at `indices` can hold + /// data: every other chunk is a hole and is not read. A chunk equal + /// to the one `known` holds at its offset is not uploaded. + pub async fn upload_only( + store: &ChunkStore, + from: &Volume, + indices: Vec, + known: &Manifest, + ) -> io::Result { + let chunk_size = from.chunk_size; + let listed: std::collections::BTreeSet = + indices.iter().map(|i| i.0 * chunk_size).collect(); + let holes = ManifestDelta { + write: Vec::new(), + zero: known + .chunks + .iter() + .map(|c| c.offset) + .filter(|offset| !listed.contains(offset)) + .collect(), + }; + upload_over(store, from, indices, &known.apply(&holes)).await } } @@ -270,6 +312,45 @@ mod tests { assert_eq!(bytes_of(&over).await, bytes_of(&c2).await); } + /// Only the listed chunks are read: an unlisted chunk is a hole, also + /// where the volume holds data or `known` lists a chunk. + #[tokio::test] + async fn an_upload_of_only_some_chunks_makes_every_other_chunk_a_hole() { + let r = rig(); + let live = r.disk.create(&name("live"), CHUNK * 4).await.unwrap(); + for (i, byte) in [1, 2, 3, 4].into_iter().enumerate() { + live.write_chunk(ChunkIndex(i as u64), fill(byte)) + .await + .unwrap(); + } + let (snap, known) = checkpoint(&r.disk, &r.store, &live, &name("c1"), None) + .await + .unwrap(); + + let manifest = checkpoint_split::upload_only( + &r.store, + &snap, + vec![ChunkIndex(1), ChunkIndex(2)], + &known, + ) + .await + .unwrap(); + + assert_eq!( + manifest.chunks, + vec![ + ChunkRef { + offset: CHUNK, + hash: ChunkHash::of(&fill(2)), + }, + ChunkRef { + offset: 2 * CHUNK, + hash: ChunkHash::of(&fill(3)), + }, + ] + ); + } + #[tokio::test] async fn stage_refuses_a_manifest_whose_shape_differs_from_its_base() { let r = rig(); diff --git a/crates/engram-host-core/src/disk_sources.rs b/crates/engram-host-core/src/disk_sources.rs new file mode 100644 index 000000000..44c0dabc7 --- /dev/null +++ b/crates/engram-host-core/src/disk_sources.rs @@ -0,0 +1,241 @@ +//! Sources: read-only block devices that each serve one disk manifest, +//! for a cloned drive to read the regions it has not copied (ADR 0127). +//! A role has at most one source, from the lazy restore to the cutover. +//! +//! A drive's I/O waits on its source, and the kernel cannot remove a +//! device that has a request nothing will finish. So a source is stopped +//! only after every drive and view over it is gone, and a source that died +//! is recovered before any drive over it is suspended or removed. + +use std::io; +use std::path::PathBuf; + +use async_trait::async_trait; +use engram_chunk_store::ChunkStore; +use engram_core::types::ids::SandboxId; +use engram_core::types::manifest::ManifestRef; +use engram_core::types::snapshot::DiskRole; +use uuid::Uuid; + +use crate::disk::{FileDisk, Volume, VolumeName}; +use crate::disk_flows::stage; + +/// What a role's source serves, as [`DiskSources::start`] was asked. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Served { + pub device: PathBuf, + pub manifest: ManifestRef, + /// The manifest of the staged base volume, when it had one. + pub base: Option, +} + +#[async_trait] +pub trait DiskSources: Send + Sync { + /// Whether [`Self::start`] can work on this host now. A host that + /// cannot start a source still recovers, serves and stops the + /// sources it has. + fn can_start(&self) -> bool; + + /// Start a source of `role` that serves `manifest`, and return its + /// device path. `base`, a staged volume and the manifest it holds, + /// serves the chunks it shares with `manifest`. `AlreadyExists` when + /// the role has a source. + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option<(&Volume, ManifestRef)>, + ) -> io::Result; + + /// Make the role's source serve I/O, starting it again on the same + /// device when it died. `NotFound` when the role has no source. + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()>; + + /// The role's source. `NotFound` when the role has none. + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> io::Result; + + /// Stop the role's source. No drive or view may read it. A role with + /// no source is left as it is. The device can stay held for a moment + /// after the last drive over it is removed (a udev probe, for one); + /// `stop` waits a bounded time for that, then fails with + /// `ResourceBusy`. + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()>; + + /// Every role that has a source. + async fn list(&self) -> io::Result>; +} + +/// Sources as files the manifest is staged into, one per role in one +/// directory: for [`FileDisk`], whose drives copy their source in full +/// before they return. A file has no process that can die. Beside each +/// file, `.served` holds the manifest and the base, the file's +/// existence holds that the source is started. +pub struct FileSources { + root: PathBuf, + store: ChunkStore, +} + +impl FileSources { + pub fn new(root: impl Into, store: ChunkStore) -> io::Result { + let root = root.into(); + std::fs::create_dir_all(&root)?; + Ok(Self { root, store }) + } + + fn name(sandbox: SandboxId, role: DiskRole) -> VolumeName { + VolumeName::new(format!("{}-{role}", sandbox.as_uuid().simple())) + .expect("a source name is 37 bytes of [a-z0-9-]") + } + + fn path(&self, sandbox: SandboxId, role: DiskRole) -> PathBuf { + self.root.join(Self::name(sandbox, role).as_str()) + } + + fn served_path(&self, sandbox: SandboxId, role: DiskRole) -> PathBuf { + self.root + .join(format!("{}.served", Self::name(sandbox, role))) + } +} + +fn invalid(what: String) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, what) +} + +#[async_trait] +impl DiskSources for FileSources { + fn can_start(&self) -> bool { + true + } + + /// Reads every chunk from the store; `base` saves nothing here. + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option<(&Volume, ManifestRef)>, + ) -> io::Result { + let contents = self + .store + .get_manifest(manifest) + .await + .map_err(io::Error::other)?; + let name = Self::name(sandbox, role); + if self.path(sandbox, role).exists() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("source {name} already exists"), + )); + } + let files = FileDisk::new(&self.root, contents.chunk_size.as_u64())?; + let base = base.map_or(String::new(), |(_, m)| format!(" {m}")); + std::fs::write(self.served_path(sandbox, role), format!("{manifest}{base}"))?; + let staged = stage(&files, &self.store, &name, &contents, None).await; + if staged.is_err() { + let _ = self.stop(sandbox, role).await; + } + Ok(staged?.path) + } + + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + let device = self.path(sandbox, role); + std::fs::metadata(&device)?; + let text = std::fs::read_to_string(self.served_path(sandbox, role))?; + let mut refs = text.split(' ').map(|r| { + r.parse::() + .map_err(|e| invalid(format!("source record {text:?}: {e}"))) + }); + let manifest = refs + .next() + .ok_or_else(|| invalid(format!("source record {text:?} names no manifest")))??; + let base = refs.next().transpose()?; + Ok(Served { + device, + manifest, + base, + }) + } + + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + std::fs::metadata(self.path(sandbox, role)).map(drop) + } + + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + for path in [self.path(sandbox, role), self.served_path(sandbox, role)] { + match std::fs::remove_file(path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + _ => {} + } + } + Ok(()) + } + + async fn list(&self) -> io::Result> { + let mut roles = Vec::new(); + for entry in std::fs::read_dir(&self.root)? { + let name = entry?.file_name(); + let Some((sandbox, role)) = name.to_str().and_then(|n| n.split_once('-')) else { + continue; + }; + if let (Ok(sandbox), Ok(role)) = (Uuid::try_parse(sandbox), role.parse()) { + roles.push((SandboxId::from(sandbox), role)); + } + } + Ok(roles) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use engram_chunk_store::{ChunkSize, Manifest, ManifestKind}; + use engram_storage_local::LocalBlobStorage; + + use super::*; + + #[tokio::test] + async fn a_file_source_holds_the_manifest_until_it_stops() { + let dir = tempfile::tempdir().unwrap(); + let store = ChunkStore::new(Arc::new(LocalBlobStorage::new(dir.path().join("blobs")))); + let manifest = ManifestRef::new(); + let empty = Manifest { + chunk_size: ChunkSize::bytes(4096), + ..Manifest::empty(ManifestKind::Disk, 8192) + }; + store.put_manifest(manifest, &empty).await.unwrap(); + let sources = FileSources::new(dir.path().join("sources"), store).unwrap(); + let (sandbox, role) = (SandboxId::new(), DiskRole::Swap); + + let path = sources.start(sandbox, role, manifest, None).await.unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), vec![0; 8192]); + let again = sources.start(sandbox, role, manifest, None).await; + assert_eq!(again.unwrap_err().kind(), io::ErrorKind::AlreadyExists); + assert!(path.exists(), "a refused start keeps the running source"); + sources.ensure_alive(sandbox, role).await.unwrap(); + assert_eq!( + sources.served(sandbox, role).await.unwrap(), + Served { + device: path.clone(), + manifest, + base: None, + } + ); + assert_eq!(sources.list().await.unwrap(), vec![(sandbox, role)]); + + sources.stop(sandbox, role).await.unwrap(); + sources.stop(sandbox, role).await.unwrap(); + assert!(sources.list().await.unwrap().is_empty()); + let gone = sources.ensure_alive(sandbox, role).await.unwrap_err(); + assert_eq!(gone.kind(), io::ErrorKind::NotFound); + let gone = sources.served(sandbox, role).await.unwrap_err(); + assert_eq!(gone.kind(), io::ErrorKind::NotFound); + assert_eq!( + std::fs::read_dir(dir.path().join("sources")) + .unwrap() + .count(), + 0 + ); + } +} diff --git a/crates/engram-host-core/src/eager_estimate.rs b/crates/engram-host-core/src/eager_estimate.rs new file mode 100644 index 000000000..40aad9599 --- /dev/null +++ b/crates/engram-host-core/src/eager_estimate.rs @@ -0,0 +1,182 @@ +//! How long an eager restore of a disk writes before the guest boots +//! (ADR 0127 Decision 1), and which disks of a restore boot over a source +//! instead. The estimate reads manifests only: the chunks the eager path +//! writes, split by whether the host holds each one locally, at a rate +//! for each kind. + +use engram_chunk_store::{ChunkHash, ChunkStoreError, Manifest}; + +/// Write rates of an eager restore, in bytes per second. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Rates { + /// A chunk the host holds locally. + pub local: f64, + /// A chunk fetched from the chunk store. + pub remote: f64, +} + +/// The bytes an eager restore writes and the time it takes. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Estimate { + pub local_bytes: u64, + pub remote_bytes: u64, + pub seconds: f64, +} + +impl Estimate { + /// The eager restore writes at least one byte. + pub fn writes(&self) -> bool { + self.local_bytes + self.remote_bytes > 0 + } +} + +/// Which disks of one restore boot over a source, from the estimate of +/// each. An eager restore writes its disks one after the other, so the +/// estimates share one budget: every disk is eager when their sum is +/// within it; otherwise disks are lazy, largest estimate first, until the +/// sum of the eager ones is within it. A disk with no estimate stays +/// eager and counts as nothing. +pub fn lazy_disks(estimates: &[Option], budget_seconds: f64) -> Vec { + let mut lazy = vec![false; estimates.len()]; + let mut eager_seconds: f64 = estimates.iter().flatten().sum(); + let mut largest_first: Vec<(usize, f64)> = estimates + .iter() + .enumerate() + .filter_map(|(i, e)| e.map(|seconds| (i, seconds))) + .collect(); + largest_first.sort_by(|a, b| b.1.total_cmp(&a.1)); + for (i, seconds) in largest_first { + if eager_seconds <= budget_seconds { + break; + } + lazy[i] = true; + eager_seconds -= seconds; + } + lazy +} + +/// The estimate of writing `wanted` onto a volume that holds `base`, or +/// onto an empty volume. `is_local` says whether the host holds a chunk. +pub fn estimate( + wanted: &Manifest, + base: Option<&Manifest>, + is_local: impl Fn(ChunkHash) -> bool, + rates: Rates, +) -> Result { + let writes = match base { + Some(base) => wanted.delta_from(base)?.write, + None => wanted.chunks.clone(), + }; + let chunk = wanted.chunk_size.as_u64(); + let (mut local_bytes, mut remote_bytes) = (0, 0); + for c in writes { + let bytes = chunk.min(wanted.total_bytes.saturating_sub(c.offset)); + if is_local(c.hash) { + local_bytes += bytes; + } else { + remote_bytes += bytes; + } + } + Ok(Estimate { + local_bytes, + remote_bytes, + seconds: local_bytes as f64 / rates.local + remote_bytes as f64 / rates.remote, + }) +} + +#[cfg(test)] +mod tests { + use engram_chunk_store::{ChunkRef, ChunkSize, ManifestKind}; + + use super::*; + + const CHUNK: u64 = 1000; + const RATES: Rates = Rates { + local: 4000.0, + remote: 1000.0, + }; + + fn hash(byte: u8) -> ChunkHash { + ChunkHash::from_bytes([byte; 32]) + } + + /// A manifest of `total` bytes whose chunk `i` has hash `pattern[i]`; + /// 0 is a hole. + fn manifest(total: u64, pattern: &[u8]) -> Manifest { + Manifest { + chunk_size: ChunkSize::bytes(CHUNK), + chunks: pattern + .iter() + .enumerate() + .filter(|(_, b)| **b != 0) + .map(|(i, b)| ChunkRef { + offset: i as u64 * CHUNK, + hash: hash(*b), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, total) + } + } + + #[test] + fn with_no_base_every_chunk_is_written_and_a_short_last_chunk_counts_its_bytes() { + let wanted = manifest(3500, &[1, 0, 3, 4]); + let e = estimate(&wanted, None, |_| false, RATES).unwrap(); + assert_eq!((e.local_bytes, e.remote_bytes), (0, 2500)); + assert_eq!(e.seconds, 2.5); + } + + #[test] + fn over_a_base_only_the_chunks_that_differ_are_written() { + let base = manifest(4000, &[1, 2, 3, 4]); + let wanted = manifest(4000, &[1, 9, 3, 0]); + let e = estimate(&wanted, Some(&base), |_| false, RATES).unwrap(); + assert_eq!((e.local_bytes, e.remote_bytes), (0, 1000)); + assert_eq!(e.seconds, 1.0); + let other_shape = manifest(5000, &[1]); + assert!(estimate(&wanted, Some(&other_shape), |_| false, RATES).is_err()); + } + + #[test] + fn local_chunks_cost_the_local_rate() { + let wanted = manifest(4000, &[1, 2, 3, 4]); + let e = estimate(&wanted, None, |_| true, RATES).unwrap(); + assert_eq!((e.local_bytes, e.remote_bytes), (4000, 0)); + assert_eq!(e.seconds, 1.0); + let e = estimate(&wanted, None, |h| h == hash(1), RATES).unwrap(); + assert_eq!((e.local_bytes, e.remote_bytes), (1000, 3000)); + assert_eq!(e.seconds, 3.25); + } + + #[test] + fn an_estimate_of_no_chunks_writes_nothing() { + let nothing = estimate(&manifest(2000, &[]), None, |_| false, RATES).unwrap(); + assert_eq!(nothing.seconds, 0.0); + assert!(!nothing.writes()); + let base = manifest(2000, &[1, 2]); + let same = estimate(&base, Some(&base), |_| false, RATES).unwrap(); + assert!(!same.writes()); + assert!(estimate(&base, None, |_| false, RATES).unwrap().writes()); + } + + #[test] + fn disks_whose_sum_is_within_the_budget_stay_eager() { + assert_eq!(lazy_disks(&[Some(1.5), Some(0.5)], 2.0), vec![false, false]); + assert_eq!(lazy_disks(&[Some(2.0)], 2.0), vec![false]); + assert_eq!(lazy_disks(&[], 2.0), Vec::::new()); + } + + #[test] + fn the_largest_disk_goes_lazy_first_until_the_rest_fits() { + assert_eq!(lazy_disks(&[Some(1.5), Some(1.0)], 2.0), vec![true, false]); + assert_eq!(lazy_disks(&[Some(1.0), Some(1.5)], 2.0), vec![false, true]); + assert_eq!(lazy_disks(&[Some(3.0), Some(2.5)], 2.0), vec![true, true]); + assert_eq!(lazy_disks(&[Some(0.0), Some(0.1)], 0.0), vec![false, true]); + } + + #[test] + fn a_disk_with_no_estimate_stays_eager_and_counts_as_nothing() { + assert_eq!(lazy_disks(&[None, Some(1.0)], 2.0), vec![false, false]); + assert_eq!(lazy_disks(&[None, Some(3.0)], 2.0), vec![false, true]); + } +} diff --git a/crates/engram-host-core/src/lib.rs b/crates/engram-host-core/src/lib.rs index fbf344ca2..f83b595ac 100644 --- a/crates/engram-host-core/src/lib.rs +++ b/crates/engram-host-core/src/lib.rs @@ -21,15 +21,17 @@ //! rename / sync_dir / read / read_dir / remove_file) so a crash injector //! can fail between each. //! - [`disk::SessionDisk`] — volumes the kernel owns (create, O(1) -//! snapshot, delete, dirty-chunk diff); [`disk_flows`] moves chunks -//! between a volume and the chunk store; [`image_volumes`] keeps the -//! enabled images staged and [`session_volumes`] owns every session's -//! volumes. +//! snapshot, delete, dirty-chunk diff, cloned drives); [`disk_flows`] +//! moves chunks between a volume and the chunk store; [`image_volumes`] +//! keeps the enabled images staged and [`session_volumes`] owns every +//! session's volumes. pub mod checkpoint; pub mod coord; pub mod disk; pub mod disk_flows; +pub mod disk_sources; +pub mod eager_estimate; pub mod finalize; pub mod fs; pub mod image_volumes; @@ -40,8 +42,12 @@ mod test_support; pub use checkpoint::checkpoint_tail_admits_publish; pub use coord::{CoordControlPlane, CoordError}; -pub use disk::{is_zero_chunk, Capacity, ChunkIndex, FileDisk, SessionDisk, Volume, VolumeName}; +pub use disk::{ + is_zero_chunk, Capacity, ChunkIndex, CloneState, FileDisk, Hydration, SessionDisk, Volume, + VolumeName, +}; pub use disk_flows::{checkpoint, checkpoint_split, stage}; +pub use disk_sources::{DiskSources, FileSources}; pub use finalize::{plan_finalize_retry, FinalizeRetry, FinalizeStage}; pub use fs::{HostFs, TokioFs}; pub use session_volumes::SessionVolumes; diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 746e8f36e..a737ae726 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -6,8 +6,26 @@ //! sb-- live //! sb---ckpt--v the last checkpoint; holds that manifest //! tmp-sb-- a stage or a checkpoint in flight +//! tmp-sb---pre a live move's snapshot while the guest runs //! ``` //! +//! A role restored before its bytes are local is a cloned drive (ADR +//! 0127). Its live name is then a drive, not a volume, over these: +//! +//! ```text +//! sb---dest--v the destination; holds the lineage like a checkpoint +//! sb---bitmap the drive's copy bitmap +//! tmp-sb---bm the bitmap snapshot of a checkpoint in flight +//! tmp-sb---view a drive that reads that checkpoint +//! ``` +//! +//! A lazy provision makes a cloned role over a source from +//! [`DiskSources`]. A checkpoint of one reads through a view until the +//! cutover ([`SessionVolumes::cut_over`]), which also deletes the bitmap +//! and stops the source; the first checkpoint after it is a `-ckpt-` +//! volume, and from then on the role checkpoints as any other. A live +//! move of a cloned role is refused. +//! //! A checkpoint volume is named after a manifest the chunk store holds, //! so the name is written last: a crash leaves a `tmp-` volume, never a //! name that lies. The manifest id in the name is the session's lineage @@ -19,8 +37,9 @@ //! storage. [`SessionVolumes::reap`] deletes kept checkpoints, oldest //! release first, when the pool needs the space. -use std::collections::VecDeque; +use std::collections::{HashMap, HashSet, VecDeque}; use std::io; +use std::ops::Range; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -30,14 +49,22 @@ use engram_core::types::manifest::ManifestRef; use engram_core::types::snapshot::DiskRole; use uuid::Uuid; -use crate::disk::{Capacity, SessionDisk, Volume, VolumeName}; -use crate::disk_flows::{checkpoint_split, stage}; +use crate::disk::{ + clone_bitmap_bytes, Capacity, CloneState, Hydration, SessionDisk, Volume, VolumeName, + CLONE_REGION_BYTES, +}; +use crate::disk_flows::{checkpoint_split, shape_matches, stage}; +use crate::disk_sources::DiskSources; use crate::image_volumes; const LIVE_PREFIX: &str = "sb-"; const TRANSIENT_PREFIX: &str = "tmp-sb-"; -const CHECKPOINT_INFIX: &str = "-ckpt-"; -const PRECOPY_SUFFIX: &str = "-pre"; +const CHECKPOINT_INFIX: &str = "ckpt-"; +const DEST_INFIX: &str = "dest-"; +const BITMAP_SUFFIX: &str = "bitmap"; +const PRECOPY_SUFFIX: &str = "pre"; +const BITMAP_SNAPSHOT_SUFFIX: &str = "bm"; +const VIEW_SUFFIX: &str = "view"; /// What a volume name says. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -45,6 +72,40 @@ pub enum Kind { Live, Checkpoint(ManifestRef), Transient, + Dest(ManifestRef), + Bitmap, + BitmapSnapshot, + View, +} + +impl Kind { + /// A volume or drive that only a cloned role has. + fn is_clone_part(self) -> bool { + match self { + Self::Dest(_) | Self::Bitmap | Self::BitmapSnapshot | Self::View => true, + Self::Live | Self::Checkpoint(_) | Self::Transient => false, + } + } + + /// The manifest a volume that holds the role's lineage names, and + /// which kind of holder it is. + fn holder(self) -> Option<(ManifestRef, Holder)> { + match self { + Self::Checkpoint(m) => Some((m, Holder::Checkpoint)), + Self::Dest(m) => Some((m, Holder::Dest)), + Self::Live | Self::Transient | Self::Bitmap | Self::BitmapSnapshot | Self::View => None, + } + } +} + +/// The volume that holds a role's lineage. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Holder { + /// A complete snapshot. + Checkpoint, + /// A cloned drive's destination, which is not complete before the + /// cutover. + Dest, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -65,44 +126,78 @@ fn store_err(e: engram_chunk_store::ChunkStoreError) -> io::Error { } } +/// `-[-]`. The longest, a checkpoint or +/// destination name with a 20-digit version, is 100 bytes: that leaves +/// 27 of device-mapper's 128 for a pool's prefix on the device name. +fn role_name(prefix: &str, sandbox: SandboxId, role: DiskRole, suffix: Option<&str>) -> VolumeName { + let sandbox = sandbox.as_uuid().simple(); + let name = match suffix { + Some(suffix) => format!("{prefix}{sandbox}-{role}-{suffix}"), + None => format!("{prefix}{sandbox}-{role}"), + }; + VolumeName::new(name).expect("a session volume name is at most 100 bytes of [a-z0-9-]") +} + +fn manifest_suffix(infix: &str, manifest: ManifestRef) -> String { + format!( + "{infix}{}-v{}", + manifest.manifest_id.simple(), + manifest.version + ) +} + pub fn live_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { - VolumeName::new(format!( - "{LIVE_PREFIX}{}-{}", - sandbox.as_uuid().simple(), - role - )) - .expect("a live volume name is 40 bytes of [a-z0-9-]") + role_name(LIVE_PREFIX, sandbox, role, None) } pub fn checkpoint_name(sandbox: SandboxId, role: DiskRole, manifest: ManifestRef) -> VolumeName { - VolumeName::new(format!( - "{LIVE_PREFIX}{}-{}{CHECKPOINT_INFIX}{}-v{}", - sandbox.as_uuid().simple(), + role_name( + LIVE_PREFIX, + sandbox, role, - manifest.manifest_id.simple(), - manifest.version - )) - .expect("a checkpoint volume name is at most 100 bytes of [a-z0-9-]") + Some(&manifest_suffix(CHECKPOINT_INFIX, manifest)), + ) +} + +/// The volume a cloned drive writes into, named after its lineage's +/// newest manifest. +pub fn dest_name(sandbox: SandboxId, role: DiskRole, manifest: ManifestRef) -> VolumeName { + role_name( + LIVE_PREFIX, + sandbox, + role, + Some(&manifest_suffix(DEST_INFIX, manifest)), + ) +} + +pub fn bitmap_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { + role_name(LIVE_PREFIX, sandbox, role, Some(BITMAP_SUFFIX)) } fn transient_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { - VolumeName::new(format!( - "{TRANSIENT_PREFIX}{}-{}", - sandbox.as_uuid().simple(), - role - )) - .expect("a transient volume name is 44 bytes of [a-z0-9-]") + role_name(TRANSIENT_PREFIX, sandbox, role, None) } /// The snapshot a live move takes while the guest runs; it lives /// beside the transient one the capture takes under the pause. fn precopy_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { - VolumeName::new(format!( - "{TRANSIENT_PREFIX}{}-{}{PRECOPY_SUFFIX}", - sandbox.as_uuid().simple(), - role - )) - .expect("a precopy volume name is 48 bytes of [a-z0-9-]") + role_name(TRANSIENT_PREFIX, sandbox, role, Some(PRECOPY_SUFFIX)) +} + +/// The bitmap half of a cloned drive's checkpoint; the transient +/// volume is the destination half. +pub fn bitmap_snapshot_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { + role_name( + TRANSIENT_PREFIX, + sandbox, + role, + Some(BITMAP_SNAPSHOT_SUFFIX), + ) +} + +/// The read-only drive over a cloned drive's checkpoint. +pub fn view_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { + role_name(TRANSIENT_PREFIX, sandbox, role, Some(VIEW_SUFFIX)) } /// What `name` says; `None` for a volume that is not a session's. @@ -113,25 +208,32 @@ pub fn parse(name: &VolumeName) -> Option { }; let (sandbox, rest) = rest.split_once('-')?; let sandbox = SandboxId::from(Uuid::try_parse(sandbox).ok()?); - let (role, rest) = match rest.split_once(CHECKPOINT_INFIX) { - Some((role, manifest)) => (role, Some(manifest)), + let (role, suffix) = match rest.split_once('-') { + Some((role, suffix)) => (role, Some(suffix)), None => (rest, None), }; - let role = match (transient, role.strip_suffix(PRECOPY_SUFFIX)) { - (true, Some(role)) => role, - _ => role, - }; let role: DiskRole = role.parse().ok()?; - let kind = match (transient, rest) { - (true, None) => Kind::Transient, + let manifest = |text: &str| { + let (id, version) = text.split_once("-v")?; + Some(ManifestRef { + manifest_id: Uuid::try_parse(id).ok()?, + version: version.parse().ok()?, + }) + }; + let kind = match (transient, suffix) { (false, None) => Kind::Live, - (false, Some(manifest)) => { - let (id, version) = manifest.split_once("-v")?; - Kind::Checkpoint(ManifestRef { - manifest_id: Uuid::try_parse(id).ok()?, - version: version.parse().ok()?, - }) - } + (false, Some(BITMAP_SUFFIX)) => Kind::Bitmap, + (false, Some(suffix)) => match ( + suffix.strip_prefix(CHECKPOINT_INFIX), + suffix.strip_prefix(DEST_INFIX), + ) { + (Some(m), _) => Kind::Checkpoint(manifest(m)?), + (_, Some(m)) => Kind::Dest(manifest(m)?), + (None, None) => return None, + }, + (true, None | Some(PRECOPY_SUFFIX)) => Kind::Transient, + (true, Some(BITMAP_SNAPSHOT_SUFFIX)) => Kind::BitmapSnapshot, + (true, Some(VIEW_SUFFIX)) => Kind::View, (true, Some(_)) => return None, }; Some(Parsed { @@ -141,10 +243,124 @@ pub fn parse(name: &VolumeName) -> Option { }) } +/// The role has a volume or drive that only a cloned role has. +fn cloned_in(names: &[VolumeName], sandbox: SandboxId, role: DiskRole) -> bool { + names + .iter() + .filter_map(parse) + .any(|p| p.sandbox == sandbox && p.role == role && p.kind.is_clone_part()) +} + +/// The roles among `parsed` that are cloned, each once. +fn cloned_roles<'a>(parsed: impl IntoIterator) -> Vec<(SandboxId, DiskRole)> { + let mut roles = Vec::new(); + for p in parsed { + if p.kind.is_clone_part() && !roles.contains(&(p.sandbox, p.role)) { + roles.push((p.sandbox, p.role)); + } + } + roles +} + +/// A role's newest lineage holder, or the manifest ids of two lineages. +type Lineage = Result<(ManifestRef, Holder), [Uuid; 2]>; + +/// The lineage of each role among `parsed`. Two lineages for one role +/// is an error: the names are the only record, so nothing says which +/// one is the session's. +fn holders<'a>( + parsed: impl IntoIterator, +) -> HashMap<(SandboxId, DiskRole), Lineage> { + let mut holders = HashMap::new(); + for p in parsed { + let Some((m, holder)) = p.kind.holder() else { + continue; + }; + let entry = holders + .entry((p.sandbox, p.role)) + .or_insert(Ok((m, holder))); + *entry = match *entry { + Ok((newest, _)) if newest.manifest_id != m.manifest_id => { + Err([newest.manifest_id, m.manifest_id]) + } + Ok((newest, _)) if m.version > newest.version => Ok((m, holder)), + kept => kept, + }; + } + holders +} + +fn two_lineages(sandbox: SandboxId, role: DiskRole, ids: [Uuid; 2]) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "sandbox {sandbox} has two {role} lineages, {} and {}", + ids[0], ids[1] + ), + ) +} + +/// The manifest the role's lineage holds and its holder: the newest of +/// its checkpoint volumes and destinations. +fn lineage_in( + names: &[VolumeName], + sandbox: SandboxId, + role: DiskRole, +) -> io::Result<(ManifestRef, Holder)> { + let parsed: Vec = names + .iter() + .filter_map(parse) + .filter(|p| p.sandbox == sandbox && p.role == role) + .collect(); + match holders(&parsed).remove(&(sandbox, role)) { + Some(holder) => holder.map_err(|ids| two_lineages(sandbox, role, ids)), + None => Err(io::Error::new( + io::ErrorKind::NotFound, + format!("sandbox {sandbox} has no {role} checkpoint volume"), + )), + } +} + +/// The name of the role's destination. +fn dest_in(names: &[VolumeName], sandbox: SandboxId, role: DiskRole) -> io::Result { + names + .iter() + .find(|name| { + parse(name).is_some_and(|p| { + p.sandbox == sandbox && p.role == role && matches!(p.kind, Kind::Dest(_)) + }) + }) + .cloned() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("sandbox {sandbox} has no {role} destination"), + ) + }) +} + +/// The refusal of `op` on a cloned role. +fn cloned_refused(sandbox: SandboxId, role: DiskRole, op: &str) -> io::Error { + io::Error::new( + io::ErrorKind::Unsupported, + format!("the {role} disk of sandbox {sandbox} is a cloned drive, which cannot {op}"), + ) +} + +/// `e`, from the source of the role, with the role named. +fn source_err(sandbox: SandboxId, role: DiskRole, e: io::Error) -> io::Error { + let what = format!("source of the {role} disk of sandbox {sandbox}: {e}"); + io::Error::new(e.kind(), what) +} + /// Where a live volume's contents come from. A manifest is written in /// full before the sandbox starts; when `base` names an image staged on /// this host, that volume is the origin and only the difference is /// written. A manifest that is itself a staged image is snapshotted. +/// +/// A lazy source makes the role a cloned drive instead: the sandbox +/// starts at once, reads what is not yet local from a source that serves +/// the manifest, and the drive copies the rest in the background. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Source { /// A manifest many sandboxes share: the volume starts a new lineage @@ -160,8 +376,102 @@ pub enum Source { }, /// Zeros, in a new lineage. Empty { size_bytes: u64 }, + /// [`Self::Fork`] over a cloned drive. + LazyFork { + manifest: ManifestRef, + base: Option, + }, + /// [`Self::Continue`] over a cloned drive. + LazyContinue { + manifest: ManifestRef, + base: Option, + }, +} + +impl Source { + /// The lazy counterpart: a fork or a continue over a cloned drive. + /// Zeros stay zeros. + pub fn into_lazy(self) -> Self { + match self { + Self::Fork { manifest, base } => Self::LazyFork { manifest, base }, + Self::Continue { manifest, base } => Self::LazyContinue { manifest, base }, + other => other, + } + } +} + +/// The byte ranges of a cloned drive whose destination already holds +/// `wanted` there, from the manifests alone. `origin` is the manifest the +/// destination holds (a snapshot of a staged volume), `None` for a new +/// volume, which reads as zeros. A chunk qualifies when both list the +/// same hash or neither lists it. Each run of such chunks is rounded +/// inward to whole regions, the unit a skip marks. +fn hydration_skips(wanted: &Manifest, origin: Option<&Manifest>) -> io::Result>> { + let zeros = Manifest { + chunk_size: wanted.chunk_size, + ..Manifest::empty(ManifestKind::Disk, wanted.total_bytes) + }; + let delta = wanted + .delta_from(origin.unwrap_or(&zeros)) + .map_err(store_err)?; + let differs: HashSet = delta + .write + .iter() + .map(|c| c.offset) + .chain(delta.zero) + .collect(); + let chunk = wanted.chunk_size.as_u64(); + let mut runs: Vec> = Vec::new(); + for offset in (0..wanted.expected_chunk_count()).map(|i| i * chunk) { + if differs.contains(&offset) { + continue; + } + let end = (offset + chunk).min(wanted.total_bytes); + match runs.last_mut() { + Some(run) if run.end == offset => run.end = end, + _ => runs.push(offset..end), + } + } + Ok(runs + .into_iter() + .map(|run| { + run.start.next_multiple_of(CLONE_REGION_BYTES) + ..run.end / CLONE_REGION_BYTES * CLONE_REGION_BYTES + }) + .filter(|run| run.start < run.end) + .collect()) +} + +/// The writes of an eager provision: `wanted`, as its difference from +/// `base` when the provision starts from a volume that holds it. +#[derive(Clone, Debug)] +pub struct EagerPlan { + pub wanted: Manifest, + pub base: Option, +} + +/// Where an eager provision starts. +enum Origin { + /// The staged image volume that holds `wanted`, to snapshot. + Staged { volume: Volume, wanted: Manifest }, + /// The checkpoint `owner` kept at the manifest, to adopt. + Kept { owner: SandboxId, volume: Volume }, + /// `wanted` staged into a new volume, onto `base` (a volume and the + /// manifest it holds) when there is one. + Stage { + wanted: Manifest, + base: Option<(Volume, Manifest)>, + }, } +/// The kernel's default copy depth for a cloned drive. The drive is made +/// with hydration off, so it copies nothing before its skips are marked. +const LAZY_HYDRATION: Hydration = Hydration { + enabled: false, + threshold_regions: 1, + batch_regions: 1, +}; + /// What `provision` made: the device paths a backend opens and the /// lineage each volume checkpoints under. #[derive(Clone, Debug, PartialEq, Eq)] @@ -173,7 +483,7 @@ pub struct Provisioned { } /// A snapshot taken under the guest pause, not yet published. Dropping -/// it leaves the transient volume for the next `take`, `pending` or +/// it leaves the transient volumes for the next `take`, `pending` or /// `release` to find. #[derive(Debug)] pub struct PendingCheckpoint { @@ -181,6 +491,70 @@ pub struct PendingCheckpoint { role: DiskRole, snapshot: Volume, previous: ManifestRef, + shape: Shape, +} + +/// What a pending checkpoint snapshotted, which says how it publishes. +#[derive(Debug)] +enum Shape { + /// A volume next to the checkpoint volume of `previous`: the upload + /// is the difference from that volume. + Volume, + /// A cloned drive's destination with this bitmap snapshot, taken + /// together before the cutover. The destination snapshot reads zeros + /// where the drive had not copied, so the upload reads through a view. + Hydrating { bitmap: Volume }, + /// A cloned drive's destination after the cutover, with no complete + /// volume of `previous`: the upload hashes every chunk the snapshot + /// maps. + CutOver, +} + +/// The regions a cloned drive has copied. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Copied { + pub hydrated_regions: u64, + pub total_regions: u64, +} + +/// Where a cloned role is on its way to a finished cutover. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Hydrating { + /// Regions are still to copy. + Copying(Copied), + /// Every region is copied, or the drive is cut over and its bitmap + /// or source is left: [`SessionVolumes::cut_over`] finishes it. + Ready, +} + +/// What [`SessionVolumes::cut_over`] did. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Cutover { + /// The drive maps its destination alone, and the bitmap and the + /// source are gone. + Done, + /// Nothing changed; the cutover can go ahead later. + Waiting(Waiting), +} + +/// Why a cutover waits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Waiting { + /// Regions are still to copy. + Copying(Copied), + /// A checkpoint of the role is in flight, or a crash left one: its + /// publish reads through a view over the source. + Checkpoint, +} + +/// Where a role's writes are. +enum Written { + /// Its live volume. + Live, + /// The destination, named here, under a cut-over drive. + CutOver(VolumeName), + /// The destination, named here, under a drive that copies. + Copying(VolumeName), } impl PendingCheckpoint { @@ -205,6 +579,7 @@ impl PendingCheckpoint { #[derive(Debug, Default)] pub struct Report { + /// The volumes deleted and the drives removed. pub deleted: Vec, pub failed: Vec<(VolumeName, io::Error)>, /// Sandboxes that still have a checkpoint volume. @@ -215,6 +590,8 @@ pub struct Report { pub struct SessionVolumes { disk: Arc, store: ChunkStore, + /// `None` refuses a lazy source. + sources: Option>, /// Released sandboxes whose checkpoint volumes stay, oldest release /// first. A sandbox `reconcile` finds with no live volume joins at /// the front: its age is unknown, so it goes first. @@ -226,10 +603,16 @@ impl SessionVolumes { Self { disk, store, + sources: None, kept: Mutex::new(VecDeque::new()), } } + pub fn with_sources(mut self, sources: Arc) -> Self { + self.sources = Some(sources); + self + } + pub fn disk(&self) -> &dyn SessionDisk { self.disk.as_ref() } @@ -273,10 +656,11 @@ impl SessionVolumes { provisioned } - /// The checkpoint volume first, under the transient name until its - /// manifest is in the store; then the live volume as its snapshot. - /// A `Continue` of a checkpoint this host kept for the lineage skips - /// the stage; one of a newer manifest stages only the delta onto it. + /// Eager: the checkpoint volume first, under the transient name until + /// its manifest is in the store; then the live volume as its + /// snapshot. A `Continue` of a checkpoint this host kept for the + /// lineage skips the stage; one of a newer manifest stages only the + /// delta onto it. Lazy: a cloned drive, see [`Self::clone_role`]. async fn provision_role( &self, sandbox: SandboxId, @@ -285,66 +669,225 @@ impl SessionVolumes { ) -> io::Result<(PathBuf, ManifestRef)> { let tmp = transient_name(sandbox, role); self.delete_if_present(&tmp).await?; - let (checkpoint, lineage) = match source { - Source::Fork { manifest, base } => { - let wanted = self.store.get_manifest(manifest).await.map_err(store_err)?; - match self.staged(manifest).await { - Some((origin, _)) => self.disk.snapshot(&origin, &tmp).await?, - None => self.stage_into(&tmp, &wanted, base).await?, - }; - let lineage = ManifestRef::new(); - self.store - .put_manifest(lineage, &wanted) - .await - .map_err(store_err)?; - (self.commit(&tmp, sandbox, role, lineage).await?, lineage) - } - Source::Continue { manifest, base } => { - match self.kept(sandbox, role, manifest).await? { - Some((owner, volume, _, held)) if held == manifest => ( - self.adopt(owner, volume, sandbox, role, held).await?, - manifest, - ), - kept => { - let wanted = self.store.get_manifest(manifest).await.map_err(store_err)?; - match kept { - Some((_, volume, held_manifest, _)) - if volume.size_bytes == wanted.total_bytes => - { - stage( - self.disk.as_ref(), - &self.store, - &tmp, - &wanted, - Some((&volume, &held_manifest)), - ) - .await? - } - _ => self.stage_into(&tmp, &wanted, base).await?, - }; - (self.commit(&tmp, sandbox, role, manifest).await?, manifest) + Ok(match source { + Source::Fork { manifest, base } | Source::Continue { manifest, base } => { + let continues = matches!(source, Source::Continue { .. }); + let wanted = match self + .eager_origin(sandbox, role, manifest, base, continues) + .await? + { + Origin::Kept { owner, volume } => { + let checkpoint = self.adopt(owner, volume, sandbox, role, manifest).await?; + return Ok((self.live_over(&checkpoint, sandbox, role).await?, manifest)); } - } + Origin::Staged { volume, wanted } => { + self.disk.snapshot(&volume, &tmp).await?; + wanted + } + Origin::Stage { wanted, base } => { + let base = base.as_ref().map(|(volume, held)| (volume, held)); + stage(self.disk.as_ref(), &self.store, &tmp, &wanted, base).await?; + wanted + } + }; + let lineage = if continues { + manifest + } else { + self.new_lineage(&wanted).await? + }; + let checkpoint = self.commit(&tmp, sandbox, role, lineage).await?; + (self.live_over(&checkpoint, sandbox, role).await?, lineage) } Source::Empty { size_bytes } => { let volume = self.disk.create(&tmp, size_bytes).await?; - let lineage = ManifestRef::new(); let manifest = Manifest { chunk_size: ChunkSize::bytes(volume.chunk_size), ..Manifest::empty(ManifestKind::Disk, size_bytes) }; - self.store - .put_manifest(lineage, &manifest) - .await - .map_err(store_err)?; - (self.commit(&tmp, sandbox, role, lineage).await?, lineage) + let lineage = self.new_lineage(&manifest).await?; + let checkpoint = self.commit(&tmp, sandbox, role, lineage).await?; + (self.live_over(&checkpoint, sandbox, role).await?, lineage) + } + Source::LazyFork { manifest, base } => { + let wanted = self.store.get_manifest(manifest).await.map_err(store_err)?; + let lineage = self.new_lineage(&wanted).await?; + let drive = self + .clone_role(sandbox, role, manifest, &wanted, base, lineage) + .await?; + (drive, lineage) + } + Source::LazyContinue { manifest, base } => { + let wanted = self.store.get_manifest(manifest).await.map_err(store_err)?; + let drive = self + .clone_role(sandbox, role, manifest, &wanted, base, manifest) + .await?; + (drive, manifest) + } + }) + } + + /// Where an eager provision of `manifest` starts. A fork snapshots the + /// staged image of the manifest when this host has it; a continue + /// adopts the checkpoint this host kept at the manifest, or stages the + /// difference onto a kept checkpoint of the lineage. Otherwise the + /// manifest is staged onto the staged image of `base`, or in full. + async fn eager_origin( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option, + continues: bool, + ) -> io::Result { + let kept = if continues { + self.kept(sandbox, role, manifest).await? + } else { + if let Some((volume, wanted)) = self.staged(manifest).await { + return Ok(Origin::Staged { volume, wanted }); + } + None + }; + let kept = match kept { + Some((owner, volume, _, held)) if held == manifest => { + return Ok(Origin::Kept { owner, volume }) + } + kept => kept, + }; + let wanted = self.store.get_manifest(manifest).await.map_err(store_err)?; + let base = match kept { + Some((_, volume, held_manifest, _)) if volume.size_bytes == wanted.total_bytes => { + Some((volume, held_manifest)) + } + _ => match base { + Some(image) => self.staged(image).await, + None => None, + }, + }; + Ok(Origin::Stage { wanted, base }) + } + + /// What an eager provision of `source` would write, read from the + /// manifests and the volume names only. `None` when it writes nothing: + /// it snapshots a staged image, adopts a kept checkpoint, or makes + /// zeros. A lazy source is planned as its eager counterpart. + pub async fn eager_plan( + &self, + sandbox: SandboxId, + role: DiskRole, + source: Source, + ) -> io::Result> { + let (manifest, base, continues) = match source { + Source::Fork { manifest, base } | Source::LazyFork { manifest, base } => { + (manifest, base, false) } + Source::Continue { manifest, base } | Source::LazyContinue { manifest, base } => { + (manifest, base, true) + } + Source::Empty { .. } => return Ok(None), }; + Ok( + match self + .eager_origin(sandbox, role, manifest, base, continues) + .await? + { + Origin::Stage { wanted, base } => Some(EagerPlan { + wanted, + base: base.map(|(_, held)| held), + }), + Origin::Staged { .. } | Origin::Kept { .. } => None, + }, + ) + } + + /// A lazy provision can start the source it needs. + pub fn can_start_sources(&self) -> bool { + self.sources.as_ref().is_some_and(|s| s.can_start()) + } + + /// A new lineage that holds a copy of `manifest`. + async fn new_lineage(&self, manifest: &Manifest) -> io::Result { + let lineage = ManifestRef::new(); + self.store + .put_manifest(lineage, manifest) + .await + .map_err(store_err)?; + Ok(lineage) + } + + /// The live volume of a role, as a snapshot of its checkpoint. + async fn live_over( + &self, + checkpoint: &Volume, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result { let live = self .disk - .snapshot(&checkpoint, &live_name(sandbox, role)) + .snapshot(checkpoint, &live_name(sandbox, role)) + .await?; + Ok(live.path) + } + + /// Make the role a cloned drive that reads `wanted` from a source + /// serving `served`, and return the drive's path. The destination is + /// a snapshot of the staged volume of `served` or of `base` when this + /// host has one, else a new volume; it is named after `lineage`. + /// Hydration skips the regions the destination already holds as + /// `wanted` ([`hydration_skips`]). A checkpoint takes the source + /// manifest's chunk wherever the destination was not written, which + /// there is the destination's bytes. A failure leaves the rest to the + /// caller's purge. + async fn clone_role( + &self, + sandbox: SandboxId, + role: DiskRole, + served: ManifestRef, + wanted: &Manifest, + base: Option, + lineage: ManifestRef, + ) -> io::Result { + let sources = self.sources(role)?; + let bitmap = self + .disk + .create( + &bitmap_name(sandbox, role), + clone_bitmap_bytes(wanted.total_bytes), + ) .await?; - Ok((live.path, lineage)) + let staged = match self.staged(served).await { + Some((volume, held)) => Some((volume, served, held)), + None => match base { + Some(image) => self + .staged(image) + .await + .map(|(volume, held)| (volume, image, held)), + None => None, + }, + }; + let name = dest_name(sandbox, role, lineage); + let dest = match &staged { + Some((origin, _, _)) => self.disk.snapshot(origin, &name).await?, + None => self.disk.create(&name, wanted.total_bytes).await?, + }; + shape_matches(&dest, wanted)?; + let skips = hydration_skips(wanted, staged.as_ref().map(|(_, _, held)| held))?; + let source = sources + .start( + sandbox, + role, + served, + staged.as_ref().map(|(volume, image, _)| (volume, *image)), + ) + .await?; + let drive = live_name(sandbox, role); + let path = self + .disk + .clone_drive(&drive, &dest, &bitmap, &source, LAZY_HYDRATION) + .await? + .path; + self.disk.skip_hydration(&drive, &skips).await?; + self.disk.enable_hydration(&drive).await?; + Ok(path) } /// Give the staged transient volume its checkpoint name and delete @@ -441,28 +984,6 @@ impl SessionVolumes { self.disk.open(&name).await } - /// Write `wanted` into a new volume `name`, from the staged volume of - /// `base` when this host has it. - async fn stage_into( - &self, - name: &VolumeName, - wanted: &Manifest, - base: Option, - ) -> io::Result { - let base = match base { - Some(image) => self.staged(image).await, - None => None, - }; - stage( - self.disk.as_ref(), - &self.store, - name, - wanted, - base.as_ref().map(|(v, m)| (v, m)), - ) - .await - } - /// The staged volume of an image and its manifest, when this host /// has both. async fn staged(&self, image: ManifestRef) -> Option<(Volume, Manifest)> { @@ -475,77 +996,293 @@ impl SessionVolumes { } } - /// The roles the sandbox has a live volume for. + /// The roles the sandbox has a live volume or a cloned drive for. + /// [`SessionDisk::list`] does not list drives; a drive holds its + /// destination, so a cloned role always has a listed volume. pub async fn roles(&self, sandbox: SandboxId) -> io::Result> { let names = self.disk.list().await?; Ok(DiskRole::ALL .into_iter() - .filter(|role| names.contains(&live_name(sandbox, *role))) + .filter(|role| { + names.contains(&live_name(sandbox, *role)) || cloned_in(&names, sandbox, *role) + }) .collect()) } - /// A live volume, the one the hypervisor opens. + /// A live volume, the one the hypervisor opens: for a cloned role, + /// the drive once it is cut over. `Unsupported` while it copies. pub async fn live(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { - self.disk.open(&live_name(sandbox, role)).await + let name = live_name(sandbox, role); + match self.disk.open(&name).await { + Err(_) if self.cloned(sandbox, role).await? => { + match self.disk.clone_state(&name).await? { + CloneState::CutOver => self.disk.open_clone(&name).await, + _ => Err(cloned_refused( + sandbox, + role, + "open as a volume while it copies", + )), + } + } + opened => opened, + } } - /// The device path of a live volume. + /// The device path the hypervisor opens: the live volume's, or the + /// drive's for a cloned role. pub async fn path(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { - Ok(self.live(sandbox, role).await?.path) + let name = live_name(sandbox, role); + match self.disk.open(&name).await { + Err(_) if self.cloned(sandbox, role).await? => { + Ok(self.disk.open_clone(&name).await?.path) + } + opened => Ok(opened?.path), + } + } + + async fn cloned(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + Ok(cloned_in(&self.disk.list().await?, sandbox, role)) + } + + /// The role's lineage, its holder, and where its writes are. Only the + /// kernel knows whether a drive still copies; after its cutover the + /// role snapshots its destination as a plain role snapshots its live + /// volume. + async fn written( + &self, + names: &[VolumeName], + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result<(ManifestRef, Holder, Written)> { + let (previous, holder) = lineage_in(names, sandbox, role)?; + if !cloned_in(names, sandbox, role) { + return Ok((previous, holder, Written::Live)); + } + let drive = live_name(sandbox, role); + let dest = dest_in(names, sandbox, role)?; + let written = match (self.disk.clone_state(&drive).await?, holder) { + (CloneState::Copying { .. }, Holder::Dest) => Written::Copying(dest), + (CloneState::CutOver, _) => Written::CutOver(dest), + (state, holder) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{drive} is {state:?} with its lineage on a {holder:?}"), + )) + } + }; + Ok((previous, holder, written)) + } + + /// Sync a cut-over drive and snapshot its destination `dest` as + /// `name`. + async fn snapshot_under_drive( + &self, + sandbox: SandboxId, + role: DiskRole, + dest: &VolumeName, + name: &VolumeName, + ) -> io::Result { + self.disk + .open_clone(&live_name(sandbox, role)) + .await? + .sync() + .await?; + let dest = self.disk.open(dest).await?; + self.disk.snapshot(&dest, name).await } - /// Under the guest pause: sync the live volume and snapshot it. + /// Under the guest pause: sync the live volume and snapshot it. A + /// cloned role syncs its drive and snapshots the destination, with + /// the bitmap while the drive copies. pub async fn take(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { - let previous = self.checkpoint_of(sandbox, role).await?; + let names = self.disk.list().await?; + let (previous, holder, written) = self.written(&names, sandbox, role).await?; let tmp = transient_name(sandbox, role); - self.delete_if_present(&tmp).await?; - let live = self.live(sandbox, role).await?; - let snapshot = checkpoint_split::take(self.disk.as_ref(), &live, &tmp).await?; - Ok(PendingCheckpoint { + let pending = |snapshot, shape| PendingCheckpoint { sandbox, role, snapshot, previous, - }) + shape, + }; + match written { + Written::Live => { + self.delete_if_present(&tmp).await?; + let live = self.live(sandbox, role).await?; + let snapshot = checkpoint_split::take(self.disk.as_ref(), &live, &tmp).await?; + Ok(pending(snapshot, Shape::Volume)) + } + Written::CutOver(dest) => { + self.discard_pair(sandbox, role).await?; + let snapshot = self + .snapshot_under_drive(sandbox, role, &dest, &tmp) + .await?; + let shape = match holder { + Holder::Checkpoint => Shape::Volume, + Holder::Dest => Shape::CutOver, + }; + Ok(pending(snapshot, shape)) + } + Written::Copying(dest) => { + self.discard_pair(sandbox, role).await?; + // A suspend waits for the source's reads in flight. + self.sources(role)?.ensure_alive(sandbox, role).await?; + let drive = live_name(sandbox, role); + self.disk.open_clone(&drive).await?.sync().await?; + let dest = self.disk.open(&dest).await?; + let bitmap = self.disk.open(&bitmap_name(sandbox, role)).await?; + let (snapshot, bitmap) = self + .disk + .snapshot_clone( + &drive, + &dest, + &bitmap, + &tmp, + &bitmap_snapshot_name(sandbox, role), + ) + .await?; + Ok(pending(snapshot, Shape::Hydrating { bitmap })) + } + } } /// The checkpoint a `take` left unpublished, found by name: a new /// process redrives the publish after a crash. `NotFound` when - /// there is none. + /// there is none. A destination snapshot without its bitmap snapshot + /// while the drive copies is `InvalidData`: alone it reads zeros + /// where the drive had not copied. pub async fn pending( &self, sandbox: SandboxId, role: DiskRole, ) -> io::Result { - let previous = self.checkpoint_of(sandbox, role).await?; - let snapshot = self.disk.open(&transient_name(sandbox, role)).await?; + let names = self.disk.list().await?; + let (previous, holder) = lineage_in(&names, sandbox, role)?; + let tmp = transient_name(sandbox, role); + let snapshot = self.disk.open(&tmp).await?; + let shape = match holder { + Holder::Checkpoint => Shape::Volume, + Holder::Dest => match self.disk.open(&bitmap_snapshot_name(sandbox, role)).await { + Ok(bitmap) => Shape::Hydrating { bitmap }, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let drive = live_name(sandbox, role); + match self.disk.clone_state(&drive).await { + Ok(CloneState::CutOver) => Shape::CutOver, + state => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{tmp} has no bitmap snapshot and {drive} is {state:?}; \ + take the checkpoint again" + ), + )) + } + } + } + Err(e) => return Err(e), + }, + }; Ok(PendingCheckpoint { sandbox, role, snapshot, previous, + shape, }) } - /// A snapshot of the live volume while the guest runs, for a live - /// move to carry before the pause. It is independent of the - /// transient snapshot [`Self::take`] makes, so the capture can take - /// its own beside it. A precopy left by an earlier move is replaced. - pub async fn take_precopy( + /// Delete what a cloned role's earlier checkpoint left: the view and + /// the snapshot pair. + async fn discard_pair(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + self.remove_clone_of(sandbox, role, &view_name(sandbox, role)) + .await?; + self.delete_if_present(&transient_name(sandbox, role)) + .await?; + self.delete_if_present(&bitmap_snapshot_name(sandbox, role)) + .await + } + + /// Remove the drive or view `name` of the role; `false` when the name + /// is not one. The role's source is recovered first: the kernel + /// cannot remove a device that holds a read no source will finish. + async fn remove_clone_of( &self, sandbox: SandboxId, role: DiskRole, - ) -> io::Result { - let previous = self.checkpoint_of(sandbox, role).await?; - let name = precopy_name(sandbox, role); - self.delete_if_present(&name).await?; - let live = self.live(sandbox, role).await?; - let snapshot = checkpoint_split::take(self.disk.as_ref(), &live, &name).await?; + name: &VolumeName, + ) -> io::Result { + if let Some(sources) = &self.sources { + match sources.ensure_alive(sandbox, role).await { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(source_err(sandbox, role, e)); + } + _ => {} + } + } + match self.disk.remove_clone(name).await { + Ok(()) => Ok(true), + // `InvalidInput`: the name is a volume. + Err(e) + if matches!( + e.kind(), + io::ErrorKind::NotFound | io::ErrorKind::InvalidInput + ) => + { + Ok(false) + } + Err(e) => Err(e), + } + } + + fn sources(&self, role: DiskRole) -> io::Result<&dyn DiskSources> { + self.sources.as_deref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::Unsupported, + format!("a cloned {role} disk needs disk sources, which this host does not have"), + ) + }) + } + + /// A snapshot of the live volume while the guest runs, for a live + /// move to carry before the pause. It is independent of the + /// transient snapshot [`Self::take`] makes, so the capture can take + /// its own beside it. A precopy left by an earlier move is replaced. + /// A cloned role moves once a checkpoint volume newer than its + /// destination exists, which needs the cutover; before that the move + /// has no complete volume to carry a difference from. + pub async fn take_precopy( + &self, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result { + let names = self.disk.list().await?; + let (previous, holder, written) = self.written(&names, sandbox, role).await?; + let name = precopy_name(sandbox, role); + let snapshot = match (written, holder) { + (Written::Live, _) => { + self.delete_if_present(&name).await?; + let live = self.live(sandbox, role).await?; + checkpoint_split::take(self.disk.as_ref(), &live, &name).await? + } + (Written::CutOver(dest), Holder::Checkpoint) => { + self.delete_if_present(&name).await?; + self.snapshot_under_drive(sandbox, role, &dest, &name) + .await? + } + (Written::CutOver(_) | Written::Copying(_), _) => { + return Err(cloned_refused( + sandbox, + role, + "move live before its first checkpoint after the cutover", + )) + } + }; Ok(PendingCheckpoint { sandbox, role, snapshot, previous, + shape: Shape::Volume, }) } @@ -584,27 +1321,47 @@ impl SessionVolumes { .await } - /// Outside the pause: upload what changed since the last checkpoint, - /// publish the manifest at the lineage's next free version, and make - /// the snapshot the sandbox's checkpoint volume. + /// Outside the pause: upload what changed since the last checkpoint + /// and publish the manifest at the lineage's next free version. The + /// snapshot becomes the sandbox's checkpoint volume; a cloned role's + /// destination takes the new version's name until the cutover. pub async fn publish(&self, pending: PendingCheckpoint) -> io::Result { let PendingCheckpoint { sandbox, role, snapshot, previous, + shape, } = pending; let previous_name = checkpoint_name(sandbox, role, previous); - let previous_volume = self.disk.open(&previous_name).await?; - let previous_manifest = self.store.get_manifest(previous).await.map_err(store_err)?; - let manifest = checkpoint_split::upload( - self.disk.as_ref(), - &self.store, - &snapshot, - Some((&previous_volume, &previous_manifest)), - ) - .await?; - drop(previous_volume); + let manifest = match &shape { + Shape::Volume => { + let previous_volume = self.disk.open(&previous_name).await?; + let previous_manifest = + self.store.get_manifest(previous).await.map_err(store_err)?; + checkpoint_split::upload( + self.disk.as_ref(), + &self.store, + &snapshot, + Some((&previous_volume, &previous_manifest)), + ) + .await? + } + Shape::Hydrating { bitmap } => { + self.upload_through_view(sandbox, role, &snapshot, bitmap) + .await? + } + // A chunk the snapshot does not map reads as zeros, so only the + // mapped chunks are read; one equal to the durable previous + // manifest's chunk is not uploaded. + Shape::CutOver => { + let previous_manifest = + self.store.get_manifest(previous).await.map_err(store_err)?; + let mapped = self.disk.written_chunks(None, &snapshot).await?; + checkpoint_split::upload_only(&self.store, &snapshot, mapped, &previous_manifest) + .await? + } + }; // The store, not the volume name, says which versions exist: a // crash after `put_manifest` and before the rename leaves a // version the name never reached. @@ -624,13 +1381,79 @@ impl SessionVolumes { .map_err(store_err)?; let tmp = snapshot.name.clone(); drop(snapshot); - self.disk - .rename(&tmp, &checkpoint_name(sandbox, role, next)) - .await?; - self.disk.delete(&previous_name).await?; + match shape { + Shape::Volume => { + self.disk + .rename(&tmp, &checkpoint_name(sandbox, role, next)) + .await?; + self.disk.delete(&previous_name).await?; + } + Shape::CutOver => { + self.disk + .rename(&tmp, &checkpoint_name(sandbox, role, next)) + .await?; + } + // A crash after the rename leaves the pair, which publishes + // again under the version after `next`, with the same bytes. + Shape::Hydrating { bitmap } => { + drop(bitmap); + self.disk + .rename( + &dest_name(sandbox, role, previous), + &dest_name(sandbox, role, next), + ) + .await?; + self.discard_pair(sandbox, role).await?; + } + } Ok(next) } + /// The source's manifest with every chunk the destination snapshot + /// can have written since its origin replaced by what the view over + /// the pair reads there. A chunk the destination never wrote is a + /// chunk the drive had not copied, which the source serves. + async fn upload_through_view( + &self, + sandbox: SandboxId, + role: DiskRole, + snapshot: &Volume, + bitmap: &Volume, + ) -> io::Result { + let sources = self.sources(role)?; + sources.ensure_alive(sandbox, role).await?; + let served = sources.served(sandbox, role).await?; + let name = view_name(sandbox, role); + self.remove_clone_of(sandbox, role, &name).await?; + let view = self + .disk + .clone_view(&name, snapshot, bitmap, &served.device) + .await?; + let uploaded = async { + let origin = match served.base { + Some(base) => match self.disk.open(&image_volumes::volume_name(base)).await { + Ok(origin) => Some(origin).filter(|o| o.size_bytes == snapshot.size_bytes), + Err(e) if e.kind() == io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }, + None => None, + }; + let written = self.disk.written_chunks(origin.as_ref(), snapshot).await?; + let source = self + .store + .get_manifest(served.manifest) + .await + .map_err(store_err)?; + checkpoint_split::upload_over(&self.store, &view, written, &source).await + } + .await; + drop(view); + let removed = self.remove_clone_of(sandbox, role, &name).await; + let manifest = uploaded?; + removed?; + Ok(manifest) + } + /// [`Self::take`] then [`Self::publish`], for a guest that is paused /// throughout or has no guest. pub async fn checkpoint(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { @@ -638,18 +1461,178 @@ impl SessionVolumes { self.publish(pending).await } - /// Delete a pending checkpoint's snapshot without publishing it. + /// Delete a pending checkpoint's snapshots without publishing them. pub async fn discard(&self, pending: PendingCheckpoint) -> io::Result<()> { - let name = pending.snapshot.name.clone(); - drop(pending); - self.disk.delete(&name).await + let PendingCheckpoint { + sandbox, + role, + snapshot, + shape, + .. + } = pending; + let name = snapshot.name.clone(); + drop(snapshot); + match shape { + Shape::Hydrating { bitmap } => { + drop(bitmap); + self.discard_pair(sandbox, role).await + } + Shape::Volume | Shape::CutOver => self.disk.delete(&name).await, + } + } + + /// Where the role is on its way to a finished cutover; `None` when it + /// is not a cloned role or its cutover is finished. While the drive + /// copies, a source that died is started again first: the guest's + /// reads of uncopied regions wait on it. + pub async fn hydration( + &self, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result> { + let names = self.disk.list().await?; + if !cloned_in(&names, sandbox, role) { + return Ok(None); + } + Ok(match self.clone_progress(sandbox, role).await? { + Some(copied) => { + self.sources(role)? + .ensure_alive(sandbox, role) + .await + .map_err(|e| source_err(sandbox, role, e))?; + Some(if copied.hydrated_regions < copied.total_regions { + Hydrating::Copying(copied) + } else { + Hydrating::Ready + }) + } + None => { + let sourced = match &self.sources { + Some(sources) => sources.list().await?.contains(&(sandbox, role)), + None => false, + }; + (sourced || names.contains(&bitmap_name(sandbox, role))).then_some(Hydrating::Ready) + } + }) + } + + /// Cut a hydrated cloned role over to its destination, then delete + /// its bitmap. The caller serializes this with the role's + /// checkpoints, from `take` to `publish`, and then stops the source + /// with [`Self::stop_source`] outside that lock. A crash at any step + /// leaves a state the next call finishes: [`Self::hydration`] reports + /// `Ready` while the bitmap or the source is left. `InvalidInput` when + /// the role is not cloned. + pub async fn cut_over(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + let names = self.disk.list().await?; + if !cloned_in(&names, sandbox, role) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("the {role} disk of sandbox {sandbox} is not a cloned drive"), + )); + } + let progress = self.clone_progress(sandbox, role).await?; + if let Some(copied) = progress.filter(|c| c.hydrated_regions < c.total_regions) { + return Ok(Cutover::Waiting(Waiting::Copying(copied))); + } + if self.checkpoint_in_flight(&names, sandbox, role).await? { + return Ok(Cutover::Waiting(Waiting::Checkpoint)); + } + if progress.is_some() { + // A suspend waits for the source's reads in flight. The caller + // recovers a dead source before it takes the lock (through + // `hydration`), so this check is cheap unless the source died + // since. + self.sources(role)? + .ensure_alive(sandbox, role) + .await + .map_err(|e| source_err(sandbox, role, e))?; + self.disk.cut_over(&live_name(sandbox, role)).await?; + } + self.delete_if_present(&bitmap_name(sandbox, role)).await?; + Ok(Cutover::Done) + } + + /// Stop the source of a role that is cut over. No drive reads it: the + /// live drive is linear, and a checkpoint of a cut-over role makes no + /// view. `InvalidInput` while the drive copies or a view exists. A + /// role with no source is left as it is. + pub async fn stop_source(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + let Some(sources) = &self.sources else { + return Ok(()); + }; + let names = self.disk.list().await?; + let copies = + cloned_in(&names, sandbox, role) && self.clone_progress(sandbox, role).await?.is_some(); + let viewed = match self.disk.clone_state(&view_name(sandbox, role)).await { + Ok(_) => true, + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => return Err(e), + }; + if copies || viewed { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("a drive of the {role} disk of sandbox {sandbox} still reads its source"), + )); + } + sources + .stop(sandbox, role) + .await + .map_err(|e| source_err(sandbox, role, e)) + } + + /// The regions the role's drive has copied; `None` once it is cut + /// over. + async fn clone_progress( + &self, + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result> { + let drive = live_name(sandbox, role); + match self.disk.clone_state(&drive).await? { + CloneState::Copying { + hydrated_regions, + total_regions, + .. + } => Ok(Some(Copied { + hydrated_regions, + total_regions, + })), + CloneState::CutOver => Ok(None), + CloneState::Incomplete => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{drive} has no table"), + )), + } + } + + /// A part of a checkpoint of the role exists: the snapshot pair, or + /// the view over it. + async fn checkpoint_in_flight( + &self, + names: &[VolumeName], + sandbox: SandboxId, + role: DiskRole, + ) -> io::Result { + if names.contains(&transient_name(sandbox, role)) + || names.contains(&bitmap_snapshot_name(sandbox, role)) + { + return Ok(true); + } + match self.disk.clone_state(&view_name(sandbox, role)).await { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } } - /// Delete the sandbox's live and transient volumes and every - /// checkpoint volume but the newest of each role. The kept + /// Delete the sandbox's drives, its live and transient volumes and + /// every checkpoint volume but the newest of each role. The kept /// checkpoint makes a resume on this host a snapshot instead of a /// stage; [`Self::reap`] deletes it when the pool needs the space. - /// Fails while a process holds a volume open. + /// A cloned role whose destination is newer than its checkpoint + /// keeps none: the destination is not complete. Fails while a + /// process holds a volume open. pub async fn release(&self, sandbox: SandboxId) -> io::Result<()> { let report = self.sweep(|s| s == sandbox, &[]).await?; if let Some((_, e)) = report.failed.into_iter().next() { @@ -661,16 +1644,27 @@ impl SessionVolumes { Ok(()) } - /// Delete every volume of the sandbox. A volume a process holds - /// open stays and is reported as failed. + /// Delete every drive and volume of the sandbox. One a process + /// holds open stays and is reported as failed. pub async fn purge(&self, sandbox: SandboxId) -> io::Result { let mut report = Report::default(); - for name in self.disk.list().await? { - if parse(&name).is_some_and(|p| p.sandbox == sandbox) { - match self.disk.delete(&name).await { - Ok(()) => report.deleted.push(name), - Err(e) => report.failed.push((name, e)), - } + let parsed: Vec<(VolumeName, Parsed)> = self + .disk + .list() + .await? + .into_iter() + .filter_map(|name| parse(&name).map(|p| (name, p))) + .filter(|(_, p)| p.sandbox == sandbox) + .collect(); + let roles = self.cloned_or_sourced(&parsed, |s| s == sandbox).await?; + let stuck = self.take_down(roles, &mut report).await; + for (name, p) in parsed { + if stuck.contains(&(p.sandbox, p.role)) { + continue; + } + match self.disk.delete(&name).await { + Ok(()) => report.deleted.push(name), + Err(e) => report.failed.push((name, e)), } } if report.failed.is_empty() { @@ -679,11 +1673,12 @@ impl SessionVolumes { Ok(report) } - /// Delete what no sandbox needs: the live and transient volumes of - /// every sandbox not in `live` and every checkpoint volume but the - /// newest of its role. A live sandbox keeps its transient volume: - /// it may be a checkpoint whose publish is being redriven. Run once - /// after the live set is known, before any sandbox is provisioned. + /// Delete what no sandbox needs: the drives and the live and + /// transient volumes of every sandbox not in `live` and every + /// checkpoint volume but the newest of its role. A live sandbox + /// keeps its transient volume: it may be a checkpoint whose publish + /// is being redriven. Run once after the live set is known, before + /// any sandbox is provisioned. pub async fn reconcile(&self, live: &[SandboxId]) -> io::Result { let report = self.sweep(|_| true, live).await?; for sandbox in &report.kept { @@ -716,15 +1711,17 @@ impl SessionVolumes { } /// Among the volumes of the sandboxes `within` selects, delete the - /// live and transient volumes of sandboxes outside `live` and every - /// checkpoint volume but the newest of its role. + /// drives and the live and transient volumes of sandboxes outside + /// `live`, and every checkpoint volume older than the newest + /// checkpoint or destination of its role. A role whose drives or + /// source could not be taken down keeps its volumes, and so does a + /// role with two lineages, which is reported. async fn sweep( &self, within: impl Fn(SandboxId) -> bool, live: &[SandboxId], ) -> io::Result { let mut report = Report::default(); - let mut newest: std::collections::HashMap<(SandboxId, DiskRole), u64> = Default::default(); let parsed: Vec<(VolumeName, Parsed)> = self .disk .list() @@ -733,16 +1730,31 @@ impl SessionVolumes { .filter_map(|name| parse(&name).map(|p| (name, p))) .filter(|(_, p)| within(p.sandbox)) .collect(); - for (_, p) in &parsed { - if let Kind::Checkpoint(m) = p.kind { - let v = newest.entry((p.sandbox, p.role)).or_insert(0); - *v = (*v).max(m.version); + let holders = holders(parsed.iter().map(|(_, p)| p)); + let dead = self + .cloned_or_sourced(&parsed, |s| within(s) && !live.contains(&s)) + .await?; + let mut stuck = self.take_down(dead, &mut report).await; + for (&(sandbox, role), holder) in &holders { + if let Err(ids) = holder { + report + .failed + .push((live_name(sandbox, role), two_lineages(sandbox, role, *ids))); + stuck.push((sandbox, role)); } } for (name, p) in parsed { let unwanted = match p.kind { - Kind::Live | Kind::Transient => !live.contains(&p.sandbox), - Kind::Checkpoint(m) => m.version < newest[&(p.sandbox, p.role)], + Kind::Live + | Kind::Transient + | Kind::Dest(_) + | Kind::Bitmap + | Kind::BitmapSnapshot + | Kind::View => !live.contains(&p.sandbox), + Kind::Checkpoint(m) => match holders[&(p.sandbox, p.role)] { + Ok((newest, _)) => m.version < newest.version, + Err(_) => false, + }, }; if !unwanted { if matches!(p.kind, Kind::Checkpoint(_)) && !report.kept.contains(&p.sandbox) { @@ -750,6 +1762,9 @@ impl SessionVolumes { } continue; } + if stuck.contains(&(p.sandbox, p.role)) { + continue; + } match self.disk.delete(&name).await { Ok(()) => report.deleted.push(name), Err(e) => report.failed.push((name, e)), @@ -758,6 +1773,68 @@ impl SessionVolumes { Ok(report) } + /// The cloned roles among `parsed` and the roles with a source, of + /// the sandboxes `select` picks. + async fn cloned_or_sourced( + &self, + parsed: &[(VolumeName, Parsed)], + select: impl Fn(SandboxId) -> bool, + ) -> io::Result> { + let mut roles = cloned_roles(parsed.iter().map(|(_, p)| p).filter(|p| select(p.sandbox))); + if let Some(sources) = &self.sources { + for (sandbox, role) in sources.list().await? { + if select(sandbox) && !roles.contains(&(sandbox, role)) { + roles.push((sandbox, role)); + } + } + } + Ok(roles) + } + + /// Take down each role's drives and source before its volumes go: + /// they hold the destination, bitmap and checkpoint snapshots. A + /// role whose step fails keeps the steps after it, and is returned + /// so its volumes stay too. + async fn take_down( + &self, + roles: Vec<(SandboxId, DiskRole)>, + report: &mut Report, + ) -> Vec<(SandboxId, DiskRole)> { + let mut stuck = Vec::new(); + for (sandbox, role) in roles { + if let Err(failed) = self.take_down_role(sandbox, role, report).await { + report.failed.push(failed); + stuck.push((sandbox, role)); + } + } + stuck + } + + /// Recover the source, remove the view and the drive, stop the + /// source: no drive is removed over a dead source, and no source + /// stops under a drive. + async fn take_down_role( + &self, + sandbox: SandboxId, + role: DiskRole, + report: &mut Report, + ) -> Result<(), (VolumeName, io::Error)> { + for name in [view_name(sandbox, role), live_name(sandbox, role)] { + match self.remove_clone_of(sandbox, role, &name).await { + Ok(true) => report.deleted.push(name), + Ok(false) => {} + Err(e) => return Err((name, e)), + } + } + if let Some(sources) = &self.sources { + sources + .stop(sandbox, role) + .await + .map_err(|e| (live_name(sandbox, role), source_err(sandbox, role, e)))?; + } + Ok(()) + } + /// The released sandboxes whose kept checkpoints a resume may take. fn released(&self) -> Vec { self.kept.lock().unwrap().iter().copied().collect() @@ -774,32 +1851,14 @@ impl SessionVolumes { } } - /// The manifest of the sandbox's checkpoint volume: what the last - /// publish made durable. + /// What the last publish of the role made durable: the manifest of + /// its newest checkpoint volume or destination. pub async fn checkpoint_of( &self, sandbox: SandboxId, role: DiskRole, ) -> io::Result { - let mut found: Option = None; - for name in self.disk.list().await? { - if let Some(Parsed { - sandbox: s, - role: r, - kind: Kind::Checkpoint(m), - }) = parse(&name) - { - if s == sandbox && r == role && found.is_none_or(|f| m.version > f.version) { - found = Some(m); - } - } - } - found.ok_or_else(|| { - io::Error::new( - io::ErrorKind::NotFound, - format!("sandbox {sandbox} has no {} checkpoint volume", role), - ) - }) + Ok(lineage_in(&self.disk.list().await?, sandbox, role)?.0) } async fn delete_if_present(&self, name: &VolumeName) -> io::Result<()> { @@ -812,11 +1871,14 @@ impl SessionVolumes { #[cfg(test)] mod tests { + use std::path::Path; + use engram_storage_local::LocalBlobStorage; use super::*; - use crate::disk::{ChunkIndex, FileDisk}; + use crate::disk::{ChunkIndex, CloneState, FileDisk, Hydration, CLONE_HEADER_BYTES}; use crate::disk_flows::checkpoint; + use crate::disk_sources::{FileSources, Served}; use crate::test_support::CountingBlob; const CHUNK: u64 = 4096; @@ -826,6 +1888,7 @@ mod tests { disk: Arc, store: ChunkStore, volumes: SessionVolumes, + sources: Arc, blob: Arc, } @@ -850,10 +1913,13 @@ mod tests { dir.path().join("blobs"), ))); let store = ChunkStore::new(blob.clone()); + let sources = + Arc::new(FileSources::new(dir.path().join("sources"), store.clone()).unwrap()); Rig { - volumes: SessionVolumes::new(disk.clone(), store.clone()), + volumes: SessionVolumes::new(disk.clone(), store.clone()).with_sources(sources.clone()), disk, store, + sources, blob, _dir: dir, } @@ -893,6 +1959,63 @@ mod tests { metadata_total_bytes: POOL_TOTAL, }) } + async fn clone_drive( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + source: &Path, + hydration: Hydration, + ) -> io::Result { + self.0 + .clone_drive(drive, dest, bitmap, source, hydration) + .await + } + async fn open_clone(&self, drive: &VolumeName) -> io::Result { + self.0.open_clone(drive).await + } + async fn clone_state(&self, drive: &VolumeName) -> io::Result { + self.0.clone_state(drive).await + } + async fn skip_hydration( + &self, + drive: &VolumeName, + ranges: &[Range], + ) -> io::Result<()> { + self.0.skip_hydration(drive, ranges).await + } + async fn enable_hydration(&self, drive: &VolumeName) -> io::Result<()> { + self.0.enable_hydration(drive).await + } + async fn cut_over(&self, drive: &VolumeName) -> io::Result<()> { + self.0.cut_over(drive).await + } + async fn snapshot_clone( + &self, + drive: &VolumeName, + dest: &Volume, + bitmap: &Volume, + dest_snapshot: &VolumeName, + bitmap_snapshot: &VolumeName, + ) -> io::Result<(Volume, Volume)> { + self.0 + .snapshot_clone(drive, dest, bitmap, dest_snapshot, bitmap_snapshot) + .await + } + async fn clone_view( + &self, + view: &VolumeName, + dest_snapshot: &Volume, + bitmap_snapshot: &Volume, + source: &Path, + ) -> io::Result { + self.0 + .clone_view(view, dest_snapshot, bitmap_snapshot, source) + .await + } + async fn remove_clone(&self, drive: &VolumeName) -> io::Result<()> { + self.0.remove_clone(drive).await + } } fn fork(image: ManifestRef) -> Source { @@ -992,6 +2115,87 @@ mod tests { out } + /// A disk manifest of `total` bytes in 16 MiB chunks; `hashes[i]` + /// lists chunk `i` with that hash byte, `None` leaves it a hole. + fn manifest_of(total: u64, hashes: &[Option]) -> Manifest { + const MIB16: u64 = 16 << 20; + Manifest { + chunk_size: ChunkSize::bytes(MIB16), + chunks: hashes + .iter() + .enumerate() + .filter_map(|(i, hash)| { + hash.map(|h| engram_chunk_store::ChunkRef { + offset: i as u64 * MIB16, + hash: engram_chunk_store::ChunkHash::from_bytes([h; 32]), + }) + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, total) + } + } + + const MIB: u64 = 1 << 20; + + #[test] + fn a_new_destination_skips_the_chunks_the_manifest_does_not_list() { + let wanted = manifest_of(64 * MIB, &[None, Some(1), None, None]); + assert_eq!( + hydration_skips(&wanted, None).unwrap(), + vec![0..16 * MIB, 32 * MIB..64 * MIB] + ); + let full = manifest_of(32 * MIB, &[Some(1), Some(2)]); + assert_eq!(hydration_skips(&full, None).unwrap(), vec![]); + let empty = manifest_of(32 * MIB, &[]); + assert_eq!(hydration_skips(&empty, None).unwrap(), vec![0..32 * MIB]); + } + + #[test] + fn a_base_snapshot_skips_the_chunks_equal_to_the_base() { + // Equal, different, a hole in both, listed only in the base, + // listed only in the wanted manifest, equal. + let base = manifest_of(96 * MIB, &[Some(1), Some(2), None, Some(4), None, Some(6)]); + let wanted = manifest_of(96 * MIB, &[Some(1), Some(9), None, None, Some(5), Some(6)]); + assert_eq!( + hydration_skips(&wanted, Some(&base)).unwrap(), + vec![0..16 * MIB, 32 * MIB..48 * MIB, 80 * MIB..96 * MIB] + ); + assert_eq!( + hydration_skips(&wanted, Some(&wanted)).unwrap(), + vec![0..96 * MIB] + ); + let other_size = manifest_of(80 * MIB, &[Some(1)]); + assert!(hydration_skips(&wanted, Some(&other_size)).is_err()); + } + + #[test] + fn a_short_last_chunk_is_skipped_up_to_its_last_whole_region() { + let region = CLONE_REGION_BYTES; + let total = 32 * MIB + 3 * region + 512; + let wanted = manifest_of(total, &[Some(1), None, None]); + assert_eq!( + hydration_skips(&wanted, None).unwrap(), + vec![16 * MIB..32 * MIB + 3 * region] + ); + // A chunk smaller than a region: only regions whose every chunk + // qualifies are skipped. + let small = Manifest { + chunk_size: ChunkSize::bytes(region / 4), + chunks: [1u64, 9] + .iter() + .map(|i| engram_chunk_store::ChunkRef { + offset: i * region / 4, + hash: engram_chunk_store::ChunkHash::from_bytes([1; 32]), + }) + .collect(), + ..Manifest::empty(ManifestKind::Disk, 4 * region) + }; + assert_eq!( + hydration_skips(&small, None).unwrap(), + vec![region..2 * region, 3 * region..4 * region] + ); + } + #[test] fn names_say_what_a_volume_is() { let sandbox = SandboxId::new(); @@ -1012,19 +2216,83 @@ mod tests { kind: Kind::Checkpoint(manifest) }) ); - for name in [ - transient_name(sandbox, DiskRole::Root), - precopy_name(sandbox, DiskRole::Root), + for (name, kind) in [ + (transient_name(sandbox, DiskRole::Root), Kind::Transient), + (precopy_name(sandbox, DiskRole::Root), Kind::Transient), + ( + dest_name(sandbox, DiskRole::Root, manifest), + Kind::Dest(manifest), + ), + (bitmap_name(sandbox, DiskRole::Root), Kind::Bitmap), + ( + bitmap_snapshot_name(sandbox, DiskRole::Root), + Kind::BitmapSnapshot, + ), + (view_name(sandbox, DiskRole::Root), Kind::View), ] { assert_eq!( parse(&name), Some(Parsed { sandbox, role: DiskRole::Root, - kind: Kind::Transient - }) + kind + }), + "{name}" ); } + let s = sandbox.as_uuid().simple(); + let m = manifest.manifest_id.simple(); + for (name, want) in [ + (live_name(sandbox, DiskRole::Root), format!("sb-{s}-root")), + ( + checkpoint_name(sandbox, DiskRole::Root, manifest), + format!("sb-{s}-root-ckpt-{m}-v1"), + ), + ( + dest_name(sandbox, DiskRole::Root, manifest), + format!("sb-{s}-root-dest-{m}-v1"), + ), + ( + bitmap_name(sandbox, DiskRole::Root), + format!("sb-{s}-root-bitmap"), + ), + ( + transient_name(sandbox, DiskRole::Root), + format!("tmp-sb-{s}-root"), + ), + ( + precopy_name(sandbox, DiskRole::Root), + format!("tmp-sb-{s}-root-pre"), + ), + ( + bitmap_snapshot_name(sandbox, DiskRole::Root), + format!("tmp-sb-{s}-root-bm"), + ), + ( + view_name(sandbox, DiskRole::Root), + format!("tmp-sb-{s}-root-view"), + ), + ] { + assert_eq!(name.as_str(), want); + } + let longest = dest_name( + sandbox, + DiskRole::Root, + ManifestRef { + version: u64::MAX, + ..manifest + }, + ); + assert_eq!(longest.as_str().len(), 100); + for bad in [ + format!("sb-{s}-root-pre"), + format!("sb-{s}-root-bm"), + format!("tmp-sb-{s}-root-bitmap"), + format!("tmp-sb-{s}-root-dest-{m}-v1"), + format!("sb-{s}-root-dest-{m}"), + ] { + assert_eq!(parse(&VolumeName::new(bad.clone()).unwrap()), None, "{bad}"); + } assert_eq!(parse(&image_volumes::volume_name(manifest)), None); assert_eq!(parse(&VolumeName::new("sb-nothex-root").unwrap()), None); assert_eq!( @@ -1883,4 +3151,1057 @@ mod tests { assert_eq!(reaped.deleted.len(), 1); assert!(checkpoints_of(&r, dead, DiskRole::Root).await.is_empty()); } + + /// Make `role` of `sandbox` cloned as a lazy restore does: a + /// destination named after `lineage`, a bitmap and the drive over + /// them. Returns the drive. + async fn cloned_role( + r: &Rig, + sandbox: SandboxId, + role: DiskRole, + lineage: ManifestRef, + ) -> Volume { + let dest = r + .disk + .create(&dest_name(sandbox, role, lineage), 2 * CHUNK) + .await + .unwrap(); + let bitmap = r + .disk + .create(&bitmap_name(sandbox, role), CLONE_HEADER_BYTES) + .await + .unwrap(); + let source = r._dir.path().join(format!("source-{sandbox}-{role}")); + std::fs::write(&source, vec![7u8; 2 * CHUNK as usize]).unwrap(); + let hydration = Hydration { + enabled: true, + threshold_regions: 1, + batch_regions: 1, + }; + r.disk + .clone_drive( + &live_name(sandbox, role), + &dest, + &bitmap, + &source, + hydration, + ) + .await + .unwrap() + } + + /// A cloned role's checkpoint in flight: the snapshot pair and the + /// view over it. + async fn cloned_checkpoint(r: &Rig, sandbox: SandboxId, role: DiskRole, lineage: ManifestRef) { + let dest = r + .disk + .open(&dest_name(sandbox, role, lineage)) + .await + .unwrap(); + let bitmap = r.disk.open(&bitmap_name(sandbox, role)).await.unwrap(); + let (dest_snap, bitmap_snap) = r + .disk + .snapshot_clone( + &live_name(sandbox, role), + &dest, + &bitmap, + &transient_name(sandbox, role), + &bitmap_snapshot_name(sandbox, role), + ) + .await + .unwrap(); + r.disk + .clone_view( + &view_name(sandbox, role), + &dest_snap, + &bitmap_snap, + Path::new("/dev/null"), + ) + .await + .unwrap(); + } + + fn version(lineage: ManifestRef, version: u64) -> ManifestRef { + ManifestRef { version, ..lineage } + } + + #[tokio::test] + async fn a_cloned_role_is_a_role_whose_drive_the_hypervisor_opens() { + let r = rig(); + let sandbox = SandboxId::new(); + let lineage = ManifestRef::new(); + r.disk + .create( + &checkpoint_name(sandbox, DiskRole::Root, lineage), + 2 * CHUNK, + ) + .await + .unwrap(); + let drive = cloned_role(&r, sandbox, DiskRole::Root, version(lineage, 2)).await; + + assert_eq!( + r.volumes.roles(sandbox).await.unwrap(), + vec![DiskRole::Root] + ); + assert_eq!( + r.volumes.path(sandbox, DiskRole::Root).await.unwrap(), + drive.path + ); + assert_eq!( + r.volumes + .checkpoint_of(sandbox, DiskRole::Root) + .await + .unwrap(), + version(lineage, 2) + ); + assert_eq!( + r.volumes + .live(sandbox, DiskRole::Root) + .await + .unwrap_err() + .kind(), + io::ErrorKind::Unsupported + ); + } + + #[tokio::test] + async fn a_cloned_role_refuses_a_live_move_and_finds_its_pending_pair() { + let r = rig(); + let sandbox = SandboxId::new(); + let lineage = ManifestRef::new(); + cloned_role(&r, sandbox, DiskRole::Root, lineage).await; + assert_eq!( + r.volumes + .pending(sandbox, DiskRole::Root) + .await + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + cloned_checkpoint(&r, sandbox, DiskRole::Root, lineage).await; + let before = names(&r).await; + + let refused = r + .volumes + .take_precopy(sandbox, DiskRole::Root) + .await + .unwrap_err(); + assert_eq!(refused.kind(), io::ErrorKind::Unsupported, "{refused}"); + assert_eq!(names(&r).await, before); + let pending = r.volumes.pending(sandbox, DiskRole::Root).await.unwrap(); + assert!(matches!(pending.shape, Shape::Hydrating { .. })); + assert_eq!(pending.previous(), lineage); + } + + #[tokio::test] + async fn release_of_a_cloned_role_removes_its_drives_and_keeps_no_checkpoint() { + let r = rig(); + let sandbox = SandboxId::new(); + let lineage = ManifestRef::new(); + r.disk + .create( + &checkpoint_name(sandbox, DiskRole::Root, lineage), + 2 * CHUNK, + ) + .await + .unwrap(); + cloned_role(&r, sandbox, DiskRole::Root, version(lineage, 2)).await; + cloned_checkpoint(&r, sandbox, DiskRole::Root, version(lineage, 2)).await; + + r.volumes.release(sandbox).await.unwrap(); + + assert!(names(&r).await.is_empty(), "{:?}", names(&r).await); + for drive in [ + live_name(sandbox, DiskRole::Root), + view_name(sandbox, DiskRole::Root), + ] { + assert_eq!( + r.disk.open_clone(&drive).await.unwrap_err().kind(), + io::ErrorKind::NotFound + ); + } + assert!(r.volumes.released().is_empty()); + } + + /// After the cutover a checkpoint volume newer than the destination + /// is complete, so a release keeps it. + #[tokio::test] + async fn release_keeps_a_checkpoint_newer_than_the_destination() { + let r = rig(); + let sandbox = SandboxId::new(); + let lineage = ManifestRef::new(); + cloned_role(&r, sandbox, DiskRole::Root, lineage).await; + let newer = checkpoint_name(sandbox, DiskRole::Root, version(lineage, 2)); + r.disk.create(&newer, 2 * CHUNK).await.unwrap(); + + r.volumes.release(sandbox).await.unwrap(); + + assert_eq!(names(&r).await, vec![newer.as_str().to_string()]); + assert_eq!(r.volumes.released(), vec![sandbox]); + } + + #[tokio::test] + async fn reconcile_keeps_a_live_cloned_role_and_purge_removes_it() { + let r = rig(); + let (live, dead) = (SandboxId::new(), SandboxId::new()); + for sandbox in [live, dead] { + let lineage = ManifestRef::new(); + cloned_role(&r, sandbox, DiskRole::Root, lineage).await; + cloned_checkpoint(&r, sandbox, DiskRole::Root, lineage).await; + } + let mut kept: Vec = names(&r).await; + kept.retain(|n| parse(&VolumeName::new(n.clone()).unwrap()).unwrap().sandbox == live); + + let report = r.volumes.reconcile(&[live]).await.unwrap(); + + assert!(report.failed.is_empty(), "{:?}", report.failed); + assert_eq!(names(&r).await, kept); + r.disk + .open_clone(&live_name(live, DiskRole::Root)) + .await + .unwrap(); + r.disk + .open_clone(&view_name(live, DiskRole::Root)) + .await + .unwrap(); + assert_eq!( + r.disk + .open_clone(&live_name(dead, DiskRole::Root)) + .await + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + + let report = r.volumes.purge(live).await.unwrap(); + + assert!(report.failed.is_empty(), "{:?}", report.failed); + assert!(names(&r).await.is_empty(), "{:?}", names(&r).await); + assert_eq!( + r.disk + .open_clone(&live_name(live, DiskRole::Root)) + .await + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + } + + /// A manifest of the sandbox's own lineage with chunk `i` holding + /// `pattern[i]`, published over a fork of `image`. + async fn published(r: &Rig, image: ManifestRef, pattern: &[u8]) -> ManifestRef { + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, fork(image), None) + .await + .unwrap(); + for (i, byte) in pattern.iter().enumerate() { + write_byte(&p.root, i as u64, *byte).await; + } + let manifest = r.volumes.checkpoint(sandbox, DiskRole::Root).await.unwrap(); + r.volumes.purge(sandbox).await.unwrap(); + manifest + } + + fn lazy_fork(image: ManifestRef) -> Source { + Source::LazyFork { + manifest: image, + base: None, + } + } + + #[tokio::test] + async fn a_lazy_provision_reads_the_manifest_through_a_drive() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0]).await; + let forked = SandboxId::new(); + + let p = r + .volumes + .provision(forked, lazy_fork(image), None) + .await + .unwrap(); + + assert_eq!(bytes_at(&p.root).await, vec![1, 2, 0]); + assert_ne!(p.root_manifest, image); + assert_eq!( + r.store.get_manifest(p.root_manifest).await.unwrap(), + r.store.get_manifest(image).await.unwrap() + ); + assert_eq!( + r.volumes.path(forked, DiskRole::Root).await.unwrap(), + p.root + ); + assert_eq!( + r.volumes + .checkpoint_of(forked, DiskRole::Root) + .await + .unwrap(), + p.root_manifest + ); + assert_eq!( + r.sources.list().await.unwrap(), + vec![(forked, DiskRole::Root)] + ); + + let manifest = published(&r, image, &[1, 9, 0]).await; + let continued = SandboxId::new(); + let p = r + .volumes + .provision( + continued, + Source::LazyContinue { + manifest, + base: Some(image), + }, + None, + ) + .await + .unwrap(); + assert_eq!(bytes_at(&p.root).await, vec![1, 9, 0]); + assert_eq!(p.root_manifest, manifest); + assert!(names(&r).await.contains( + &dest_name(continued, DiskRole::Root, manifest) + .as_str() + .to_string() + )); + } + + #[tokio::test] + async fn a_failed_lazy_provision_leaves_nothing_of_the_sandbox() { + let r = rig(); + let image = staged_image(&r, &[1]).await; + let only_image = vec![image_volumes::volume_name(image).as_str().to_string()]; + let sandbox = SandboxId::new(); + // The swap drive's name is taken, so the swap fails after its + // source started. + r.disk + .create(&live_name(sandbox, DiskRole::Swap), CHUNK) + .await + .unwrap(); + + let err = r + .volumes + .provision(sandbox, lazy_fork(image), Some(lazy_fork(image))) + .await + .unwrap_err(); + + assert_eq!(err.kind(), io::ErrorKind::AlreadyExists, "{err}"); + assert_eq!(names(&r).await, only_image); + assert!(r.sources.list().await.unwrap().is_empty()); + assert_eq!( + r.disk + .open_clone(&live_name(sandbox, DiskRole::Root)) + .await + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + + let without = SessionVolumes::new(r.disk.clone(), r.store.clone()); + let err = without + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::Unsupported, "{err}"); + assert_eq!(names(&r).await, only_image); + } + + /// [`FileSources`] that logs each recovery and stop, and whether the + /// role's drive still existed at the stop. + struct Watched { + inner: FileSources, + disk: Arc, + log: Mutex>, + } + + #[async_trait::async_trait] + impl DiskSources for Watched { + fn can_start(&self) -> bool { + true + } + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option<(&Volume, ManifestRef)>, + ) -> io::Result { + self.inner.start(sandbox, role, manifest, base).await + } + async fn ensure_alive(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + self.log.lock().unwrap().push(format!("alive {role}")); + self.inner.ensure_alive(sandbox, role).await + } + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + self.inner.served(sandbox, role).await + } + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + let drive = self + .disk + .open_clone(&live_name(sandbox, role)) + .await + .is_ok(); + self.log + .lock() + .unwrap() + .push(format!("stop {role} drive={drive}")); + self.inner.stop(sandbox, role).await + } + async fn list(&self) -> io::Result> { + self.inner.list().await + } + } + + #[tokio::test] + async fn release_recovers_the_source_removes_the_drive_then_stops_the_source() { + let r = rig(); + let image = staged_image(&r, &[1]).await; + let watched = Arc::new(Watched { + inner: FileSources::new(r._dir.path().join("watched"), r.store.clone()).unwrap(), + disk: r.disk.clone(), + log: Mutex::new(Vec::new()), + }); + let volumes = + SessionVolumes::new(r.disk.clone(), r.store.clone()).with_sources(watched.clone()); + let sandbox = SandboxId::new(); + volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + + volumes.release(sandbox).await.unwrap(); + + assert_eq!( + *watched.log.lock().unwrap(), + vec![ + "alive root".to_string(), + "alive root".to_string(), + "stop root drive=false".to_string() + ] + ); + assert!(watched.inner.list().await.unwrap().is_empty()); + assert_eq!( + names(&r).await, + vec![image_volumes::volume_name(image).as_str().to_string()] + ); + assert!(volumes.released().is_empty()); + } + + /// The bytes of `manifest`, staged into a scratch volume. + async fn staged_bytes(r: &Rig, manifest: ManifestRef) -> Vec { + let contents = r.store.get_manifest(manifest).await.unwrap(); + let name = VolumeName::new(format!("check-{}", manifest.version)).unwrap(); + let volume = stage(r.disk.as_ref(), &r.store, &name, &contents, None) + .await + .unwrap(); + let bytes = std::fs::read(&volume.path).unwrap(); + drop(volume); + r.disk.delete(&name).await.unwrap(); + bytes + } + + fn next(manifest: ManifestRef) -> ManifestRef { + version(manifest, manifest.version + 1) + } + + #[tokio::test] + async fn a_checkpoint_while_hydrating_publishes_what_the_drive_reads() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0, 3]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + write_half(&p.root, 0, 1, 7).await; + write_byte(&p.root, 2, 9).await; + + let published = r.volumes.checkpoint(sandbox, DiskRole::Root).await.unwrap(); + + assert_eq!(published, next(p.root_manifest)); + assert_eq!( + staged_bytes(&r, published).await, + std::fs::read(&p.root).unwrap() + ); + let names = names(&r).await; + assert!(names.contains(&dest_name(sandbox, DiskRole::Root, published).to_string())); + for gone in [ + dest_name(sandbox, DiskRole::Root, p.root_manifest), + transient_name(sandbox, DiskRole::Root), + bitmap_snapshot_name(sandbox, DiskRole::Root), + ] { + assert!(!names.contains(&gone.to_string()), "{gone}"); + } + let view = r.disk.open_clone(&view_name(sandbox, DiskRole::Root)).await; + assert_eq!(view.unwrap_err().kind(), io::ErrorKind::NotFound); + } + + /// A process death after the take, and one inside the publish that + /// left the view, are redriven from the names. + #[tokio::test] + async fn a_hydrating_checkpoint_cut_after_the_take_publishes_from_pending() { + let r = rig(); + let image = staged_image(&r, &[1, 2]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + write_byte(&p.root, 1, 8).await; + drop(r.volumes.take(sandbox, DiskRole::Root).await.unwrap()); + write_byte(&p.root, 0, 4).await; + let tmp = r + .disk + .open(&transient_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + let bm = r + .disk + .open(&bitmap_snapshot_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + r.disk + .clone_view( + &view_name(sandbox, DiskRole::Root), + &tmp, + &bm, + Path::new("/dev/null"), + ) + .await + .unwrap(); + drop((tmp, bm)); + + let pending = r.volumes.pending(sandbox, DiskRole::Root).await.unwrap(); + let published = r.volumes.publish(pending).await.unwrap(); + + assert_eq!(published, next(p.root_manifest)); + assert_eq!( + staged_bytes(&r, published).await[..CHUNK as usize], + vec![1; CHUNK as usize] + ); + assert_eq!( + staged_bytes(&r, published).await[CHUNK as usize..], + vec![8; CHUNK as usize] + ); + assert_eq!( + r.volumes + .pending(sandbox, DiskRole::Root) + .await + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + } + + #[tokio::test] + async fn after_the_cutover_a_checkpoint_hashes_the_disk_and_the_next_one_diffs() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0, 3]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + r.disk + .cut_over(&live_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + write_byte(&p.root, 1, 5).await; + + let first = r.volumes.checkpoint(sandbox, DiskRole::Root).await.unwrap(); + + assert_eq!(first, next(p.root_manifest)); + assert_eq!( + staged_bytes(&r, first).await, + std::fs::read(&p.root).unwrap() + ); + assert_eq!( + r.volumes + .checkpoint_of(sandbox, DiskRole::Root) + .await + .unwrap(), + first + ); + let names_now = names(&r).await; + assert!(names_now.contains(&checkpoint_name(sandbox, DiskRole::Root, first).to_string())); + assert!( + names_now.contains(&dest_name(sandbox, DiskRole::Root, p.root_manifest).to_string()) + ); + + write_byte(&p.root, 3, 6).await; + let pending = r.volumes.take(sandbox, DiskRole::Root).await.unwrap(); + assert!( + matches!(pending.shape, Shape::Volume), + "{:?}", + pending.shape + ); + assert_eq!( + r.volumes.changed_blocks(&pending, CHUNK).await.unwrap(), + vec![3] + ); + let second = r.volumes.publish(pending).await.unwrap(); + + assert_eq!(second, next(first)); + assert_eq!( + staged_bytes(&r, second).await, + std::fs::read(&p.root).unwrap() + ); + assert!(!names(&r) + .await + .contains(&checkpoint_name(sandbox, DiskRole::Root, first).to_string())); + + r.volumes.release(sandbox).await.unwrap(); + + assert_eq!( + names(&r).await, + vec![ + image_volumes::volume_name(image).to_string(), + checkpoint_name(sandbox, DiskRole::Root, second).to_string(), + ] + ); + assert_eq!(r.volumes.released(), vec![sandbox]); + } + + /// A destination snapshot left without its bitmap snapshot (a + /// cleanup cut between the two) reads zeros where the drive had not + /// copied, so it never publishes; the next take replaces it. + #[tokio::test] + async fn a_lone_destination_snapshot_while_copying_is_refused() { + let r = rig(); + let image = staged_image(&r, &[1, 2]).await; + let sandbox = SandboxId::new(); + r.volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + drop(r.volumes.take(sandbox, DiskRole::Root).await.unwrap()); + r.disk + .delete(&bitmap_snapshot_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + + let err = r + .volumes + .pending(sandbox, DiskRole::Root) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData, "{err}"); + + let pending = r.volumes.take(sandbox, DiskRole::Root).await.unwrap(); + assert!(matches!(pending.shape, Shape::Hydrating { .. })); + } + + /// After the cutover and a checkpoint volume newer than the + /// destination, the role opens and moves as a plain role does; its + /// live name stays a drive. + #[tokio::test] + async fn a_cut_over_role_moves_live_after_its_first_checkpoint() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + let refused = r.volumes.take_precopy(sandbox, DiskRole::Root).await; + assert_eq!(refused.unwrap_err().kind(), io::ErrorKind::Unsupported); + let refused = r.volumes.live(sandbox, DiskRole::Root).await; + assert_eq!(refused.unwrap_err().kind(), io::ErrorKind::Unsupported); + r.disk + .cut_over(&live_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + let refused = r.volumes.take_precopy(sandbox, DiskRole::Root).await; + assert_eq!(refused.unwrap_err().kind(), io::ErrorKind::Unsupported); + + r.volumes.checkpoint(sandbox, DiskRole::Root).await.unwrap(); + write_byte(&p.root, 2, 7).await; + + assert_eq!( + r.volumes.roles(sandbox).await.unwrap(), + vec![DiskRole::Root] + ); + assert_eq!( + r.volumes.live(sandbox, DiskRole::Root).await.unwrap().path, + p.root + ); + assert_eq!( + r.volumes.path(sandbox, DiskRole::Root).await.unwrap(), + p.root + ); + let precopy = r + .volumes + .take_precopy(sandbox, DiskRole::Root) + .await + .unwrap(); + assert_eq!( + r.volumes.changed_blocks(&precopy, CHUNK).await.unwrap(), + vec![2] + ); + r.volumes.discard(precopy).await.unwrap(); + } + + async fn sourced(r: &Rig, sandbox: SandboxId) -> bool { + r.sources + .list() + .await + .unwrap() + .contains(&(sandbox, DiskRole::Root)) + } + + /// A checkpoint's publish reads through a view over the source, so a + /// cutover waits while its pair or view exists, and goes ahead once it + /// is published. + #[tokio::test] + async fn a_cutover_waits_for_a_checkpoint_in_flight() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + write_byte(&p.root, 2, 5).await; + let drive = live_name(sandbox, DiskRole::Root); + let pending = r.volumes.take(sandbox, DiskRole::Root).await.unwrap(); + let tmp = r.disk.open(&transient_name(sandbox, DiskRole::Root)).await; + let bm = r + .disk + .open(&bitmap_snapshot_name(sandbox, DiskRole::Root)) + .await; + r.disk + .clone_view( + &view_name(sandbox, DiskRole::Root), + &tmp.unwrap(), + &bm.unwrap(), + Path::new("/dev/null"), + ) + .await + .unwrap(); + let before = names(&r).await; + + for _ in 0..2 { + assert_eq!( + r.volumes.hydration(sandbox, DiskRole::Root).await.unwrap(), + Some(Hydrating::Ready) + ); + assert_eq!( + r.volumes.cut_over(sandbox, DiskRole::Root).await.unwrap(), + Cutover::Waiting(Waiting::Checkpoint) + ); + assert_eq!(names(&r).await, before); + assert!(sourced(&r, sandbox).await); + assert!(matches!( + r.disk.clone_state(&drive).await.unwrap(), + CloneState::Copying { .. } + )); + // A crash after the take leaves the pair alone. + r.volumes + .remove_clone_of(sandbox, DiskRole::Root, &view_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + } + let published = r.volumes.publish(pending).await.unwrap(); + let err = r + .volumes + .stop_source(sandbox, DiskRole::Root) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput, "the drive copies"); + + assert_eq!( + r.volumes.cut_over(sandbox, DiskRole::Root).await.unwrap(), + Cutover::Done + ); + assert_eq!( + r.disk.clone_state(&drive).await.unwrap(), + CloneState::CutOver + ); + // The source stops outside the cutover, which leaves it. + assert!(sourced(&r, sandbox).await); + assert_eq!( + r.volumes.hydration(sandbox, DiskRole::Root).await.unwrap(), + Some(Hydrating::Ready) + ); + r.volumes + .stop_source(sandbox, DiskRole::Root) + .await + .unwrap(); + assert!(!sourced(&r, sandbox).await); + assert!(!names(&r) + .await + .contains(&bitmap_name(sandbox, DiskRole::Root).to_string())); + assert_eq!( + r.volumes.hydration(sandbox, DiskRole::Root).await.unwrap(), + None + ); + assert_eq!( + r.volumes.cut_over(sandbox, DiskRole::Root).await.unwrap(), + Cutover::Done + ); + assert_eq!(bytes_at(&p.root).await, vec![1, 2, 5]); + assert_eq!( + staged_bytes(&r, published).await, + std::fs::read(&p.root).unwrap() + ); + + let plain = SandboxId::new(); + r.volumes.provision(plain, fork(image), None).await.unwrap(); + assert_eq!( + r.volumes.hydration(plain, DiskRole::Root).await.unwrap(), + None + ); + let err = r.volumes.cut_over(plain, DiskRole::Root).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput, "{err}"); + } + + /// A crash after the drive is cut over leaves the bitmap and the + /// source, and one after the bitmap delete leaves the source. The + /// next call finishes either; until then the role checkpoints and + /// releases as a cut-over role. + #[tokio::test] + async fn a_cutover_cut_short_is_finished_by_the_next_call() { + let r = rig(); + let image = staged_image(&r, &[1, 2]).await; + let (whole, unfinished) = (SandboxId::new(), SandboxId::new()); + for sandbox in [whole, unfinished] { + r.volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + r.disk + .cut_over(&live_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + } + r.disk + .delete(&bitmap_name(whole, DiskRole::Root)) + .await + .unwrap(); + + for sandbox in [whole, unfinished] { + assert_eq!( + r.volumes.hydration(sandbox, DiskRole::Root).await.unwrap(), + Some(Hydrating::Ready) + ); + } + assert_eq!( + r.volumes.cut_over(whole, DiskRole::Root).await.unwrap(), + Cutover::Done + ); + r.volumes.stop_source(whole, DiskRole::Root).await.unwrap(); + assert!(!sourced(&r, whole).await); + assert_eq!( + r.volumes.hydration(whole, DiskRole::Root).await.unwrap(), + None + ); + + let first = r + .volumes + .checkpoint(unfinished, DiskRole::Root) + .await + .unwrap(); + assert!(names(&r) + .await + .contains(&checkpoint_name(unfinished, DiskRole::Root, first).to_string())); + assert!(sourced(&r, unfinished).await); + for sandbox in [whole, unfinished] { + r.volumes.release(sandbox).await.unwrap(); + } + assert!(r.sources.list().await.unwrap().is_empty()); + assert_eq!( + names(&r).await, + vec![ + image_volumes::volume_name(image).to_string(), + checkpoint_name(unfinished, DiskRole::Root, first).to_string(), + ] + ); + } + + /// After a cutover through the volumes the first checkpoint hashes + /// the disk into a checkpoint volume, and a release keeps it. + #[tokio::test] + async fn after_a_cutover_the_first_checkpoint_is_a_checkpoint_volume() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0, 3]).await; + let sandbox = SandboxId::new(); + let p = r + .volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + write_byte(&p.root, 1, 5).await; + assert_eq!( + r.volumes.cut_over(sandbox, DiskRole::Root).await.unwrap(), + Cutover::Done + ); + + let first = r.volumes.checkpoint(sandbox, DiskRole::Root).await.unwrap(); + + assert_eq!(first, next(p.root_manifest)); + assert_eq!( + staged_bytes(&r, first).await, + std::fs::read(&p.root).unwrap() + ); + r.volumes.release(sandbox).await.unwrap(); + assert_eq!( + names(&r).await, + vec![ + image_volumes::volume_name(image).to_string(), + checkpoint_name(sandbox, DiskRole::Root, first).to_string(), + ] + ); + } + + /// The plan of an eager provision names the volume the provision + /// would start from, so it lists exactly the chunks it would write. + #[tokio::test] + async fn an_eager_plan_starts_where_the_provision_would() { + let r = rig(); + let image = staged_image(&r, &[1, 2, 0]).await; + let sandbox = SandboxId::new(); + let plan = |source| r.volumes.eager_plan(sandbox, DiskRole::Root, source); + let image_manifest = r.store.get_manifest(image).await.unwrap(); + + assert!(plan(fork(image)).await.unwrap().is_none()); + let empty = plan(Source::Empty { size_bytes: CHUNK }).await; + assert!(empty.unwrap().is_none()); + let manifest = published(&r, image, &[1, 9, 0]).await; + let wanted = r.store.get_manifest(manifest).await.unwrap(); + for (source, base) in [ + ( + Source::Fork { + manifest, + base: None, + }, + None, + ), + ( + Source::LazyContinue { + manifest, + base: Some(image), + }, + Some(&image_manifest), + ), + ] { + let p = plan(source).await.unwrap().unwrap(); + assert_eq!(p.wanted, wanted); + assert_eq!(p.base.as_ref(), base); + } + + // A released sandbox keeps its checkpoint of the lineage: a resume + // at that version adopts it, one at a newer version stages onto it. + let continued = |manifest| Source::Continue { + manifest, + base: Some(image), + }; + r.volumes + .provision(sandbox, continued(manifest), None) + .await + .unwrap(); + r.volumes.release(sandbox).await.unwrap(); + let resumed = SandboxId::new(); + let plan = r + .volumes + .eager_plan(resumed, DiskRole::Root, continued(manifest)); + assert!(plan.await.unwrap().is_none()); + let newer = next(manifest); + r.store.put_manifest(newer, &image_manifest).await.unwrap(); + let plan = r + .volumes + .eager_plan(resumed, DiskRole::Root, continued(newer)); + let plan = plan.await.unwrap().unwrap(); + assert_eq!(plan.wanted, image_manifest); + assert_eq!(plan.base, Some(wanted)); + } + + /// [`FileSources`] whose recovery always fails. + struct DeadSources(FileSources); + + #[async_trait::async_trait] + impl DiskSources for DeadSources { + fn can_start(&self) -> bool { + true + } + async fn start( + &self, + sandbox: SandboxId, + role: DiskRole, + manifest: ManifestRef, + base: Option<(&Volume, ManifestRef)>, + ) -> io::Result { + self.0.start(sandbox, role, manifest, base).await + } + async fn ensure_alive(&self, _: SandboxId, _: DiskRole) -> io::Result<()> { + Err(io::Error::other("the source does not start")) + } + async fn served(&self, sandbox: SandboxId, role: DiskRole) -> io::Result { + self.0.served(sandbox, role).await + } + async fn stop(&self, sandbox: SandboxId, role: DiskRole) -> io::Result<()> { + self.0.stop(sandbox, role).await + } + async fn list(&self) -> io::Result> { + self.0.list().await + } + } + + /// A role whose drive cannot be removed keeps every volume the drive + /// holds, for the next sweep. + #[tokio::test] + async fn a_failed_take_down_keeps_the_role_volumes() { + let r = rig(); + let image = staged_image(&r, &[1]).await; + let dead = + DeadSources(FileSources::new(r._dir.path().join("dead"), r.store.clone()).unwrap()); + let volumes = + SessionVolumes::new(r.disk.clone(), r.store.clone()).with_sources(Arc::new(dead)); + let sandbox = SandboxId::new(); + volumes + .provision(sandbox, lazy_fork(image), None) + .await + .unwrap(); + let before = names(&r).await; + + assert!(volumes.release(sandbox).await.is_err()); + let report = volumes.purge(sandbox).await.unwrap(); + + assert_eq!(report.failed.len(), 1, "{:?}", report.failed); + assert_eq!(names(&r).await, before); + r.disk + .open_clone(&live_name(sandbox, DiskRole::Root)) + .await + .unwrap(); + } + + #[tokio::test] + async fn two_lineages_for_one_role_are_an_error_and_keep_their_volumes() { + let r = rig(); + let sandbox = SandboxId::new(); + let (a, b) = (ManifestRef::new(), ManifestRef::new()); + for m in [a, b] { + r.disk + .create(&checkpoint_name(sandbox, DiskRole::Root, m), CHUNK) + .await + .unwrap(); + } + let before = names(&r).await; + + let err = r + .volumes + .checkpoint_of(sandbox, DiskRole::Root) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData, "{err}"); + let err = r.volumes.release(sandbox).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData, "{err}"); + assert_eq!(names(&r).await, before); + } } diff --git a/crates/engram-protocol/src/heartbeat.rs b/crates/engram-protocol/src/heartbeat.rs index f6803b301..b31a0606f 100644 --- a/crates/engram-protocol/src/heartbeat.rs +++ b/crates/engram-protocol/src/heartbeat.rs @@ -87,6 +87,12 @@ pub struct Heartbeat { /// must still decode). #[serde(default)] pub capture_job_reports: Vec, + /// Sandboxes whose disks still copy from a source, or whose cutover + /// has steps left (ADR 0127 Decision 9). A fact the host restates + /// every heartbeat, not an event. Absent from an older host's + /// heartbeat, which decodes as none. + #[serde(default)] + pub hydrating: Vec, } /// Issue #529: distinguishes a periodic (ADR 0028 Fix A) checkpoint @@ -294,6 +300,7 @@ mod tests { checkpoints: Vec::new(), utilization: Default::default(), capture_job_reports: Vec::new(), + hydrating: Vec::new(), } } @@ -596,6 +603,25 @@ mod tests { assert!(ack.acked_capture_jobs.is_empty()); } + #[test] + fn hydrating_round_trips_and_defaults_to_none_for_an_older_host() { + let mut h = sample(); + h.hydrating = vec![SandboxId::new()]; + let back: Heartbeat = serde_json::from_str(&serde_json::to_string(&h).unwrap()).unwrap(); + assert_eq!(back.hydrating, h.hydrating); + + let older = r#"{ + "host_id": "00000000-0000-0000-0000-000000000000", + "sent_at": "2026-11-05T00:00:00Z", + "capacity": {"total_mib": 1024, "used_mib": 0, "running_sandboxes": 0}, + "running_sandboxes": [], + "draining": false, + "capture_job_reports": [] + }"#; + let hb: Heartbeat = serde_json::from_str(older).expect("decode without hydrating"); + assert!(hb.hydrating.is_empty()); + } + #[test] fn ready_images_serializes_as_array() { let mut h = sample(); diff --git a/crates/engram-sandbox-firecracker/src/lib.rs b/crates/engram-sandbox-firecracker/src/lib.rs index df3272a34..43756241f 100644 --- a/crates/engram-sandbox-firecracker/src/lib.rs +++ b/crates/engram-sandbox-firecracker/src/lib.rs @@ -4003,21 +4003,17 @@ fn persist_sandbox_manifest( .map_err(|e| SandboxError::Vm(format!("write sandbox manifest: {e}").into())) } -/// ADR 0044 K2: move the VM's processes (FC + any uffd-handler) into a -/// node-level cgroup-v2 leaf `//`, OUT of the -/// host-agent's own (pod) cgroup, so a pod-scope `cgroup.kill` on -/// teardown doesn't reap the microVM. `hostPID` alone is insufficient — -/// it shares the PID namespace, but the FC processes stay in the pod's -/// cgroup and die with it. Best-effort: returns the error so the caller -/// can log loudly, but it never aborts create/restore (the VM still -/// runs; it just won't survive a pod restart on this host). +/// ADR 0044 K2: move processes into the node-level cgroup-v2 leaf +/// `//`, OUT of the host-agent's own (pod) cgroup, so a +/// pod-scope `cgroup.kill` on teardown doesn't reap them. `hostPID` +/// alone is insufficient — it shares the PID namespace, but the +/// processes stay in the pod's cgroup and die with it. A VM's leaf +/// (FC + any uffd-handler) is its sandbox id. Callers treat a failure as +/// best-effort: the process still runs; it just won't survive a pod +/// restart on this host. #[cfg(target_os = "linux")] -fn escape_to_node_cgroup( - parent: &Path, - sandbox_id: SandboxId, - pids: &[u32], -) -> std::io::Result<()> { - let dir = parent.join(sandbox_id.to_string()); +pub fn escape_to_node_cgroup(parent: &Path, leaf: &str, pids: &[u32]) -> std::io::Result<()> { + let dir = parent.join(leaf); std::fs::create_dir_all(&dir)?; let procs = dir.join("cgroup.procs"); for &pid in pids { @@ -4030,11 +4026,7 @@ fn escape_to_node_cgroup( } #[cfg(not(target_os = "linux"))] -fn escape_to_node_cgroup( - _parent: &Path, - _sandbox_id: SandboxId, - _pids: &[u32], -) -> std::io::Result<()> { +pub fn escape_to_node_cgroup(_parent: &Path, _leaf: &str, _pids: &[u32]) -> std::io::Result<()> { Ok(()) } @@ -4046,7 +4038,7 @@ fn place_vm_in_node_cgroup(parent: Option<&Path>, sandbox_id: SandboxId, pids: & let Some(parent) = parent else { return; // dev / tests: no pod scope to escape. }; - match escape_to_node_cgroup(parent, sandbox_id, pids) { + match escape_to_node_cgroup(parent, &sandbox_id.to_string(), pids) { Ok(()) => tracing::debug!( %sandbox_id, parent = %parent.display(), @@ -4064,21 +4056,21 @@ fn place_vm_in_node_cgroup(parent: Option<&Path>, sandbox_id: SandboxId, pids: & } } -/// Remove the per-VM node cgroup leaf [`escape_to_node_cgroup`] created. -/// `rmdir` of a cgroup only succeeds once it's empty, so `destroy()` -/// calls this AFTER killing FC + the uffd handler. Best-effort. -fn remove_node_cgroup(parent: Option<&Path>, sandbox_id: SandboxId) { +/// Remove a node cgroup leaf [`escape_to_node_cgroup`] created. `rmdir` +/// of a cgroup only succeeds once it's empty, so callers run this AFTER +/// its processes exit (`destroy()`: after killing FC + the uffd +/// handler). Best-effort. +pub fn remove_node_cgroup(parent: Option<&Path>, leaf: &str) { let Some(parent) = parent else { return; }; - let dir = parent.join(sandbox_id.to_string()); + let dir = parent.join(leaf); if let Err(e) = std::fs::remove_dir(&dir) { if e.kind() != std::io::ErrorKind::NotFound { tracing::debug!( - %sandbox_id, path = %dir.display(), error = %e, - "ADR 0044 K2: could not remove per-VM node cgroup (non-empty or already gone)" + "ADR 0044 K2: could not remove node cgroup leaf (non-empty or already gone)" ); } } @@ -6574,7 +6566,7 @@ async fn destroy_teardown( // ADR 0044 K2: FC + the uffd handler are gone, so the per-VM node // cgroup is now empty — remove it (rmdir of a cgroup only succeeds // once empty). No-op when no `vm_cgroup_parent` is configured. - remove_node_cgroup(vm_cgroup_parent.as_deref(), id); + remove_node_cgroup(vm_cgroup_parent.as_deref(), &id.to_string()); // Tear down per-VM networking: yank iptables rules tagged // with this sandbox's chain comment, delete the TAP, return @@ -7360,9 +7352,9 @@ mod tests { #[test] fn node_cgroup_escape_creates_leaf_and_writes_pid() { let tmp = tempfile::tempdir().unwrap(); - let id = SandboxId::new(); - escape_to_node_cgroup(tmp.path(), id, &[4242]).expect("escape"); - let leaf = tmp.path().join(id.to_string()); + let id = SandboxId::new().to_string(); + escape_to_node_cgroup(tmp.path(), &id, &[4242]).expect("escape"); + let leaf = tmp.path().join(&id); assert!(leaf.is_dir(), "per-VM leaf cgroup dir created"); let procs = std::fs::read_to_string(leaf.join("cgroup.procs")).unwrap(); assert_eq!(procs.trim(), "4242", "pid written to cgroup.procs"); @@ -7373,13 +7365,13 @@ mod tests { fn node_cgroup_remove_drops_empty_leaf_idempotently() { // Models destroy() after FC + the handler are killed (empty cgroup). let tmp = tempfile::tempdir().unwrap(); - let id = SandboxId::new(); - let leaf = tmp.path().join(id.to_string()); + let id = SandboxId::new().to_string(); + let leaf = tmp.path().join(&id); std::fs::create_dir_all(&leaf).unwrap(); - remove_node_cgroup(Some(tmp.path()), id); + remove_node_cgroup(Some(tmp.path()), &id); assert!(!leaf.exists(), "empty leaf removed"); - remove_node_cgroup(Some(tmp.path()), id); // gone → no-op - remove_node_cgroup(None, id); // unconfigured → no-op + remove_node_cgroup(Some(tmp.path()), &id); // gone → no-op + remove_node_cgroup(None, &id); // unconfigured → no-op } /// Default backend for negative-path unit tests: kernel/firecracker diff --git a/deploy/helm/engram-host-fleet/files/thin-pool.sh b/deploy/helm/engram-host-fleet/files/thin-pool.sh index 18ad84993..961f9b469 100755 --- a/deploy/helm/engram-host-fleet/files/thin-pool.sh +++ b/deploy/helm/engram-host-fleet/files/thin-pool.sh @@ -1,6 +1,7 @@ #!/bin/sh # Build or re-activate the node's dm-thin pool over two preallocated -# files attached as loop devices. Idempotent: safe on every boot. +# files attached as loop devices, and load the ublk driver. Idempotent: +# safe on every boot. # Exits non-zero when thin_check fails, so a node never comes up with a # pool it cannot trust. Runs in the host mount namespace. # @@ -31,6 +32,12 @@ DATA="$POOL_DIR/data.img" log() { echo "[thin-pool] $*"; } +# The disk sources of lazy restores are ublk devices. Loaded before the +# pool check, so a node whose pool is already active loads it too. A +# kernel without the driver keeps the node: the host-agent then refuses +# to start a source and logs why. +modprobe ublk_drv 2>/dev/null || log "WARNING: no ublk driver; lazy restore cannot run here" + if dmsetup info "$POOL_NAME" >/dev/null 2>&1; then log "$POOL_NAME is active" dmsetup status "$POOL_NAME" diff --git a/docker/host-agent.Dockerfile b/docker/host-agent.Dockerfile index 04994ceec..30a76e751 100644 --- a/docker/host-agent.Dockerfile +++ b/docker/host-agent.Dockerfile @@ -30,9 +30,11 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ && echo "egress-proxyd fingerprint: $FP" \ && ENGRAM_EGRESS_PROXYD_FINGERPRINT="$FP" \ cargo build --release -p engram-host-agent -p engram-uffd-handler -p engram-egress-proxyd \ + -p engram-disk-source \ && cp target/release/engram-host-agent /tmp/engram-host-agent \ && cp target/release/engram-uffd-handler /tmp/engram-uffd-handler \ - && cp target/release/engram-egress-proxyd /tmp/engram-egress-proxyd + && cp target/release/engram-egress-proxyd /tmp/engram-egress-proxyd \ + && cp target/release/engram-disk-source /tmp/engram-disk-source # Trixie matches the builder's glibc — bookworm (2.36) refuses # binaries linked against trixie's glibc 2.39+. @@ -80,4 +82,7 @@ COPY --from=builder /tmp/engram-uffd-handler /usr/local/bin/engram-uffd-handler # text keeps the running process alive after the image is unlinked — # the FC precedent), so in-flight guest egress survives rolls. COPY --from=builder /tmp/engram-egress-proxyd /usr/local/bin/engram-egress-proxyd +# ADR 0127: the ublk server a lazy restore reads its disk through. Spawned +# by host-agent by a PATH lookup; like the daemon above, it outlives the pod. +COPY --from=builder /tmp/engram-disk-source /usr/local/bin/engram-disk-source ENTRYPOINT ["/usr/local/bin/engram-host-agent"] diff --git a/docs/adr/0127-lazy-session-disk.md b/docs/adr/0127-lazy-session-disk.md new file mode 100644 index 000000000..bfccc7ca4 --- /dev/null +++ b/docs/adr/0127-lazy-session-disk.md @@ -0,0 +1,455 @@ +# ADR 0127: The lazy session disk — a restore boots before its disk is complete + +- Status: Proposed +- Date: 2026-11-06 +- Amends: ADR 0125 (the kernel-owned session disk), its Resume + lifecycle ("the dirty chunks are written before the VM starts; + nothing is faulted lazily from the chunk store on the disk path") and + its Tradeoff "staging precedes the first boot on a node … there is no + lazy fallback". Every other decision of ADR 0125 stands. +- Related: ADR 0007 (chunk store), ADR 0036 (image readiness and + prestage), ADR 0039 / 0092 (lazy memory), ADR 0044 K2 (VM processes + outlive the host-agent pod), ADR 0079 / 0116 (ownership), ADR 0098 / + 0099 (simulation and invariants), ADR 0123 (teleport). + +## Terms + +- **Source**: a read-only block device whose bytes are one published + disk manifest (ADR 0007). It holds no writes. +- **Source process**: the userspace process that serves one source + through ublk. One per sandbox while that sandbox copies. +- **dm-clone**: a device-mapper target that presents a writable copy of + a read-only source device. Reads of a region it has not copied go to + the source; writes go to the destination; a background task copies + the remaining regions. A **region** is its unit of copy (64 KiB here, + the default block size of a production pool). Its **bitmap** records + the copied regions, on a metadata device. +- **Hydration**: the background copy, from the source into the + destination, until every region is copied. +- **Cutover**: replacing the dm-clone table with a `linear` table onto + the destination, under a device-mapper suspend. The device node and + every open handle stay valid. +- **Eager restore**: ADR 0125's restore. Write every chunk the volume + needs, then boot. +- **Lazy restore**: this ADR's restore. Boot over dm-clone, then hydrate. + +## Summary, in plain English + +ADR 0125 made the kernel the only owner of a session disk. It also made +a restore write the whole disk before the guest boots, unless the host +already holds the session's base image version. In production that is +the usual case, not the rare one: images are baked often, hosts stage +only the enabled version, and most idle sessions point at an older +version. A restore of such a session writes the full disk from the +chunk store inside the restore deadline. On a busy or new host it does +not finish, each retry starts again from zero, and the session becomes +dead after the retry limit. + +The target for every session operation is under 10 seconds in the worst +case. No eager design meets that: a 26 GiB root disk at 1 GB/s is 26 +seconds before any contention. Time to boot must not grow with disk +size. + +After this change, a restore that cannot finish eagerly in about two +seconds boots over dm-clone at once. The guest reads uncopied regions +through a read-only source and writes into a thin volume as before. +Hydration copies the rest in the background. When it ends, the device +is cut over to a plain thin volume and the session is exactly an ADR +0125 session. The only userspace process in the path serves read-only +data, holds no writes, and exists only while its sandbox copies. + +## Context + +### What failed + +The first production deploy of ADR 0125 had three properties that +combined: + +1. A restore with no staged base for the session's image version wrote + every chunk of the root and swap disks before boot. For a large + development image that is tens of GiB. +2. The restore ran inside the restore RPC deadline (240 s). The deadline + cancels the host's work, so a retry starts the copy again. Memory + prefetch that the failed attempt spawned kept running. +3. The host that took the restore was new and was still staging images + and bundles. It pulled far less than its network and disk could + carry, and its chunk cache evicted chunks under the combined load. + +Each cause can be fixed. None of the fixes gives a time to boot that is +independent of disk size. + +### What ADR 0125 protects + +ADR 0125's lesson: "a process that restarts several times a week must +not own a resource whose lifetime is a session." Its incident table has +three classes: + +| Class | Failure | Root | +|---|---|---| +| Durability | acknowledged writes lost on a restart | userspace held write state | +| Liveness | a healthy VM killed after a failed reconfigure | the kernel device was bound to the old process, and recovery reacted | +| Event channel | a recovery event reached the coordinator before the recovery finished | a side channel beside ownership | + +A lazy disk must not bring any of the three back. The design below +addresses each one explicitly. + +### What the kernel can do (spike, 2026-11-05) + +A spike on the Linux dev VM (kernel 6.8) measured dm-clone with a dm-thin +destination over a read-only source behind dm-delay, 2 GiB, region 64 +KiB. Every sample includes about 4–5 ms of tool overhead. + +| Measure | 80 ms source | 1 ms source | +|---|---|---| +| create to first read | 175 ms | 12 ms | +| read of an uncopied region | 91 ms | 3.4 ms | +| second read of the same region (reads do not copy) | 85 ms | 3.0 ms | +| 4 KiB write to an uncopied region (copy, then write) | 95 ms | 6.4 ms | +| 64 KiB write to an uncopied region (no copy) | 5.3 ms | 5.4 ms | +| 4 KiB write to a copied region | 4.8 ms | 4.7 ms | +| 4 KiB write + fsync, dm-clone / plain thin | 11.3 / 8.2 ms | 10.9 / 8.1 ms | +| hydration of 2 GiB | 49.6 s (41 MiB/s) | 6.2 s (332 MiB/s) | +| cutover to linear with a handle open | 6.4 ms | 5.9 ms | +| destination equals source plus every write | pass | pass | + +`ublk_drv` loads on the same kernel from the distribution's extra +modules. The production node image (Container-Optimized OS, kernel +6.12) builds dm-clone into the kernel (`CONFIG_DM_CLONE=y`) and ships +`ublk_drv` as a signed module (`CONFIG_BLK_DEV_UBLK=m`) that is not +loaded until something loads it. + +A first source on the same dev VM (`engram-disk-source`, libublk) +served a manifest's exact bytes and survived a `SIGKILL` mid-read: a +new process attached with user recovery, the kernel reissued the +queued reads, and no read failed. The same work found a kernel rule +the design must respect: stopping or deleting a ublk device that has +a request in flight and no live server to finish it blocks in the +kernel until the node reboots. + +### Where the fetch time goes + +A host-agent profile of a cold resume on the development cluster found +that the chunk fetch paths drive every fetch from one task: TLS, +response parsing, the hash check and the copies for every chunk run on +one core. Raising the number of requests in flight did not raise the +throughput; it raised the latency of each request. A production host +in the failure above pulled 100–220 MB/s with its CPU, network and disk +far below their limits, which is the same single-core bound. A fetch +path that spreads this work across cores is a precondition for any +estimate in Decision 1. + +With one task per fetch, the same profile moves the limit off the +network. Memory chunks then arrive at about 1 GB/s, and the disk diff +is bound by writes: of 64 fetches in flight, on average 3 wait on the +store and the rest wait on direct writes into the thin volume. The +memory prefetch writes its chunks to the chunk cache on the same disk, +so a faster memory stream makes the disk diff slower (about 130 MB/s +into the volume while both run). The rates in Decision 1 are therefore +write rates of the host's disk while a memory restore runs, not +network rates, and the estimate counts the bytes of both streams. + +## Decision + +1. **A restore picks eager or lazy before it starts, and can fall back + to lazy.** The host computes the bytes the volume needs from the + manifests (`Manifest::delta_from` against the staged base when one + exists) and splits them into local bytes (a staged volume or the + chunk cache holds them) and remote bytes. The estimate is + `local / local_rate + remote / remote_rate`, with rates the host + measures from its own recent stages and a conservative floor; no + rate is a constant. At or under the eager budget, the restore is + eager. The budget starts at 2 s and is set from measurement once + the fetch path uses every core. Over it, the restore is lazy. An eager restore that passes + its budget continues as a lazy restore over the partly written + volume: the eager path writes exactly the manifest's bytes, so + hydration rewrites those regions with identical data and the result + is the same volume. A wrong estimate costs time, never data. Every + restore records the path, the estimate and the actual time. +2. **A lazy restore boots over dm-clone.** The destination is a thin + volume: a snapshot of the staged base when there is one, otherwise a + fresh volume. The bitmap is on a small thin volume of its own. The + source is a ublk device. The guest's drive is the dm-clone device, + made under `/dev/engram/` as ADR 0125 makes every drive. The guest + boots as soon as the table is loaded. + + Hydration copies only what the destination lacks. dm-clone copies + every region it has not marked, and it has no zero detection, so + without help it would write the whole volume: every zero region + becomes allocated pool blocks and every region equal to the base + stops sharing the base's blocks. The drive is loaded with hydration + off and `no_discard_passdown`. The host discards every whole region + in which the destination already holds the manifest's bytes (a chunk + absent from both the manifest and the destination's origin, or with + the same hash in both), which marks those regions copied without + touching the destination. Then it turns hydration on. The set comes + from manifests only. The guest cannot discard (Decision 8), so these + host discards are the only ones. +3. **The source is read-only and stateless.** It serves one manifest. + For each read it uses the first tier that holds the chunk: a staged + volume whose chunk has the same hash, the chunk cache, a peer, then + a ranged read from the chunk store followed by a fill of the whole + chunk. Guest reads go before hydration reads. The fetch, the hash + check and the copies of each chunk run in their own task, so the + source uses every core and one slow fetch holds up only the reads + that need its chunk. The source holds no + writes; every byte it can serve is in the chunk store. Its state is + a function of the manifest id. +4. **One source process per copying sandbox, outside the host-agent.** + The host-agent starts it, moves it to a leaf of the node's cgroup of + its own (ADR 0044 K2; not the Firecracker leaf, which goes with the + VM while the source can outlive it), and records it in a durable + record of its own. The record is not `sandbox.json`: that file + belongs to the Firecracker backend, and the orphan reap deletes it + with the jail. The source exits after the cutover. A host-agent + restart does not touch it. A recovery runs the build the host has + then, so the serve arguments and the device geometry are a contract + across one release. +5. **A source failure stalls reads; it never ends a session.** The ublk + device uses user recovery with reissue: when the process dies the + kernel keeps the device and queues its I/O, and a new process + attaches and the kernel reissues what was in flight. The host-agent + restarts a dead source from its record. No code path kills, + evicts or rewinds a VM because its source failed. + + A suspend of a drive waits for the requests in flight, so a dead + source blocks it. Every suspend (a checkpoint, the cutover) first + makes sure the source is alive, and a recovery never waits on a + lock that a suspend holds. + + A read the source cannot serve is retried with backoff while the + failure is transient (the store is unreachable, slow or throttled). + A definitive failure, a chunk the store reports as absent or whose + bytes fail the hash check again after a fresh fetch, is data loss: + the source returns an I/O error for that read and logs an error + with a fixed message that alerting matches. Waiting cannot repair + it, and a panic would only make recovery reissue the same read. + + A source is stopped only while it can finish every request it + holds: the host-agent recovers a dead source before it stops it, + and a stopping source first fails any read it cannot serve. The + kernel waits for requests in flight when a ublk device stops; a + request that nothing will finish blocks that stop, and every later + device operation on the node, until the node reboots. +6. **Hydration belongs to the sandbox.** It is a host-owned job tied to + the sandbox's lifetime: destroyed with the sandbox, resumed after a + host-agent restart from the bitmap (which the kernel owns), never + restarted from zero. The restore RPC returns when the VM runs, not + when hydration ends. Memory prefetch is tied to the sandbox the same + way. +7. **Cutover is the only exit.** When the bitmap shows every region + copied and no checkpoint of the drive is still uploading, the + host-agent suspends the device, loads a `linear` table onto the + destination, resumes it, and deletes the bitmap volume. It stops the + source process when the last view over it is gone: an eviction's + upload reads through a view after the VM has stopped. The session is + then an ADR 0125 session in every respect. + + While a sandbox hydrates, a live move of it is refused and the + coordinator moves it by snapshot. A release during hydration keeps + no checkpoint volume, because no volume of the role is complete. +8. **A checkpoint during hydration is correct without waiting.** Under + the pause the checkpoint syncs the drive, which commits the bitmap + with every region the guest wrote. It then suspends the drive, which + stops the background copy, snapshots the bitmap volume and then the + destination, and resumes the drive. The copy does not stop for the + guest pause. With the bitmap snapshot taken first and the drive + synced, the pair is consistent: the kernel records a region in the + bitmap only after its copy is in the destination, so the later + destination snapshot holds every region the bitmap snapshot records. + The suspend makes the pair consistent in either order, and costs + milliseconds inside the pause that already exists. A destination + snapshot reads zeros in regions not yet + copied, so the upload never reads it directly. It reads through a + view: a second dm-clone table over the two snapshots and the same + source, with hydration off and loaded read-only, which the kernel + composes exactly as the guest saw it. The dm-thin root lane proves + this on kernel 6.8 with the copy running and guest writes before and + after the snapshots. The upload hashes every 16 MiB window in which the + destination snapshot differs from its origin (the staged base when + the destination is a snapshot of it, otherwise an empty volume), and + publishes the windows whose hash differs from the source manifest; + every other window is the source manifest's chunk. With no staged + base, every copied window differs from the origin, so such a + checkpoint reads and hashes the whole disk, outside the pause. The + rule holds while the guest cannot discard: dm-clone records a + discarded region as copied without writing it. Copied windows hash equal to the source, so + they cost a read and a hash, not an upload. The view and both + snapshots are deleted once the manifest is durable. +9. **The heartbeat carries a fact, not an event.** Per sandbox: whether + it is hydrating. A host roll waits, within a bound, until no sandbox + on the host is hydrating; past the bound it rolls and the sources + survive it (Decision 4). +10. **Disk readiness stops gating placement.** A host that has not + staged an image can take a session of that image: the first session + is a lazy restore from the image manifest. Staging continues in the + background and later sessions are eager. Memory readiness (ADR 0092) + is unchanged. + +## Design + +### Objects on a node + +```text +/dev/engram/engram-pool.sb--root the guest's drive: dm-clone while hydrating, then linear +/dev/engram/engram-pool.sb--root-dest--v the destination thin; names the lineage's newest manifest +/dev/engram/engram-pool.sb--root-bitmap the dm-clone metadata thin volume +/dev/engram/engram-pool.tmp-sb--root{,-bm,-view} a checkpoint's pair and its read-only view, while it uploads +/dev/ublkbN the source, one per hydrating role +/disk-sources/-root.json the source process, its device and its serve arguments +``` + +Swap follows root: a role whose manifest is not local restores lazily +the same way. + +`SessionDisk` carries the drive operations: `clone_drive`, +`open_clone`, `clone_state`, `enable_hydration`, `cut_over`, +`snapshot_clone`, `clone_view` and `remove_clone`. `FileDisk` (macOS and +the simulator) has no dm-clone. It hydrates eagerly: `clone_drive` +copies the source into the destination before it returns, so a drive +is hydrated from the start and `cut_over` only records the state. + +A drive takes a bitmap that no drive has used (its first 4 KiB, where +the kernel writes the dm-clone superblock, reads as zeros), and refuses +a destination or bitmap that another drive or view uses. Operations on +one drive are serialized, and a suspend runs to its resume in a task +that a cancelled caller cannot stop. + +The names stay the only record, as in ADR 0125. A checkpoint published +during hydration renames the destination to the new manifest version. +After the cutover, the first checkpoint publishes a `-ckpt-` volume, a +complete snapshot of the destination, which is newer than the +destination's name; from then on the role has the ADR 0125 names. + +The region size is part of the on-disk format. The bitmap's superblock +records it, and the kernel refuses a table over that bitmap with any +other size, so a change of `CLONE_REGION_BYTES` is a format change. A +view reads the region size from the superblock of its bitmap snapshot, +so it composes with whatever size the drive used. + +### Lifecycles + +**Lazy restore.** Create the destination and bitmap volumes. Start the +source process for the manifest and wait for its ublk device. Load the +dm-clone table (region 128 sectors, hydration enabled, threshold and +batch sized from the measured source rate). Record the source. Start +Firecracker on the dm-clone device. + +**Hydration.** The kernel copies. The host-agent reads the bitmap +counters from `dmsetup status` on each heartbeat and starts the cutover +when they are complete. + +**Host-agent restart.** Opening the pool resumes every device of the +pool that a death between a suspend and its resume left suspended: +thins first, then the drives and views on them. Read the source +records, before the volume reconcile, which removes drives. For a +hydrating sandbox, confirm the dm-clone device and the source process. Restart the source if it is gone (Decision 5). A drive with no +live table is a `clone_drive` that did not finish: remove it and make +it again on a new bitmap. Nothing is copied again. + +**Checkpoint, evict, destroy.** As ADR 0125, with Decision 8 for the +upload. Destroy also stops the source and deletes the bitmap. + +**Teleport target.** The target boots on a lazy source for the moved +checkpoint instead of staging it first. The three-tier block move of +ADR 0123 is unchanged for the blocks the source wrote after the +checkpoint. + +### Placement + +The pool floor (ADR 0125) stays a hard gate. A lazy restore writes the +chunks that differ from its destination's origin, as an eager restore +does, so placement counts the same bytes for both paths. + +## Tradeoffs + +- **A userspace process is in the read path while a sandbox + hydrates.** It is read-only and bounded in time, and its failure mode + is a read stall. It is not in the write path at any time. +- **First reads during hydration cost the source's latency.** A GCS + miss is tens of milliseconds for a ranged read. The page cache, the + restored guest memory, the chunk cache and the read priority keep + this to the first touch of each chunk. +- **A partial write into an uncopied region waits for one region + copy.** Measured at one source read. A whole-region write does not + wait. +- **An fsync during hydration commits the bitmap.** Measured at about 3 + ms more than a plain thin volume. +- **Two restore paths.** Decision 1 decides between them from manifests + before the restore, and the fallback makes a wrong decision safe. + With the skipped regions, both paths write the same bytes into the + pool; they differ in when the bytes arrive. A later change can make + every restore lazy if the read latency during hydration allows it. +- **A ublk device can block a node.** A device stopped with a request + that nothing will finish blocks in the kernel until reboot (Decision + 5 states the rule that prevents it). The rule is enforced in the + source and in the host-agent, and tested. +- **dm-clone is less used than dm-thin.** It has been in mainline Linux + since 5.4. The tests below cut every step, and the production node + image is checked before any code depends on it. + +## Testing + +- **Unit (`engram-disk-source`):** the source serves a manifest's exact + bytes for every offset and length; tier order; guest reads ahead of + hydration reads; a zero chunk reads as zeros without a fetch. +- **dm-thin root lane (KVM runners, `ci.yml`):** the spike's checks as a + test: reads, partial and whole-region writes, hydration, cutover with + a handle open, and destination equals source plus every write. Cut at + every step of the lazy restore and the cutover, then converge on the + next pass to the same bytes, keeping every write of the earlier pass; + a drive left with no table is removed and made again. A drive left + suspended resumes when the pool is opened again and copies on to the + cutover. Kill the source mid-hydration; the reads + stall and complete after recovery with no error. Stop a source that + holds a read it cannot serve; the read fails and the stop completes. + A chunk the store reports absent fails its read with an I/O error; a + store that is only slow does not. +- **Firecracker integration (KVM runners, `ci.yml`):** resume from a + manifest with no staged base: the guest runs in seconds and its disk + is correct; a checkpoint during hydration reproduces through `stage`; + a host-agent restart during hydration loses no write and copies + nothing again. Each sized to its property. +- **Decision 1:** the estimate picks eager for a small diff over a + staged base and lazy for an unstaged image; a forced fallback from + eager to lazy produces the same bytes. +- **Simulation (engram-sim):** the heartbeat's hydrating fact; the + invariant that no source failure produces an eviction or a rewind. +- **Production evidence for acceptance:** a resume of an idle session on + an image version no host stages, on a new host, boots in under 10 s; + restore-deadline failures at zero for a week. + +## Rollout + +Each step is one PR and retires what it replaces. + +0. **Gate.** The production node image has dm-clone built in and + `ublk_drv` as a module (checked 2026-11-05). Node-prep loads + `ublk_drv` once per boot beside `dm_thin_pool`, so no source process + loads modules. Turn on host-agent tracing in production so restores + can be measured. Make the chunk fetch paths spread their work across + cores (Context: where the fetch time goes) and measure the eager + rates Decision 1 uses. +1. **The source.** The `engram-disk-source` crate and binary: the ublk + server over a manifest, with its tests. +2. **dm-clone in `engram-disk-thin`.** `SessionDisk` gains the clone, + status and cutover operations, `FileDisk` gains their file + equivalents for macOS and the simulator, with the root-lane tests. +3. **Lazy restore.** Decision 1 and the lazy path in + `provision_for_restore`; hydration and cutover owned by the sandbox; + the restore RPC returns at boot; memory prefetch tied to the sandbox. + A host-agent flag (`off`, `force`, `auto`; off by default) turns it + on. The build that first understands the lazy volume names ships + before the flag is on anywhere, so a rollback never meets a lazy + sandbox that it cannot see. A rollback to a build without source + support while sandboxes still hydrate removes their drives without + first recovering a dead source, which is the case Decision 5 says + can block the node; turn the flag off and let hydration finish + before such a rollback. + + The chunk selection of Decision 8 depends on a guest that cannot + discard. Firecracker gives the guest no discard today; a change that + gives it one changes this design first. +4. **Placement and teleport.** Decision 10, the pool accounting for + lazy restores, and the teleport target. +5. **Accept.** Write the facts into the DESIGN.md "Session disk" page + and flip this ADR. diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1159481e9..63cd54843 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -57,6 +57,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["http1" hyper-util = { version = "0.1", features = ["client-legacy", "client-proxy", "server-auto", "service"] } idna = { version = "1" } indexmap = { version = "2", features = ["serde"] } +itertools = { version = "0.14" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits"] } log = { version = "0.4", default-features = false, features = ["std"] } @@ -137,6 +138,7 @@ hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15" } hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17" } idna = { version = "1" } indexmap = { version = "2", features = ["serde"] } +itertools = { version = "0.14" } log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } num-traits = { version = "0.2", features = ["i128", "libm"] } @@ -181,44 +183,58 @@ winnow = { version = "1" } zeroize = { version = "1", features = ["derive"] } [target.x86_64-unknown-linux-gnu.dependencies] +concurrent-queue = { version = "2" } +event-listener-strategy = { version = "0.5" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } libc = { version = "0.2", default-features = false, features = ["use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } -nix = { version = "0.31", features = ["hostname", "ioctl", "mount", "socket"] } +nix = { version = "0.31", features = ["event", "hostname", "ioctl", "mount", "socket"] } nom = { version = "7" } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } tower = { version = "0.5", default-features = false, features = ["timeout"] } [target.x86_64-unknown-linux-gnu.build-dependencies] +bindgen = { version = "0.69" } +concurrent-queue = { version = "2" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nom = { version = "7" } +prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } [target.x86_64-unknown-linux-musl.dependencies] +concurrent-queue = { version = "2" } +event-listener-strategy = { version = "0.5" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } libc = { version = "0.2", default-features = false, features = ["use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } -nix = { version = "0.31", features = ["hostname", "ioctl", "mount", "socket"] } +nix = { version = "0.31", features = ["event", "hostname", "ioctl", "mount", "socket"] } nom = { version = "7" } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } tower = { version = "0.5", default-features = false, features = ["timeout"] } [target.x86_64-unknown-linux-musl.build-dependencies] +bindgen = { version = "0.69" } +concurrent-queue = { version = "2" } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } +linux-raw-sys = { version = "0.12", default-features = false, features = ["auxvec", "elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "xdp"] } mio = { version = "1", features = ["net", "os-ext"] } nom = { version = "7" } +prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } prost = { version = "0.14", default-features = false, features = ["no-recursion-limit"] } -rustix = { version = "1", features = ["event", "fs", "stdio", "termios"] } +rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "stdio", "termios", "time"] } [target.x86_64-apple-darwin.dependencies] errno = { version = "0.3" }