From cb6f6e1bda26cb1721c6052dc3c07930c761fba1 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 8 Sep 2026 18:14:46 +0200 Subject: [PATCH 1/6] ostree-ext: Write full digests in generate_derived_oci_from_tar The diff_id appended for the generated layer was the bare hex encoding rather than the `sha256:...` form the spec requires. Our own importer does not look at it, so nothing noticed, but any other consumer of an image built by this helper trips over it. Assisted-by: AI Signed-off-by: Alexander Larsson --- crates/ostree-ext/src/integrationtest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ostree-ext/src/integrationtest.rs b/crates/ostree-ext/src/integrationtest.rs index 8c05c961e0..28d7234197 100644 --- a/crates/ostree-ext/src/integrationtest.rs +++ b/crates/ostree-ext/src/integrationtest.rs @@ -105,7 +105,7 @@ where config .rootfs_mut() .diff_ids_mut() - .push(new_layer.uncompressed_sha256.digest().to_string()); + .push(new_layer.uncompressed_sha256_as_digest().to_string()); let new_config_desc = src.write_config(config)?; manifest.set_config(new_config_desc); From 99043f8a13070411144cf952e76f4db523a3e296 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 8 Sep 2026 14:16:11 +0200 Subject: [PATCH 2/6] Add --from-delta for the composefs backend to support oci-delta This adds a `--from-delta PATH` flag to `bootc upgrade` and `bootc switch` that applies an oci-delta in one step. This commit add support only for the composefs backend, and uses the native delta pull support in composefs-rs. Later commits add ostree backend support. Basic parsing of the delta file is done by the oci-delta rust bindings. See https://github.com/containers/oci-delta for details of the format. Unified storage cannot apply a delta yet and will fail. Real support for this would require delta support in containers/storage. Generated-by: AI Signed-off-by: Alexander Larsson --- Cargo.lock | 14 + crates/lib/Cargo.toml | 1 + crates/lib/src/bootc_composefs/repo.rs | 20 +- crates/lib/src/bootc_composefs/status.rs | 9 +- crates/lib/src/bootc_composefs/switch.rs | 32 +- crates/lib/src/bootc_composefs/update.rs | 109 +++- crates/lib/src/cli.rs | 59 ++ crates/lib/src/delta.rs | 657 +++++++++++++++++++++++ crates/lib/src/install.rs | 2 +- crates/lib/src/lib.rs | 2 + docs/src/man/bootc-switch.8.md | 4 + docs/src/man/bootc-upgrade.8.md | 4 + 12 files changed, 883 insertions(+), 30 deletions(-) create mode 100644 crates/lib/src/delta.rs diff --git a/Cargo.lock b/Cargo.lock index b5d4d1c7d0..8b20c8827c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,6 +391,7 @@ dependencies = [ "linkme", "linux-kernel-cmdline", "nom 8.0.0", + "oci-delta", "ocidir", "openssl", "ostree-ext", @@ -2419,6 +2420,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-delta" +version = "0.1.0" +source = "git+https://github.com/containers/oci-delta?tag=rust-v0.1.0#1a3ca4633dedcb731272c7f2ff83cd8053472b9e" +dependencies = [ + "anyhow", + "flate2", + "hex", + "oci-spec", + "openssl", + "zstd", +] + [[package]] name = "oci-spec" version = "0.10.0" diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index 27698065be..109e26758f 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -13,6 +13,7 @@ rust-version = "1.85.0" include = ["/src", "/build.rs", "LICENSE-APACHE", "LICENSE-MIT"] [dependencies] +oci-delta = { git = "https://github.com/containers/oci-delta", tag = "rust-v0.1.0" } # Internal crates bootc-blockdev = { package = "bootc-internal-blockdev", path = "../blockdev", version = "1.16.12" } linux-kernel-cmdline = { workspace = true } diff --git a/crates/lib/src/bootc_composefs/repo.rs b/crates/lib/src/bootc_composefs/repo.rs index 9a12624aaf..5141ae44da 100644 --- a/crates/lib/src/bootc_composefs/repo.rs +++ b/crates/lib/src/bootc_composefs/repo.rs @@ -316,10 +316,17 @@ async fn pull_composefs_unified( /// When `use_unified` is false (the default), the image is pulled directly /// into the composefs repo via skopeo. /// +/// With a `delta`, the content is read from the delta file instead and no +/// network access happens at all; `spec_imgref` then only names what the +/// resulting deployment tracks. composefs-ctl recognises the delta artifact +/// from the layout's manifest and reconstructs the changed layers against the +/// source image in the repository, failing if that image is absent. +/// /// Checks for boot entries in the image and returns them. #[context("Pulling composefs repository")] pub(crate) async fn pull_composefs_repo( spec_imgref: &crate::spec::ImageReference, + delta: Option<&crate::delta::Delta>, allow_missing_fsverity: bool, use_unified: bool, quiet: bool, @@ -327,7 +334,17 @@ pub(crate) async fn pull_composefs_repo( ) -> Result { const COMPOSEFS_PULL_JOURNAL_ID: &str = "4c3b2a1f0e9d8c7b6a5f4e3d2c1b0a9f8"; - let imgref = spec_imgref.to_image_proxy_ref()?; + let imgref = match delta { + Some(delta) => { + delta.validate_image_reference(spec_imgref)?; + crate::delta::reject_unified_storage(delta, use_unified)?; + if !quiet { + println!("Applying delta {}", delta.describe()); + } + delta.pull_ref()? + } + None => spec_imgref.to_image_proxy_ref()?, + }; tracing::info!( message_id = COMPOSEFS_PULL_JOURNAL_ID, @@ -336,6 +353,7 @@ pub(crate) async fn pull_composefs_repo( bootc.transport = %imgref.transport, bootc.allow_missing_fsverity = allow_missing_fsverity, bootc.unified_storage = use_unified, + bootc.delta = delta.map(|d| d.path.as_str()), "Pulling composefs image {imgref}", ); diff --git a/crates/lib/src/bootc_composefs/status.rs b/crates/lib/src/bootc_composefs/status.rs index 33c323bd07..945158b468 100644 --- a/crates/lib/src/bootc_composefs/status.rs +++ b/crates/lib/src/bootc_composefs/status.rs @@ -412,10 +412,13 @@ pub(crate) fn list_bootloader_entries(storage: &Storage) -> Result Result { +) -> Result<(ImgConfigManifest, String)> { let mut config = crate::deploy::new_proxy_config(); ostree_ext::container::merge_default_container_proxy_opts(&mut config)?; @@ -427,7 +430,7 @@ pub(crate) async fn get_container_manifest_and_config( .await .with_context(|| format!("Opening image {imgref}"))?; - let (_, manifest) = proxy.fetch_manifest(&img).await?; + let (manifest_digest, manifest) = proxy.fetch_manifest(&img).await?; let (mut reader, driver) = proxy.get_descriptor(&img, manifest.config()).await?; let mut buf = Vec::with_capacity(manifest.config().size() as usize); @@ -437,7 +440,7 @@ pub(crate) async fn get_container_manifest_and_config( let config: oci_spec::image::ImageConfiguration = serde_json::from_slice(&buf)?; - Ok(ImgConfigManifest { manifest, config }) + Ok((ImgConfigManifest { manifest, config }, manifest_digest)) } /// Directory where BLS-compatible bootloaders expect Type 1 boot entries. diff --git a/crates/lib/src/bootc_composefs/switch.rs b/crates/lib/src/bootc_composefs/switch.rs index 0cb0b9ebea..3a580a6285 100644 --- a/crates/lib/src/bootc_composefs/switch.rs +++ b/crates/lib/src/bootc_composefs/switch.rs @@ -6,7 +6,8 @@ use crate::{ status::get_composefs_status, update::{ DoUpgradeOpts, UpdateAction, apply_upgrade_from_downloaded, do_upgrade, - is_image_pulled, validate_update, + ensure_delta_source_present, is_image_pulled, lookup_config_splitstream, + validate_update, }, }, cli::{SwitchOpts, imgref_for_switch}, @@ -25,6 +26,8 @@ pub(crate) async fn switch_composefs( .await .context("Getting composefs deployment status")?; + let delta = crate::delta::open_opt(opts.from_delta.as_deref()).await?; + let prog: ProgressWriter = opts.progress.clone().try_into()?; let mut do_upgrade_opts = DoUpgradeOpts { @@ -34,6 +37,7 @@ pub(crate) async fn switch_composefs( use_unified: false, quiet: opts.quiet, prog, + delta: delta.as_ref(), }; if opts.download_opts.from_downloaded { @@ -41,6 +45,9 @@ pub(crate) async fn switch_composefs( } let target = imgref_for_switch(&opts)?; + if let Some(delta) = delta.as_ref() { + delta.validate_image_reference(&target)?; + } let new_spec = { let mut new_spec = host.spec.clone(); @@ -92,14 +99,29 @@ pub(crate) async fn switch_composefs( booted_unified || target_unified }; - let (image, img_config) = is_image_pulled(repo, &target_imgref).await?; + // With a delta the target is whatever the delta says it is, and we can look + // it up locally; without one we have to ask the registry. + let (image, manifest) = match &delta { + Some(delta) => { + crate::delta::reject_unified_storage(delta, do_upgrade_opts.use_unified)?; + ensure_delta_source_present(repo, delta)?; + ( + lookup_config_splitstream(repo, delta.target_manifest().config().digest())?, + delta.target_manifest().clone(), + ) + } + None => { + let (image, img_config, _) = is_image_pulled(repo, &target_imgref).await?; + (image, img_config.manifest) + } + }; if let Some(cfg_verity) = image { let action = validate_update( storage, booted_cfs, &host, - img_config.manifest.config().digest().as_ref(), + manifest.config().digest().as_ref(), &cfg_verity, true, )?; @@ -117,7 +139,7 @@ pub(crate) async fn switch_composefs( &host, &target_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await; } @@ -130,7 +152,7 @@ pub(crate) async fn switch_composefs( &host, &target_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await?; diff --git a/crates/lib/src/bootc_composefs/update.rs b/crates/lib/src/bootc_composefs/update.rs index c2b43c8b86..9239638c64 100644 --- a/crates/lib/src/bootc_composefs/update.rs +++ b/crates/lib/src/bootc_composefs/update.rs @@ -14,6 +14,7 @@ use ostree_ext::container::ManifestDiff; use crate::bootc_composefs::finalize::get_etc_diff; use crate::bootc_composefs::gc::GCOpts; +use crate::delta::Delta; use crate::spec::BootloaderKind; use crate::{ bootc_composefs::{ @@ -59,23 +60,51 @@ use crate::{ /// * `Some` if the image is pulled/available locally, `None` otherwise /// * The container image manifest /// * The container image configuration +/// * The digest of the manifest, as the registry reported it #[context("Checking if image {} is pulled", imgref.image)] pub(crate) async fn is_image_pulled( repo: &ComposefsRepository, imgref: &ImageReference, -) -> Result<(Option, ImgConfigManifest)> { +) -> Result<(Option, ImgConfigManifest, String)> { let imgref_repr = imgref.to_image_proxy_ref()?; - let img_config_manifest = get_container_manifest_and_config(&imgref_repr).await?; + let (img_config_manifest, manifest_digest) = + get_container_manifest_and_config(&imgref_repr).await?; - let img_digest = img_config_manifest.manifest.config().digest().digest(); + let container_pulled = + lookup_config_splitstream(repo, img_config_manifest.manifest.config().digest())?; + + Ok((container_pulled, img_config_manifest, manifest_digest)) +} + +/// Look up the config splitstream for an image config digest, which is present +/// exactly when that image has been imported into `repo`. +pub(crate) fn lookup_config_splitstream( + repo: &ComposefsRepository, + config_digest: &composefs_oci::OciDigest, +) -> Result> { + let img_digest = config_digest.digest(); // TODO: export config_identifier function from composefs-oci/src/lib.rs and use it here let img_id = format!("oci-config-sha256:{img_digest}"); // NB: add deep checking? - let container_pulled = repo.has_stream(&img_id).context("Checking stream")?; + repo.has_stream(&img_id).context("Checking stream") +} - Ok((container_pulled, img_config_manifest)) +/// A delta only carries the layers that changed; the rest have to come from the +/// image it was built against, which must therefore already be in `repo`. +/// +/// composefs would notice this eventually, but only once it starts importing, which is +/// pretty late for a nice experience. +pub(crate) fn ensure_delta_source_present(repo: &ComposefsRepository, delta: &Delta) -> Result<()> { + let source = delta.source_config_digest(); + anyhow::ensure!( + lookup_config_splitstream(repo, source)?.is_some(), + "Delta {} was built against the image with config {source}, which is not present in \ + this system's composefs repository.", + delta.path, + ); + Ok(()) } fn rm_staged_type1_ent(boot_dir: &Dir) -> Result<()> { @@ -210,7 +239,7 @@ pub(crate) fn validate_update( } /// This is just an intersection of SwitchOpts and UpgradeOpts -pub(crate) struct DoUpgradeOpts { +pub(crate) struct DoUpgradeOpts<'a> { pub(crate) apply: bool, pub(crate) soft_reboot: Option, pub(crate) download_only: bool, @@ -220,13 +249,15 @@ pub(crate) struct DoUpgradeOpts { pub(crate) quiet: bool, /// Structured (JSON-Lines) progress sink; see `--progress-fd`. pub(crate) prog: ProgressWriter, + /// Take the image content from this local delta rather than the network. + pub(crate) delta: Option<&'a Delta>, } async fn apply_upgrade( storage: &Storage, booted_cfs: &BootedComposefs, depl_id: &String, - opts: &DoUpgradeOpts, + opts: &DoUpgradeOpts<'_>, ) -> Result<()> { if let Some(soft_reboot_mode) = opts.soft_reboot { return prepare_soft_reboot_composefs( @@ -253,7 +284,7 @@ pub(crate) async fn do_upgrade( booted_cfs: &BootedComposefs, host: &Host, imgref: &ImageReference, - opts: &DoUpgradeOpts, + opts: &DoUpgradeOpts<'_>, manifest: &ostree_ext::oci_spec::image::ImageManifest, ) -> Result<()> { // Pre-flight disk space check before pulling. @@ -269,6 +300,7 @@ pub(crate) async fn do_upgrade( fs: oci_fs, } = pull_composefs_repo( imgref, + opts.delta, booted_cfs.cmdline.allow_missing_fsverity, opts.use_unified, opts.quiet, @@ -276,6 +308,17 @@ pub(crate) async fn do_upgrade( ) .await?; + // We validated the delta by reading the file ourselves; composefs read it + // independently. Validate they are the same. + if let Some(delta) = opts.delta { + let expected = delta.target_manifest_digest().to_string(); + anyhow::ensure!( + manifest_digest == expected, + "Delta {} imported manifest {manifest_digest}, but it records {expected} as its target", + delta.path, + ); + } + // If the target image produces the same fs-verity digest as any existing // deployment (booted, staged, rollback, or pinned), error out. Two images // from different sources can have identical content; we cannot silently reuse @@ -400,7 +443,7 @@ pub(crate) async fn apply_upgrade_from_downloaded( storage: &Storage, composefs: &BootedComposefs, host: &Host, - do_upgrade_opts: &DoUpgradeOpts, + do_upgrade_opts: &DoUpgradeOpts<'_>, ) -> Result<()> { let staged = host .status @@ -481,6 +524,8 @@ pub(crate) async fn upgrade_composefs( None }; + let delta = crate::delta::open_opt(opts.from_delta.as_deref()).await?; + let prog: ProgressWriter = opts.progress.try_into()?; let mut do_upgrade_opts = DoUpgradeOpts { @@ -490,6 +535,7 @@ pub(crate) async fn upgrade_composefs( use_unified: false, quiet: opts.quiet, prog, + delta: delta.as_ref(), }; if opts.download_opts.from_downloaded { @@ -498,6 +544,9 @@ pub(crate) async fn upgrade_composefs( let imgref = derived_image.as_ref().or(current_image); let mut booted_imgref = imgref.ok_or_else(|| anyhow::anyhow!("No image source specified"))?; + if let Some(delta) = delta.as_ref() { + delta.validate_image_reference(booted_imgref)?; + } // Auto-detect unified storage: use the unified path if the target image is // already in bootc-owned containers-storage, OR if the booted image is — @@ -513,17 +562,37 @@ pub(crate) async fn upgrade_composefs( let repo = &*composefs.repo; - let (img_pulled, mut img_config) = is_image_pulled(&repo, booted_imgref).await?; - let booted_img_digest = img_config.manifest.config().digest().to_string(); + // With a delta the target is whatever the delta says it is, and we can look + // it up locally; without one we have to ask the registry. + let (img_pulled, mut manifest, manifest_digest) = match &delta { + Some(delta) => { + crate::delta::reject_unified_storage(delta, do_upgrade_opts.use_unified)?; + ensure_delta_source_present(repo, delta)?; + ( + lookup_config_splitstream(repo, delta.target_manifest().config().digest())?, + delta.target_manifest().clone(), + delta.target_manifest_digest().to_string(), + ) + } + None => is_image_pulled(&repo, booted_imgref) + .await + .map(|(pulled, img, digest)| (pulled, img.manifest, digest))?, + }; // Check if we already have this update staged // Or if we have another staged deployment with a different image let staged_image = host.status.staged.as_ref().and_then(|i| i.image.as_ref()); + // When applying deltas, only handle the staged image if it matches the delta, + // because with deltas we always want the specific version, and don't want to do + // any network resolve here. + let staged_image = + staged_image.filter(|staged| delta.is_none() || staged.image_digest == manifest_digest); + if let Some(staged_image) = staged_image { // We have a staged image and it has the same digest as the currently booted image's latest // digest - if staged_image.image_digest == booted_img_digest { + if staged_image.image_digest == manifest_digest { if opts.apply { return crate::reboot::reboot(); } @@ -538,15 +607,15 @@ pub(crate) async fn upgrade_composefs( // Switch takes precedence over update, so we change the imgref booted_imgref = &staged_image.image; - let (img_pulled, staged_img_config) = is_image_pulled(&repo, booted_imgref).await?; - img_config = staged_img_config; + let (img_pulled, staged_img_config, _) = is_image_pulled(&repo, booted_imgref).await?; + manifest = staged_img_config.manifest; if let Some(cfg_verity) = img_pulled { let action = validate_update( storage, composefs, &host, - img_config.manifest.config().digest().as_ref(), + manifest.config().digest().as_ref(), &cfg_verity, false, )?; @@ -564,7 +633,7 @@ pub(crate) async fn upgrade_composefs( &host, booted_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await; } @@ -578,7 +647,7 @@ pub(crate) async fn upgrade_composefs( storage, composefs, &host, - &booted_img_digest, + manifest.config().digest().as_ref(), &cfg_verity, false, )?; @@ -596,7 +665,7 @@ pub(crate) async fn upgrade_composefs( &host, booted_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await; } @@ -605,7 +674,7 @@ pub(crate) async fn upgrade_composefs( if opts.check { let (current_manifest, _) = get_imginfo(storage, &*composefs.cmdline.digest)?; - let diff = ManifestDiff::new(¤t_manifest.manifest, &img_config.manifest); + let diff = ManifestDiff::new(¤t_manifest.manifest, &manifest); diff.print(); return Ok(()); } @@ -616,7 +685,7 @@ pub(crate) async fn upgrade_composefs( &host, booted_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await?; diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 97b9a9e778..957444ad5b 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -138,6 +138,13 @@ pub(crate) struct UpgradeOpts { #[clap(long)] pub(crate) tag: Option, + /// Upgrade from a local oci-delta artifact instead of the network. + /// + /// The delta must have been created against an image that is already + /// present on this system. + #[clap(long, value_name = "PATH", conflicts_with_all = ["check", "from_downloaded", "tag"])] + pub(crate) from_delta: Option, + #[clap(flatten)] pub(crate) progress: ProgressOptions, } @@ -199,6 +206,19 @@ pub(crate) struct SwitchOpts { #[clap(long = "experimental-unified-storage", hide = true)] pub(crate) unified_storage_exp: bool, + /// Switch using a local oci-delta artifact instead of the network. + /// + /// The delta must have been created against an image that is already + /// present on this system. + #[clap(long, value_name = "PATH", conflicts_with_all = [ + "from_downloaded", + "transport", + "mutate_in_place", + "unified_storage_exp", + "enforce_container_sigpolicy", + ])] + pub(crate) from_delta: Option, + /// Target image to use for the next boot. /// Required unless `--from-downloaded` is present. #[clap( @@ -1264,6 +1284,8 @@ async fn upgrade( storage: &Storage, booted_ostree: &BootedOstree<'_>, ) -> Result<()> { + crate::delta::reject_unsupported(opts.from_delta.as_deref())?; + let repo = &booted_ostree.repo(); let host = crate::status::get_status(booted_ostree)?.1; @@ -1490,6 +1512,8 @@ async fn switch_ostree( storage: &Storage, booted_ostree: &BootedOstree<'_>, ) -> Result<()> { + crate::delta::reject_unsupported(opts.from_delta.as_deref())?; + let (_, host) = crate::status::get_status(booted_ostree)?; if opts.download_opts.from_downloaded { @@ -2652,6 +2676,41 @@ mod tests { } } + #[test] + fn test_parse_from_delta() { + let o = + Opt::try_parse_from(["bootc", "switch", "--from-delta", "/d", "quay.io/e/x"]).unwrap(); + match o { + Opt::Switch(o) => assert_eq!(o.from_delta.as_deref().unwrap(), "/d"), + o => panic!("Expected switch opts, not {o:?}"), + } + + // A delta carries its own image content; anything that says where else + // to get that content, or that asks for a check we can't perform, must + // be rejected rather than silently ignored. + let conflicting: &[&[&str]] = &[ + &["switch", "--transport=oci-archive", "quay.io/e/x"], + &["switch", "--mutate-in-place", "quay.io/e/x"], + &["switch", "--experimental-unified-storage", "quay.io/e/x"], + &["switch", "--enforce-container-sigpolicy", "quay.io/e/x"], + &["upgrade", "--check"], + &["upgrade", "--tag=other"], + ]; + for args in conflicting { + let (subcommand, rest) = args.split_first().unwrap(); + let mut full = vec!["bootc", subcommand, "--from-delta", "/d"]; + full.extend_from_slice(rest); + let err = Opt::try_parse_from(&full) + .err() + .unwrap_or_else(|| panic!("{args:?}: expected a conflict")); + assert_eq!( + err.kind(), + clap::error::ErrorKind::ArgumentConflict, + "{args:?}: {err}" + ); + } + } + #[test] fn test_parse_install_args() { // Verify we still process the legacy --target-no-signature-verification diff --git a/crates/lib/src/delta.rs b/crates/lib/src/delta.rs new file mode 100644 index 0000000000..c25f44e1d3 --- /dev/null +++ b/crates/lib/src/delta.rs @@ -0,0 +1,657 @@ +//! Support for oci-delta artifacts (`--from-delta`). +//! +//! An oci-delta is an OCI layout (normally packed as an uncompressed tar) +//! whose single manifest is an artifact rather than an image. It carries the +//! target image manifest and config verbatim, plus binary patches for the +//! layers that changed relative to a source image. Layers that did not change +//! are omitted entirely and are expected to already be present locally. +//! +//! See for the format. +//! +//! # Trust model +//! +//! Everything in a delta is verified against a single value, the target +//! manifest digest: the embedded manifest and config must hash to the digests +//! the delta records, the config must be the one the manifest references, and +//! each reconstructed layer is checked against the corresponding diff_id in +//! that config. +//! +//! What is not currently established is that the digest is the one you meant to +//! deploy: it comes from the file rather than from a registry, and nothing +//! signs it. Future work may use signatures to complete this. + +use std::collections::HashSet; + +use anyhow::{Context, Result, bail, ensure}; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std_ext::cap_std; +use fn_error_context::context; +use ocidir::oci_spec::image::{ + Descriptor, Digest, DigestAlgorithm, ImageConfiguration, ImageManifest, MediaType, +}; +use ocidir::prelude::*; +use ocidir::{OciArchive, OciDir}; +use ostree_ext::containers_image_proxy; + +use oci_delta::BlobStream; +use oci_delta::{ + BlobStreamFuture, DeltaBlobReader, MEDIA_TYPE_DELTA, ParsedDelta, is_delta_artifact, + parse_delta_manifest, +}; + +/// The OCI layout a delta was read from. +/// +/// Kept open for the lifetime of the [`Delta`] so that patch blobs can be read +/// back when the layers are applied, without parsing the layout again. +#[derive(Debug)] +enum Layout { + Dir(OciDir), + Archive(OciArchive), +} + +impl Layout { + fn open(path: &Utf8Path) -> Result { + if path.is_dir() { + let dir = cap_std::fs::Dir::open_ambient_dir(path, cap_std::ambient_authority()) + .context("Opening OCI layout directory")?; + Ok(Self::Dir( + OciDir::open(dir).context("Reading OCI layout directory")?, + )) + } else { + Ok(Self::Archive( + OciArchive::open(path).context("Reading OCI archive")?, + )) + } + } + + fn read_index(&self) -> Result { + match self { + Self::Dir(o) => o.read_index(), + Self::Archive(o) => o.read_index(), + } + .map_err(Into::into) + } + + fn read_blob(&self, desc: &Descriptor) -> Result> { + let blob: Box = match self { + Self::Dir(o) => Box::new(o.read_blob(desc)?), + Self::Archive(o) => Box::new(o.read_blob(desc)?), + }; + Ok(blob) + } +} + +impl DeltaBlobReader for Layout { + fn open_blob(&self, desc: &Descriptor) -> BlobStreamFuture<'_> { + let result = self + .read_blob(desc) + .with_context(|| format!("Reading blob {}", desc.digest())); + Box::pin(std::future::ready(result)) + } +} + +/// A validated oci-delta artifact, ready to be applied. +#[derive(Debug)] +pub(crate) struct Delta { + /// Where this was loaded from; used in diagnostics. + pub(crate) path: Utf8PathBuf, + /// Parsed information about the delta. + pub(crate) parsed: ParsedDelta, +} + +impl Delta { + /// Open and validate the delta at `path`, which may be an OCI layout + /// directory or (usually) an uncompressed tar of one. + #[context("Opening delta {path}")] + pub(crate) async fn open(path: &Utf8Path) -> Result { + let layout = Layout::open(path)?; + let parsed = parse(&layout).await?; + + validate(&parsed)?; + Ok(Self { + path: path.to_owned(), + parsed, + }) + } + + /// The digest of the target image's manifest, as recorded in the delta. + pub(crate) fn target_manifest_digest(&self) -> &Digest { + self.parsed.target_manifest_descriptor.digest() + } + + pub(crate) fn validate_image_reference( + &self, + imgref: &crate::spec::ImageReference, + ) -> Result<()> { + if imgref.transport()? != containers_image_proxy::Transport::Registry { + return Ok(()); + } + let reference: ocidir::oci_spec::distribution::Reference = imgref + .image + .parse() + .with_context(|| format!("Parsing image reference {}", imgref.image))?; + if let Some(digest) = reference.digest() { + let digest: Digest = digest.parse()?; + ensure!( + &digest == self.target_manifest_digest(), + "Image reference {} names digest {digest}, but delta {} targets {}", + imgref.image, + self.path, + self.target_manifest_digest(), + ); + } + Ok(()) + } + + /// The target image's manifest. + pub(crate) fn target_manifest(&self) -> &ImageManifest { + &self.parsed.target_manifest + } + + /// The config digest of the image this delta was built against. + pub(crate) fn source_config_digest(&self) -> &Digest { + &self.parsed.source_config_digest + } + + /// An image reference naming this delta as a local OCI layout, for handing + /// to the image import machinery. + pub(crate) fn pull_ref(&self) -> Result { + // An `oci:` reference is `path[:tag]`, so a colon in the path would be + // taken as a tag separator. + ensure!( + !self.path.as_str().contains(':'), + "Delta path {} contains a colon, which cannot be expressed as an image reference", + self.path, + ); + let transport = if self.path.is_dir() { + "oci" + } else { + "oci-archive" + }; + format!("{transport}:{}", self.path) + .as_str() + .try_into() + .map_err(|e| anyhow::anyhow!("Building image reference for {}: {e}", self.path)) + } + + /// A one-line description of what applying this delta would do. + pub(crate) fn describe(&self) -> String { + let total = self.parsed.target_manifest.layers().len(); + let patched = self.parsed.delta_layer_by_to.len(); + format!( + "{}: target manifest {}, {total} layers ({patched} patched, {} reused from source config {})", + self.path, + self.target_manifest_digest(), + total - patched, + self.parsed.source_config_digest, + ) + } +} + +/// Open the delta named by a `--from-delta` argument, if there is one. +pub(crate) async fn open_opt(path: Option<&Utf8Path>) -> Result> { + match path { + Some(path) => Ok(Some(Delta::open(path).await?)), + None => Ok(None), + } +} + +/// Reject `--from-delta` on a storage backend that cannot apply one yet. +pub(crate) fn reject_unsupported(path: Option<&Utf8Path>) -> Result<()> { + match path { + Some(path) => bail!("--from-delta ({path}) is not supported by the ostree backend yet"), + None => Ok(()), + } +} + +/// Reject `--from-delta` for an image that also has to be in containers-storage. +pub(crate) fn reject_unified_storage(delta: &Delta, use_unified: bool) -> Result<()> { + ensure!( + !use_unified, + "Cannot apply delta {}: deltas with unified storage not supported.", + delta.path, + ); + Ok(()) +} + +/// Check the delta's internal consistency, to fail early +fn validate(p: &ParsedDelta) -> Result<()> { + verify_digest( + "Embedded target manifest", + &p.target_manifest_raw, + p.target_manifest_descriptor.digest(), + )?; + verify_digest( + "Embedded target config", + &p.target_config_raw, + p.target_config_descriptor.digest(), + )?; + + ensure!( + p.target_manifest.config().digest() == p.target_config_descriptor.digest(), + "Delta target manifest references config {}, but the embedded config is {}", + p.target_manifest.config().digest(), + p.target_config_descriptor.digest(), + ); + + let config = ImageConfiguration::from_reader(&p.target_config_raw[..]) + .context("Parsing embedded target config")?; + let layers = p.target_manifest.layers(); + ensure!(!layers.is_empty(), "Delta target image has no layers"); + ensure!( + config.rootfs().diff_ids().len() == layers.len(), + "Delta target image has {} diff_ids but {} layers", + config.rootfs().diff_ids().len(), + layers.len(), + ); + + let target_layers: HashSet<&Digest> = layers.iter().map(|l| l.digest()).collect(); + for to in p.delta_layer_by_to.keys() { + ensure!( + target_layers.contains(to), + "Delta contains a patch for layer {to}, which is not part of the target image", + ); + } + + Ok(()) +} + +/// Read the single manifest out of an OCI layout and parse it as a delta. +async fn parse(oci: &Layout) -> Result { + let index = oci.read_index().context("Reading index")?; + let [desc] = index.manifests().as_slice() else { + bail!( + "Expected an OCI layout with a single manifest, found {}; this is not a delta", + index.manifests().len() + ); + }; + ensure!( + desc.media_type() == &MediaType::ImageManifest, + "Expected an image manifest, found {}; this is not a delta", + desc.media_type(), + ); + + let mut raw = Vec::new(); + std::io::Read::read_to_end( + &mut oci.read_blob(desc).context("Reading delta manifest")?, + &mut raw, + ) + .context("Reading delta manifest")?; + verify_digest("Delta manifest", &raw, desc.digest())?; + let manifest: ImageManifest = serde_json::from_slice(&raw).context("Parsing delta manifest")?; + + ensure!( + is_delta_artifact(&manifest), + "Not a delta: expected artifactType {MEDIA_TYPE_DELTA}, found {}.", + manifest + .artifact_type() + .as_ref() + .map(|t| t.to_string()) + .unwrap_or_else(|| "none".into()), + ); + + parse_delta_manifest(&manifest, oci).await +} + +/// Verify that `data` hashes to `expected`. +fn verify_digest(what: &str, data: &[u8], expected: &Digest) -> Result<()> { + let algorithm = match expected.algorithm() { + DigestAlgorithm::Sha256 => openssl::hash::MessageDigest::sha256(), + DigestAlgorithm::Sha384 => openssl::hash::MessageDigest::sha384(), + DigestAlgorithm::Sha512 => openssl::hash::MessageDigest::sha512(), + other => bail!("{what} uses unsupported digest algorithm {other}"), + }; + let found = hex::encode(openssl::hash::hash(algorithm, data)?); + ensure!( + found == expected.digest(), + "{what} does not match its digest: expected {expected}, got {found}", + ); + Ok(()) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use ocidir::oci_spec::image::{ImageConfigurationBuilder, ImageManifestBuilder, RootFsBuilder}; + use std::collections::HashMap; + + const DELTA_CONTENT: &str = "io.github.containers.delta.content"; + const DELTA_TO: &str = "io.github.containers.delta.to"; + const DELTA_SOURCE_CONFIG: &str = "io.github.containers.delta.source-config"; + const TAR_DIFF: &str = "application/vnd.tar-diff"; + const ZERO_DIGEST: &str = + "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + + fn blob(oci: &OciDir, data: &[u8], media_type: MediaType) -> Descriptor { + let mut w = oci.create_blob().unwrap(); + std::io::Write::write_all(&mut w, data).unwrap(); + w.complete() + .unwrap() + .descriptor() + .media_type(media_type) + .build() + .unwrap() + } + + fn image_config(diff_ids: &[&str]) -> ImageConfiguration { + ImageConfigurationBuilder::default() + .rootfs( + RootFsBuilder::default() + .typ("layers") + .diff_ids(diff_ids.iter().map(|s| s.to_string()).collect::>()) + .build() + .unwrap(), + ) + .build() + .unwrap() + } + + fn annotate(mut desc: Descriptor, annotations: &[(&str, &str)]) -> Descriptor { + desc.set_annotations(Some( + annotations + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + )); + desc + } + + /// A minimal but structurally valid target image, of which the last layer + /// is the one the delta patches. + pub(crate) struct TestDelta { + oci: OciDir, + tmp: tempfile::TempDir, + manifest_desc: Descriptor, + config_desc: Descriptor, + layer_descs: Vec, + } + + impl TestDelta { + pub(crate) fn new() -> Self { + Self::with_diff_ids(2) + } + + /// Two layers, but `n_diff_ids` diff_ids, to check invalid combinations + fn with_diff_ids(n_diff_ids: usize) -> Self { + let tmp = tempfile::tempdir().unwrap(); + let dir = cap_std::fs::Dir::open_ambient_dir(tmp.path(), cap_std::ambient_authority()) + .unwrap(); + let oci = OciDir::ensure(dir).unwrap(); + + let layer_descs: Vec = ["layer-one", "layer-two"] + .iter() + .map(|d| blob(&oci, d.as_bytes(), MediaType::ImageLayerGzip)) + .collect(); + let diff_ids = (0..n_diff_ids) + .map(|i| format!("sha256:{}", i.to_string().repeat(64))) + .collect::>(); + let config = image_config(&diff_ids.iter().map(|s| s.as_str()).collect::>()); + let config_desc = blob( + &oci, + serde_json::to_vec(&config).unwrap().as_slice(), + MediaType::ImageConfig, + ); + + let manifest = ImageManifestBuilder::default() + .schema_version(2u32) + .media_type(MediaType::ImageManifest) + .config(config_desc.clone()) + .layers(layer_descs.clone()) + .build() + .unwrap(); + let manifest_desc = blob( + &oci, + serde_json::to_vec(&manifest).unwrap().as_slice(), + MediaType::ImageManifest, + ); + + Self { + oci, + tmp, + manifest_desc, + config_desc, + layer_descs, + } + } + + pub(crate) fn path(&self) -> &Utf8Path { + Utf8Path::from_path(self.tmp.path()).unwrap() + } + + /// Write out the delta manifest with the given layers and artifactType. + fn finish(&self, artifact_type: Option<&str>, layers: Vec) { + let empty = blob(&self.oci, b"{}", MediaType::EmptyJSON); + let mut b = ImageManifestBuilder::default() + .schema_version(2u32) + .media_type(MediaType::ImageManifest) + .config(empty) + .layers(layers) + .annotations(HashMap::from([( + DELTA_SOURCE_CONFIG.to_string(), + ZERO_DIGEST.to_string(), + )])); + if let Some(t) = artifact_type { + b = b.artifact_type(MediaType::Other(t.to_string())); + } + self.oci + .replace_with_single_manifest(b.build().unwrap(), Default::default()) + .unwrap(); + } + + /// Write out the standard delta: the embedded manifest, the embedded + /// config, and a patch for the last image layer. + pub(crate) fn finish_default(&self) { + self.finish(Some(MEDIA_TYPE_DELTA), self.default_layers()); + } + + fn patch_for(&self, to: &str) -> Descriptor { + let patch = blob( + &self.oci, + b"patch-data", + MediaType::Other(TAR_DIFF.to_string()), + ); + annotate(patch, &[(DELTA_CONTENT, "image-layer"), (DELTA_TO, to)]) + } + + fn default_layers(&self) -> Vec { + vec![ + annotate( + self.manifest_desc.clone(), + &[(DELTA_CONTENT, "image-manifest")], + ), + annotate(self.config_desc.clone(), &[(DELTA_CONTENT, "image-config")]), + self.patch_for(self.layer_descs.last().unwrap().digest().as_ref()), + ] + } + + /// Rewrite a blob in place, keeping its size so that ocidir's own size + /// check isn't what catches the alteration. + fn corrupt_blob(&self, desc: &Descriptor) { + let name = desc.digest().digest(); + let mut content = Vec::new(); + std::io::Read::read_to_end(&mut self.oci.blobs_dir().open(name).unwrap(), &mut content) + .unwrap(); + // Flip the last hex character of the first digest, which keeps the + // bytes both the same length and valid JSON. + let pos = content + .windows(7) + .position(|w| w == b"sha256:") + .expect("no digest in blob") + + 7 + + 63; + content[pos] = if content[pos] == b'0' { b'1' } else { b'0' }; + self.oci.blobs_dir().write(name, &content).unwrap(); + } + } + + #[tokio::test] + async fn test_valid_delta() { + let t = TestDelta::new(); + t.finish_default(); + + let delta = Delta::open(t.path()).await.unwrap(); + assert_eq!(delta.target_manifest_digest(), t.manifest_desc.digest()); + assert_eq!(delta.parsed.target_manifest.layers().len(), 2); + assert_eq!(delta.parsed.delta_layer_by_to.len(), 1); + assert!(delta.describe().contains("1 patched, 1 reused")); + + let pull_ref = delta.pull_ref().unwrap(); + assert_eq!( + pull_ref.transport, + ostree_ext::containers_image_proxy::Transport::OciDir + ); + assert_eq!(pull_ref.name, t.path().as_str()); + } + + #[tokio::test] + async fn test_rejects_invalid() { + struct Case { + name: &'static str, + prepare: fn(&TestDelta), + expected: &'static str, + } + let cases = [ + Case { + name: "plain image, no artifactType", + prepare: |t| t.finish(None, t.layer_descs.clone()), + expected: "Not a delta", + }, + Case { + name: "patch for a layer not in the target image", + prepare: |t| { + let mut layers = t.default_layers(); + layers.pop(); + layers.push(t.patch_for(ZERO_DIGEST)); + t.finish(Some(MEDIA_TYPE_DELTA), layers); + }, + expected: "not part of the target image", + }, + Case { + name: "embedded manifest altered after the fact", + prepare: |t| { + t.finish_default(); + t.corrupt_blob(&t.manifest_desc); + }, + expected: "does not match its digest", + }, + Case { + name: "delta manifest altered after the fact", + prepare: |t| { + t.finish_default(); + let index = t.oci.read_index().unwrap(); + t.corrupt_blob(&index.manifests()[0]); + }, + expected: "does not match its digest", + }, + Case { + name: "embedded config is not the one the manifest references", + prepare: |t| { + let other = blob( + &t.oci, + &serde_json::to_vec(&image_config(&["sha256:aa", "sha256:bb"])).unwrap(), + MediaType::ImageConfig, + ); + let mut layers = t.default_layers(); + layers[1] = annotate(other, &[(DELTA_CONTENT, "image-config")]); + t.finish(Some(MEDIA_TYPE_DELTA), layers); + }, + expected: "references config", + }, + ]; + for case in cases { + let t = TestDelta::new(); + (case.prepare)(&t); + let err = Delta::open(t.path()) + .await + .err() + .unwrap_or_else(|| panic!("{}: expected failure", case.name)); + assert!( + format!("{err:#}").contains(case.expected), + "{}: unexpected error: {err:#}", + case.name + ); + } + } + + #[tokio::test] + async fn test_rejects_diff_id_mismatch() { + let t = TestDelta::with_diff_ids(1); + t.finish_default(); + + let err = Delta::open(t.path()).await.unwrap_err(); + assert!( + format!("{err:#}").contains("1 diff_ids but 2 layers"), + "unexpected error: {err:#}" + ); + } + + #[tokio::test] + async fn test_pull_ref() { + let t = TestDelta::new(); + t.finish_default(); + let mut delta = Delta::open(t.path()).await.unwrap(); + + // A path that is not a directory is taken to be an archive. + delta.path = "/var/tmp/update.oci-delta".into(); + let pull_ref = delta.pull_ref().unwrap(); + assert_eq!( + pull_ref.transport, + ostree_ext::containers_image_proxy::Transport::OciArchive + ); + assert_eq!(pull_ref.name, "/var/tmp/update.oci-delta"); + + delta.path = "/var/tmp/a:b.oci-delta".into(); + let err = delta.pull_ref().unwrap_err(); + assert!( + format!("{err:#}").contains("contains a colon"), + "unexpected error: {err:#}" + ); + } + + #[tokio::test] + async fn test_validate_image_reference() { + let t = TestDelta::new(); + t.finish_default(); + let delta = Delta::open(t.path()).await.unwrap(); + let target = delta.target_manifest_digest(); + for (image, valid) in [ + ("quay.io/example/os:latest".to_owned(), true), + (format!("quay.io/example/os@{target}"), true), + (format!("quay.io/example/os:latest@{target}"), true), + (format!("quay.io/example/os@{ZERO_DIGEST}"), false), + (format!("quay.io/example/os:latest@{ZERO_DIGEST}"), false), + ] { + let imgref = crate::spec::ImageReference { + image, + transport: "registry".into(), + signature: None, + }; + let result = delta.validate_image_reference(&imgref); + if valid { + result.unwrap(); + } else { + let error = result.unwrap_err().to_string(); + assert!(error.contains(ZERO_DIGEST), "{error}"); + assert!(error.contains(&target.to_string()), "{error}"); + } + } + } + + #[test] + fn test_reject_unsupported() { + reject_unsupported(None).unwrap(); + let err = reject_unsupported(Some(Utf8Path::new("/d"))).unwrap_err(); + assert!(format!("{err:#}").contains("ostree backend")); + } + + #[test] + fn test_verify_digest() { + let expected: Digest = + "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + .parse() + .unwrap(); + verify_digest("test", b"hello", &expected).unwrap(); + let err = verify_digest("test", b"goodbye", &expected).unwrap_err(); + assert!(format!("{err:#}").contains("does not match its digest")); + } +} diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index c82d857f7e..3fe8cc2cf4 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -2025,7 +2025,7 @@ async fn install_to_filesystem_impl( // Pre-flight disk space check for native composefs install path. { let imgref = &state.source.imageref; - let img_manifest_config = get_container_manifest_and_config(&imgref).await?; + let (img_manifest_config, _) = get_container_manifest_and_config(&imgref).await?; crate::store::ensure_composefs_dir(&rootfs.physical_root)?; // Use init_path since the repo may not exist yet during install let config = diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index d9eccc0c81..8616512c7f 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -33,6 +33,7 @@ //! ## Container and Image Handling //! //! - [`image`] - Image operations and queries +//! - [`delta`] - oci-delta artifacts (`--from-delta`) //! - [`boundimage`] - Logically Bound Images (LBIs) //! - [`podstorage`] - bootc-owned container storage (`/usr/lib/bootc/storage`) //! - [`podman`] - Podman command helpers @@ -71,6 +72,7 @@ pub mod cli; mod composefs_consts; mod container_export; mod containerenv; +pub(crate) mod delta; pub(crate) mod deploy; mod discoverable_partition_specification; pub(crate) mod fsck; diff --git a/docs/src/man/bootc-switch.8.md b/docs/src/man/bootc-switch.8.md index 407aa0bc09..fd80baf5be 100644 --- a/docs/src/man/bootc-switch.8.md +++ b/docs/src/man/bootc-switch.8.md @@ -79,6 +79,10 @@ Soft reboot allows faster system restart by avoiding full hardware reboot when p Retain reference to currently booted image +**--from-delta**=*PATH* + + Switch using a local oci-delta artifact instead of the network + # EXAMPLES diff --git a/docs/src/man/bootc-upgrade.8.md b/docs/src/man/bootc-upgrade.8.md index b1b3f3b694..91b97b1f7f 100644 --- a/docs/src/man/bootc-upgrade.8.md +++ b/docs/src/man/bootc-upgrade.8.md @@ -73,6 +73,10 @@ Soft reboot allows faster system restart by avoiding full hardware reboot when p Upgrade to a different tag of the currently booted image +**--from-delta**=*PATH* + + Upgrade from a local oci-delta artifact instead of the network + # EXAMPLES From 60bc5ac6174496a90bc11373f0fffcca4f99a411 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 8 Sep 2026 16:37:28 +0200 Subject: [PATCH 3/6] ostree-ext: Add a LayerSource abstraction and prepare_from_manifest() The importer has always fetched both the image metadata and the layer bytes over the same containers-image-proxy connection. To apply an OCI delta we need to split those apart: the manifest and config come out of the delta file, and each layer is either already in the repository or reconstructed locally from a binary patch. No network access at all. Introduce a `LayerSource` trait, which is just the existing `fetch_layer` signature behind a trait object, and a `ProxyLayerSource` implementing today's behaviour. `PreparedImport` now carries a LayerSource instead of a bare `OpenedImage`, and `ImageImporter::prepare_from_manifest` builds a `PreparedImport` from a manifest and config the caller already has, with the layers coming from whatever source it passes in. `ImageImporter::new_without_proxy` skips spawning skopeo entirely, so an importer that will never touch the network does not need it installed. Along the way, `prepare_internal` splits into `check_sigverify` and `diff_previous_state` so both prepare paths share the "is this already imported?" logic, and `cache_pending` moves into `create_prepared_import` where both paths reach it. `ProxyLayerSource` also fetches the proxy's layer info once and caches it, where before both `unencapsulate_base()` and `import()` asked for it separately. Assisted-by: AI Signed-off-by: Alexander Larsson --- .../ostree-ext/src/container/layer_source.rs | 101 ++++++ crates/ostree-ext/src/container/mod.rs | 2 + crates/ostree-ext/src/container/store.rs | 343 ++++++++++++------ crates/ostree-ext/tests/it/main.rs | 97 ++++- 4 files changed, 433 insertions(+), 110 deletions(-) create mode 100644 crates/ostree-ext/src/container/layer_source.rs diff --git a/crates/ostree-ext/src/container/layer_source.rs b/crates/ostree-ext/src/container/layer_source.rs new file mode 100644 index 0000000000..d89967a620 --- /dev/null +++ b/crates/ostree-ext/src/container/layer_source.rs @@ -0,0 +1,101 @@ +//! Abstraction over where the bytes of the image layers come from. + +use anyhow::Result; +use containers_image_proxy::oci_spec::image as oci_image; +use containers_image_proxy::{ConvertedLayerInfo, ImageProxy, OpenedImage, Transport}; +use futures_util::future::BoxFuture; +use tokio::io::AsyncBufRead; +use tokio::sync::{OnceCell, watch::Sender}; + +use super::store::LayerProgress; +use super::unencapsulate::fetch_layer; + +/// A layer opened for reading: its bytes, a driver future which must be polled +/// alongside reading them, and the media type of those bytes. The media type is +/// not necessarily the one in the layer descriptor, as some transports store +/// layers uncompressed regardless of what the manifest says. +pub type FetchedLayer<'a> = ( + Box, + BoxFuture<'a, Result<()>>, + oci_image::MediaType, +); + +/// Abstraction for layer blob data. +/// +/// Typically ProxyLayerSource which pulls from a registry, but for deltas +/// we reconstruct data directly from the delta. +pub trait LayerSource: std::fmt::Debug + Send + Sync { + /// Open one layer of `manifest` for reading. + fn fetch_layer<'a>( + &'a self, + manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + progress: Option<&'a Sender>>, + ) -> BoxFuture<'a, Result>>; + + /// Release the source, at most once and after the last + /// [`Self::fetch_layer`]. + /// + /// This is only reached when the import succeeds; an import that fails + /// part way through drops the source instead. Anything that must be + /// cleaned up either way belongs in a [`Drop`] impl. + fn finish(self: Box) -> BoxFuture<'static, Result<()>>; +} + +/// Implementation of LayerSource via containers-image-proxy. +#[derive(Debug)] +pub(crate) struct ProxyLayerSource { + proxy: ImageProxy, + img: OpenedImage, + transport: Transport, + layer_info: OnceCell>>, +} + +impl ProxyLayerSource { + pub(crate) fn new(proxy: ImageProxy, img: OpenedImage, transport: Transport) -> Self { + Self { + proxy, + img, + transport, + layer_info: OnceCell::new(), + } + } +} + +impl LayerSource for ProxyLayerSource { + fn fetch_layer<'a>( + &'a self, + manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + progress: Option<&'a Sender>>, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let layer_info = self + .layer_info + .get_or_try_init(|| self.proxy.get_layer_info(&self.img)) + .await?; + let (blob, driver, media_type) = fetch_layer( + &self.proxy, + &self.img, + manifest, + layer, + progress, + layer_info.as_ref(), + self.transport, + ) + .await?; + Ok((blob, Box::pin(driver) as BoxFuture<'a, _>, media_type)) + }) + } + + fn finish(self: Box) -> BoxFuture<'static, Result<()>> { + Box::pin(async move { + let Self { proxy, img, .. } = *self; + // TODO change the imageproxy API to ensure this happens automatically when + // the image reference is dropped + proxy.close_image(&img).await?; + proxy.finalize().await?; + Ok(()) + }) + } +} diff --git a/crates/ostree-ext/src/container/mod.rs b/crates/ostree-ext/src/container/mod.rs index 97d9c2fc0e..3e5109f63c 100644 --- a/crates/ostree-ext/src/container/mod.rs +++ b/crates/ostree-ext/src/container/mod.rs @@ -468,6 +468,8 @@ pub fn version_for_config(config: &oci_spec::image::ImageConfiguration) -> Optio pub mod deploy; mod encapsulate; pub use encapsulate::*; +mod layer_source; +pub use layer_source::*; mod unencapsulate; pub use unencapsulate::*; pub mod skopeo; diff --git a/crates/ostree-ext/src/container/store.rs b/crates/ostree-ext/src/container/store.rs index 449d3db2ec..a8b9e47a98 100644 --- a/crates/ostree-ext/src/container/store.rs +++ b/crates/ostree-ext/src/container/store.rs @@ -41,6 +41,15 @@ //! - The merge commit overlays all layers, processing whiteouts //! - Image metadata (manifest, config) is stored in commit metadata //! +//! ## Importing without a registry +//! +//! Steps 1 and 2 above assume the manifest and the layer content both come +//! over the registry connection. A caller that already has the manifest and +//! can produce the layer content itself - by reconstructing it from an OCI +//! delta, say - instead uses [`ImageImporter::new_without_proxy`] and +//! [`ImageImporter::prepare_from_manifest`], passing a +//! [`crate::container::LayerSource`] for the layers. Step 3 is unchanged. +//! //! ## Layer Types //! //! The manifest layout is parsed to identify different layer types: @@ -139,7 +148,7 @@ use cap_std_ext::cap_std; use cap_std_ext::cap_std::fs::{Dir, MetadataExt}; use cap_std_ext::dirext::CapStdExtDirExt; -use containers_image_proxy::{ImageProxy, OpenedImage}; +use containers_image_proxy::ImageProxy; use flate2::Compression; use fn_error_context::context; use futures_util::TryFutureExt; @@ -292,6 +301,15 @@ impl CachedImageUpdate { } } +/// What a target manifest amounts to, relative to what we imported before. +enum PreviousState { + /// The target image is already imported; there is nothing to do. + UpToDate(Box), + /// The target differs; carries the previous import to diff against and its + /// image ID, if there was a previous import at all. + Outdated(Option<(Box, String)>), +} + /// A layer in the ostree repo, identified by its ref and commit checksum. struct LayerRef { ostree_ref: String, @@ -304,7 +322,7 @@ pub struct ImageImporter { repo: ostree::Repo, /// The root filesystem directory, used for policy lookups. root: Dir, - pub(crate) proxy: ImageProxy, + proxy: Option, imgref: OstreeImageReference, target_imgref: Option, no_imgref: bool, // If true, do not write final image ref @@ -376,8 +394,8 @@ pub struct PreparedImport { pub layers: Vec, /// OSTree remote signature verification text, if enabled. pub verify_text: Option, - /// Our open image reference - proxy_img: OpenedImage, + /// Where to read the layer content from. + layer_source: Box, } impl PreparedImport { @@ -646,6 +664,22 @@ impl ImageImporter { &format!("Fetching {imgref}"), ); + Self::new_impl(repo, imgref, Some(proxy)) + } + + /// Create an importer with no image proxy, and hence no way to fetch + /// anything itself; the caller supplies the image metadata and the layer + /// content via [`Self::prepare_from_manifest`]. + #[context("Creating importer")] + pub fn new_without_proxy(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + Self::new_impl(repo, imgref, None) + } + + fn new_impl( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + proxy: Option, + ) -> Result { let repo = repo.clone(); let diffid_to_digest = Self::build_diffid_to_digest_map(&repo)?; @@ -739,7 +773,7 @@ impl ImageImporter { /// Serialize the metadata about a pending fetch as detached metadata on the commit object, /// so it can be retrieved later offline #[context("Writing cached pending manifest")] - pub(crate) async fn cache_pending( + async fn cache_pending( &self, commit: &str, manifest_digest: &Digest, @@ -875,17 +909,32 @@ impl ImageImporter { Ok(()) } - /// Given existing metadata (manifest, config, previous image statE) generate a PreparedImport structure + /// Given existing metadata (manifest, config, previous image state) generate a PreparedImport structure /// which e.g. includes a diff of the layers. - fn create_prepared_import( + /// + /// If there is a previous image, this also caches the new manifest and + /// config against it. + async fn create_prepared_import( &mut self, manifest_digest: Digest, manifest: ImageManifest, config: ImageConfiguration, previous_state: Option>, previous_imageid: Option, - proxy_img: OpenedImage, + layer_source: Box, ) -> Result> { + // If there is a currently fetched image, cache the new pending manifest+config + // as detached commit metadata, so that future fetches can query it offline. + if let Some(previous_state) = previous_state.as_ref() { + self.cache_pending( + previous_state.merge_commit.as_str(), + &manifest_digest, + &manifest, + &config, + ) + .await?; + } + let config_labels = super::labels_of(&config); if self.require_bootable { let bootable_key = ostree::METADATA_KEY_BOOTABLE; @@ -931,29 +980,134 @@ impl ImageImporter { ostree_commit_layer: commit_layer, layers: remaining_layers, verify_text: None, - proxy_img, + layer_source, }; Ok(Box::new(imp)) } - /// Determine if there is a new manifest, and if so return its digest. - #[context("Fetching manifest")] - pub(crate) async fn prepare_internal(&mut self, verify_layers: bool) -> Result { + /// Verify that our signature source is usable at all. + fn check_sigverify(&self, verify_layers: bool) -> Result<()> { match &self.imgref.sigverify { SignatureSource::ContainerPolicy if skopeo::container_policy_is_default_insecure(&self.root)? => { - return Err(anyhow!( + Err(anyhow!( "containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage" - )); - } - SignatureSource::OstreeRemote(_) if verify_layers => { - return Err(anyhow!( - "Cannot currently verify layered containers via ostree remote" - )); + )) } - _ => {} + SignatureSource::OstreeRemote(_) if verify_layers => Err(anyhow!( + "Cannot currently verify layered containers via ostree remote" + )), + _ => Ok(()), } + } + + /// Compare a target manifest against any previous import of the same image + /// reference. + fn diff_previous_state( + previous_state: Option>, + manifest_digest: &Digest, + new_imageid: &Digest, + ) -> PreviousState { + let Some(previous_state) = previous_state else { + return PreviousState::Outdated(None); + }; + // If the manifest digests match, we're done. + if &previous_state.manifest_digest == manifest_digest { + return PreviousState::UpToDate(previous_state); + } + // Failing that, if they have the same imageID, we're also done. + let previous_imageid = previous_state.manifest.config().digest(); + if previous_imageid == new_imageid { + return PreviousState::UpToDate(previous_state); + } + let previous_imageid = previous_imageid.to_string(); + PreviousState::Outdated(Some((previous_state, previous_imageid))) + } + + /// Prepare an import of a manifest the caller already has, reading the + /// layer content from `layer_source` instead of over the network. + /// + /// `manifest_digest` is taken on trust: it is what future update checks for + /// this image reference will compare against, and nothing here can verify + /// it, as re-serializing a parsed manifest does not necessarily reproduce + /// the bytes it was parsed from. The caller must have checked the raw + /// manifest and config bytes against their digests, and must have + /// established that this is the image `self.imgref` names, before calling + /// this. + /// + /// [`SignatureSource::ContainerPolicy`] is rejected: enforcing + /// containers-policy.json is skopeo's job, and there is no registry + /// interaction here for it to enforce anything on. Importing anyway would + /// produce a merge commit indistinguishable from a policy-verified pull. + /// [`SignatureSource::OstreeRemote`] is fine, as that signature is on the + /// ostree commit and is still checked while committing it. + #[context("Preparing import from manifest")] + pub async fn prepare_from_manifest( + &mut self, + manifest_digest: Digest, + manifest: ImageManifest, + config: ImageConfiguration, + layer_source: Box, + ) -> Result { + if matches!(self.imgref.sigverify, SignatureSource::ContainerPolicy) { + anyhow::bail!( + "Cannot verify {} against containers-policy.json without fetching it; refusing usage", + self.imgref, + ); + } + + // We have no proxy to fetch anything with, so make sure we do not leave + // a skopeo process behind if the caller made one anyway. + if let Some(proxy) = self.proxy.take() { + proxy.finalize().await?; + } + + // A digested pull spec is the one part of the caller's claim we can + // check ourselves. + let target_reference = self.imgref.imgref.name.parse::().ok(); + if let Some(target_digest) = target_reference + .as_ref() + .and_then(|v| v.digest()) + .map(Digest::from_str) + .transpose()? + && target_digest != manifest_digest + { + anyhow::bail!( + "Image reference {} names digest {target_digest}, but the manifest is {manifest_digest}", + self.imgref, + ); + } + + // Check if we have an image already pulled + let previous_state = try_query_image(&self.repo, &self.imgref.imgref)?; + + let (previous_state, previous_imageid) = match Self::diff_previous_state( + previous_state, + &manifest_digest, + manifest.config().digest(), + ) { + PreviousState::UpToDate(state) => return Ok(PrepareResult::AlreadyPresent(state)), + PreviousState::Outdated(previous) => previous.unzip(), + }; + + let imp = self + .create_prepared_import( + manifest_digest, + manifest, + config, + previous_state, + previous_imageid, + layer_source, + ) + .await?; + Ok(PrepareResult::Ready(imp)) + } + + /// Determine if there is a new manifest, and if so return its digest. + #[context("Fetching manifest")] + pub(crate) async fn prepare_internal(&mut self, verify_layers: bool) -> Result { + self.check_sigverify(verify_layers)?; // Check if we have an image already pulled let previous_state = try_query_image(&self.repo, &self.imgref.imgref)?; @@ -984,55 +1138,48 @@ impl ImageImporter { anyhow::bail!("Manifest fetch required in offline mode"); } - let proxy_img = self - .proxy - .open_image(&self.imgref.imgref.to_string()) - .await?; + let proxy = self.proxy.as_ref().ok_or_else(|| { + anyhow!( + "This importer was created without an image proxy; use prepare_from_manifest() instead" + ) + })?; + let proxy_img = proxy.open_image(&self.imgref.imgref.to_string()).await?; - let (manifest_digest, manifest) = self.proxy.fetch_manifest(&proxy_img).await?; + let (manifest_digest, manifest) = proxy.fetch_manifest(&proxy_img).await?; let manifest_digest = Digest::from_str(&manifest_digest)?; - let new_imageid = manifest.config().digest(); // Query for previous stored state - let (previous_state, previous_imageid) = if let Some(previous_state) = previous_state { - // If the manifest digests match, we're done. - if previous_state.manifest_digest == manifest_digest { - return Ok(PrepareResult::AlreadyPresent(previous_state)); - } - // Failing that, if they have the same imageID, we're also done. - let previous_imageid = previous_state.manifest.config().digest(); - if previous_imageid == new_imageid { - return Ok(PrepareResult::AlreadyPresent(previous_state)); - } - let previous_imageid = previous_imageid.to_string(); - (Some(previous_state), Some(previous_imageid)) - } else { - (None, None) + let (previous_state, previous_imageid) = match Self::diff_previous_state( + previous_state, + &manifest_digest, + manifest.config().digest(), + ) { + PreviousState::UpToDate(state) => return Ok(PrepareResult::AlreadyPresent(state)), + PreviousState::Outdated(previous) => previous.unzip(), }; - let config = self.proxy.fetch_config(&proxy_img).await?; + let config = proxy.fetch_config(&proxy_img).await?; - // If there is a currently fetched image, cache the new pending manifest+config - // as detached commit metadata, so that future fetches can query it offline. - if let Some(previous_state) = previous_state.as_ref() { - self.cache_pending( - previous_state.merge_commit.as_str(), - &manifest_digest, - &manifest, - &config, - ) - .await?; - } + // The layer source consumes the proxy, so only take it once the checks + // that can return `AlreadyPresent` are done. + let proxy = self.proxy.take().unwrap(); - let imp = self.create_prepared_import( - manifest_digest, - manifest, - config, - previous_state, - previous_imageid, + let layer_source = Box::new(ProxyLayerSource::new( + proxy, proxy_img, - )?; + self.imgref.imgref.transport, + )); + let imp = self + .create_prepared_import( + manifest_digest, + manifest, + config, + previous_state, + previous_imageid, + layer_source, + ) + .await?; Ok(PrepareResult::Ready(imp)) } @@ -1045,13 +1192,7 @@ impl ImageImporter { write_refs: bool, ) -> Result<()> { tracing::debug!("Fetching base"); - if matches!(self.imgref.sigverify, SignatureSource::ContainerPolicy) - && skopeo::container_policy_is_default_insecure(&self.root)? - { - return Err(anyhow!( - "containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage" - )); - } + self.check_sigverify(false)?; let remote = match &self.imgref.sigverify { SignatureSource::OstreeRemote(remote) => Some(remote.clone()), SignatureSource::ContainerPolicy | SignatureSource::ContainerPolicyAllowInsecure => { @@ -1066,7 +1207,6 @@ impl ImageImporter { } return Ok(()); }; - let des_layers = self.proxy.get_layer_info(&import.proxy_img).await?; for layer in import.ostree_layers.iter_mut() { if let Some(commit) = layer.commit.as_ref() { if write_refs { @@ -1078,16 +1218,14 @@ impl ImageImporter { p.send(ImportProgress::OstreeChunkStarted(layer.layer.clone())) .await?; } - let (blob, driver, media_type) = fetch_layer( - &self.proxy, - &import.proxy_img, - &import.manifest, - &layer.layer, - self.layer_byte_progress.as_ref(), - des_layers.as_ref(), - self.imgref.imgref.transport, - ) - .await?; + let (blob, driver, media_type) = import + .layer_source + .fetch_layer( + &import.manifest, + &layer.layer, + self.layer_byte_progress.as_ref(), + ) + .await?; let repo = self.repo.clone(); let target_ref = layer.ostree_ref.clone(); let import_task = @@ -1129,16 +1267,14 @@ impl ImageImporter { )) .await?; } - let (blob, driver, media_type) = fetch_layer( - &self.proxy, - &import.proxy_img, - &import.manifest, - &commit_layer.layer, - self.layer_byte_progress.as_ref(), - des_layers.as_ref(), - self.imgref.imgref.transport, - ) - .await?; + let (blob, driver, media_type) = import + .layer_source + .fetch_layer( + &import.manifest, + &commit_layer.layer, + self.layer_byte_progress.as_ref(), + ) + .await?; let repo = self.repo.clone(); let target_ref = commit_layer.ostree_ref.clone(); let import_task = @@ -1189,9 +1325,7 @@ impl ImageImporter { } let deprecated_warning = prep.deprecated_warning().map(ToOwned::to_owned); self.unencapsulate_base(&mut prep, true, false).await?; - // TODO change the imageproxy API to ensure this happens automatically when - // the image reference is dropped - self.proxy.close_image(&prep.proxy_img).await?; + prep.layer_source.finish().await?; // SAFETY: We know we have a commit let ostree_commit = prep.ostree_commit_layer.unwrap().commit.unwrap(); let image_digest = prep.manifest_digest; @@ -1455,8 +1589,6 @@ impl ImageImporter { // First download all layers for the base image (if necessary) - we need the SELinux policy // there to label all following layers. self.unencapsulate_base(&mut import, false, true).await?; - let des_layers = self.proxy.get_layer_info(&import.proxy_img).await?; - let proxy = self.proxy; let target_imgref = self.target_imgref.as_ref().unwrap_or(&self.imgref); let base_commit = import .ostree_commit_layer @@ -1493,16 +1625,14 @@ impl ImageImporter { p.send(ImportProgress::DerivedLayerStarted(layer.layer.clone())) .await?; } - let (blob, driver, media_type) = super::unencapsulate::fetch_layer( - &proxy, - &import.proxy_img, - &import.manifest, - &layer.layer, - self.layer_byte_progress.as_ref(), - des_layers.as_ref(), - self.imgref.imgref.transport, - ) - .await?; + let (blob, driver, media_type) = import + .layer_source + .fetch_layer( + &import.manifest, + &layer.layer, + self.layer_byte_progress.as_ref(), + ) + .await?; // SELinux label derived layers using the base policy. For non-ostree // containers (base_commit is None), fall back to the caller-provided // sepolicy_commit (typically the booted deployment's commit). @@ -1553,13 +1683,8 @@ impl ImageImporter { } } - // TODO change the imageproxy API to ensure this happens automatically when - // the image reference is dropped - proxy.close_image(&import.proxy_img).await?; - - // We're done with the proxy, make sure it didn't have any errors. - proxy.finalize().await?; - tracing::debug!("finalized proxy"); + import.layer_source.finish().await?; + tracing::debug!("finished layer source"); // Disconnect progress notifiers to signal we're done with fetching. let _ = self.layer_byte_progress.take(); diff --git a/crates/ostree-ext/tests/it/main.rs b/crates/ostree-ext/tests/it/main.rs index e43aab027b..8f973d3c44 100644 --- a/crates/ostree-ext/tests/it/main.rs +++ b/crates/ostree-ext/tests/it/main.rs @@ -5,6 +5,7 @@ use camino::Utf8Path; use cap_std::fs::{Dir, DirBuilder, DirBuilderExt}; use cap_std_ext::cap_std; use containers_image_proxy::oci_spec; +use futures_util::future::BoxFuture; use gvariant::aligned_bytes::TryAsAligned; use gvariant::{Marker, Structure}; use oci_image::ImageManifest; @@ -14,7 +15,8 @@ use ocidir::oci_spec::distribution::Reference; use ocidir::oci_spec::image::{Arch, DigestAlgorithm}; use ostree_ext::chunking::ObjectMetaSized; use ostree_ext::container::{ - Config, ExportOpts, ImageReference, OstreeImageReference, SignatureSource, Transport, + Config, ExportOpts, FetchedLayer, ImageReference, LayerSource, OstreeImageReference, + SignatureSource, Transport, }; use ostree_ext::container::{ManifestDiff, OSTREE_COMMIT_LABEL, store}; use ostree_ext::prelude::{Cast, FileExt}; @@ -763,6 +765,99 @@ async fn test_no_fetch_digested() -> Result<()> { Ok(()) } +/// A [`LayerSource`] which reads blobs straight out of an OCI directory, +/// standing in for anything that produces layer content without a registry. +#[derive(Debug)] +struct OciDirLayerSource(ocidir::OciDir); + +impl LayerSource for OciDirLayerSource { + fn fetch_layer<'a>( + &'a self, + _manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + _progress: Option<&'a tokio::sync::watch::Sender>>, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let blob = tokio::fs::File::from_std(self.0.read_blob(layer)?); + let blob: Box = + Box::new(tokio::io::BufReader::new(blob)); + let driver = Box::pin(std::future::ready(Ok(()))); + Ok((blob, driver as BoxFuture<'a, _>, layer.media_type().clone())) + }) + } + + fn finish(self: Box) -> BoxFuture<'static, Result<()>> { + Box::pin(std::future::ready(Ok(()))) + } +} + +/// Import an image from a manifest and a caller-provided layer source, with no +/// image proxy in play at all. +#[tokio::test] +async fn test_prepare_from_manifest() -> Result<()> { + let fixture = Fixture::new_v1()?; + let (src_imgref, expected_digest) = fixture.export_container().await.unwrap(); + + let open_source = || -> Result> { + let dir = Dir::open_ambient_dir( + Utf8Path::new(src_imgref.name.as_str()), + cap_std::ambient_authority(), + )?; + Ok(Box::new(OciDirLayerSource(ocidir::OciDir::open(dir)?))) + }; + + let ocidir = ocidir::OciDir::open(Dir::open_ambient_dir( + Utf8Path::new(src_imgref.name.as_str()), + cap_std::ambient_authority(), + )?)?; + let index = ocidir.read_index()?; + let manifest: oci_image::ImageManifest = + ocidir.read_json_blob(index.manifests().first().unwrap())?; + let config: oci_image::ImageConfiguration = ocidir.read_json_blob(manifest.config())?; + + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: src_imgref.clone(), + }; + + let mut imp = store::ImageImporter::new_without_proxy(fixture.destrepo(), &imgref)?; + let prep = match imp + .prepare_from_manifest( + expected_digest.clone(), + manifest.clone(), + config.clone(), + open_source()?, + ) + .await? + { + store::PrepareResult::AlreadyPresent(_) => panic!("Image should not be present yet"), + store::PrepareResult::Ready(prep) => prep, + }; + assert_eq!(prep.manifest_digest, expected_digest); + assert_eq!(prep.all_layers().count(), LAYERS_V0_LEN); + let state = imp.import(prep).await?; + assert_eq!(state.manifest_digest, expected_digest); + + // The layers we read out of the OCI directory reassemble into the content + // the fixture started from. + let (commitdata, _) = fixture.destrepo().load_commit(&state.base_commit)?; + assert_eq!( + CONTENTS_CHECKSUM_V0, + ostree::commit_get_content_checksum(&commitdata) + .unwrap() + .as_str() + ); + + // And now it is present, established without ever contacting a registry. + let mut imp = store::ImageImporter::new_without_proxy(fixture.destrepo(), &imgref)?; + let prep = imp + .prepare_from_manifest(expected_digest, manifest, config, open_source()?) + .await?; + assert!(matches!(prep, store::PrepareResult::AlreadyPresent(_))); + + Ok(()) +} + #[tokio::test] async fn test_export_as_container_derived() -> Result<()> { if !check_skopeo() { From 654b65912c743e1c45c3d43a3a8f2348ae07d99f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2026 11:32:45 +0200 Subject: [PATCH 4/6] ostree-ext: Add list_container_deployment_commits() We will need this later. We're reusing most of the old list_container_deployment_manifests() for this. Signed-off-by: Alexander Larsson --- crates/ostree-ext/src/container/store.rs | 38 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/crates/ostree-ext/src/container/store.rs b/crates/ostree-ext/src/container/store.rs index a8b9e47a98..9316d85470 100644 --- a/crates/ostree-ext/src/container/store.rs +++ b/crates/ostree-ext/src/container/store.rs @@ -2160,13 +2160,13 @@ pub async fn export( .await } -/// Iterate over deployment commits, returning the manifests from +/// Iterate over deployment commits, returning the commit id and commit object for /// commits which point to a container image. -#[context("Listing deployment manifests")] -fn list_container_deployment_manifests( +#[context("Listing deployment commits")] +fn list_container_deployment_commits_full( repo: &ostree::Repo, cancellable: Option<&gio::Cancellable>, -) -> Result> { +) -> Result> { // Gather all refs which start with ostree/0/ or ostree/1/ or rpmostree/base/ // and create a set of the commits which they reference. let commits = OSTREE_BASE_DEPLOYMENT_REFS @@ -2197,13 +2197,39 @@ fn list_container_deployment_manifests( .is_some() { tracing::trace!("Commit {commit} is a container image"); - let manifest = manifest_data_from_commitmeta(commit_meta)?.0; - r.push(manifest); + r.push((commit.to_string(), commit_obj)); } } Ok(r) } +/// List container image commits retained by deployment or protected base-image refs. +pub fn list_container_deployment_commits( + repo: &ostree::Repo, + cancellable: Option<&gio::Cancellable>, +) -> Result> { + list_container_deployment_commits_full(repo, cancellable)? + .into_iter() + .map(|(commit, _commit_obj)| Ok(commit)) + .collect() +} + +/// Iterate over deployment commits, returning the manifests from +/// commits which point to a container image. +#[context("Listing deployment manifests")] +fn list_container_deployment_manifests( + repo: &ostree::Repo, + cancellable: Option<&gio::Cancellable>, +) -> Result> { + list_container_deployment_commits_full(repo, cancellable)? + .into_iter() + .map(|(_commit, commit_obj)| { + let metadata = glib::VariantDict::new(Some(&commit_obj.child_value(0))); + Ok(manifest_data_from_commitmeta(&metadata)?.0) + }) + .collect() +} + /// Garbage collect unused image layer references. /// /// This function assumes no transaction is active on the repository. From 9d3cb1e57193a6f63c2eefd435d9db532cc1b731 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2026 11:30:28 +0200 Subject: [PATCH 5/6] Apply OCI deltas with the ostree backend `bootc upgrade --from-delta` and `bootc switch --from-delta` worked only with the composefs backend. Wire them up for ostree too. A tar-diff patch reads the source image by *path*, not from the original tar byte stream, so the source does not have to be a container layer: an ostree commit holding the same filesystem works just as well. `OstreeDataSource` serves file content out of a commit, and `DeltaLayerSource` plugs that into the container importer in place of the image proxy, so nothing is fetched. ostree does not store an image's root filesystem verbatim - the tar importer moves `/etc` to `/usr/etc`, and `/var` to `/usr/share/factory/var` on ostree older than v2024.3 - so a source path is tried at each of the locations that importer could have put it. Each reconstructed layer is streamed to the importer over a pipe, with the reconstruction itself running as the driver future that `join_fetch` already polls concurrently, so a multi-GB base layer never has to be spooled to disk. The source image is looked up by config digest among the images in the repository, and an absent source is fatal: there is no guarantee of a network connection at the point a delta is applied, so falling back to the registry would defeat the purpose. Byte-level progress is not reported for delta layers: the reconstructed bytes are uncompressed, so counting them against the descriptor's compressed size would overshoot. Per-layer start/completion still is. A layer's ref is written as the layer is unpacked, i.e. before the driver future has had the chance to report a diff_id mismatch, so a failed apply can leave a ref to unverified content that a retry would reuse. Prune the unreferenced layers after a failed delta apply. Also, `switch --from-delta` no longer short-circuits when the image specification is unchanged - a delta names one specific target digest, and switching to the reference you are already tracking is exactly how it is normally used. Assisted-by: AI Signed-off-by: Alexander Larsson --- crates/lib/src/cli.rs | 38 ++- crates/lib/src/delta.rs | 37 +-- crates/lib/src/delta_ostree.rs | 441 +++++++++++++++++++++++++++++++++ crates/lib/src/deploy.rs | 115 +++++++-- crates/lib/src/lib.rs | 1 + 5 files changed, 594 insertions(+), 38 deletions(-) create mode 100644 crates/lib/src/delta_ostree.rs diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 957444ad5b..a85b936395 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -1284,7 +1284,9 @@ async fn upgrade( storage: &Storage, booted_ostree: &BootedOstree<'_>, ) -> Result<()> { - crate::delta::reject_unsupported(opts.from_delta.as_deref())?; + let delta = crate::delta::open_opt(opts.from_delta.as_deref()) + .await? + .map(std::sync::Arc::new); let repo = &booted_ostree.repo(); @@ -1349,6 +1351,9 @@ async fn upgrade( // needs this for update_mtime() and the non-check path needs it for // unified pull detection. let use_unified = crate::deploy::image_exists_in_unified_storage(storage, imgref).await?; + if let Some(delta) = delta.as_deref() { + crate::delta::reject_unified_storage(delta, use_unified)?; + } if opts.check { let ostree_imgref = imgref.clone().into(); @@ -1375,7 +1380,17 @@ async fn upgrade( } } } else { - let fetched = if use_unified { + let fetched = if let Some(delta) = delta.clone() { + crate::deploy::pull_delta( + repo, + imgref, + delta, + opts.quiet, + prog.clone(), + Some(&booted_ostree.deployment), + ) + .await? + } else if use_unified { crate::deploy::pull_unified( repo, imgref, @@ -1512,7 +1527,9 @@ async fn switch_ostree( storage: &Storage, booted_ostree: &BootedOstree<'_>, ) -> Result<()> { - crate::delta::reject_unsupported(opts.from_delta.as_deref())?; + let delta = crate::delta::open_opt(opts.from_delta.as_deref()) + .await? + .map(std::sync::Arc::new); let (_, host) = crate::status::get_status(booted_ostree)?; @@ -1577,8 +1594,21 @@ async fn switch_ostree( } else { crate::deploy::image_exists_in_unified_storage(storage, &target).await? }; + if let Some(delta) = delta.as_deref() { + crate::delta::reject_unified_storage(delta, use_unified)?; + } - let fetched = if use_unified { + let fetched = if let Some(delta) = delta.clone() { + crate::deploy::pull_delta( + repo, + &target, + delta, + opts.quiet, + prog.clone(), + Some(&booted_ostree.deployment), + ) + .await? + } else if use_unified { crate::deploy::pull_unified( repo, &target, diff --git a/crates/lib/src/delta.rs b/crates/lib/src/delta.rs index c25f44e1d3..ef3a566031 100644 --- a/crates/lib/src/delta.rs +++ b/crates/lib/src/delta.rs @@ -97,6 +97,8 @@ pub(crate) struct Delta { pub(crate) path: Utf8PathBuf, /// Parsed information about the delta. pub(crate) parsed: ParsedDelta, + config: ImageConfiguration, + layout: Layout, } impl Delta { @@ -107,13 +109,22 @@ impl Delta { let layout = Layout::open(path)?; let parsed = parse(&layout).await?; - validate(&parsed)?; + let config = validate(&parsed)?; Ok(Self { path: path.to_owned(), parsed, + config, + layout, }) } + /// Open the patch blob `desc` + pub(crate) fn read_patch(&self, desc: &Descriptor) -> Result> { + self.layout + .read_blob(desc) + .with_context(|| format!("Reading delta patch {}", desc.digest())) + } + /// The digest of the target image's manifest, as recorded in the delta. pub(crate) fn target_manifest_digest(&self) -> &Digest { self.parsed.target_manifest_descriptor.digest() @@ -148,6 +159,11 @@ impl Delta { &self.parsed.target_manifest } + /// The target image's config. + pub(crate) fn target_config(&self) -> &ImageConfiguration { + &self.config + } + /// The config digest of the image this delta was built against. pub(crate) fn source_config_digest(&self) -> &Digest { &self.parsed.source_config_digest @@ -196,14 +212,6 @@ pub(crate) async fn open_opt(path: Option<&Utf8Path>) -> Result> { } } -/// Reject `--from-delta` on a storage backend that cannot apply one yet. -pub(crate) fn reject_unsupported(path: Option<&Utf8Path>) -> Result<()> { - match path { - Some(path) => bail!("--from-delta ({path}) is not supported by the ostree backend yet"), - None => Ok(()), - } -} - /// Reject `--from-delta` for an image that also has to be in containers-storage. pub(crate) fn reject_unified_storage(delta: &Delta, use_unified: bool) -> Result<()> { ensure!( @@ -215,7 +223,7 @@ pub(crate) fn reject_unified_storage(delta: &Delta, use_unified: bool) -> Result } /// Check the delta's internal consistency, to fail early -fn validate(p: &ParsedDelta) -> Result<()> { +fn validate(p: &ParsedDelta) -> Result { verify_digest( "Embedded target manifest", &p.target_manifest_raw, @@ -253,7 +261,7 @@ fn validate(p: &ParsedDelta) -> Result<()> { ); } - Ok(()) + Ok(config) } /// Read the single manifest out of an OCI layout and parse it as a delta. @@ -637,13 +645,6 @@ pub(crate) mod tests { } } - #[test] - fn test_reject_unsupported() { - reject_unsupported(None).unwrap(); - let err = reject_unsupported(Some(Utf8Path::new("/d"))).unwrap_err(); - assert!(format!("{err:#}").contains("ostree backend")); - } - #[test] fn test_verify_digest() { let expected: Digest = diff --git a/crates/lib/src/delta_ostree.rs b/crates/lib/src/delta_ostree.rs new file mode 100644 index 0000000000..d5b6a725f2 --- /dev/null +++ b/crates/lib/src/delta_ostree.rs @@ -0,0 +1,441 @@ +//! Applying an oci-delta with the ostree backend. +//! +//! A tar-diff patch reads the source image by *path*, so the source does not +//! have to be a container layer at all - an ostree commit holding the same +//! filesystem works just as well. [`OstreeDataSource`] serves file content out +//! of a commit, and [`DeltaLayerSource`] plugs that into the container importer +//! in place of the image proxy. +//! +//! The one wrinkle is that ostree does not store an image's root filesystem +//! verbatim: the tar importer moves `/etc` to `/usr/etc`, moves `/var` to +//! `/usr/share/factory/var` on ostree older than v2024.3, and drops anything +//! outside those unless the image was imported with `allow_nonusr`. See +//! [`source_path_candidates`]. +//! +//! An ostree-native (chunked) image additionally has its layers made of repo +//! objects under `sysroot/ostree/`, which are nowhere to be found in the +//! commit's own file tree. That is not a problem here only because oci-delta +//! passes `IgnoreSourcePrefixes=["sysroot/ostree/"]` when building a delta, so +//! no patch ever asks for one. + +use std::io::{BufWriter, Read, Seek, SeekFrom, Write}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cap_std_ext::cap_std::fs::Dir; +use fn_error_context::context; +use futures_util::future::BoxFuture; +use ostree_ext::container as ostree_container; +use ostree_ext::oci_spec::image as oci_image; +use ostree_ext::prelude::*; +use ostree_ext::{gio, ostree}; +use tokio::io::AsyncBufRead; +use tokio::sync::OnceCell; +use tokio_util::io::SyncIoBridge; + +use oci_delta::BlobStream; +use oci_delta::{DeltaDataSource, reconstruct_layer_to}; +use ostree_container::{FetchedLayer, LayerSource}; + +use crate::delta::Delta; + +/// How much reconstructed layer data may sit between the worker and the +/// importer before the worker blocks. +const PIPE_BUFFER: usize = 128 * 1024; + +/// Where in an ostree commit the content for a source image path may be found. +/// +/// The first hit wins, and only the remaps the tar importer performs are tried. +/// For `etc` that is unambiguous - an image cannot have both `/etc/passwd` and +/// a distinct `/usr/etc/passwd` once imported, because the first becomes the +/// second. For `var` it is merely very unlikely: on ostree v2024.3 and newer +/// `/var` is kept as-is, so an image shipping both `/var/lib/x` and +/// `/usr/share/factory/var/lib/x` would have the fallback read the wrong one if +/// the former had been filtered out. A wrong read is caught by the diff_id +/// check on the reconstructed layer. +fn source_path_candidates(path: &str) -> Vec { + let mut candidates = vec![path.to_owned()]; + if let Some(rest) = path.strip_prefix("etc/") { + candidates.push(format!("usr/etc/{rest}")); + } else if let Some(rest) = path.strip_prefix("var/") { + candidates.push(format!("usr/share/factory/var/{rest}")); + } + candidates +} + +trait ReadSeek: Read + Seek {} +impl ReadSeek for T {} + +/// Serves file content from an ostree commit to a tar-diff patch. +struct OstreeDataSource { + root: ostree::RepoFile, + objects: Dir, + current: Option>, +} + +impl OstreeDataSource { + #[context("Opening delta source commit {commit}")] + fn new(repo: ostree::Repo, commit: &str) -> Result { + let (root, _) = repo.read_commit(commit, gio::Cancellable::NONE)?; + let root = root.downcast::().expect("downcast"); + let objects = Dir::reopen_dir(&repo.dfd_borrow())? + .open_dir("objects") + .context("Opening repo objects directory")?; + ensure!( + repo.mode() != ostree::RepoMode::Archive, + "OSTree archive repo mode is not supported" + ); + Ok(Self { + root, + objects, + current: None, + }) + } + + fn open_object(&self, checksum: &str) -> Result> { + let (prefix, rest) = checksum.split_at(2); + let f = self.objects.open(format!("{prefix}/{rest}.file"))?; + return Ok(Box::new(f.into_std())); + } + + fn current(&mut self) -> Result<&mut Box> { + self.current + .as_mut() + .context("No current file set in data source") + } +} + +impl DeltaDataSource for OstreeDataSource { + fn set_current_file(&mut self, path: &str) -> Result<()> { + self.current = None; + let path = path.trim_start_matches("./").trim_start_matches('/'); + let cancellable = gio::Cancellable::NONE; + for candidate in source_path_candidates(path) { + let f = self.root.resolve_relative_path(&candidate); + let f = f.downcast::().expect("downcast"); + if f.query_file_type(gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, cancellable) + != gio::FileType::Regular + { + continue; + } + f.ensure_resolved()?; + self.current = Some( + self.open_object(f.checksum().as_str()) + .with_context(|| format!("Opening delta source file {candidate}"))?, + ); + return Ok(()); + } + bail!("Delta source file not found in source commit: {path}"); + } + + fn read_exact_current(&mut self, buf: &mut [u8]) -> Result<()> { + Ok(self.current()?.read_exact(buf)?) + } + + fn seek_current(&mut self, offset: u64) -> Result { + Ok(self.current()?.seek(SeekFrom::Start(offset))?) + } + + fn read_current_to_end(&mut self, max_size: u64) -> Result> { + let current = self.current()?; + let size = current.seek(SeekFrom::End(0))?; + ensure!( + size <= max_size, + "Source file too large: {size} > {max_size}" + ); + current.seek(SeekFrom::Start(0))?; + let mut data = Vec::with_capacity(size as usize); + current.read_to_end(&mut data)?; + Ok(data) + } + + fn copy_to(&mut self, dst: &mut dyn Write, n: u64) -> Result<()> { + let current = self.current()?; + let copied = std::io::copy(&mut Read::by_ref(current).take(n), dst)?; + if copied != n { + bail!("Short read from delta source: expected {n}, got {copied}"); + } + Ok(()) + } +} + +/// Produces the target image's layers from an oci-delta and a source commit +/// already in the repository, without any network access. +#[derive(Debug)] +pub(crate) struct DeltaLayerSource { + delta: Arc, + /// `ostree::Repo` is `Send` but not `Sync`, and [`LayerSource`] is both. + repo: Mutex, + source_commit: OnceCell, +} + +impl DeltaLayerSource { + pub(crate) fn new(delta: Arc, repo: &ostree::Repo) -> Self { + Self { + delta, + repo: Mutex::new(repo.clone()), + source_commit: OnceCell::new(), + } + } + + fn repo(&self) -> ostree::Repo { + self.repo.lock().unwrap().clone() + } + + /// The commit to read source content from. + /// + /// Resolved on first use rather than up front: an import that turns out to + /// need no layers at all needs no source either, and this still runs before + /// anything is written. + async fn source_commit(&self) -> Result<&str> { + self.source_commit + .get_or_try_init(|| async { find_source_commit(&self.repo(), &self.delta) }) + .await + .map(|s| s.as_str()) + } + + /// The patch for `layer`, its media type, and the diff_id the reconstructed + /// content must hash to. + fn open_patch( + &self, + manifest: &oci_image::ImageManifest, + layer: &oci_image::Descriptor, + ) -> Result<(Box, oci_image::MediaType, oci_image::Digest)> { + let Some(patch) = self.delta.parsed.delta_layer_by_to.get(layer.digest()) else { + bail!( + "Delta {} carries no patch for layer {}, and it is not present locally; \ + there is nothing to reconstruct it from", + self.delta.path, + layer.digest(), + ); + }; + let index = manifest + .layers() + .iter() + .position(|l| l == layer) + .ok_or_else(|| anyhow!("Layer {} is not part of the target image", layer.digest()))?; + let diff_id = self + .delta + .target_config() + .rootfs() + .diff_ids() + .get(index) + .ok_or_else(|| anyhow!("Target image has no diff_id for layer {index}"))?; + let diff_id = oci_image::Digest::from_str(diff_id) + .with_context(|| format!("Parsing diff_id for layer {index}"))?; + Ok(( + self.delta.read_patch(patch)?, + patch.media_type().clone(), + diff_id, + )) + } +} + +impl LayerSource for DeltaLayerSource { + fn fetch_layer<'a>( + &'a self, + manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + // The reconstructed bytes are uncompressed, so counting them against + // the descriptor's compressed size would overshoot; the importer still + // reports per-layer start and completion. + _progress: Option< + &'a tokio::sync::watch::Sender>, + >, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let (blob, media_type, diff_id) = self.open_patch(manifest, layer)?; + let source_commit = self.source_commit().await?.to_owned(); + let repo = self.repo(); + let (writer, reader) = tokio::io::duplex(PIPE_BUFFER); + + let worker = tokio::task::spawn_blocking(move || -> Result<()> { + let mut source = OstreeDataSource::new(repo, &source_commit)?; + let mut dst = BufWriter::with_capacity(PIPE_BUFFER, SyncIoBridge::new(writer)); + reconstruct_layer_to(blob, &media_type, &mut source, &diff_id, &mut dst)?; + dst.into_inner() + .map_err(|e| anyhow!("Flushing reconstructed layer: {e}"))? + .shutdown()?; + Ok(()) + }); + let driver = async move { worker.await.context("Delta worker")? }; + + Ok(( + Box::new(tokio::io::BufReader::new(reader)) as Box, + Box::pin(driver) as BoxFuture<'a, _>, + oci_image::MediaType::ImageLayer, + )) + }) + } + + fn finish(self: Box) -> BoxFuture<'static, Result<()>> { + Box::pin(std::future::ready(Ok(()))) + } +} + +/// Find the ostree commit holding the image this delta was built against. +/// +/// Deltas are applied offline, so an absent source is fatal rather than a +/// reason to go to the registry. +#[context("Finding delta source image")] +pub(crate) fn find_source_commit(repo: &ostree::Repo, delta: &Delta) -> Result { + let wanted = delta.source_config_digest(); + for image in ostree_container::store::list_images(repo)? { + let imgref = ostree_container::ImageReference::try_from(image.as_str()) + .with_context(|| format!("Parsing stored image reference {image}"))?; + let Some(state) = ostree_container::store::query_image(repo, &imgref)? else { + continue; + }; + if state.manifest.config().digest() == wanted { + tracing::debug!("Delta source {wanted} is {imgref} ({})", state.merge_commit); + return Ok(state.merge_commit); + } + } + for commit in + ostree_container::store::list_container_deployment_commits(repo, gio::Cancellable::NONE)? + { + let state = ostree_container::store::query_image_commit(repo, &commit)?; + if state.manifest.config().digest() == wanted { + return Ok(commit); + } + } + bail!( + "Delta {} was built against the image with config {wanted}, which is not present in this \ + system's ostree repository. A delta can only be applied on top of its source image.", + delta.path, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_std_ext::cap_std::ambient_authority; + use std::os::fd::{AsFd, AsRawFd}; + + #[test] + fn test_source_path_candidates() { + assert_eq!(source_path_candidates("usr/bin/ls"), ["usr/bin/ls"]); + assert_eq!( + source_path_candidates("etc/passwd"), + ["etc/passwd", "usr/etc/passwd"] + ); + assert_eq!( + source_path_candidates("var/lib/foo"), + ["var/lib/foo", "usr/share/factory/var/lib/foo"] + ); + // Only the first component is remapped. + assert_eq!( + source_path_candidates("usr/share/etc/x"), + ["usr/share/etc/x"] + ); + } + + /// Commit `dir` as `testref` and return an [`OstreeDataSource`] for it. + fn data_source_for(repo: &ostree::Repo, dir: &Dir) -> Result { + let cancellable = gio::Cancellable::NONE; + let txn = repo.auto_transaction(cancellable)?; + let mt = ostree::MutableTree::new(); + let modifier = + ostree::RepoCommitModifier::new(ostree::RepoCommitModifierFlags::SKIP_XATTRS, None); + repo.write_dfd_to_mtree( + dir.as_fd().as_raw_fd(), + ".", + &mt, + Some(&modifier), + cancellable, + )?; + let root = repo.write_mtree(&mt, cancellable)?; + let root = root.downcast::().unwrap(); + let commit = repo.write_commit(None, None, None, None, &root, cancellable)?; + txn.commit(cancellable)?; + + OstreeDataSource::new(repo.clone(), commit.as_str()) + } + + fn read_all(source: &mut OstreeDataSource, path: &str) -> Result> { + source.set_current_file(path)?; + let mut out = Vec::new(); + source.current()?.read_to_end(&mut out)?; + Ok(out) + } + + #[test] + fn test_ostree_data_source() -> Result<()> { + let td = cap_std_ext::cap_tempfile::TempDir::new(ambient_authority())?; + td.create_dir("repo")?; + let repo = ostree::Repo::create_at( + td.as_fd().as_raw_fd(), + "repo", + ostree::RepoMode::BareUser, + None, + gio::Cancellable::NONE, + )?; + + td.create_dir_all("rootfs/usr/bin")?; + td.create_dir_all("rootfs/usr/etc")?; + td.create_dir_all("rootfs/usr/share/factory/var/lib")?; + td.write("rootfs/usr/bin/ls", b"binary")?; + td.write("rootfs/usr/etc/passwd", b"root:x:0:0")?; + td.write("rootfs/usr/share/factory/var/lib/state", b"stateful")?; + let rootfs = td.open_dir("rootfs")?; + + let mut source = data_source_for(&repo, &rootfs)?; + + assert_eq!(read_all(&mut source, "usr/bin/ls")?, b"binary"); + // The remapped locations are found under their pre-import paths. + assert_eq!(read_all(&mut source, "etc/passwd")?, b"root:x:0:0"); + assert_eq!(read_all(&mut source, "var/lib/state")?, b"stateful"); + // As are leading-`./` forms, which is how tar names entries. + assert_eq!(read_all(&mut source, "./etc/passwd")?, b"root:x:0:0"); + + // Partial reads from an offset, which is what a tar-diff mostly does. + source.set_current_file("usr/etc/passwd")?; + source.seek_current(5)?; + let mut buf = [0u8; 3]; + source.read_exact_current(&mut buf)?; + assert_eq!(&buf, b"x:0"); + + // A directory is not content, and neither is an absent path. + for missing in ["usr/bin", "usr/bin/nope", "etc/nope"] { + let err = source.set_current_file(missing).unwrap_err(); + assert!( + format!("{err:#}").contains("not found in source commit"), + "{missing}: unexpected error: {err:#}" + ); + } + + // A short read is an error rather than a silent truncation. + source.set_current_file("usr/bin/ls")?; + let err = source.copy_to(&mut Vec::new(), 100).unwrap_err(); + assert!(format!("{err:#}").contains("Short read"), "{err:#}"); + + Ok(()) + } + + /// A delta whose source image is not in the repository must fail, not fall + /// back to fetching it: there may well be no network at this point. + #[tokio::test] + async fn test_find_source_commit_absent() -> Result<()> { + let t = crate::delta::tests::TestDelta::new(); + t.finish_default(); + let delta = crate::delta::Delta::open(t.path()).await?; + + let td = cap_std_ext::cap_tempfile::TempDir::new(ambient_authority())?; + td.create_dir("repo")?; + let repo = ostree::Repo::create_at( + td.as_fd().as_raw_fd(), + "repo", + ostree::RepoMode::BareUser, + None, + gio::Cancellable::NONE, + )?; + + let err = find_source_commit(&repo, &delta).unwrap_err(); + assert!( + format!("{err:#}").contains("not present in this system's ostree repository"), + "{err:#}" + ); + Ok(()) + } +} diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index b361a3b79f..b5b9ae76c5 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -48,6 +48,7 @@ use std::collections::HashSet; use std::io::{BufRead, Write}; use std::os::fd::AsFd; use std::process::Command; +use std::sync::Arc; use anyhow::{Context, Result, anyhow}; use bootc_utils::skopeo_bin; @@ -136,21 +137,26 @@ impl ImageState { } } -/// Wrapper for pulling a container image, wiring up status output. -pub(crate) async fn new_importer( - repo: &ostree::Repo, - imgref: &ostree_container::OstreeImageReference, +/// The importer settings every bootc pull uses, however the layers arrive. +fn configure_importer( + imp: &mut ostree_container::store::ImageImporter, booted_deployment: Option<&ostree::Deployment>, -) -> Result { - let config = new_proxy_config(); - let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?; +) { imp.require_bootable(); // We do our own GC/prune in deploy::prune(), so skip the importer's internal one. imp.disable_gc(); if let Some(deployment) = booted_deployment { imp.set_sepolicy_commit(deployment.csum().to_string()); } - Ok(imp) +} + +/// Wrapper for pulling a container image, wiring up status output. +pub(crate) async fn new_importer( + repo: &ostree::Repo, + imgref: &ostree_container::OstreeImageReference, + booted_deployment: Option<&ostree::Deployment>, +) -> Result { + new_importer_with_config(repo, imgref, new_proxy_config(), booted_deployment).await } /// Wrapper for pulling a container image with a custom proxy config (e.g. for unified storage). @@ -161,12 +167,7 @@ pub(crate) async fn new_importer_with_config( booted_deployment: Option<&ostree::Deployment>, ) -> Result { let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?; - imp.require_bootable(); - // We do our own GC/prune in deploy::prune(), so skip the importer's internal one. - imp.disable_gc(); - if let Some(deployment) = booted_deployment { - imp.set_sepolicy_commit(deployment.csum().to_string()); - } + configure_importer(&mut imp, booted_deployment); Ok(imp) } @@ -525,6 +526,17 @@ pub(crate) async fn prepare_for_pull( } PrepareResult::Ready(p) => p, }; + Ok(PreparedPullResult::Ready(Box::new( + prepared_import_meta(imp, prep).await?, + ))) +} + +/// Report on what a prepared import is going to do, and tally it up for the +/// progress display. +async fn prepared_import_meta( + imp: ImageImporter, + prep: Box, +) -> Result { check_bootc_label(&prep.config); if let Some(warning) = prep.deprecated_warning() { ostree_ext::cli::print_deprecated_warning(warning).await; @@ -532,7 +544,7 @@ pub(crate) async fn prepare_for_pull( ostree_ext::cli::print_layer_status(&prep); let layers_to_fetch = prep.layers_to_fetch().collect::>>()?; - let prepared_image = PreparedImportMeta { + Ok(PreparedImportMeta { imp, n_layers_to_fetch: layers_to_fetch.len(), layers_total: prep.all_layers().count(), @@ -540,9 +552,80 @@ pub(crate) async fn prepare_for_pull( bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(), digest: prep.manifest_digest.clone(), prep, + }) +} + +/// Prepare an import whose metadata and layer content both come from a delta +/// file plus the source image already in `repo`; nothing is fetched. +pub(crate) async fn prepare_for_pull_delta( + repo: &ostree::Repo, + imgref: &ImageReference, + delta: Arc, + booted_deployment: Option<&ostree::Deployment>, +) -> Result { + delta.validate_image_reference(imgref)?; + let imgref_canonicalized = imgref.clone().canonicalize()?; + tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}"); + let ostree_imgref = &OstreeImageReference::from(imgref_canonicalized); + + let mut imp = ImageImporter::new_without_proxy(repo, ostree_imgref)?; + configure_importer(&mut imp, booted_deployment); + + let layer_source = Box::new(crate::delta_ostree::DeltaLayerSource::new( + Arc::clone(&delta), + repo, + )); + let prep = match imp + .prepare_from_manifest( + delta.target_manifest_digest().clone(), + delta.target_manifest().clone(), + delta.target_config().clone(), + layer_source, + ) + .await? + { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {imgref:#} => {}", c.manifest_digest); + return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into()))); + } + PrepareResult::Ready(p) => p, }; - Ok(PreparedPullResult::Ready(Box::new(prepared_image))) + Ok(PreparedPullResult::Ready(Box::new( + prepared_import_meta(imp, prep).await?, + ))) +} + +/// Wrapper for applying a delta, wiring up status output. +pub(crate) async fn pull_delta( + repo: &ostree::Repo, + imgref: &ImageReference, + delta: Arc, + quiet: bool, + prog: ProgressWriter, + booted_deployment: Option<&ostree::Deployment>, +) -> Result> { + if !quiet { + println!("Applying delta {}", delta.describe()); + } + match prepare_for_pull_delta(repo, imgref, delta, booted_deployment).await? { + PreparedPullResult::AlreadyPresent(existing) => Ok(existing), + PreparedPullResult::Ready(prepared_image_meta) => { + check_disk_space_ostree(repo, &prepared_image_meta, imgref)?; + let r = pull_from_prepared(imgref, quiet, prog, *prepared_image_meta).await; + if r.is_err() { + // The importer publishes a layer's ref before the reconstruction + // that feeds it has finished being verified, so a failed apply + // can leave a ref to content that never passed its diff_id + // check. Drop the unreferenced layers so a retry cannot pick one + // up; failing to do so is not worth masking the real error. + if let Err(e) = ostree_container::store::gc_image_layers(repo) { + tracing::warn!("Pruning layers after failed delta apply: {e:#}"); + } + } + r + } + } } /// Check whether the image exists in bootc's unified container storage. diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 8616512c7f..7a77da91ec 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -73,6 +73,7 @@ mod composefs_consts; mod container_export; mod containerenv; pub(crate) mod delta; +pub(crate) mod delta_ostree; pub(crate) mod deploy; mod discoverable_partition_specification; pub(crate) mod fsck; From 033dc0d1996bff07b301f8fb8c94a143eb6ca447 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 8 Sep 2026 18:14:56 +0200 Subject: [PATCH 6/6] Test applying OCI deltas with the ostree backend Three tests covering the whole path from a delta on disk to an imported image, at increasing degrees of realism: `test_apply_delta_whole_layers` builds a delta whose patches are the target layers carried verbatim. The format allows that, and the layer reconstruction dispatches on media type, so this exercises parsing, validation, source lookup and the import while needing no external tooling - it runs everywhere. `test_apply_delta_chunked` and `test_apply_delta_derived` use the real `oci-delta` to build the delta, and skip themselves when it is not installed. The first covers an ostree-native image, whose layers are made of repo objects under `sysroot/ostree/` that do not exist in the commit's file tree - it passes only because oci-delta is told never to ask for one. The second covers a derived layer, an ordinary root filesystem tar, which is the case that actually drives `OstreeDataSource`; the file it patches lives at `/etc/bigconf`, i.e. at a path the importer relocates, and the test asserts both that it lands at `/usr/etc/bigconf` and that the patch is far smaller than the layer, which it can only be if the content really was read back out of the source commit. `oci-delta` goes in a new optional package list, because it is not available on every distribution we build on. Assisted-by: AI Signed-off-by: Alexander Larsson --- contrib/packaging/fedora-extra-optional.txt | 7 + contrib/packaging/install-buildroot | 5 + crates/lib/Cargo.toml | 2 + crates/lib/src/delta.rs | 10 +- crates/lib/src/delta_ostree.rs | 435 ++++++++++++++++++++ 5 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 contrib/packaging/fedora-extra-optional.txt diff --git a/contrib/packaging/fedora-extra-optional.txt b/contrib/packaging/fedora-extra-optional.txt new file mode 100644 index 0000000000..72597cf21e --- /dev/null +++ b/contrib/packaging/fedora-extra-optional.txt @@ -0,0 +1,7 @@ +# Packages that are only needed by some tests, and that we do not require to +# be available everywhere we build. Anything listed here must be optional: +# install-buildroot carries on without it, and the tests that use it skip +# themselves when it is absent. +# +# Generates the deltas the `--from-delta` tests apply. +oci-delta diff --git a/contrib/packaging/install-buildroot b/contrib/packaging/install-buildroot index 1bde1a2d28..161354edcc 100755 --- a/contrib/packaging/install-buildroot +++ b/contrib/packaging/install-buildroot @@ -21,3 +21,8 @@ dnf -y distro-sync ostree{,-libs} systemd dnf -y builddep bootc.spec # And extra packages grep -Ev -e '^#' fedora-extra.txt | xargs dnf -y install +# Packages that only enable extra tests, and are not in every distribution we +# build on. The tests that want these skip themselves when they are missing. +for pkg in $(grep -Ev -e '^#' fedora-extra-optional.txt); do + dnf -y install "$pkg" || echo "warning: optional package $pkg is unavailable" +done diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index 109e26758f..9d5ec1e7fa 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -79,6 +79,8 @@ uuid = { version = "1.8.0", features = ["v4"] } uapi-version = "0.4.0" [dev-dependencies] +# For the ostree/container test fixtures +ostree-ext = { path = "../ostree-ext", features = ["bootc", "internal-testing-api"] } similar-asserts = { workspace = true } static_assertions = { workspace = true } diff --git a/crates/lib/src/delta.rs b/crates/lib/src/delta.rs index ef3a566031..9256136dc2 100644 --- a/crates/lib/src/delta.rs +++ b/crates/lib/src/delta.rs @@ -323,14 +323,14 @@ pub(crate) mod tests { use ocidir::oci_spec::image::{ImageConfigurationBuilder, ImageManifestBuilder, RootFsBuilder}; use std::collections::HashMap; - const DELTA_CONTENT: &str = "io.github.containers.delta.content"; - const DELTA_TO: &str = "io.github.containers.delta.to"; - const DELTA_SOURCE_CONFIG: &str = "io.github.containers.delta.source-config"; + pub(crate) const DELTA_CONTENT: &str = "io.github.containers.delta.content"; + pub(crate) const DELTA_TO: &str = "io.github.containers.delta.to"; + pub(crate) const DELTA_SOURCE_CONFIG: &str = "io.github.containers.delta.source-config"; const TAR_DIFF: &str = "application/vnd.tar-diff"; const ZERO_DIGEST: &str = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; - fn blob(oci: &OciDir, data: &[u8], media_type: MediaType) -> Descriptor { + pub(crate) fn blob(oci: &OciDir, data: &[u8], media_type: MediaType) -> Descriptor { let mut w = oci.create_blob().unwrap(); std::io::Write::write_all(&mut w, data).unwrap(); w.complete() @@ -354,7 +354,7 @@ pub(crate) mod tests { .unwrap() } - fn annotate(mut desc: Descriptor, annotations: &[(&str, &str)]) -> Descriptor { + pub(crate) fn annotate(mut desc: Descriptor, annotations: &[(&str, &str)]) -> Descriptor { desc.set_annotations(Some( annotations .iter() diff --git a/crates/lib/src/delta_ostree.rs b/crates/lib/src/delta_ostree.rs index d5b6a725f2..fcff49f09f 100644 --- a/crates/lib/src/delta_ostree.rs +++ b/crates/lib/src/delta_ostree.rs @@ -310,8 +310,16 @@ pub(crate) fn find_source_commit(repo: &ostree::Repo, delta: &Delta) -> Result bool { + Command::new(name) + .arg("--help") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok() + } + + fn copy_dir(src: &Utf8Path, dst: &Utf8Path) -> Result<()> { + let st = Command::new("cp") + .args(["-a", src.as_str(), dst.as_str()]) + .status()?; + ensure!(st.success(), "cp -a {src} {dst} failed: {st}"); + Ok(()) + } + + fn open_oci(path: &Utf8Path) -> Result { + Ok(OciDir::open(Dir::open_ambient_dir( + path, + ambient_authority(), + )?)?) + } + + fn single_manifest(oci: &OciDir) -> Result<(oci_image::Descriptor, oci_image::ImageManifest)> { + let index = oci.read_index()?; + let desc = index + .manifests() + .first() + .cloned() + .context("Layout has no manifest")?; + let manifest = oci.read_json_blob(&desc)?; + Ok((desc, manifest)) + } + + fn read_blob(oci: &OciDir, desc: &oci_image::Descriptor) -> Result> { + let mut buf = Vec::new(); + oci.read_blob(desc)?.read_to_end(&mut buf)?; + Ok(buf) + } + + /// Import `path` as the image a delta will be applied on top of. + async fn import_source(fixture: &Fixture, path: &Utf8Path) -> Result<()> { + fixture + .must_import(&ostree_container::ImageReference { + transport: ostree_container::Transport::OciDir, + name: path.to_string(), + }) + .await?; + Ok(()) + } + + /// Apply the delta at `delta_path` as an update to `target_path`. + async fn apply_delta( + fixture: &Fixture, + target_path: &Utf8Path, + delta_path: &Utf8Path, + ) -> Result<(Arc, Box)> { + let delta = Arc::new(crate::delta::Delta::open(delta_path).await?); + let imgref = crate::spec::ImageReference { + image: target_path.to_string(), + transport: "oci".into(), + signature: None, + }; + let state = crate::deploy::pull_delta( + fixture.destrepo(), + &imgref, + Arc::clone(&delta), + true, + Default::default(), + None, + ) + .await?; + Ok((delta, state)) + } + + /// The delta reconstructed the target image exactly: the same content as + /// the fixture's own commit, recorded under the target's real digest. + fn assert_applied( + fixture: &Fixture, + state: &crate::deploy::ImageState, + target: &oci_image::Digest, + ) { + assert_eq!(&state.manifest_digest, target); + let expected = fixture.srcrepo().require_rev(fixture.testref()).unwrap(); + let layered = + ostree_container::store::query_image_commit(fixture.destrepo(), &state.ostree_commit) + .unwrap(); + ostree_ext::fixture::assert_commits_content_equal( + fixture.destrepo(), + &layered.base_commit, + fixture.srcrepo(), + &expected, + ); + } + + /// Export the fixture as a container, change it, and export it again: a + /// pair of ostree-native (chunked) images, as a bootc base image update + /// looks. The source is imported into the destination repo. + async fn chunked_source_and_target() + -> Result<(Fixture, Utf8PathBuf, Utf8PathBuf, oci_image::Digest)> { + let mut fixture = Fixture::new_v1()?; + let (exported, _) = fixture.export_container().await?; + let source = fixture.path.join("source-oci"); + copy_dir(Utf8Path::new(&exported.name), &source)?; + + fixture.update( + // Both of these are paths the fixture has an owning "package" for, + // which its chunked export requires. + FileDef::iter_from( + "r usr/bin/bash the-bash-shell-v2\nr usr/etc/someconfig.conf someconfig-v2\n", + ), + std::iter::empty(), + )?; + let (exported, digest) = fixture.export_container().await?; + let target = Utf8PathBuf::from(exported.name); + + import_source(&fixture, &source).await?; + Ok((fixture, source, target, digest)) + } + + /// 64 KiB of incompressible but deterministic data. `v2` differs from `v1` + /// in eight bytes, so a binary diff of the two is tiny - which is how the + /// tests tell whether the patch really read from the source. + fn big_file(v2: bool) -> Vec { + let mut state = 0x1234_5678u32; + let mut data: Vec = std::iter::repeat_with(|| { + state = state.wrapping_mul(1664525).wrapping_add(1013904223); + (state >> 24) as u8 + }) + .take(64 * 1024) + .collect(); + if v2 { + data[32 * 1024..32 * 1024 + 8].fill(0xff); + } + data + } + + /// Append a derived layer holding [`big_file`] at `/etc/bigconf`, i.e. at a + /// path the ostree importer relocates. + fn derive(oci: &Utf8Path, v2: bool) -> Result<()> { + let content = big_file(v2); + ostree_ext::integrationtest::generate_derived_oci_from_tar( + oci, + move |w| { + let mut tar = tar::Builder::new(w); + let mut dir = tar::Header::new_gnu(); + dir.set_entry_type(tar::EntryType::Directory); + dir.set_mode(0o755); + dir.set_size(0); + tar.append_data(&mut dir, "etc/", std::io::empty())?; + let mut file = tar::Header::new_gnu(); + file.set_mode(0o644); + file.set_size(content.len() as u64); + tar.append_data(&mut file, "etc/bigconf", content.as_slice())?; + tar.finish()?; + Ok(()) + }, + None, + None, + ) + } + + /// A pair of images that share a base and differ only in one derived layer, + /// as a bootc image built from a Containerfile does. The source is imported + /// into the destination repo. + async fn derived_source_and_target() + -> Result<(Fixture, Utf8PathBuf, Utf8PathBuf, oci_image::Digest)> { + let fixture = Fixture::new_v1()?; + let (exported, _) = fixture.export_container().await?; + let source = fixture.path.join("source-oci"); + let target = fixture.path.join("target-oci"); + copy_dir(Utf8Path::new(&exported.name), &source)?; + copy_dir(Utf8Path::new(&exported.name), &target)?; + derive(&source, false)?; + derive(&target, true)?; + let digest = single_manifest(&open_oci(&target)?)?.0.digest().clone(); + + import_source(&fixture, &source).await?; + Ok((fixture, source, target, digest)) + } + + /// Write a delta from `source` to `target` that carries each changed layer + /// whole rather than as a tar-diff. + /// + /// The format permits either, so this covers everything but the patch + /// application itself while needing no external tooling; the tests below + /// that use the real `oci-delta` skip themselves when it is absent. + fn build_whole_layer_delta(source: &Utf8Path, target: &Utf8Path, out: &Utf8Path) -> Result<()> { + use crate::delta::tests::{DELTA_CONTENT, DELTA_SOURCE_CONFIG, DELTA_TO, annotate, blob}; + + let source = open_oci(source)?; + let target = open_oci(target)?; + let (_, source_manifest) = single_manifest(&source)?; + let (target_manifest_desc, target_manifest) = single_manifest(&target)?; + + std::fs::create_dir_all(out)?; + let out = OciDir::ensure(Dir::open_ambient_dir(out, ambient_authority())?)?; + + let mut layers = vec![ + annotate( + blob( + &out, + &read_blob(&target, &target_manifest_desc)?, + oci_image::MediaType::ImageManifest, + ), + &[(DELTA_CONTENT, "image-manifest")], + ), + annotate( + blob( + &out, + &read_blob(&target, target_manifest.config())?, + oci_image::MediaType::ImageConfig, + ), + &[(DELTA_CONTENT, "image-config")], + ), + ]; + let shared: HashSet<_> = source_manifest + .layers() + .iter() + .map(|l| l.digest()) + .collect(); + for layer in target_manifest.layers() { + if shared.contains(layer.digest()) { + continue; + } + let patch = blob( + &out, + &read_blob(&target, layer)?, + layer.media_type().clone(), + ); + let to = layer.digest().to_string(); + layers.push(annotate( + patch, + &[(DELTA_CONTENT, "image-layer"), (DELTA_TO, to.as_str())], + )); + } + ensure!( + layers.len() > 2, + "Source and target images share every layer" + ); + + let empty = blob(&out, b"{}", oci_image::MediaType::EmptyJSON); + let manifest = oci_image::ImageManifestBuilder::default() + .schema_version(2u32) + .media_type(oci_image::MediaType::ImageManifest) + .artifact_type(oci_image::MediaType::Other(MEDIA_TYPE_DELTA.to_string())) + .config(empty) + .layers(layers) + .annotations(std::collections::HashMap::from([( + DELTA_SOURCE_CONFIG.to_string(), + source_manifest.config().digest().to_string(), + )])) + .build()?; + out.replace_with_single_manifest(manifest, Default::default())?; + Ok(()) + } + + fn create_delta(source: &Utf8Path, target: &Utf8Path, out: &Utf8Path) -> Result<()> { + let out = Command::new("oci-delta") + .arg("create") + .arg(format!("oci:{source}")) + .arg(format!("oci:{target}")) + .arg(format!("oci:{out}")) + .output()?; + ensure!( + out.status.success(), + "oci-delta create failed: {}\n{}", + out.status, + String::from_utf8_lossy(&out.stderr), + ); + Ok(()) + } + + /// The whole pipeline - parse and validate the delta, find the source image + /// in the repository, reconstruct the changed layers, import - with the + /// patches degenerate so that no external tooling is needed. + #[tokio::test] + async fn test_apply_delta_whole_layers() -> Result<()> { + let (fixture, source, target, digest) = chunked_source_and_target().await?; + let delta_path = fixture.path.join("delta"); + build_whole_layer_delta(&source, &target, &delta_path)?; + + let (_, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_applied(&fixture, &state, &digest); + Ok(()) + } + + /// Construct and apply a real delta between two ostree-native (chunked) images. + #[tokio::test] + async fn test_apply_delta_chunked() -> Result<()> { + if !have_tool("oci-delta") { + eprintln!("skipping: oci-delta not found in PATH"); + return Ok(()); + } + let (fixture, source, target, digest) = chunked_source_and_target().await?; + let delta_path = fixture.path.join("delta"); + create_delta(&source, &target, &delta_path)?; + + let (_, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_applied(&fixture, &state, &digest); + Ok(()) + } + + /// A real delta over a derived layer, which unlike a chunked one is an + /// ordinary root filesystem tar. This is the case that actually drives + /// [`OstreeDataSource`], and the file it patches is one the importer + /// relocated from `/etc` to `/usr/etc`. + #[tokio::test] + async fn test_apply_delta_derived() -> Result<()> { + if !have_tool("oci-delta") { + eprintln!("skipping: oci-delta not found in PATH"); + return Ok(()); + } + let (fixture, source, target, digest) = derived_source_and_target().await?; + let delta_path = fixture.path.join("delta"); + create_delta(&source, &target, &delta_path)?; + + let (delta, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_eq!(&state.manifest_digest, &digest); + + // Only the derived layer changed, and the patch for it is a fraction of + // its size - which it can only be if the reconstruction read the bulk + // of the content back out of the source commit. + let patched = delta.parsed.delta_layer_by_to.iter().collect::>(); + let [(to, patch)] = patched.as_slice() else { + panic!("expected one patched layer, got {}", patched.len()); + }; + let layer = delta + .target_manifest() + .layers() + .iter() + .find(|l| l.digest() == *to) + .unwrap(); + assert!( + patch.size() * 4 < layer.size(), + "patch is {} bytes against a {} byte layer, so nothing was reused from the source", + patch.size(), + layer.size(), + ); + + let root = ostree_ext::fixture::ostree_ls(fixture.destrepo(), &state.ostree_commit)?; + assert!( + root.contains(&format!("r /usr/etc/bigconf {}\n", 64 * 1024)), + "/usr/etc/bigconf is missing from the applied image:\n{root}" + ); + Ok(()) + } + + /// A deployed source remains usable after staging advances to its image ref. + #[tokio::test] + async fn test_delta_source_retained_by_deployment() -> Result<()> { + let (mut fixture, source, target, _) = chunked_source_and_target().await?; + let repo = fixture.destrepo().clone(); + let source_ref = ostree_container::ImageReference { + transport: ostree_container::Transport::OciDir, + name: source.to_string(), + }; + let original = ostree_container::store::query_image(&repo, &source_ref)?.unwrap(); + let newer = fixture + .must_import(&ostree_container::ImageReference { + transport: ostree_container::Transport::OciDir, + name: target.to_string(), + }) + .await?; + // Model staging B over A under the same image reference. + for (name, commit) in repo.list_refs_ext( + Some("ostree/container/image"), + ostree::RepoListRefsExtFlags::empty(), + gio::Cancellable::NONE, + )? { + if commit.as_str() == original.merge_commit { + repo.set_ref_immediate( + None, + &name, + Some(&newer.merge_commit), + gio::Cancellable::NONE, + )?; + } + } + + // Build A→C; no image ref now identifies A. + fixture.update( + FileDef::iter_from("r usr/bin/bash the-bash-shell-v3\n"), + std::iter::empty(), + )?; + let (_, digest) = fixture.export_container().await?; + let delta_path = fixture.path.join("delta-retained-source"); + build_whole_layer_delta(&source, &target, &delta_path)?; + let delta = Delta::open(&delta_path).await?; + assert!(find_source_commit(&repo, &delta).is_err()); + + // Ordinary ostree deployments have no container metadata. + repo.set_ref_immediate( + None, + "ostree/0/0/1", + Some(&original.base_commit), + gio::Cancellable::NONE, + )?; + // Each supported deployment/base-image ref can retain A independently. + for name in [ + "ostree/0/0/0", + "ostree/1/0/0", + "rpmostree/base/test", + "ostree/container/baseimage/test", + ] { + repo.set_ref_immediate( + None, + name, + Some(&original.merge_commit), + gio::Cancellable::NONE, + )?; + assert_eq!(find_source_commit(&repo, &delta)?, original.merge_commit); + repo.set_ref_immediate(None, name, None, gio::Cancellable::NONE)?; + } + repo.set_ref_immediate( + None, + "ostree/0/0/0", + Some(&original.merge_commit), + gio::Cancellable::NONE, + )?; + // Apply A→C using A retained solely by its deployment ref. + let (_, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_applied(&fixture, &state, &digest); + Ok(()) + } }