diff --git a/Cargo.lock b/Cargo.lock index b5d4d1c7d0..47f6ff27c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,6 +391,7 @@ dependencies = [ "linkme", "linux-kernel-cmdline", "nom 8.0.0", + "oci-delta", "ocidir", "openssl", "ostree-ext", @@ -2419,6 +2420,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-delta" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc185feabe86ac78e5c4ae8b800078f257621ba652cd426d516a846aed93b543" +dependencies = [ + "anyhow", + "flate2", + "hex", + "oci-spec", + "openssl", + "zstd", +] + [[package]] name = "oci-spec" version = "0.10.0" diff --git a/contrib/packaging/fedora-extra-optional.txt b/contrib/packaging/fedora-extra-optional.txt new file mode 100644 index 0000000000..72597cf21e --- /dev/null +++ b/contrib/packaging/fedora-extra-optional.txt @@ -0,0 +1,7 @@ +# Packages that are only needed by some tests, and that we do not require to +# be available everywhere we build. Anything listed here must be optional: +# install-buildroot carries on without it, and the tests that use it skip +# themselves when it is absent. +# +# Generates the deltas the `--from-delta` tests apply. +oci-delta diff --git a/contrib/packaging/install-buildroot b/contrib/packaging/install-buildroot index 1bde1a2d28..161354edcc 100755 --- a/contrib/packaging/install-buildroot +++ b/contrib/packaging/install-buildroot @@ -21,3 +21,8 @@ dnf -y distro-sync ostree{,-libs} systemd dnf -y builddep bootc.spec # And extra packages grep -Ev -e '^#' fedora-extra.txt | xargs dnf -y install +# Packages that only enable extra tests, and are not in every distribution we +# build on. The tests that want these skip themselves when they are missing. +for pkg in $(grep -Ev -e '^#' fedora-extra-optional.txt); do + dnf -y install "$pkg" || echo "warning: optional package $pkg is unavailable" +done diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index 27698065be..e89ac0a956 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -13,6 +13,7 @@ rust-version = "1.85.0" include = ["/src", "/build.rs", "LICENSE-APACHE", "LICENSE-MIT"] [dependencies] +oci-delta = "0.1.0" # Internal crates bootc-blockdev = { package = "bootc-internal-blockdev", path = "../blockdev", version = "1.16.12" } linux-kernel-cmdline = { workspace = true } @@ -78,6 +79,8 @@ uuid = { version = "1.8.0", features = ["v4"] } uapi-version = "0.4.0" [dev-dependencies] +# For the ostree/container test fixtures +ostree-ext = { path = "../ostree-ext", features = ["bootc", "internal-testing-api"] } similar-asserts = { workspace = true } static_assertions = { workspace = true } diff --git a/crates/lib/src/bootc_composefs/repo.rs b/crates/lib/src/bootc_composefs/repo.rs index 9a12624aaf..5141ae44da 100644 --- a/crates/lib/src/bootc_composefs/repo.rs +++ b/crates/lib/src/bootc_composefs/repo.rs @@ -316,10 +316,17 @@ async fn pull_composefs_unified( /// When `use_unified` is false (the default), the image is pulled directly /// into the composefs repo via skopeo. /// +/// With a `delta`, the content is read from the delta file instead and no +/// network access happens at all; `spec_imgref` then only names what the +/// resulting deployment tracks. composefs-ctl recognises the delta artifact +/// from the layout's manifest and reconstructs the changed layers against the +/// source image in the repository, failing if that image is absent. +/// /// Checks for boot entries in the image and returns them. #[context("Pulling composefs repository")] pub(crate) async fn pull_composefs_repo( spec_imgref: &crate::spec::ImageReference, + delta: Option<&crate::delta::Delta>, allow_missing_fsverity: bool, use_unified: bool, quiet: bool, @@ -327,7 +334,17 @@ pub(crate) async fn pull_composefs_repo( ) -> Result { const COMPOSEFS_PULL_JOURNAL_ID: &str = "4c3b2a1f0e9d8c7b6a5f4e3d2c1b0a9f8"; - let imgref = spec_imgref.to_image_proxy_ref()?; + let imgref = match delta { + Some(delta) => { + delta.validate_image_reference(spec_imgref)?; + crate::delta::reject_unified_storage(delta, use_unified)?; + if !quiet { + println!("Applying delta {}", delta.describe()); + } + delta.pull_ref()? + } + None => spec_imgref.to_image_proxy_ref()?, + }; tracing::info!( message_id = COMPOSEFS_PULL_JOURNAL_ID, @@ -336,6 +353,7 @@ pub(crate) async fn pull_composefs_repo( bootc.transport = %imgref.transport, bootc.allow_missing_fsverity = allow_missing_fsverity, bootc.unified_storage = use_unified, + bootc.delta = delta.map(|d| d.path.as_str()), "Pulling composefs image {imgref}", ); diff --git a/crates/lib/src/bootc_composefs/status.rs b/crates/lib/src/bootc_composefs/status.rs index 33c323bd07..945158b468 100644 --- a/crates/lib/src/bootc_composefs/status.rs +++ b/crates/lib/src/bootc_composefs/status.rs @@ -412,10 +412,13 @@ pub(crate) fn list_bootloader_entries(storage: &Storage) -> Result Result { +) -> Result<(ImgConfigManifest, String)> { let mut config = crate::deploy::new_proxy_config(); ostree_ext::container::merge_default_container_proxy_opts(&mut config)?; @@ -427,7 +430,7 @@ pub(crate) async fn get_container_manifest_and_config( .await .with_context(|| format!("Opening image {imgref}"))?; - let (_, manifest) = proxy.fetch_manifest(&img).await?; + let (manifest_digest, manifest) = proxy.fetch_manifest(&img).await?; let (mut reader, driver) = proxy.get_descriptor(&img, manifest.config()).await?; let mut buf = Vec::with_capacity(manifest.config().size() as usize); @@ -437,7 +440,7 @@ pub(crate) async fn get_container_manifest_and_config( let config: oci_spec::image::ImageConfiguration = serde_json::from_slice(&buf)?; - Ok(ImgConfigManifest { manifest, config }) + Ok((ImgConfigManifest { manifest, config }, manifest_digest)) } /// Directory where BLS-compatible bootloaders expect Type 1 boot entries. diff --git a/crates/lib/src/bootc_composefs/switch.rs b/crates/lib/src/bootc_composefs/switch.rs index 0cb0b9ebea..3a580a6285 100644 --- a/crates/lib/src/bootc_composefs/switch.rs +++ b/crates/lib/src/bootc_composefs/switch.rs @@ -6,7 +6,8 @@ use crate::{ status::get_composefs_status, update::{ DoUpgradeOpts, UpdateAction, apply_upgrade_from_downloaded, do_upgrade, - is_image_pulled, validate_update, + ensure_delta_source_present, is_image_pulled, lookup_config_splitstream, + validate_update, }, }, cli::{SwitchOpts, imgref_for_switch}, @@ -25,6 +26,8 @@ pub(crate) async fn switch_composefs( .await .context("Getting composefs deployment status")?; + let delta = crate::delta::open_opt(opts.from_delta.as_deref()).await?; + let prog: ProgressWriter = opts.progress.clone().try_into()?; let mut do_upgrade_opts = DoUpgradeOpts { @@ -34,6 +37,7 @@ pub(crate) async fn switch_composefs( use_unified: false, quiet: opts.quiet, prog, + delta: delta.as_ref(), }; if opts.download_opts.from_downloaded { @@ -41,6 +45,9 @@ pub(crate) async fn switch_composefs( } let target = imgref_for_switch(&opts)?; + if let Some(delta) = delta.as_ref() { + delta.validate_image_reference(&target)?; + } let new_spec = { let mut new_spec = host.spec.clone(); @@ -92,14 +99,29 @@ pub(crate) async fn switch_composefs( booted_unified || target_unified }; - let (image, img_config) = is_image_pulled(repo, &target_imgref).await?; + // With a delta the target is whatever the delta says it is, and we can look + // it up locally; without one we have to ask the registry. + let (image, manifest) = match &delta { + Some(delta) => { + crate::delta::reject_unified_storage(delta, do_upgrade_opts.use_unified)?; + ensure_delta_source_present(repo, delta)?; + ( + lookup_config_splitstream(repo, delta.target_manifest().config().digest())?, + delta.target_manifest().clone(), + ) + } + None => { + let (image, img_config, _) = is_image_pulled(repo, &target_imgref).await?; + (image, img_config.manifest) + } + }; if let Some(cfg_verity) = image { let action = validate_update( storage, booted_cfs, &host, - img_config.manifest.config().digest().as_ref(), + manifest.config().digest().as_ref(), &cfg_verity, true, )?; @@ -117,7 +139,7 @@ pub(crate) async fn switch_composefs( &host, &target_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await; } @@ -130,7 +152,7 @@ pub(crate) async fn switch_composefs( &host, &target_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await?; diff --git a/crates/lib/src/bootc_composefs/update.rs b/crates/lib/src/bootc_composefs/update.rs index c2b43c8b86..9239638c64 100644 --- a/crates/lib/src/bootc_composefs/update.rs +++ b/crates/lib/src/bootc_composefs/update.rs @@ -14,6 +14,7 @@ use ostree_ext::container::ManifestDiff; use crate::bootc_composefs::finalize::get_etc_diff; use crate::bootc_composefs::gc::GCOpts; +use crate::delta::Delta; use crate::spec::BootloaderKind; use crate::{ bootc_composefs::{ @@ -59,23 +60,51 @@ use crate::{ /// * `Some` if the image is pulled/available locally, `None` otherwise /// * The container image manifest /// * The container image configuration +/// * The digest of the manifest, as the registry reported it #[context("Checking if image {} is pulled", imgref.image)] pub(crate) async fn is_image_pulled( repo: &ComposefsRepository, imgref: &ImageReference, -) -> Result<(Option, ImgConfigManifest)> { +) -> Result<(Option, ImgConfigManifest, String)> { let imgref_repr = imgref.to_image_proxy_ref()?; - let img_config_manifest = get_container_manifest_and_config(&imgref_repr).await?; + let (img_config_manifest, manifest_digest) = + get_container_manifest_and_config(&imgref_repr).await?; - let img_digest = img_config_manifest.manifest.config().digest().digest(); + let container_pulled = + lookup_config_splitstream(repo, img_config_manifest.manifest.config().digest())?; + + Ok((container_pulled, img_config_manifest, manifest_digest)) +} + +/// Look up the config splitstream for an image config digest, which is present +/// exactly when that image has been imported into `repo`. +pub(crate) fn lookup_config_splitstream( + repo: &ComposefsRepository, + config_digest: &composefs_oci::OciDigest, +) -> Result> { + let img_digest = config_digest.digest(); // TODO: export config_identifier function from composefs-oci/src/lib.rs and use it here let img_id = format!("oci-config-sha256:{img_digest}"); // NB: add deep checking? - let container_pulled = repo.has_stream(&img_id).context("Checking stream")?; + repo.has_stream(&img_id).context("Checking stream") +} - Ok((container_pulled, img_config_manifest)) +/// A delta only carries the layers that changed; the rest have to come from the +/// image it was built against, which must therefore already be in `repo`. +/// +/// composefs would notice this eventually, but only once it starts importing, which is +/// pretty late for a nice experience. +pub(crate) fn ensure_delta_source_present(repo: &ComposefsRepository, delta: &Delta) -> Result<()> { + let source = delta.source_config_digest(); + anyhow::ensure!( + lookup_config_splitstream(repo, source)?.is_some(), + "Delta {} was built against the image with config {source}, which is not present in \ + this system's composefs repository.", + delta.path, + ); + Ok(()) } fn rm_staged_type1_ent(boot_dir: &Dir) -> Result<()> { @@ -210,7 +239,7 @@ pub(crate) fn validate_update( } /// This is just an intersection of SwitchOpts and UpgradeOpts -pub(crate) struct DoUpgradeOpts { +pub(crate) struct DoUpgradeOpts<'a> { pub(crate) apply: bool, pub(crate) soft_reboot: Option, pub(crate) download_only: bool, @@ -220,13 +249,15 @@ pub(crate) struct DoUpgradeOpts { pub(crate) quiet: bool, /// Structured (JSON-Lines) progress sink; see `--progress-fd`. pub(crate) prog: ProgressWriter, + /// Take the image content from this local delta rather than the network. + pub(crate) delta: Option<&'a Delta>, } async fn apply_upgrade( storage: &Storage, booted_cfs: &BootedComposefs, depl_id: &String, - opts: &DoUpgradeOpts, + opts: &DoUpgradeOpts<'_>, ) -> Result<()> { if let Some(soft_reboot_mode) = opts.soft_reboot { return prepare_soft_reboot_composefs( @@ -253,7 +284,7 @@ pub(crate) async fn do_upgrade( booted_cfs: &BootedComposefs, host: &Host, imgref: &ImageReference, - opts: &DoUpgradeOpts, + opts: &DoUpgradeOpts<'_>, manifest: &ostree_ext::oci_spec::image::ImageManifest, ) -> Result<()> { // Pre-flight disk space check before pulling. @@ -269,6 +300,7 @@ pub(crate) async fn do_upgrade( fs: oci_fs, } = pull_composefs_repo( imgref, + opts.delta, booted_cfs.cmdline.allow_missing_fsverity, opts.use_unified, opts.quiet, @@ -276,6 +308,17 @@ pub(crate) async fn do_upgrade( ) .await?; + // We validated the delta by reading the file ourselves; composefs read it + // independently. Validate they are the same. + if let Some(delta) = opts.delta { + let expected = delta.target_manifest_digest().to_string(); + anyhow::ensure!( + manifest_digest == expected, + "Delta {} imported manifest {manifest_digest}, but it records {expected} as its target", + delta.path, + ); + } + // If the target image produces the same fs-verity digest as any existing // deployment (booted, staged, rollback, or pinned), error out. Two images // from different sources can have identical content; we cannot silently reuse @@ -400,7 +443,7 @@ pub(crate) async fn apply_upgrade_from_downloaded( storage: &Storage, composefs: &BootedComposefs, host: &Host, - do_upgrade_opts: &DoUpgradeOpts, + do_upgrade_opts: &DoUpgradeOpts<'_>, ) -> Result<()> { let staged = host .status @@ -481,6 +524,8 @@ pub(crate) async fn upgrade_composefs( None }; + let delta = crate::delta::open_opt(opts.from_delta.as_deref()).await?; + let prog: ProgressWriter = opts.progress.try_into()?; let mut do_upgrade_opts = DoUpgradeOpts { @@ -490,6 +535,7 @@ pub(crate) async fn upgrade_composefs( use_unified: false, quiet: opts.quiet, prog, + delta: delta.as_ref(), }; if opts.download_opts.from_downloaded { @@ -498,6 +544,9 @@ pub(crate) async fn upgrade_composefs( let imgref = derived_image.as_ref().or(current_image); let mut booted_imgref = imgref.ok_or_else(|| anyhow::anyhow!("No image source specified"))?; + if let Some(delta) = delta.as_ref() { + delta.validate_image_reference(booted_imgref)?; + } // Auto-detect unified storage: use the unified path if the target image is // already in bootc-owned containers-storage, OR if the booted image is — @@ -513,17 +562,37 @@ pub(crate) async fn upgrade_composefs( let repo = &*composefs.repo; - let (img_pulled, mut img_config) = is_image_pulled(&repo, booted_imgref).await?; - let booted_img_digest = img_config.manifest.config().digest().to_string(); + // With a delta the target is whatever the delta says it is, and we can look + // it up locally; without one we have to ask the registry. + let (img_pulled, mut manifest, manifest_digest) = match &delta { + Some(delta) => { + crate::delta::reject_unified_storage(delta, do_upgrade_opts.use_unified)?; + ensure_delta_source_present(repo, delta)?; + ( + lookup_config_splitstream(repo, delta.target_manifest().config().digest())?, + delta.target_manifest().clone(), + delta.target_manifest_digest().to_string(), + ) + } + None => is_image_pulled(&repo, booted_imgref) + .await + .map(|(pulled, img, digest)| (pulled, img.manifest, digest))?, + }; // Check if we already have this update staged // Or if we have another staged deployment with a different image let staged_image = host.status.staged.as_ref().and_then(|i| i.image.as_ref()); + // When applying deltas, only handle the staged image if it matches the delta, + // because with deltas we always want the specific version, and don't want to do + // any network resolve here. + let staged_image = + staged_image.filter(|staged| delta.is_none() || staged.image_digest == manifest_digest); + if let Some(staged_image) = staged_image { // We have a staged image and it has the same digest as the currently booted image's latest // digest - if staged_image.image_digest == booted_img_digest { + if staged_image.image_digest == manifest_digest { if opts.apply { return crate::reboot::reboot(); } @@ -538,15 +607,15 @@ pub(crate) async fn upgrade_composefs( // Switch takes precedence over update, so we change the imgref booted_imgref = &staged_image.image; - let (img_pulled, staged_img_config) = is_image_pulled(&repo, booted_imgref).await?; - img_config = staged_img_config; + let (img_pulled, staged_img_config, _) = is_image_pulled(&repo, booted_imgref).await?; + manifest = staged_img_config.manifest; if let Some(cfg_verity) = img_pulled { let action = validate_update( storage, composefs, &host, - img_config.manifest.config().digest().as_ref(), + manifest.config().digest().as_ref(), &cfg_verity, false, )?; @@ -564,7 +633,7 @@ pub(crate) async fn upgrade_composefs( &host, booted_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await; } @@ -578,7 +647,7 @@ pub(crate) async fn upgrade_composefs( storage, composefs, &host, - &booted_img_digest, + manifest.config().digest().as_ref(), &cfg_verity, false, )?; @@ -596,7 +665,7 @@ pub(crate) async fn upgrade_composefs( &host, booted_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await; } @@ -605,7 +674,7 @@ pub(crate) async fn upgrade_composefs( if opts.check { let (current_manifest, _) = get_imginfo(storage, &*composefs.cmdline.digest)?; - let diff = ManifestDiff::new(¤t_manifest.manifest, &img_config.manifest); + let diff = ManifestDiff::new(¤t_manifest.manifest, &manifest); diff.print(); return Ok(()); } @@ -616,7 +685,7 @@ pub(crate) async fn upgrade_composefs( &host, booted_imgref, &do_upgrade_opts, - &img_config.manifest, + &manifest, ) .await?; diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 97b9a9e778..a85b936395 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -138,6 +138,13 @@ pub(crate) struct UpgradeOpts { #[clap(long)] pub(crate) tag: Option, + /// Upgrade from a local oci-delta artifact instead of the network. + /// + /// The delta must have been created against an image that is already + /// present on this system. + #[clap(long, value_name = "PATH", conflicts_with_all = ["check", "from_downloaded", "tag"])] + pub(crate) from_delta: Option, + #[clap(flatten)] pub(crate) progress: ProgressOptions, } @@ -199,6 +206,19 @@ pub(crate) struct SwitchOpts { #[clap(long = "experimental-unified-storage", hide = true)] pub(crate) unified_storage_exp: bool, + /// Switch using a local oci-delta artifact instead of the network. + /// + /// The delta must have been created against an image that is already + /// present on this system. + #[clap(long, value_name = "PATH", conflicts_with_all = [ + "from_downloaded", + "transport", + "mutate_in_place", + "unified_storage_exp", + "enforce_container_sigpolicy", + ])] + pub(crate) from_delta: Option, + /// Target image to use for the next boot. /// Required unless `--from-downloaded` is present. #[clap( @@ -1264,6 +1284,10 @@ async fn upgrade( storage: &Storage, booted_ostree: &BootedOstree<'_>, ) -> Result<()> { + let delta = crate::delta::open_opt(opts.from_delta.as_deref()) + .await? + .map(std::sync::Arc::new); + let repo = &booted_ostree.repo(); let host = crate::status::get_status(booted_ostree)?.1; @@ -1327,6 +1351,9 @@ async fn upgrade( // needs this for update_mtime() and the non-check path needs it for // unified pull detection. let use_unified = crate::deploy::image_exists_in_unified_storage(storage, imgref).await?; + if let Some(delta) = delta.as_deref() { + crate::delta::reject_unified_storage(delta, use_unified)?; + } if opts.check { let ostree_imgref = imgref.clone().into(); @@ -1353,7 +1380,17 @@ async fn upgrade( } } } else { - let fetched = if use_unified { + let fetched = if let Some(delta) = delta.clone() { + crate::deploy::pull_delta( + repo, + imgref, + delta, + opts.quiet, + prog.clone(), + Some(&booted_ostree.deployment), + ) + .await? + } else if use_unified { crate::deploy::pull_unified( repo, imgref, @@ -1490,6 +1527,10 @@ async fn switch_ostree( storage: &Storage, booted_ostree: &BootedOstree<'_>, ) -> Result<()> { + let delta = crate::delta::open_opt(opts.from_delta.as_deref()) + .await? + .map(std::sync::Arc::new); + let (_, host) = crate::status::get_status(booted_ostree)?; if opts.download_opts.from_downloaded { @@ -1553,8 +1594,21 @@ async fn switch_ostree( } else { crate::deploy::image_exists_in_unified_storage(storage, &target).await? }; + if let Some(delta) = delta.as_deref() { + crate::delta::reject_unified_storage(delta, use_unified)?; + } - let fetched = if use_unified { + let fetched = if let Some(delta) = delta.clone() { + crate::deploy::pull_delta( + repo, + &target, + delta, + opts.quiet, + prog.clone(), + Some(&booted_ostree.deployment), + ) + .await? + } else if use_unified { crate::deploy::pull_unified( repo, &target, @@ -2652,6 +2706,41 @@ mod tests { } } + #[test] + fn test_parse_from_delta() { + let o = + Opt::try_parse_from(["bootc", "switch", "--from-delta", "/d", "quay.io/e/x"]).unwrap(); + match o { + Opt::Switch(o) => assert_eq!(o.from_delta.as_deref().unwrap(), "/d"), + o => panic!("Expected switch opts, not {o:?}"), + } + + // A delta carries its own image content; anything that says where else + // to get that content, or that asks for a check we can't perform, must + // be rejected rather than silently ignored. + let conflicting: &[&[&str]] = &[ + &["switch", "--transport=oci-archive", "quay.io/e/x"], + &["switch", "--mutate-in-place", "quay.io/e/x"], + &["switch", "--experimental-unified-storage", "quay.io/e/x"], + &["switch", "--enforce-container-sigpolicy", "quay.io/e/x"], + &["upgrade", "--check"], + &["upgrade", "--tag=other"], + ]; + for args in conflicting { + let (subcommand, rest) = args.split_first().unwrap(); + let mut full = vec!["bootc", subcommand, "--from-delta", "/d"]; + full.extend_from_slice(rest); + let err = Opt::try_parse_from(&full) + .err() + .unwrap_or_else(|| panic!("{args:?}: expected a conflict")); + assert_eq!( + err.kind(), + clap::error::ErrorKind::ArgumentConflict, + "{args:?}: {err}" + ); + } + } + #[test] fn test_parse_install_args() { // Verify we still process the legacy --target-no-signature-verification diff --git a/crates/lib/src/delta.rs b/crates/lib/src/delta.rs new file mode 100644 index 0000000000..9256136dc2 --- /dev/null +++ b/crates/lib/src/delta.rs @@ -0,0 +1,658 @@ +//! Support for oci-delta artifacts (`--from-delta`). +//! +//! An oci-delta is an OCI layout (normally packed as an uncompressed tar) +//! whose single manifest is an artifact rather than an image. It carries the +//! target image manifest and config verbatim, plus binary patches for the +//! layers that changed relative to a source image. Layers that did not change +//! are omitted entirely and are expected to already be present locally. +//! +//! See for the format. +//! +//! # Trust model +//! +//! Everything in a delta is verified against a single value, the target +//! manifest digest: the embedded manifest and config must hash to the digests +//! the delta records, the config must be the one the manifest references, and +//! each reconstructed layer is checked against the corresponding diff_id in +//! that config. +//! +//! What is not currently established is that the digest is the one you meant to +//! deploy: it comes from the file rather than from a registry, and nothing +//! signs it. Future work may use signatures to complete this. + +use std::collections::HashSet; + +use anyhow::{Context, Result, bail, ensure}; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std_ext::cap_std; +use fn_error_context::context; +use ocidir::oci_spec::image::{ + Descriptor, Digest, DigestAlgorithm, ImageConfiguration, ImageManifest, MediaType, +}; +use ocidir::prelude::*; +use ocidir::{OciArchive, OciDir}; +use ostree_ext::containers_image_proxy; + +use oci_delta::BlobStream; +use oci_delta::{ + BlobStreamFuture, DeltaBlobReader, MEDIA_TYPE_DELTA, ParsedDelta, is_delta_artifact, + parse_delta_manifest, +}; + +/// The OCI layout a delta was read from. +/// +/// Kept open for the lifetime of the [`Delta`] so that patch blobs can be read +/// back when the layers are applied, without parsing the layout again. +#[derive(Debug)] +enum Layout { + Dir(OciDir), + Archive(OciArchive), +} + +impl Layout { + fn open(path: &Utf8Path) -> Result { + if path.is_dir() { + let dir = cap_std::fs::Dir::open_ambient_dir(path, cap_std::ambient_authority()) + .context("Opening OCI layout directory")?; + Ok(Self::Dir( + OciDir::open(dir).context("Reading OCI layout directory")?, + )) + } else { + Ok(Self::Archive( + OciArchive::open(path).context("Reading OCI archive")?, + )) + } + } + + fn read_index(&self) -> Result { + match self { + Self::Dir(o) => o.read_index(), + Self::Archive(o) => o.read_index(), + } + .map_err(Into::into) + } + + fn read_blob(&self, desc: &Descriptor) -> Result> { + let blob: Box = match self { + Self::Dir(o) => Box::new(o.read_blob(desc)?), + Self::Archive(o) => Box::new(o.read_blob(desc)?), + }; + Ok(blob) + } +} + +impl DeltaBlobReader for Layout { + fn open_blob(&self, desc: &Descriptor) -> BlobStreamFuture<'_> { + let result = self + .read_blob(desc) + .with_context(|| format!("Reading blob {}", desc.digest())); + Box::pin(std::future::ready(result)) + } +} + +/// A validated oci-delta artifact, ready to be applied. +#[derive(Debug)] +pub(crate) struct Delta { + /// Where this was loaded from; used in diagnostics. + pub(crate) path: Utf8PathBuf, + /// Parsed information about the delta. + pub(crate) parsed: ParsedDelta, + config: ImageConfiguration, + layout: Layout, +} + +impl Delta { + /// Open and validate the delta at `path`, which may be an OCI layout + /// directory or (usually) an uncompressed tar of one. + #[context("Opening delta {path}")] + pub(crate) async fn open(path: &Utf8Path) -> Result { + let layout = Layout::open(path)?; + let parsed = parse(&layout).await?; + + let config = validate(&parsed)?; + Ok(Self { + path: path.to_owned(), + parsed, + config, + layout, + }) + } + + /// Open the patch blob `desc` + pub(crate) fn read_patch(&self, desc: &Descriptor) -> Result> { + self.layout + .read_blob(desc) + .with_context(|| format!("Reading delta patch {}", desc.digest())) + } + + /// The digest of the target image's manifest, as recorded in the delta. + pub(crate) fn target_manifest_digest(&self) -> &Digest { + self.parsed.target_manifest_descriptor.digest() + } + + pub(crate) fn validate_image_reference( + &self, + imgref: &crate::spec::ImageReference, + ) -> Result<()> { + if imgref.transport()? != containers_image_proxy::Transport::Registry { + return Ok(()); + } + let reference: ocidir::oci_spec::distribution::Reference = imgref + .image + .parse() + .with_context(|| format!("Parsing image reference {}", imgref.image))?; + if let Some(digest) = reference.digest() { + let digest: Digest = digest.parse()?; + ensure!( + &digest == self.target_manifest_digest(), + "Image reference {} names digest {digest}, but delta {} targets {}", + imgref.image, + self.path, + self.target_manifest_digest(), + ); + } + Ok(()) + } + + /// The target image's manifest. + pub(crate) fn target_manifest(&self) -> &ImageManifest { + &self.parsed.target_manifest + } + + /// The target image's config. + pub(crate) fn target_config(&self) -> &ImageConfiguration { + &self.config + } + + /// The config digest of the image this delta was built against. + pub(crate) fn source_config_digest(&self) -> &Digest { + &self.parsed.source_config_digest + } + + /// An image reference naming this delta as a local OCI layout, for handing + /// to the image import machinery. + pub(crate) fn pull_ref(&self) -> Result { + // An `oci:` reference is `path[:tag]`, so a colon in the path would be + // taken as a tag separator. + ensure!( + !self.path.as_str().contains(':'), + "Delta path {} contains a colon, which cannot be expressed as an image reference", + self.path, + ); + let transport = if self.path.is_dir() { + "oci" + } else { + "oci-archive" + }; + format!("{transport}:{}", self.path) + .as_str() + .try_into() + .map_err(|e| anyhow::anyhow!("Building image reference for {}: {e}", self.path)) + } + + /// A one-line description of what applying this delta would do. + pub(crate) fn describe(&self) -> String { + let total = self.parsed.target_manifest.layers().len(); + let patched = self.parsed.delta_layer_by_to.len(); + format!( + "{}: target manifest {}, {total} layers ({patched} patched, {} reused from source config {})", + self.path, + self.target_manifest_digest(), + total - patched, + self.parsed.source_config_digest, + ) + } +} + +/// Open the delta named by a `--from-delta` argument, if there is one. +pub(crate) async fn open_opt(path: Option<&Utf8Path>) -> Result> { + match path { + Some(path) => Ok(Some(Delta::open(path).await?)), + None => Ok(None), + } +} + +/// Reject `--from-delta` for an image that also has to be in containers-storage. +pub(crate) fn reject_unified_storage(delta: &Delta, use_unified: bool) -> Result<()> { + ensure!( + !use_unified, + "Cannot apply delta {}: deltas with unified storage not supported.", + delta.path, + ); + Ok(()) +} + +/// Check the delta's internal consistency, to fail early +fn validate(p: &ParsedDelta) -> Result { + verify_digest( + "Embedded target manifest", + &p.target_manifest_raw, + p.target_manifest_descriptor.digest(), + )?; + verify_digest( + "Embedded target config", + &p.target_config_raw, + p.target_config_descriptor.digest(), + )?; + + ensure!( + p.target_manifest.config().digest() == p.target_config_descriptor.digest(), + "Delta target manifest references config {}, but the embedded config is {}", + p.target_manifest.config().digest(), + p.target_config_descriptor.digest(), + ); + + let config = ImageConfiguration::from_reader(&p.target_config_raw[..]) + .context("Parsing embedded target config")?; + let layers = p.target_manifest.layers(); + ensure!(!layers.is_empty(), "Delta target image has no layers"); + ensure!( + config.rootfs().diff_ids().len() == layers.len(), + "Delta target image has {} diff_ids but {} layers", + config.rootfs().diff_ids().len(), + layers.len(), + ); + + let target_layers: HashSet<&Digest> = layers.iter().map(|l| l.digest()).collect(); + for to in p.delta_layer_by_to.keys() { + ensure!( + target_layers.contains(to), + "Delta contains a patch for layer {to}, which is not part of the target image", + ); + } + + Ok(config) +} + +/// Read the single manifest out of an OCI layout and parse it as a delta. +async fn parse(oci: &Layout) -> Result { + let index = oci.read_index().context("Reading index")?; + let [desc] = index.manifests().as_slice() else { + bail!( + "Expected an OCI layout with a single manifest, found {}; this is not a delta", + index.manifests().len() + ); + }; + ensure!( + desc.media_type() == &MediaType::ImageManifest, + "Expected an image manifest, found {}; this is not a delta", + desc.media_type(), + ); + + let mut raw = Vec::new(); + std::io::Read::read_to_end( + &mut oci.read_blob(desc).context("Reading delta manifest")?, + &mut raw, + ) + .context("Reading delta manifest")?; + verify_digest("Delta manifest", &raw, desc.digest())?; + let manifest: ImageManifest = serde_json::from_slice(&raw).context("Parsing delta manifest")?; + + ensure!( + is_delta_artifact(&manifest), + "Not a delta: expected artifactType {MEDIA_TYPE_DELTA}, found {}.", + manifest + .artifact_type() + .as_ref() + .map(|t| t.to_string()) + .unwrap_or_else(|| "none".into()), + ); + + parse_delta_manifest(&manifest, oci).await +} + +/// Verify that `data` hashes to `expected`. +fn verify_digest(what: &str, data: &[u8], expected: &Digest) -> Result<()> { + let algorithm = match expected.algorithm() { + DigestAlgorithm::Sha256 => openssl::hash::MessageDigest::sha256(), + DigestAlgorithm::Sha384 => openssl::hash::MessageDigest::sha384(), + DigestAlgorithm::Sha512 => openssl::hash::MessageDigest::sha512(), + other => bail!("{what} uses unsupported digest algorithm {other}"), + }; + let found = hex::encode(openssl::hash::hash(algorithm, data)?); + ensure!( + found == expected.digest(), + "{what} does not match its digest: expected {expected}, got {found}", + ); + Ok(()) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use ocidir::oci_spec::image::{ImageConfigurationBuilder, ImageManifestBuilder, RootFsBuilder}; + use std::collections::HashMap; + + pub(crate) const DELTA_CONTENT: &str = "io.github.containers.delta.content"; + pub(crate) const DELTA_TO: &str = "io.github.containers.delta.to"; + pub(crate) const DELTA_SOURCE_CONFIG: &str = "io.github.containers.delta.source-config"; + const TAR_DIFF: &str = "application/vnd.tar-diff"; + const ZERO_DIGEST: &str = + "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + + pub(crate) fn blob(oci: &OciDir, data: &[u8], media_type: MediaType) -> Descriptor { + let mut w = oci.create_blob().unwrap(); + std::io::Write::write_all(&mut w, data).unwrap(); + w.complete() + .unwrap() + .descriptor() + .media_type(media_type) + .build() + .unwrap() + } + + fn image_config(diff_ids: &[&str]) -> ImageConfiguration { + ImageConfigurationBuilder::default() + .rootfs( + RootFsBuilder::default() + .typ("layers") + .diff_ids(diff_ids.iter().map(|s| s.to_string()).collect::>()) + .build() + .unwrap(), + ) + .build() + .unwrap() + } + + pub(crate) fn annotate(mut desc: Descriptor, annotations: &[(&str, &str)]) -> Descriptor { + desc.set_annotations(Some( + annotations + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + )); + desc + } + + /// A minimal but structurally valid target image, of which the last layer + /// is the one the delta patches. + pub(crate) struct TestDelta { + oci: OciDir, + tmp: tempfile::TempDir, + manifest_desc: Descriptor, + config_desc: Descriptor, + layer_descs: Vec, + } + + impl TestDelta { + pub(crate) fn new() -> Self { + Self::with_diff_ids(2) + } + + /// Two layers, but `n_diff_ids` diff_ids, to check invalid combinations + fn with_diff_ids(n_diff_ids: usize) -> Self { + let tmp = tempfile::tempdir().unwrap(); + let dir = cap_std::fs::Dir::open_ambient_dir(tmp.path(), cap_std::ambient_authority()) + .unwrap(); + let oci = OciDir::ensure(dir).unwrap(); + + let layer_descs: Vec = ["layer-one", "layer-two"] + .iter() + .map(|d| blob(&oci, d.as_bytes(), MediaType::ImageLayerGzip)) + .collect(); + let diff_ids = (0..n_diff_ids) + .map(|i| format!("sha256:{}", i.to_string().repeat(64))) + .collect::>(); + let config = image_config(&diff_ids.iter().map(|s| s.as_str()).collect::>()); + let config_desc = blob( + &oci, + serde_json::to_vec(&config).unwrap().as_slice(), + MediaType::ImageConfig, + ); + + let manifest = ImageManifestBuilder::default() + .schema_version(2u32) + .media_type(MediaType::ImageManifest) + .config(config_desc.clone()) + .layers(layer_descs.clone()) + .build() + .unwrap(); + let manifest_desc = blob( + &oci, + serde_json::to_vec(&manifest).unwrap().as_slice(), + MediaType::ImageManifest, + ); + + Self { + oci, + tmp, + manifest_desc, + config_desc, + layer_descs, + } + } + + pub(crate) fn path(&self) -> &Utf8Path { + Utf8Path::from_path(self.tmp.path()).unwrap() + } + + /// Write out the delta manifest with the given layers and artifactType. + fn finish(&self, artifact_type: Option<&str>, layers: Vec) { + let empty = blob(&self.oci, b"{}", MediaType::EmptyJSON); + let mut b = ImageManifestBuilder::default() + .schema_version(2u32) + .media_type(MediaType::ImageManifest) + .config(empty) + .layers(layers) + .annotations(HashMap::from([( + DELTA_SOURCE_CONFIG.to_string(), + ZERO_DIGEST.to_string(), + )])); + if let Some(t) = artifact_type { + b = b.artifact_type(MediaType::Other(t.to_string())); + } + self.oci + .replace_with_single_manifest(b.build().unwrap(), Default::default()) + .unwrap(); + } + + /// Write out the standard delta: the embedded manifest, the embedded + /// config, and a patch for the last image layer. + pub(crate) fn finish_default(&self) { + self.finish(Some(MEDIA_TYPE_DELTA), self.default_layers()); + } + + fn patch_for(&self, to: &str) -> Descriptor { + let patch = blob( + &self.oci, + b"patch-data", + MediaType::Other(TAR_DIFF.to_string()), + ); + annotate(patch, &[(DELTA_CONTENT, "image-layer"), (DELTA_TO, to)]) + } + + fn default_layers(&self) -> Vec { + vec![ + annotate( + self.manifest_desc.clone(), + &[(DELTA_CONTENT, "image-manifest")], + ), + annotate(self.config_desc.clone(), &[(DELTA_CONTENT, "image-config")]), + self.patch_for(self.layer_descs.last().unwrap().digest().as_ref()), + ] + } + + /// Rewrite a blob in place, keeping its size so that ocidir's own size + /// check isn't what catches the alteration. + fn corrupt_blob(&self, desc: &Descriptor) { + let name = desc.digest().digest(); + let mut content = Vec::new(); + std::io::Read::read_to_end(&mut self.oci.blobs_dir().open(name).unwrap(), &mut content) + .unwrap(); + // Flip the last hex character of the first digest, which keeps the + // bytes both the same length and valid JSON. + let pos = content + .windows(7) + .position(|w| w == b"sha256:") + .expect("no digest in blob") + + 7 + + 63; + content[pos] = if content[pos] == b'0' { b'1' } else { b'0' }; + self.oci.blobs_dir().write(name, &content).unwrap(); + } + } + + #[tokio::test] + async fn test_valid_delta() { + let t = TestDelta::new(); + t.finish_default(); + + let delta = Delta::open(t.path()).await.unwrap(); + assert_eq!(delta.target_manifest_digest(), t.manifest_desc.digest()); + assert_eq!(delta.parsed.target_manifest.layers().len(), 2); + assert_eq!(delta.parsed.delta_layer_by_to.len(), 1); + assert!(delta.describe().contains("1 patched, 1 reused")); + + let pull_ref = delta.pull_ref().unwrap(); + assert_eq!( + pull_ref.transport, + ostree_ext::containers_image_proxy::Transport::OciDir + ); + assert_eq!(pull_ref.name, t.path().as_str()); + } + + #[tokio::test] + async fn test_rejects_invalid() { + struct Case { + name: &'static str, + prepare: fn(&TestDelta), + expected: &'static str, + } + let cases = [ + Case { + name: "plain image, no artifactType", + prepare: |t| t.finish(None, t.layer_descs.clone()), + expected: "Not a delta", + }, + Case { + name: "patch for a layer not in the target image", + prepare: |t| { + let mut layers = t.default_layers(); + layers.pop(); + layers.push(t.patch_for(ZERO_DIGEST)); + t.finish(Some(MEDIA_TYPE_DELTA), layers); + }, + expected: "not part of the target image", + }, + Case { + name: "embedded manifest altered after the fact", + prepare: |t| { + t.finish_default(); + t.corrupt_blob(&t.manifest_desc); + }, + expected: "does not match its digest", + }, + Case { + name: "delta manifest altered after the fact", + prepare: |t| { + t.finish_default(); + let index = t.oci.read_index().unwrap(); + t.corrupt_blob(&index.manifests()[0]); + }, + expected: "does not match its digest", + }, + Case { + name: "embedded config is not the one the manifest references", + prepare: |t| { + let other = blob( + &t.oci, + &serde_json::to_vec(&image_config(&["sha256:aa", "sha256:bb"])).unwrap(), + MediaType::ImageConfig, + ); + let mut layers = t.default_layers(); + layers[1] = annotate(other, &[(DELTA_CONTENT, "image-config")]); + t.finish(Some(MEDIA_TYPE_DELTA), layers); + }, + expected: "references config", + }, + ]; + for case in cases { + let t = TestDelta::new(); + (case.prepare)(&t); + let err = Delta::open(t.path()) + .await + .err() + .unwrap_or_else(|| panic!("{}: expected failure", case.name)); + assert!( + format!("{err:#}").contains(case.expected), + "{}: unexpected error: {err:#}", + case.name + ); + } + } + + #[tokio::test] + async fn test_rejects_diff_id_mismatch() { + let t = TestDelta::with_diff_ids(1); + t.finish_default(); + + let err = Delta::open(t.path()).await.unwrap_err(); + assert!( + format!("{err:#}").contains("1 diff_ids but 2 layers"), + "unexpected error: {err:#}" + ); + } + + #[tokio::test] + async fn test_pull_ref() { + let t = TestDelta::new(); + t.finish_default(); + let mut delta = Delta::open(t.path()).await.unwrap(); + + // A path that is not a directory is taken to be an archive. + delta.path = "/var/tmp/update.oci-delta".into(); + let pull_ref = delta.pull_ref().unwrap(); + assert_eq!( + pull_ref.transport, + ostree_ext::containers_image_proxy::Transport::OciArchive + ); + assert_eq!(pull_ref.name, "/var/tmp/update.oci-delta"); + + delta.path = "/var/tmp/a:b.oci-delta".into(); + let err = delta.pull_ref().unwrap_err(); + assert!( + format!("{err:#}").contains("contains a colon"), + "unexpected error: {err:#}" + ); + } + + #[tokio::test] + async fn test_validate_image_reference() { + let t = TestDelta::new(); + t.finish_default(); + let delta = Delta::open(t.path()).await.unwrap(); + let target = delta.target_manifest_digest(); + for (image, valid) in [ + ("quay.io/example/os:latest".to_owned(), true), + (format!("quay.io/example/os@{target}"), true), + (format!("quay.io/example/os:latest@{target}"), true), + (format!("quay.io/example/os@{ZERO_DIGEST}"), false), + (format!("quay.io/example/os:latest@{ZERO_DIGEST}"), false), + ] { + let imgref = crate::spec::ImageReference { + image, + transport: "registry".into(), + signature: None, + }; + let result = delta.validate_image_reference(&imgref); + if valid { + result.unwrap(); + } else { + let error = result.unwrap_err().to_string(); + assert!(error.contains(ZERO_DIGEST), "{error}"); + assert!(error.contains(&target.to_string()), "{error}"); + } + } + } + + #[test] + fn test_verify_digest() { + let expected: Digest = + "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + .parse() + .unwrap(); + verify_digest("test", b"hello", &expected).unwrap(); + let err = verify_digest("test", b"goodbye", &expected).unwrap_err(); + assert!(format!("{err:#}").contains("does not match its digest")); + } +} diff --git a/crates/lib/src/delta_ostree.rs b/crates/lib/src/delta_ostree.rs new file mode 100644 index 0000000000..fcff49f09f --- /dev/null +++ b/crates/lib/src/delta_ostree.rs @@ -0,0 +1,876 @@ +//! Applying an oci-delta with the ostree backend. +//! +//! A tar-diff patch reads the source image by *path*, so the source does not +//! have to be a container layer at all - an ostree commit holding the same +//! filesystem works just as well. [`OstreeDataSource`] serves file content out +//! of a commit, and [`DeltaLayerSource`] plugs that into the container importer +//! in place of the image proxy. +//! +//! The one wrinkle is that ostree does not store an image's root filesystem +//! verbatim: the tar importer moves `/etc` to `/usr/etc`, moves `/var` to +//! `/usr/share/factory/var` on ostree older than v2024.3, and drops anything +//! outside those unless the image was imported with `allow_nonusr`. See +//! [`source_path_candidates`]. +//! +//! An ostree-native (chunked) image additionally has its layers made of repo +//! objects under `sysroot/ostree/`, which are nowhere to be found in the +//! commit's own file tree. That is not a problem here only because oci-delta +//! passes `IgnoreSourcePrefixes=["sysroot/ostree/"]` when building a delta, so +//! no patch ever asks for one. + +use std::io::{BufWriter, Read, Seek, SeekFrom, Write}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cap_std_ext::cap_std::fs::Dir; +use fn_error_context::context; +use futures_util::future::BoxFuture; +use ostree_ext::container as ostree_container; +use ostree_ext::oci_spec::image as oci_image; +use ostree_ext::prelude::*; +use ostree_ext::{gio, ostree}; +use tokio::io::AsyncBufRead; +use tokio::sync::OnceCell; +use tokio_util::io::SyncIoBridge; + +use oci_delta::BlobStream; +use oci_delta::{DeltaDataSource, reconstruct_layer_to}; +use ostree_container::{FetchedLayer, LayerSource}; + +use crate::delta::Delta; + +/// How much reconstructed layer data may sit between the worker and the +/// importer before the worker blocks. +const PIPE_BUFFER: usize = 128 * 1024; + +/// Where in an ostree commit the content for a source image path may be found. +/// +/// The first hit wins, and only the remaps the tar importer performs are tried. +/// For `etc` that is unambiguous - an image cannot have both `/etc/passwd` and +/// a distinct `/usr/etc/passwd` once imported, because the first becomes the +/// second. For `var` it is merely very unlikely: on ostree v2024.3 and newer +/// `/var` is kept as-is, so an image shipping both `/var/lib/x` and +/// `/usr/share/factory/var/lib/x` would have the fallback read the wrong one if +/// the former had been filtered out. A wrong read is caught by the diff_id +/// check on the reconstructed layer. +fn source_path_candidates(path: &str) -> Vec { + let mut candidates = vec![path.to_owned()]; + if let Some(rest) = path.strip_prefix("etc/") { + candidates.push(format!("usr/etc/{rest}")); + } else if let Some(rest) = path.strip_prefix("var/") { + candidates.push(format!("usr/share/factory/var/{rest}")); + } + candidates +} + +trait ReadSeek: Read + Seek {} +impl ReadSeek for T {} + +/// Serves file content from an ostree commit to a tar-diff patch. +struct OstreeDataSource { + root: ostree::RepoFile, + objects: Dir, + current: Option>, +} + +impl OstreeDataSource { + #[context("Opening delta source commit {commit}")] + fn new(repo: ostree::Repo, commit: &str) -> Result { + let (root, _) = repo.read_commit(commit, gio::Cancellable::NONE)?; + let root = root.downcast::().expect("downcast"); + let objects = Dir::reopen_dir(&repo.dfd_borrow())? + .open_dir("objects") + .context("Opening repo objects directory")?; + ensure!( + repo.mode() != ostree::RepoMode::Archive, + "OSTree archive repo mode is not supported" + ); + Ok(Self { + root, + objects, + current: None, + }) + } + + fn open_object(&self, checksum: &str) -> Result> { + let (prefix, rest) = checksum.split_at(2); + let f = self.objects.open(format!("{prefix}/{rest}.file"))?; + return Ok(Box::new(f.into_std())); + } + + fn current(&mut self) -> Result<&mut Box> { + self.current + .as_mut() + .context("No current file set in data source") + } +} + +impl DeltaDataSource for OstreeDataSource { + fn set_current_file(&mut self, path: &str) -> Result<()> { + self.current = None; + let path = path.trim_start_matches("./").trim_start_matches('/'); + let cancellable = gio::Cancellable::NONE; + for candidate in source_path_candidates(path) { + let f = self.root.resolve_relative_path(&candidate); + let f = f.downcast::().expect("downcast"); + if f.query_file_type(gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, cancellable) + != gio::FileType::Regular + { + continue; + } + f.ensure_resolved()?; + self.current = Some( + self.open_object(f.checksum().as_str()) + .with_context(|| format!("Opening delta source file {candidate}"))?, + ); + return Ok(()); + } + bail!("Delta source file not found in source commit: {path}"); + } + + fn read_exact_current(&mut self, buf: &mut [u8]) -> Result<()> { + Ok(self.current()?.read_exact(buf)?) + } + + fn seek_current(&mut self, offset: u64) -> Result { + Ok(self.current()?.seek(SeekFrom::Start(offset))?) + } + + fn read_current_to_end(&mut self, max_size: u64) -> Result> { + let current = self.current()?; + let size = current.seek(SeekFrom::End(0))?; + ensure!( + size <= max_size, + "Source file too large: {size} > {max_size}" + ); + current.seek(SeekFrom::Start(0))?; + let mut data = Vec::with_capacity(size as usize); + current.read_to_end(&mut data)?; + Ok(data) + } + + fn copy_to(&mut self, dst: &mut dyn Write, n: u64) -> Result<()> { + let current = self.current()?; + let copied = std::io::copy(&mut Read::by_ref(current).take(n), dst)?; + if copied != n { + bail!("Short read from delta source: expected {n}, got {copied}"); + } + Ok(()) + } +} + +/// Produces the target image's layers from an oci-delta and a source commit +/// already in the repository, without any network access. +#[derive(Debug)] +pub(crate) struct DeltaLayerSource { + delta: Arc, + /// `ostree::Repo` is `Send` but not `Sync`, and [`LayerSource`] is both. + repo: Mutex, + source_commit: OnceCell, +} + +impl DeltaLayerSource { + pub(crate) fn new(delta: Arc, repo: &ostree::Repo) -> Self { + Self { + delta, + repo: Mutex::new(repo.clone()), + source_commit: OnceCell::new(), + } + } + + fn repo(&self) -> ostree::Repo { + self.repo.lock().unwrap().clone() + } + + /// The commit to read source content from. + /// + /// Resolved on first use rather than up front: an import that turns out to + /// need no layers at all needs no source either, and this still runs before + /// anything is written. + async fn source_commit(&self) -> Result<&str> { + self.source_commit + .get_or_try_init(|| async { find_source_commit(&self.repo(), &self.delta) }) + .await + .map(|s| s.as_str()) + } + + /// The patch for `layer`, its media type, and the diff_id the reconstructed + /// content must hash to. + fn open_patch( + &self, + manifest: &oci_image::ImageManifest, + layer: &oci_image::Descriptor, + ) -> Result<(Box, oci_image::MediaType, oci_image::Digest)> { + let Some(patch) = self.delta.parsed.delta_layer_by_to.get(layer.digest()) else { + bail!( + "Delta {} carries no patch for layer {}, and it is not present locally; \ + there is nothing to reconstruct it from", + self.delta.path, + layer.digest(), + ); + }; + let index = manifest + .layers() + .iter() + .position(|l| l == layer) + .ok_or_else(|| anyhow!("Layer {} is not part of the target image", layer.digest()))?; + let diff_id = self + .delta + .target_config() + .rootfs() + .diff_ids() + .get(index) + .ok_or_else(|| anyhow!("Target image has no diff_id for layer {index}"))?; + let diff_id = oci_image::Digest::from_str(diff_id) + .with_context(|| format!("Parsing diff_id for layer {index}"))?; + Ok(( + self.delta.read_patch(patch)?, + patch.media_type().clone(), + diff_id, + )) + } +} + +impl LayerSource for DeltaLayerSource { + fn fetch_layer<'a>( + &'a self, + manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + // The reconstructed bytes are uncompressed, so counting them against + // the descriptor's compressed size would overshoot; the importer still + // reports per-layer start and completion. + _progress: Option< + &'a tokio::sync::watch::Sender>, + >, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let (blob, media_type, diff_id) = self.open_patch(manifest, layer)?; + let source_commit = self.source_commit().await?.to_owned(); + let repo = self.repo(); + let (writer, reader) = tokio::io::duplex(PIPE_BUFFER); + + let worker = tokio::task::spawn_blocking(move || -> Result<()> { + let mut source = OstreeDataSource::new(repo, &source_commit)?; + let mut dst = BufWriter::with_capacity(PIPE_BUFFER, SyncIoBridge::new(writer)); + reconstruct_layer_to(blob, &media_type, &mut source, &diff_id, &mut dst)?; + dst.into_inner() + .map_err(|e| anyhow!("Flushing reconstructed layer: {e}"))? + .shutdown()?; + Ok(()) + }); + let driver = async move { worker.await.context("Delta worker")? }; + + Ok(( + Box::new(tokio::io::BufReader::new(reader)) as Box, + Box::pin(driver) as BoxFuture<'a, _>, + oci_image::MediaType::ImageLayer, + )) + }) + } + + fn finish(self: Box) -> BoxFuture<'static, Result<()>> { + Box::pin(std::future::ready(Ok(()))) + } +} + +/// Find the ostree commit holding the image this delta was built against. +/// +/// Deltas are applied offline, so an absent source is fatal rather than a +/// reason to go to the registry. +#[context("Finding delta source image")] +pub(crate) fn find_source_commit(repo: &ostree::Repo, delta: &Delta) -> Result { + let wanted = delta.source_config_digest(); + for image in ostree_container::store::list_images(repo)? { + let imgref = ostree_container::ImageReference::try_from(image.as_str()) + .with_context(|| format!("Parsing stored image reference {image}"))?; + let Some(state) = ostree_container::store::query_image(repo, &imgref)? else { + continue; + }; + if state.manifest.config().digest() == wanted { + tracing::debug!("Delta source {wanted} is {imgref} ({})", state.merge_commit); + return Ok(state.merge_commit); + } + } + for commit in + ostree_container::store::list_container_deployment_commits(repo, gio::Cancellable::NONE)? + { + let state = ostree_container::store::query_image_commit(repo, &commit)?; + if state.manifest.config().digest() == wanted { + return Ok(commit); + } + } + bail!( + "Delta {} was built against the image with config {wanted}, which is not present in this \ + system's ostree repository. A delta can only be applied on top of its source image.", + delta.path, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::ensure; + use camino::{Utf8Path, Utf8PathBuf}; + use cap_std_ext::cap_std::ambient_authority; + use oci_delta::MEDIA_TYPE_DELTA; + use ocidir::OciDir; + use ocidir::prelude::*; + use ostree_ext::fixture::{FileDef, Fixture}; + use std::collections::HashSet; + use std::os::fd::{AsFd, AsRawFd}; + use std::process::{Command, Stdio}; + + #[test] + fn test_source_path_candidates() { + assert_eq!(source_path_candidates("usr/bin/ls"), ["usr/bin/ls"]); + assert_eq!( + source_path_candidates("etc/passwd"), + ["etc/passwd", "usr/etc/passwd"] + ); + assert_eq!( + source_path_candidates("var/lib/foo"), + ["var/lib/foo", "usr/share/factory/var/lib/foo"] + ); + // Only the first component is remapped. + assert_eq!( + source_path_candidates("usr/share/etc/x"), + ["usr/share/etc/x"] + ); + } + + /// Commit `dir` as `testref` and return an [`OstreeDataSource`] for it. + fn data_source_for(repo: &ostree::Repo, dir: &Dir) -> Result { + let cancellable = gio::Cancellable::NONE; + let txn = repo.auto_transaction(cancellable)?; + let mt = ostree::MutableTree::new(); + let modifier = + ostree::RepoCommitModifier::new(ostree::RepoCommitModifierFlags::SKIP_XATTRS, None); + repo.write_dfd_to_mtree( + dir.as_fd().as_raw_fd(), + ".", + &mt, + Some(&modifier), + cancellable, + )?; + let root = repo.write_mtree(&mt, cancellable)?; + let root = root.downcast::().unwrap(); + let commit = repo.write_commit(None, None, None, None, &root, cancellable)?; + txn.commit(cancellable)?; + + OstreeDataSource::new(repo.clone(), commit.as_str()) + } + + fn read_all(source: &mut OstreeDataSource, path: &str) -> Result> { + source.set_current_file(path)?; + let mut out = Vec::new(); + source.current()?.read_to_end(&mut out)?; + Ok(out) + } + + #[test] + fn test_ostree_data_source() -> Result<()> { + let td = cap_std_ext::cap_tempfile::TempDir::new(ambient_authority())?; + td.create_dir("repo")?; + let repo = ostree::Repo::create_at( + td.as_fd().as_raw_fd(), + "repo", + ostree::RepoMode::BareUser, + None, + gio::Cancellable::NONE, + )?; + + td.create_dir_all("rootfs/usr/bin")?; + td.create_dir_all("rootfs/usr/etc")?; + td.create_dir_all("rootfs/usr/share/factory/var/lib")?; + td.write("rootfs/usr/bin/ls", b"binary")?; + td.write("rootfs/usr/etc/passwd", b"root:x:0:0")?; + td.write("rootfs/usr/share/factory/var/lib/state", b"stateful")?; + let rootfs = td.open_dir("rootfs")?; + + let mut source = data_source_for(&repo, &rootfs)?; + + assert_eq!(read_all(&mut source, "usr/bin/ls")?, b"binary"); + // The remapped locations are found under their pre-import paths. + assert_eq!(read_all(&mut source, "etc/passwd")?, b"root:x:0:0"); + assert_eq!(read_all(&mut source, "var/lib/state")?, b"stateful"); + // As are leading-`./` forms, which is how tar names entries. + assert_eq!(read_all(&mut source, "./etc/passwd")?, b"root:x:0:0"); + + // Partial reads from an offset, which is what a tar-diff mostly does. + source.set_current_file("usr/etc/passwd")?; + source.seek_current(5)?; + let mut buf = [0u8; 3]; + source.read_exact_current(&mut buf)?; + assert_eq!(&buf, b"x:0"); + + // A directory is not content, and neither is an absent path. + for missing in ["usr/bin", "usr/bin/nope", "etc/nope"] { + let err = source.set_current_file(missing).unwrap_err(); + assert!( + format!("{err:#}").contains("not found in source commit"), + "{missing}: unexpected error: {err:#}" + ); + } + + // A short read is an error rather than a silent truncation. + source.set_current_file("usr/bin/ls")?; + let err = source.copy_to(&mut Vec::new(), 100).unwrap_err(); + assert!(format!("{err:#}").contains("Short read"), "{err:#}"); + + Ok(()) + } + + /// A delta whose source image is not in the repository must fail, not fall + /// back to fetching it: there may well be no network at this point. + #[tokio::test] + async fn test_find_source_commit_absent() -> Result<()> { + let t = crate::delta::tests::TestDelta::new(); + t.finish_default(); + let delta = crate::delta::Delta::open(t.path()).await?; + + let td = cap_std_ext::cap_tempfile::TempDir::new(ambient_authority())?; + td.create_dir("repo")?; + let repo = ostree::Repo::create_at( + td.as_fd().as_raw_fd(), + "repo", + ostree::RepoMode::BareUser, + None, + gio::Cancellable::NONE, + )?; + + let err = find_source_commit(&repo, &delta).unwrap_err(); + assert!( + format!("{err:#}").contains("not present in this system's ostree repository"), + "{err:#}" + ); + Ok(()) + } + + fn have_tool(name: &str) -> bool { + Command::new(name) + .arg("--help") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok() + } + + fn copy_dir(src: &Utf8Path, dst: &Utf8Path) -> Result<()> { + let st = Command::new("cp") + .args(["-a", src.as_str(), dst.as_str()]) + .status()?; + ensure!(st.success(), "cp -a {src} {dst} failed: {st}"); + Ok(()) + } + + fn open_oci(path: &Utf8Path) -> Result { + Ok(OciDir::open(Dir::open_ambient_dir( + path, + ambient_authority(), + )?)?) + } + + fn single_manifest(oci: &OciDir) -> Result<(oci_image::Descriptor, oci_image::ImageManifest)> { + let index = oci.read_index()?; + let desc = index + .manifests() + .first() + .cloned() + .context("Layout has no manifest")?; + let manifest = oci.read_json_blob(&desc)?; + Ok((desc, manifest)) + } + + fn read_blob(oci: &OciDir, desc: &oci_image::Descriptor) -> Result> { + let mut buf = Vec::new(); + oci.read_blob(desc)?.read_to_end(&mut buf)?; + Ok(buf) + } + + /// Import `path` as the image a delta will be applied on top of. + async fn import_source(fixture: &Fixture, path: &Utf8Path) -> Result<()> { + fixture + .must_import(&ostree_container::ImageReference { + transport: ostree_container::Transport::OciDir, + name: path.to_string(), + }) + .await?; + Ok(()) + } + + /// Apply the delta at `delta_path` as an update to `target_path`. + async fn apply_delta( + fixture: &Fixture, + target_path: &Utf8Path, + delta_path: &Utf8Path, + ) -> Result<(Arc, Box)> { + let delta = Arc::new(crate::delta::Delta::open(delta_path).await?); + let imgref = crate::spec::ImageReference { + image: target_path.to_string(), + transport: "oci".into(), + signature: None, + }; + let state = crate::deploy::pull_delta( + fixture.destrepo(), + &imgref, + Arc::clone(&delta), + true, + Default::default(), + None, + ) + .await?; + Ok((delta, state)) + } + + /// The delta reconstructed the target image exactly: the same content as + /// the fixture's own commit, recorded under the target's real digest. + fn assert_applied( + fixture: &Fixture, + state: &crate::deploy::ImageState, + target: &oci_image::Digest, + ) { + assert_eq!(&state.manifest_digest, target); + let expected = fixture.srcrepo().require_rev(fixture.testref()).unwrap(); + let layered = + ostree_container::store::query_image_commit(fixture.destrepo(), &state.ostree_commit) + .unwrap(); + ostree_ext::fixture::assert_commits_content_equal( + fixture.destrepo(), + &layered.base_commit, + fixture.srcrepo(), + &expected, + ); + } + + /// Export the fixture as a container, change it, and export it again: a + /// pair of ostree-native (chunked) images, as a bootc base image update + /// looks. The source is imported into the destination repo. + async fn chunked_source_and_target() + -> Result<(Fixture, Utf8PathBuf, Utf8PathBuf, oci_image::Digest)> { + let mut fixture = Fixture::new_v1()?; + let (exported, _) = fixture.export_container().await?; + let source = fixture.path.join("source-oci"); + copy_dir(Utf8Path::new(&exported.name), &source)?; + + fixture.update( + // Both of these are paths the fixture has an owning "package" for, + // which its chunked export requires. + FileDef::iter_from( + "r usr/bin/bash the-bash-shell-v2\nr usr/etc/someconfig.conf someconfig-v2\n", + ), + std::iter::empty(), + )?; + let (exported, digest) = fixture.export_container().await?; + let target = Utf8PathBuf::from(exported.name); + + import_source(&fixture, &source).await?; + Ok((fixture, source, target, digest)) + } + + /// 64 KiB of incompressible but deterministic data. `v2` differs from `v1` + /// in eight bytes, so a binary diff of the two is tiny - which is how the + /// tests tell whether the patch really read from the source. + fn big_file(v2: bool) -> Vec { + let mut state = 0x1234_5678u32; + let mut data: Vec = std::iter::repeat_with(|| { + state = state.wrapping_mul(1664525).wrapping_add(1013904223); + (state >> 24) as u8 + }) + .take(64 * 1024) + .collect(); + if v2 { + data[32 * 1024..32 * 1024 + 8].fill(0xff); + } + data + } + + /// Append a derived layer holding [`big_file`] at `/etc/bigconf`, i.e. at a + /// path the ostree importer relocates. + fn derive(oci: &Utf8Path, v2: bool) -> Result<()> { + let content = big_file(v2); + ostree_ext::integrationtest::generate_derived_oci_from_tar( + oci, + move |w| { + let mut tar = tar::Builder::new(w); + let mut dir = tar::Header::new_gnu(); + dir.set_entry_type(tar::EntryType::Directory); + dir.set_mode(0o755); + dir.set_size(0); + tar.append_data(&mut dir, "etc/", std::io::empty())?; + let mut file = tar::Header::new_gnu(); + file.set_mode(0o644); + file.set_size(content.len() as u64); + tar.append_data(&mut file, "etc/bigconf", content.as_slice())?; + tar.finish()?; + Ok(()) + }, + None, + None, + ) + } + + /// A pair of images that share a base and differ only in one derived layer, + /// as a bootc image built from a Containerfile does. The source is imported + /// into the destination repo. + async fn derived_source_and_target() + -> Result<(Fixture, Utf8PathBuf, Utf8PathBuf, oci_image::Digest)> { + let fixture = Fixture::new_v1()?; + let (exported, _) = fixture.export_container().await?; + let source = fixture.path.join("source-oci"); + let target = fixture.path.join("target-oci"); + copy_dir(Utf8Path::new(&exported.name), &source)?; + copy_dir(Utf8Path::new(&exported.name), &target)?; + derive(&source, false)?; + derive(&target, true)?; + let digest = single_manifest(&open_oci(&target)?)?.0.digest().clone(); + + import_source(&fixture, &source).await?; + Ok((fixture, source, target, digest)) + } + + /// Write a delta from `source` to `target` that carries each changed layer + /// whole rather than as a tar-diff. + /// + /// The format permits either, so this covers everything but the patch + /// application itself while needing no external tooling; the tests below + /// that use the real `oci-delta` skip themselves when it is absent. + fn build_whole_layer_delta(source: &Utf8Path, target: &Utf8Path, out: &Utf8Path) -> Result<()> { + use crate::delta::tests::{DELTA_CONTENT, DELTA_SOURCE_CONFIG, DELTA_TO, annotate, blob}; + + let source = open_oci(source)?; + let target = open_oci(target)?; + let (_, source_manifest) = single_manifest(&source)?; + let (target_manifest_desc, target_manifest) = single_manifest(&target)?; + + std::fs::create_dir_all(out)?; + let out = OciDir::ensure(Dir::open_ambient_dir(out, ambient_authority())?)?; + + let mut layers = vec![ + annotate( + blob( + &out, + &read_blob(&target, &target_manifest_desc)?, + oci_image::MediaType::ImageManifest, + ), + &[(DELTA_CONTENT, "image-manifest")], + ), + annotate( + blob( + &out, + &read_blob(&target, target_manifest.config())?, + oci_image::MediaType::ImageConfig, + ), + &[(DELTA_CONTENT, "image-config")], + ), + ]; + let shared: HashSet<_> = source_manifest + .layers() + .iter() + .map(|l| l.digest()) + .collect(); + for layer in target_manifest.layers() { + if shared.contains(layer.digest()) { + continue; + } + let patch = blob( + &out, + &read_blob(&target, layer)?, + layer.media_type().clone(), + ); + let to = layer.digest().to_string(); + layers.push(annotate( + patch, + &[(DELTA_CONTENT, "image-layer"), (DELTA_TO, to.as_str())], + )); + } + ensure!( + layers.len() > 2, + "Source and target images share every layer" + ); + + let empty = blob(&out, b"{}", oci_image::MediaType::EmptyJSON); + let manifest = oci_image::ImageManifestBuilder::default() + .schema_version(2u32) + .media_type(oci_image::MediaType::ImageManifest) + .artifact_type(oci_image::MediaType::Other(MEDIA_TYPE_DELTA.to_string())) + .config(empty) + .layers(layers) + .annotations(std::collections::HashMap::from([( + DELTA_SOURCE_CONFIG.to_string(), + source_manifest.config().digest().to_string(), + )])) + .build()?; + out.replace_with_single_manifest(manifest, Default::default())?; + Ok(()) + } + + fn create_delta(source: &Utf8Path, target: &Utf8Path, out: &Utf8Path) -> Result<()> { + let out = Command::new("oci-delta") + .arg("create") + .arg(format!("oci:{source}")) + .arg(format!("oci:{target}")) + .arg(format!("oci:{out}")) + .output()?; + ensure!( + out.status.success(), + "oci-delta create failed: {}\n{}", + out.status, + String::from_utf8_lossy(&out.stderr), + ); + Ok(()) + } + + /// The whole pipeline - parse and validate the delta, find the source image + /// in the repository, reconstruct the changed layers, import - with the + /// patches degenerate so that no external tooling is needed. + #[tokio::test] + async fn test_apply_delta_whole_layers() -> Result<()> { + let (fixture, source, target, digest) = chunked_source_and_target().await?; + let delta_path = fixture.path.join("delta"); + build_whole_layer_delta(&source, &target, &delta_path)?; + + let (_, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_applied(&fixture, &state, &digest); + Ok(()) + } + + /// Construct and apply a real delta between two ostree-native (chunked) images. + #[tokio::test] + async fn test_apply_delta_chunked() -> Result<()> { + if !have_tool("oci-delta") { + eprintln!("skipping: oci-delta not found in PATH"); + return Ok(()); + } + let (fixture, source, target, digest) = chunked_source_and_target().await?; + let delta_path = fixture.path.join("delta"); + create_delta(&source, &target, &delta_path)?; + + let (_, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_applied(&fixture, &state, &digest); + Ok(()) + } + + /// A real delta over a derived layer, which unlike a chunked one is an + /// ordinary root filesystem tar. This is the case that actually drives + /// [`OstreeDataSource`], and the file it patches is one the importer + /// relocated from `/etc` to `/usr/etc`. + #[tokio::test] + async fn test_apply_delta_derived() -> Result<()> { + if !have_tool("oci-delta") { + eprintln!("skipping: oci-delta not found in PATH"); + return Ok(()); + } + let (fixture, source, target, digest) = derived_source_and_target().await?; + let delta_path = fixture.path.join("delta"); + create_delta(&source, &target, &delta_path)?; + + let (delta, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_eq!(&state.manifest_digest, &digest); + + // Only the derived layer changed, and the patch for it is a fraction of + // its size - which it can only be if the reconstruction read the bulk + // of the content back out of the source commit. + let patched = delta.parsed.delta_layer_by_to.iter().collect::>(); + let [(to, patch)] = patched.as_slice() else { + panic!("expected one patched layer, got {}", patched.len()); + }; + let layer = delta + .target_manifest() + .layers() + .iter() + .find(|l| l.digest() == *to) + .unwrap(); + assert!( + patch.size() * 4 < layer.size(), + "patch is {} bytes against a {} byte layer, so nothing was reused from the source", + patch.size(), + layer.size(), + ); + + let root = ostree_ext::fixture::ostree_ls(fixture.destrepo(), &state.ostree_commit)?; + assert!( + root.contains(&format!("r /usr/etc/bigconf {}\n", 64 * 1024)), + "/usr/etc/bigconf is missing from the applied image:\n{root}" + ); + Ok(()) + } + + /// A deployed source remains usable after staging advances to its image ref. + #[tokio::test] + async fn test_delta_source_retained_by_deployment() -> Result<()> { + let (mut fixture, source, target, _) = chunked_source_and_target().await?; + let repo = fixture.destrepo().clone(); + let source_ref = ostree_container::ImageReference { + transport: ostree_container::Transport::OciDir, + name: source.to_string(), + }; + let original = ostree_container::store::query_image(&repo, &source_ref)?.unwrap(); + let newer = fixture + .must_import(&ostree_container::ImageReference { + transport: ostree_container::Transport::OciDir, + name: target.to_string(), + }) + .await?; + // Model staging B over A under the same image reference. + for (name, commit) in repo.list_refs_ext( + Some("ostree/container/image"), + ostree::RepoListRefsExtFlags::empty(), + gio::Cancellable::NONE, + )? { + if commit.as_str() == original.merge_commit { + repo.set_ref_immediate( + None, + &name, + Some(&newer.merge_commit), + gio::Cancellable::NONE, + )?; + } + } + + // Build A→C; no image ref now identifies A. + fixture.update( + FileDef::iter_from("r usr/bin/bash the-bash-shell-v3\n"), + std::iter::empty(), + )?; + let (_, digest) = fixture.export_container().await?; + let delta_path = fixture.path.join("delta-retained-source"); + build_whole_layer_delta(&source, &target, &delta_path)?; + let delta = Delta::open(&delta_path).await?; + assert!(find_source_commit(&repo, &delta).is_err()); + + // Ordinary ostree deployments have no container metadata. + repo.set_ref_immediate( + None, + "ostree/0/0/1", + Some(&original.base_commit), + gio::Cancellable::NONE, + )?; + // Each supported deployment/base-image ref can retain A independently. + for name in [ + "ostree/0/0/0", + "ostree/1/0/0", + "rpmostree/base/test", + "ostree/container/baseimage/test", + ] { + repo.set_ref_immediate( + None, + name, + Some(&original.merge_commit), + gio::Cancellable::NONE, + )?; + assert_eq!(find_source_commit(&repo, &delta)?, original.merge_commit); + repo.set_ref_immediate(None, name, None, gio::Cancellable::NONE)?; + } + repo.set_ref_immediate( + None, + "ostree/0/0/0", + Some(&original.merge_commit), + gio::Cancellable::NONE, + )?; + // Apply A→C using A retained solely by its deployment ref. + let (_, state) = apply_delta(&fixture, &target, &delta_path).await?; + assert_applied(&fixture, &state, &digest); + Ok(()) + } +} diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index b361a3b79f..b5b9ae76c5 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -48,6 +48,7 @@ use std::collections::HashSet; use std::io::{BufRead, Write}; use std::os::fd::AsFd; use std::process::Command; +use std::sync::Arc; use anyhow::{Context, Result, anyhow}; use bootc_utils::skopeo_bin; @@ -136,21 +137,26 @@ impl ImageState { } } -/// Wrapper for pulling a container image, wiring up status output. -pub(crate) async fn new_importer( - repo: &ostree::Repo, - imgref: &ostree_container::OstreeImageReference, +/// The importer settings every bootc pull uses, however the layers arrive. +fn configure_importer( + imp: &mut ostree_container::store::ImageImporter, booted_deployment: Option<&ostree::Deployment>, -) -> Result { - let config = new_proxy_config(); - let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?; +) { imp.require_bootable(); // We do our own GC/prune in deploy::prune(), so skip the importer's internal one. imp.disable_gc(); if let Some(deployment) = booted_deployment { imp.set_sepolicy_commit(deployment.csum().to_string()); } - Ok(imp) +} + +/// Wrapper for pulling a container image, wiring up status output. +pub(crate) async fn new_importer( + repo: &ostree::Repo, + imgref: &ostree_container::OstreeImageReference, + booted_deployment: Option<&ostree::Deployment>, +) -> Result { + new_importer_with_config(repo, imgref, new_proxy_config(), booted_deployment).await } /// Wrapper for pulling a container image with a custom proxy config (e.g. for unified storage). @@ -161,12 +167,7 @@ pub(crate) async fn new_importer_with_config( booted_deployment: Option<&ostree::Deployment>, ) -> Result { let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?; - imp.require_bootable(); - // We do our own GC/prune in deploy::prune(), so skip the importer's internal one. - imp.disable_gc(); - if let Some(deployment) = booted_deployment { - imp.set_sepolicy_commit(deployment.csum().to_string()); - } + configure_importer(&mut imp, booted_deployment); Ok(imp) } @@ -525,6 +526,17 @@ pub(crate) async fn prepare_for_pull( } PrepareResult::Ready(p) => p, }; + Ok(PreparedPullResult::Ready(Box::new( + prepared_import_meta(imp, prep).await?, + ))) +} + +/// Report on what a prepared import is going to do, and tally it up for the +/// progress display. +async fn prepared_import_meta( + imp: ImageImporter, + prep: Box, +) -> Result { check_bootc_label(&prep.config); if let Some(warning) = prep.deprecated_warning() { ostree_ext::cli::print_deprecated_warning(warning).await; @@ -532,7 +544,7 @@ pub(crate) async fn prepare_for_pull( ostree_ext::cli::print_layer_status(&prep); let layers_to_fetch = prep.layers_to_fetch().collect::>>()?; - let prepared_image = PreparedImportMeta { + Ok(PreparedImportMeta { imp, n_layers_to_fetch: layers_to_fetch.len(), layers_total: prep.all_layers().count(), @@ -540,9 +552,80 @@ pub(crate) async fn prepare_for_pull( bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(), digest: prep.manifest_digest.clone(), prep, + }) +} + +/// Prepare an import whose metadata and layer content both come from a delta +/// file plus the source image already in `repo`; nothing is fetched. +pub(crate) async fn prepare_for_pull_delta( + repo: &ostree::Repo, + imgref: &ImageReference, + delta: Arc, + booted_deployment: Option<&ostree::Deployment>, +) -> Result { + delta.validate_image_reference(imgref)?; + let imgref_canonicalized = imgref.clone().canonicalize()?; + tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}"); + let ostree_imgref = &OstreeImageReference::from(imgref_canonicalized); + + let mut imp = ImageImporter::new_without_proxy(repo, ostree_imgref)?; + configure_importer(&mut imp, booted_deployment); + + let layer_source = Box::new(crate::delta_ostree::DeltaLayerSource::new( + Arc::clone(&delta), + repo, + )); + let prep = match imp + .prepare_from_manifest( + delta.target_manifest_digest().clone(), + delta.target_manifest().clone(), + delta.target_config().clone(), + layer_source, + ) + .await? + { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {imgref:#} => {}", c.manifest_digest); + return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into()))); + } + PrepareResult::Ready(p) => p, }; - Ok(PreparedPullResult::Ready(Box::new(prepared_image))) + Ok(PreparedPullResult::Ready(Box::new( + prepared_import_meta(imp, prep).await?, + ))) +} + +/// Wrapper for applying a delta, wiring up status output. +pub(crate) async fn pull_delta( + repo: &ostree::Repo, + imgref: &ImageReference, + delta: Arc, + quiet: bool, + prog: ProgressWriter, + booted_deployment: Option<&ostree::Deployment>, +) -> Result> { + if !quiet { + println!("Applying delta {}", delta.describe()); + } + match prepare_for_pull_delta(repo, imgref, delta, booted_deployment).await? { + PreparedPullResult::AlreadyPresent(existing) => Ok(existing), + PreparedPullResult::Ready(prepared_image_meta) => { + check_disk_space_ostree(repo, &prepared_image_meta, imgref)?; + let r = pull_from_prepared(imgref, quiet, prog, *prepared_image_meta).await; + if r.is_err() { + // The importer publishes a layer's ref before the reconstruction + // that feeds it has finished being verified, so a failed apply + // can leave a ref to content that never passed its diff_id + // check. Drop the unreferenced layers so a retry cannot pick one + // up; failing to do so is not worth masking the real error. + if let Err(e) = ostree_container::store::gc_image_layers(repo) { + tracing::warn!("Pruning layers after failed delta apply: {e:#}"); + } + } + r + } + } } /// Check whether the image exists in bootc's unified container storage. diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index c82d857f7e..3fe8cc2cf4 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -2025,7 +2025,7 @@ async fn install_to_filesystem_impl( // Pre-flight disk space check for native composefs install path. { let imgref = &state.source.imageref; - let img_manifest_config = get_container_manifest_and_config(&imgref).await?; + let (img_manifest_config, _) = get_container_manifest_and_config(&imgref).await?; crate::store::ensure_composefs_dir(&rootfs.physical_root)?; // Use init_path since the repo may not exist yet during install let config = diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index d9eccc0c81..7a77da91ec 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -33,6 +33,7 @@ //! ## Container and Image Handling //! //! - [`image`] - Image operations and queries +//! - [`delta`] - oci-delta artifacts (`--from-delta`) //! - [`boundimage`] - Logically Bound Images (LBIs) //! - [`podstorage`] - bootc-owned container storage (`/usr/lib/bootc/storage`) //! - [`podman`] - Podman command helpers @@ -71,6 +72,8 @@ pub mod cli; mod composefs_consts; mod container_export; mod containerenv; +pub(crate) mod delta; +pub(crate) mod delta_ostree; pub(crate) mod deploy; mod discoverable_partition_specification; pub(crate) mod fsck; diff --git a/crates/ostree-ext/src/container/layer_source.rs b/crates/ostree-ext/src/container/layer_source.rs new file mode 100644 index 0000000000..d89967a620 --- /dev/null +++ b/crates/ostree-ext/src/container/layer_source.rs @@ -0,0 +1,101 @@ +//! Abstraction over where the bytes of the image layers come from. + +use anyhow::Result; +use containers_image_proxy::oci_spec::image as oci_image; +use containers_image_proxy::{ConvertedLayerInfo, ImageProxy, OpenedImage, Transport}; +use futures_util::future::BoxFuture; +use tokio::io::AsyncBufRead; +use tokio::sync::{OnceCell, watch::Sender}; + +use super::store::LayerProgress; +use super::unencapsulate::fetch_layer; + +/// A layer opened for reading: its bytes, a driver future which must be polled +/// alongside reading them, and the media type of those bytes. The media type is +/// not necessarily the one in the layer descriptor, as some transports store +/// layers uncompressed regardless of what the manifest says. +pub type FetchedLayer<'a> = ( + Box, + BoxFuture<'a, Result<()>>, + oci_image::MediaType, +); + +/// Abstraction for layer blob data. +/// +/// Typically ProxyLayerSource which pulls from a registry, but for deltas +/// we reconstruct data directly from the delta. +pub trait LayerSource: std::fmt::Debug + Send + Sync { + /// Open one layer of `manifest` for reading. + fn fetch_layer<'a>( + &'a self, + manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + progress: Option<&'a Sender>>, + ) -> BoxFuture<'a, Result>>; + + /// Release the source, at most once and after the last + /// [`Self::fetch_layer`]. + /// + /// This is only reached when the import succeeds; an import that fails + /// part way through drops the source instead. Anything that must be + /// cleaned up either way belongs in a [`Drop`] impl. + fn finish(self: Box) -> BoxFuture<'static, Result<()>>; +} + +/// Implementation of LayerSource via containers-image-proxy. +#[derive(Debug)] +pub(crate) struct ProxyLayerSource { + proxy: ImageProxy, + img: OpenedImage, + transport: Transport, + layer_info: OnceCell>>, +} + +impl ProxyLayerSource { + pub(crate) fn new(proxy: ImageProxy, img: OpenedImage, transport: Transport) -> Self { + Self { + proxy, + img, + transport, + layer_info: OnceCell::new(), + } + } +} + +impl LayerSource for ProxyLayerSource { + fn fetch_layer<'a>( + &'a self, + manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + progress: Option<&'a Sender>>, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let layer_info = self + .layer_info + .get_or_try_init(|| self.proxy.get_layer_info(&self.img)) + .await?; + let (blob, driver, media_type) = fetch_layer( + &self.proxy, + &self.img, + manifest, + layer, + progress, + layer_info.as_ref(), + self.transport, + ) + .await?; + Ok((blob, Box::pin(driver) as BoxFuture<'a, _>, media_type)) + }) + } + + fn finish(self: Box) -> BoxFuture<'static, Result<()>> { + Box::pin(async move { + let Self { proxy, img, .. } = *self; + // TODO change the imageproxy API to ensure this happens automatically when + // the image reference is dropped + proxy.close_image(&img).await?; + proxy.finalize().await?; + Ok(()) + }) + } +} diff --git a/crates/ostree-ext/src/container/mod.rs b/crates/ostree-ext/src/container/mod.rs index 97d9c2fc0e..3e5109f63c 100644 --- a/crates/ostree-ext/src/container/mod.rs +++ b/crates/ostree-ext/src/container/mod.rs @@ -468,6 +468,8 @@ pub fn version_for_config(config: &oci_spec::image::ImageConfiguration) -> Optio pub mod deploy; mod encapsulate; pub use encapsulate::*; +mod layer_source; +pub use layer_source::*; mod unencapsulate; pub use unencapsulate::*; pub mod skopeo; diff --git a/crates/ostree-ext/src/container/store.rs b/crates/ostree-ext/src/container/store.rs index 449d3db2ec..9316d85470 100644 --- a/crates/ostree-ext/src/container/store.rs +++ b/crates/ostree-ext/src/container/store.rs @@ -41,6 +41,15 @@ //! - The merge commit overlays all layers, processing whiteouts //! - Image metadata (manifest, config) is stored in commit metadata //! +//! ## Importing without a registry +//! +//! Steps 1 and 2 above assume the manifest and the layer content both come +//! over the registry connection. A caller that already has the manifest and +//! can produce the layer content itself - by reconstructing it from an OCI +//! delta, say - instead uses [`ImageImporter::new_without_proxy`] and +//! [`ImageImporter::prepare_from_manifest`], passing a +//! [`crate::container::LayerSource`] for the layers. Step 3 is unchanged. +//! //! ## Layer Types //! //! The manifest layout is parsed to identify different layer types: @@ -139,7 +148,7 @@ use cap_std_ext::cap_std; use cap_std_ext::cap_std::fs::{Dir, MetadataExt}; use cap_std_ext::dirext::CapStdExtDirExt; -use containers_image_proxy::{ImageProxy, OpenedImage}; +use containers_image_proxy::ImageProxy; use flate2::Compression; use fn_error_context::context; use futures_util::TryFutureExt; @@ -292,6 +301,15 @@ impl CachedImageUpdate { } } +/// What a target manifest amounts to, relative to what we imported before. +enum PreviousState { + /// The target image is already imported; there is nothing to do. + UpToDate(Box), + /// The target differs; carries the previous import to diff against and its + /// image ID, if there was a previous import at all. + Outdated(Option<(Box, String)>), +} + /// A layer in the ostree repo, identified by its ref and commit checksum. struct LayerRef { ostree_ref: String, @@ -304,7 +322,7 @@ pub struct ImageImporter { repo: ostree::Repo, /// The root filesystem directory, used for policy lookups. root: Dir, - pub(crate) proxy: ImageProxy, + proxy: Option, imgref: OstreeImageReference, target_imgref: Option, no_imgref: bool, // If true, do not write final image ref @@ -376,8 +394,8 @@ pub struct PreparedImport { pub layers: Vec, /// OSTree remote signature verification text, if enabled. pub verify_text: Option, - /// Our open image reference - proxy_img: OpenedImage, + /// Where to read the layer content from. + layer_source: Box, } impl PreparedImport { @@ -646,6 +664,22 @@ impl ImageImporter { &format!("Fetching {imgref}"), ); + Self::new_impl(repo, imgref, Some(proxy)) + } + + /// Create an importer with no image proxy, and hence no way to fetch + /// anything itself; the caller supplies the image metadata and the layer + /// content via [`Self::prepare_from_manifest`]. + #[context("Creating importer")] + pub fn new_without_proxy(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + Self::new_impl(repo, imgref, None) + } + + fn new_impl( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + proxy: Option, + ) -> Result { let repo = repo.clone(); let diffid_to_digest = Self::build_diffid_to_digest_map(&repo)?; @@ -739,7 +773,7 @@ impl ImageImporter { /// Serialize the metadata about a pending fetch as detached metadata on the commit object, /// so it can be retrieved later offline #[context("Writing cached pending manifest")] - pub(crate) async fn cache_pending( + async fn cache_pending( &self, commit: &str, manifest_digest: &Digest, @@ -875,17 +909,32 @@ impl ImageImporter { Ok(()) } - /// Given existing metadata (manifest, config, previous image statE) generate a PreparedImport structure + /// Given existing metadata (manifest, config, previous image state) generate a PreparedImport structure /// which e.g. includes a diff of the layers. - fn create_prepared_import( + /// + /// If there is a previous image, this also caches the new manifest and + /// config against it. + async fn create_prepared_import( &mut self, manifest_digest: Digest, manifest: ImageManifest, config: ImageConfiguration, previous_state: Option>, previous_imageid: Option, - proxy_img: OpenedImage, + layer_source: Box, ) -> Result> { + // If there is a currently fetched image, cache the new pending manifest+config + // as detached commit metadata, so that future fetches can query it offline. + if let Some(previous_state) = previous_state.as_ref() { + self.cache_pending( + previous_state.merge_commit.as_str(), + &manifest_digest, + &manifest, + &config, + ) + .await?; + } + let config_labels = super::labels_of(&config); if self.require_bootable { let bootable_key = ostree::METADATA_KEY_BOOTABLE; @@ -931,33 +980,138 @@ impl ImageImporter { ostree_commit_layer: commit_layer, layers: remaining_layers, verify_text: None, - proxy_img, + layer_source, }; Ok(Box::new(imp)) } - /// Determine if there is a new manifest, and if so return its digest. - #[context("Fetching manifest")] - pub(crate) async fn prepare_internal(&mut self, verify_layers: bool) -> Result { + /// Verify that our signature source is usable at all. + fn check_sigverify(&self, verify_layers: bool) -> Result<()> { match &self.imgref.sigverify { SignatureSource::ContainerPolicy if skopeo::container_policy_is_default_insecure(&self.root)? => { - return Err(anyhow!( + Err(anyhow!( "containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage" - )); - } - SignatureSource::OstreeRemote(_) if verify_layers => { - return Err(anyhow!( - "Cannot currently verify layered containers via ostree remote" - )); + )) } - _ => {} + SignatureSource::OstreeRemote(_) if verify_layers => Err(anyhow!( + "Cannot currently verify layered containers via ostree remote" + )), + _ => Ok(()), + } + } + + /// Compare a target manifest against any previous import of the same image + /// reference. + fn diff_previous_state( + previous_state: Option>, + manifest_digest: &Digest, + new_imageid: &Digest, + ) -> PreviousState { + let Some(previous_state) = previous_state else { + return PreviousState::Outdated(None); + }; + // If the manifest digests match, we're done. + if &previous_state.manifest_digest == manifest_digest { + return PreviousState::UpToDate(previous_state); + } + // Failing that, if they have the same imageID, we're also done. + let previous_imageid = previous_state.manifest.config().digest(); + if previous_imageid == new_imageid { + return PreviousState::UpToDate(previous_state); + } + let previous_imageid = previous_imageid.to_string(); + PreviousState::Outdated(Some((previous_state, previous_imageid))) + } + + /// Prepare an import of a manifest the caller already has, reading the + /// layer content from `layer_source` instead of over the network. + /// + /// `manifest_digest` is taken on trust: it is what future update checks for + /// this image reference will compare against, and nothing here can verify + /// it, as re-serializing a parsed manifest does not necessarily reproduce + /// the bytes it was parsed from. The caller must have checked the raw + /// manifest and config bytes against their digests, and must have + /// established that this is the image `self.imgref` names, before calling + /// this. + /// + /// [`SignatureSource::ContainerPolicy`] is rejected: enforcing + /// containers-policy.json is skopeo's job, and there is no registry + /// interaction here for it to enforce anything on. Importing anyway would + /// produce a merge commit indistinguishable from a policy-verified pull. + /// [`SignatureSource::OstreeRemote`] is fine, as that signature is on the + /// ostree commit and is still checked while committing it. + #[context("Preparing import from manifest")] + pub async fn prepare_from_manifest( + &mut self, + manifest_digest: Digest, + manifest: ImageManifest, + config: ImageConfiguration, + layer_source: Box, + ) -> Result { + if matches!(self.imgref.sigverify, SignatureSource::ContainerPolicy) { + anyhow::bail!( + "Cannot verify {} against containers-policy.json without fetching it; refusing usage", + self.imgref, + ); + } + + // We have no proxy to fetch anything with, so make sure we do not leave + // a skopeo process behind if the caller made one anyway. + if let Some(proxy) = self.proxy.take() { + proxy.finalize().await?; + } + + // A digested pull spec is the one part of the caller's claim we can + // check ourselves. + let target_reference = self.imgref.imgref.name.parse::().ok(); + if let Some(target_digest) = target_reference + .as_ref() + .and_then(|v| v.digest()) + .map(Digest::from_str) + .transpose()? + && target_digest != manifest_digest + { + anyhow::bail!( + "Image reference {} names digest {target_digest}, but the manifest is {manifest_digest}", + self.imgref, + ); } // Check if we have an image already pulled let previous_state = try_query_image(&self.repo, &self.imgref.imgref)?; + let (previous_state, previous_imageid) = match Self::diff_previous_state( + previous_state, + &manifest_digest, + manifest.config().digest(), + ) { + PreviousState::UpToDate(state) => return Ok(PrepareResult::AlreadyPresent(state)), + PreviousState::Outdated(previous) => previous.unzip(), + }; + + let imp = self + .create_prepared_import( + manifest_digest, + manifest, + config, + previous_state, + previous_imageid, + layer_source, + ) + .await?; + Ok(PrepareResult::Ready(imp)) + } + + /// Determine if there is a new manifest, and if so return its digest. + #[context("Fetching manifest")] + pub(crate) async fn prepare_internal(&mut self, verify_layers: bool) -> Result { + self.check_sigverify(verify_layers)?; + + // Check if we have an image already pulled + let previous_state = try_query_image(&self.repo, &self.imgref.imgref)?; + // Parse the target reference to see if it's a digested pull let target_reference = self.imgref.imgref.name.parse::().ok(); let previous_state = if let Some(target_digest) = target_reference @@ -984,55 +1138,48 @@ impl ImageImporter { anyhow::bail!("Manifest fetch required in offline mode"); } - let proxy_img = self - .proxy - .open_image(&self.imgref.imgref.to_string()) - .await?; + let proxy = self.proxy.as_ref().ok_or_else(|| { + anyhow!( + "This importer was created without an image proxy; use prepare_from_manifest() instead" + ) + })?; + let proxy_img = proxy.open_image(&self.imgref.imgref.to_string()).await?; - let (manifest_digest, manifest) = self.proxy.fetch_manifest(&proxy_img).await?; + let (manifest_digest, manifest) = proxy.fetch_manifest(&proxy_img).await?; let manifest_digest = Digest::from_str(&manifest_digest)?; - let new_imageid = manifest.config().digest(); // Query for previous stored state - let (previous_state, previous_imageid) = if let Some(previous_state) = previous_state { - // If the manifest digests match, we're done. - if previous_state.manifest_digest == manifest_digest { - return Ok(PrepareResult::AlreadyPresent(previous_state)); - } - // Failing that, if they have the same imageID, we're also done. - let previous_imageid = previous_state.manifest.config().digest(); - if previous_imageid == new_imageid { - return Ok(PrepareResult::AlreadyPresent(previous_state)); - } - let previous_imageid = previous_imageid.to_string(); - (Some(previous_state), Some(previous_imageid)) - } else { - (None, None) + let (previous_state, previous_imageid) = match Self::diff_previous_state( + previous_state, + &manifest_digest, + manifest.config().digest(), + ) { + PreviousState::UpToDate(state) => return Ok(PrepareResult::AlreadyPresent(state)), + PreviousState::Outdated(previous) => previous.unzip(), }; - let config = self.proxy.fetch_config(&proxy_img).await?; + let config = proxy.fetch_config(&proxy_img).await?; - // If there is a currently fetched image, cache the new pending manifest+config - // as detached commit metadata, so that future fetches can query it offline. - if let Some(previous_state) = previous_state.as_ref() { - self.cache_pending( - previous_state.merge_commit.as_str(), - &manifest_digest, - &manifest, - &config, - ) - .await?; - } + // The layer source consumes the proxy, so only take it once the checks + // that can return `AlreadyPresent` are done. + let proxy = self.proxy.take().unwrap(); - let imp = self.create_prepared_import( - manifest_digest, - manifest, - config, - previous_state, - previous_imageid, + let layer_source = Box::new(ProxyLayerSource::new( + proxy, proxy_img, - )?; + self.imgref.imgref.transport, + )); + let imp = self + .create_prepared_import( + manifest_digest, + manifest, + config, + previous_state, + previous_imageid, + layer_source, + ) + .await?; Ok(PrepareResult::Ready(imp)) } @@ -1045,13 +1192,7 @@ impl ImageImporter { write_refs: bool, ) -> Result<()> { tracing::debug!("Fetching base"); - if matches!(self.imgref.sigverify, SignatureSource::ContainerPolicy) - && skopeo::container_policy_is_default_insecure(&self.root)? - { - return Err(anyhow!( - "containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage" - )); - } + self.check_sigverify(false)?; let remote = match &self.imgref.sigverify { SignatureSource::OstreeRemote(remote) => Some(remote.clone()), SignatureSource::ContainerPolicy | SignatureSource::ContainerPolicyAllowInsecure => { @@ -1066,7 +1207,6 @@ impl ImageImporter { } return Ok(()); }; - let des_layers = self.proxy.get_layer_info(&import.proxy_img).await?; for layer in import.ostree_layers.iter_mut() { if let Some(commit) = layer.commit.as_ref() { if write_refs { @@ -1078,16 +1218,14 @@ impl ImageImporter { p.send(ImportProgress::OstreeChunkStarted(layer.layer.clone())) .await?; } - let (blob, driver, media_type) = fetch_layer( - &self.proxy, - &import.proxy_img, - &import.manifest, - &layer.layer, - self.layer_byte_progress.as_ref(), - des_layers.as_ref(), - self.imgref.imgref.transport, - ) - .await?; + let (blob, driver, media_type) = import + .layer_source + .fetch_layer( + &import.manifest, + &layer.layer, + self.layer_byte_progress.as_ref(), + ) + .await?; let repo = self.repo.clone(); let target_ref = layer.ostree_ref.clone(); let import_task = @@ -1129,16 +1267,14 @@ impl ImageImporter { )) .await?; } - let (blob, driver, media_type) = fetch_layer( - &self.proxy, - &import.proxy_img, - &import.manifest, - &commit_layer.layer, - self.layer_byte_progress.as_ref(), - des_layers.as_ref(), - self.imgref.imgref.transport, - ) - .await?; + let (blob, driver, media_type) = import + .layer_source + .fetch_layer( + &import.manifest, + &commit_layer.layer, + self.layer_byte_progress.as_ref(), + ) + .await?; let repo = self.repo.clone(); let target_ref = commit_layer.ostree_ref.clone(); let import_task = @@ -1189,9 +1325,7 @@ impl ImageImporter { } let deprecated_warning = prep.deprecated_warning().map(ToOwned::to_owned); self.unencapsulate_base(&mut prep, true, false).await?; - // TODO change the imageproxy API to ensure this happens automatically when - // the image reference is dropped - self.proxy.close_image(&prep.proxy_img).await?; + prep.layer_source.finish().await?; // SAFETY: We know we have a commit let ostree_commit = prep.ostree_commit_layer.unwrap().commit.unwrap(); let image_digest = prep.manifest_digest; @@ -1455,8 +1589,6 @@ impl ImageImporter { // First download all layers for the base image (if necessary) - we need the SELinux policy // there to label all following layers. self.unencapsulate_base(&mut import, false, true).await?; - let des_layers = self.proxy.get_layer_info(&import.proxy_img).await?; - let proxy = self.proxy; let target_imgref = self.target_imgref.as_ref().unwrap_or(&self.imgref); let base_commit = import .ostree_commit_layer @@ -1493,16 +1625,14 @@ impl ImageImporter { p.send(ImportProgress::DerivedLayerStarted(layer.layer.clone())) .await?; } - let (blob, driver, media_type) = super::unencapsulate::fetch_layer( - &proxy, - &import.proxy_img, - &import.manifest, - &layer.layer, - self.layer_byte_progress.as_ref(), - des_layers.as_ref(), - self.imgref.imgref.transport, - ) - .await?; + let (blob, driver, media_type) = import + .layer_source + .fetch_layer( + &import.manifest, + &layer.layer, + self.layer_byte_progress.as_ref(), + ) + .await?; // SELinux label derived layers using the base policy. For non-ostree // containers (base_commit is None), fall back to the caller-provided // sepolicy_commit (typically the booted deployment's commit). @@ -1553,13 +1683,8 @@ impl ImageImporter { } } - // TODO change the imageproxy API to ensure this happens automatically when - // the image reference is dropped - proxy.close_image(&import.proxy_img).await?; - - // We're done with the proxy, make sure it didn't have any errors. - proxy.finalize().await?; - tracing::debug!("finalized proxy"); + import.layer_source.finish().await?; + tracing::debug!("finished layer source"); // Disconnect progress notifiers to signal we're done with fetching. let _ = self.layer_byte_progress.take(); @@ -2035,13 +2160,13 @@ pub async fn export( .await } -/// Iterate over deployment commits, returning the manifests from +/// Iterate over deployment commits, returning the commit id and commit object for /// commits which point to a container image. -#[context("Listing deployment manifests")] -fn list_container_deployment_manifests( +#[context("Listing deployment commits")] +fn list_container_deployment_commits_full( repo: &ostree::Repo, cancellable: Option<&gio::Cancellable>, -) -> Result> { +) -> Result> { // Gather all refs which start with ostree/0/ or ostree/1/ or rpmostree/base/ // and create a set of the commits which they reference. let commits = OSTREE_BASE_DEPLOYMENT_REFS @@ -2072,13 +2197,39 @@ fn list_container_deployment_manifests( .is_some() { tracing::trace!("Commit {commit} is a container image"); - let manifest = manifest_data_from_commitmeta(commit_meta)?.0; - r.push(manifest); + r.push((commit.to_string(), commit_obj)); } } Ok(r) } +/// List container image commits retained by deployment or protected base-image refs. +pub fn list_container_deployment_commits( + repo: &ostree::Repo, + cancellable: Option<&gio::Cancellable>, +) -> Result> { + list_container_deployment_commits_full(repo, cancellable)? + .into_iter() + .map(|(commit, _commit_obj)| Ok(commit)) + .collect() +} + +/// Iterate over deployment commits, returning the manifests from +/// commits which point to a container image. +#[context("Listing deployment manifests")] +fn list_container_deployment_manifests( + repo: &ostree::Repo, + cancellable: Option<&gio::Cancellable>, +) -> Result> { + list_container_deployment_commits_full(repo, cancellable)? + .into_iter() + .map(|(_commit, commit_obj)| { + let metadata = glib::VariantDict::new(Some(&commit_obj.child_value(0))); + Ok(manifest_data_from_commitmeta(&metadata)?.0) + }) + .collect() +} + /// Garbage collect unused image layer references. /// /// This function assumes no transaction is active on the repository. diff --git a/crates/ostree-ext/src/integrationtest.rs b/crates/ostree-ext/src/integrationtest.rs index 8c05c961e0..28d7234197 100644 --- a/crates/ostree-ext/src/integrationtest.rs +++ b/crates/ostree-ext/src/integrationtest.rs @@ -105,7 +105,7 @@ where config .rootfs_mut() .diff_ids_mut() - .push(new_layer.uncompressed_sha256.digest().to_string()); + .push(new_layer.uncompressed_sha256_as_digest().to_string()); let new_config_desc = src.write_config(config)?; manifest.set_config(new_config_desc); diff --git a/crates/ostree-ext/tests/it/main.rs b/crates/ostree-ext/tests/it/main.rs index e43aab027b..8f973d3c44 100644 --- a/crates/ostree-ext/tests/it/main.rs +++ b/crates/ostree-ext/tests/it/main.rs @@ -5,6 +5,7 @@ use camino::Utf8Path; use cap_std::fs::{Dir, DirBuilder, DirBuilderExt}; use cap_std_ext::cap_std; use containers_image_proxy::oci_spec; +use futures_util::future::BoxFuture; use gvariant::aligned_bytes::TryAsAligned; use gvariant::{Marker, Structure}; use oci_image::ImageManifest; @@ -14,7 +15,8 @@ use ocidir::oci_spec::distribution::Reference; use ocidir::oci_spec::image::{Arch, DigestAlgorithm}; use ostree_ext::chunking::ObjectMetaSized; use ostree_ext::container::{ - Config, ExportOpts, ImageReference, OstreeImageReference, SignatureSource, Transport, + Config, ExportOpts, FetchedLayer, ImageReference, LayerSource, OstreeImageReference, + SignatureSource, Transport, }; use ostree_ext::container::{ManifestDiff, OSTREE_COMMIT_LABEL, store}; use ostree_ext::prelude::{Cast, FileExt}; @@ -763,6 +765,99 @@ async fn test_no_fetch_digested() -> Result<()> { Ok(()) } +/// A [`LayerSource`] which reads blobs straight out of an OCI directory, +/// standing in for anything that produces layer content without a registry. +#[derive(Debug)] +struct OciDirLayerSource(ocidir::OciDir); + +impl LayerSource for OciDirLayerSource { + fn fetch_layer<'a>( + &'a self, + _manifest: &'a oci_image::ImageManifest, + layer: &'a oci_image::Descriptor, + _progress: Option<&'a tokio::sync::watch::Sender>>, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let blob = tokio::fs::File::from_std(self.0.read_blob(layer)?); + let blob: Box = + Box::new(tokio::io::BufReader::new(blob)); + let driver = Box::pin(std::future::ready(Ok(()))); + Ok((blob, driver as BoxFuture<'a, _>, layer.media_type().clone())) + }) + } + + fn finish(self: Box) -> BoxFuture<'static, Result<()>> { + Box::pin(std::future::ready(Ok(()))) + } +} + +/// Import an image from a manifest and a caller-provided layer source, with no +/// image proxy in play at all. +#[tokio::test] +async fn test_prepare_from_manifest() -> Result<()> { + let fixture = Fixture::new_v1()?; + let (src_imgref, expected_digest) = fixture.export_container().await.unwrap(); + + let open_source = || -> Result> { + let dir = Dir::open_ambient_dir( + Utf8Path::new(src_imgref.name.as_str()), + cap_std::ambient_authority(), + )?; + Ok(Box::new(OciDirLayerSource(ocidir::OciDir::open(dir)?))) + }; + + let ocidir = ocidir::OciDir::open(Dir::open_ambient_dir( + Utf8Path::new(src_imgref.name.as_str()), + cap_std::ambient_authority(), + )?)?; + let index = ocidir.read_index()?; + let manifest: oci_image::ImageManifest = + ocidir.read_json_blob(index.manifests().first().unwrap())?; + let config: oci_image::ImageConfiguration = ocidir.read_json_blob(manifest.config())?; + + let imgref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: src_imgref.clone(), + }; + + let mut imp = store::ImageImporter::new_without_proxy(fixture.destrepo(), &imgref)?; + let prep = match imp + .prepare_from_manifest( + expected_digest.clone(), + manifest.clone(), + config.clone(), + open_source()?, + ) + .await? + { + store::PrepareResult::AlreadyPresent(_) => panic!("Image should not be present yet"), + store::PrepareResult::Ready(prep) => prep, + }; + assert_eq!(prep.manifest_digest, expected_digest); + assert_eq!(prep.all_layers().count(), LAYERS_V0_LEN); + let state = imp.import(prep).await?; + assert_eq!(state.manifest_digest, expected_digest); + + // The layers we read out of the OCI directory reassemble into the content + // the fixture started from. + let (commitdata, _) = fixture.destrepo().load_commit(&state.base_commit)?; + assert_eq!( + CONTENTS_CHECKSUM_V0, + ostree::commit_get_content_checksum(&commitdata) + .unwrap() + .as_str() + ); + + // And now it is present, established without ever contacting a registry. + let mut imp = store::ImageImporter::new_without_proxy(fixture.destrepo(), &imgref)?; + let prep = imp + .prepare_from_manifest(expected_digest, manifest, config, open_source()?) + .await?; + assert!(matches!(prep, store::PrepareResult::AlreadyPresent(_))); + + Ok(()) +} + #[tokio::test] async fn test_export_as_container_derived() -> Result<()> { if !check_skopeo() { diff --git a/docs/src/man/bootc-switch.8.md b/docs/src/man/bootc-switch.8.md index 407aa0bc09..fd80baf5be 100644 --- a/docs/src/man/bootc-switch.8.md +++ b/docs/src/man/bootc-switch.8.md @@ -79,6 +79,10 @@ Soft reboot allows faster system restart by avoiding full hardware reboot when p Retain reference to currently booted image +**--from-delta**=*PATH* + + Switch using a local oci-delta artifact instead of the network + # EXAMPLES diff --git a/docs/src/man/bootc-upgrade.8.md b/docs/src/man/bootc-upgrade.8.md index b1b3f3b694..91b97b1f7f 100644 --- a/docs/src/man/bootc-upgrade.8.md +++ b/docs/src/man/bootc-upgrade.8.md @@ -73,6 +73,10 @@ Soft reboot allows faster system restart by avoiding full hardware reboot when p Upgrade to a different tag of the currently booted image +**--from-delta**=*PATH* + + Upgrade from a local oci-delta artifact instead of the network + # EXAMPLES