From f10abd42a7acc2d21f570ae0070d71f6b87e424e Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 23 Sep 2026 10:57:05 +0100 Subject: [PATCH 1/3] Justfile: Factor out retry helper for fetch recipes Move the shell `retry` function out of `build-fetch` into a shared `_retry_fn` variable so other recipes can reuse it. No functional change. Assisted-by: AI --- Justfile | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/Justfile b/Justfile index 86f14dd278..ceaf85b052 100644 --- a/Justfile +++ b/Justfile @@ -108,23 +108,7 @@ build: package _keygen && _pull-lbi-images build-fetch: _keygen package #!/bin/bash set -euo pipefail - retries=${BOOTC_CI_RETRIES:-3} - delay=${BOOTC_CI_DELAY:-30} - retry() { - local attempt - for attempt in $(seq 1 "$retries"); do - echo "--- Attempt ${attempt}/${retries}: $*" - if "$@"; then - return 0 - fi - if [ "$attempt" -lt "$retries" ]; then - echo "--- Attempt ${attempt} failed, retrying in ${delay}s..." - sleep "$delay" - fi - done - echo "--- All ${retries} attempts failed: $*" >&2 - return 1 - } + {{_retry_fn}} # Pull the base images explicitly so failures are retried cleanly # before we even start the container build. retry podman pull -q {{base}} @@ -466,6 +450,29 @@ sysext: # Internal helpers (prefixed with _) # ============================================================================ +# Shell helper for the fetch recipes: `retry CMD...` reruns CMD on failure +# to paper over transient network errors. Tunable via BOOTC_CI_RETRIES and +# BOOTC_CI_DELAY (seconds). +_retry_fn := ''' +retries=${BOOTC_CI_RETRIES:-3} +delay=${BOOTC_CI_DELAY:-30} +retry() { + local attempt + for attempt in $(seq 1 "$retries"); do + echo "--- Attempt ${attempt}/${retries}: $*" + if "$@"; then + return 0 + fi + if [ "$attempt" -lt "$retries" ]; then + echo "--- Attempt ${attempt} failed, retrying in ${delay}s..." + sleep "$delay" + fi + done + echo "--- All ${retries} attempts failed: $*" >&2 + return 1 +} +''' + _pull-lbi-images: podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} From e410f53cf8de9d0d3a4dfa298107bd9adb5cd144 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 23 Sep 2026 10:57:05 +0100 Subject: [PATCH 2/3] Justfile: Retry all network steps reachable from build-fetch `build-fetch` is meant to front-load every network access with retries so the subsequent `just build` runs offline, but several steps were missed and show up as flakes in CI: - `package` implicitly pulled the base image and built the `buildroot` stage (install-buildroot, cargo fetch) without retries. Add a `_package-fetch` step that pulls the base and builds `--target=buildroot` with retries first, so `package` then hits the layer cache. - The `tools` stage (always built via `sdboot-signed`) runs dnf outside of `fetch`. Prefetch `--target=tools` instead, which includes `fetch`. - `_pull-lbi-images` always contacts the registry, even if the images were already pulled by `build-fetch`, relying only on podman's short internal retry. Wrap it in the retry helper as well. Assisted-by: AI --- Dockerfile | 5 +++-- Justfile | 38 +++++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd8c289a89..083c58bcf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,8 +134,9 @@ ARG boot_type ARG seal_state ARG bootloader # All network-fetching operations: package installs from distro repos, Copr, Koji. -# Separated so `just build-fetch --target=fetch` can be retried independently on -# transient network failures without re-running the configuration phase. +# Separated so `just build-fetch` can retry this (and the `tools` stage below) +# independently on transient network failures without re-running the +# configuration phase. RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \ --mount=type=bind,from=src,src=/src/hack,target=/run/hack <<-EOF set -ex diff --git a/Justfile b/Justfile index ceaf85b052..f12f902297 100644 --- a/Justfile +++ b/Justfile @@ -96,8 +96,8 @@ build: package _keygen && _pull-lbi-images # Fetch all external dependencies with a retry loop. # -# This runs `podman build --target=fetch` for both the main image and the -# upgrade-source image, retrying on transient network failures (Koji 503s, +# This pulls the base images and builds the network-fetching stages of +# both the main image and the upgrade-source image, retrying on transient network failures (Koji 503s, # Copr outages, quay.io blips, etc.). In CI this runs as its own step # before `just build` / `just test-upgrade` so that flakes don't require # re-queueing the entire PR. @@ -105,13 +105,10 @@ build: package _keygen && _pull-lbi-images # The retry parameters can be overridden via environment variables: # BOOTC_CI_RETRIES=10 BOOTC_CI_DELAY=60 just build-fetch [group('core')] -build-fetch: _keygen package +build-fetch: _keygen _package-fetch package #!/bin/bash set -euo pipefail {{_retry_fn}} - # Pull the base images explicitly so failures are retried cleanly - # before we even start the container build. - retry podman pull -q {{base}} retry podman pull -q {{buildroot_base}} # Pull LBI images (also fetched later by _pull-lbi-images, but doing it # here means a failure is retried rather than aborting the full build). @@ -121,12 +118,13 @@ build-fetch: _keygen package pkg_path=$(realpath target/packages) - # Build the network-heavy fetch stage of the main image. If this - # succeeds, `just build` will get a cache hit on the fetch layer and - # run entirely offline. + # Build the network-heavy stages of the main image: `tools` derives + # from `fetch` and additionally installs the sealing tools, and is + # always needed (via `sdboot-signed`). If this succeeds, `just build` + # will get a cache hit on these layers and run entirely offline. # Note: buildargs (not base_buildargs) is needed here because the # target-base stage requires --cap-add/--security-opt for bwrap. - retry podman build {{_nocache_arg}} --build-context "packages=${pkg_path}" --target=fetch {{buildargs}} . + retry podman build {{_nocache_arg}} --build-context "packages=${pkg_path}" --target=tools {{buildargs}} . # Same for the upgrade-source image used by test-upgrade. retry podman build {{_nocache_arg}} --build-arg=base={{base}} \ --target=fetch -f tmt/tests/Dockerfile.upgrade-source . @@ -473,8 +471,26 @@ retry() { } ''' +# Fetch the network dependencies of `package` with retries: the base image +# and the `buildroot` stage (distro packages via install-buildroot, crates +# via cargo fetch). A subsequent `just package` then gets a cache hit on +# the buildroot and only does offline work. +_package-fetch: + #!/bin/bash + set -euo pipefail + {{_retry_fn}} + retry podman pull -q {{base}} + if test -n "${BOOTC_SKIP_PACKAGE:-}"; then + exit 0 + fi + local_deps_args=$(just _local-deps-args) + retry podman build {{base_buildargs}} --target=buildroot $local_deps_args . + _pull-lbi-images: - podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} + #!/bin/bash + set -euo pipefail + {{_retry_fn}} + retry podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} _git-build-vars: #!/bin/bash From 200fd165ba1e4e738e874be20799cfcb8f42855f Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 23 Sep 2026 10:57:05 +0100 Subject: [PATCH 3/3] ci: Fetch dependencies with retry in build-and-publish The publish-images jobs ran `just build` directly and so had none of the network retries that ci.yml gets from `just build-fetch`. Nearly every failure of this workflow on main in the last month was a transient quay.io or Fedora mirror error. Run `build-fetch` first with the same retry parameters as ci.yml. Assisted-by: AI Signed-off-by: Eric Curtin --- .github/workflows/build-and-publish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index ac75eff266..06fd215795 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -12,6 +12,9 @@ env: CARGO_TERM_COLOR: always LIBVIRT_DEFAULT_URI: "qemu:///session" DEV_IMAGE: ghcr.io/bootc-dev/dev-bootc + # Retry parameters for `just build-fetch` (transient Koji/Copr/quay.io failures) + BOOTC_CI_RETRIES: "10" + BOOTC_CI_DELAY: "60" jobs: # Build and publish container images to ghcr.io @@ -41,6 +44,9 @@ jobs: echo "BOOTC_buildroot_base=${BUILDROOTBASE}" >> $GITHUB_ENV fi + - name: Fetch external dependencies (with retry) + run: just build-fetch + - name: Build container # TODO: Also consider building + publishing an image that is just "base + bootc" # as this implicitly changed to also publish our integration test images.