ci: De-flake Build and Publish Images with retried fetches

.github/workflows/build-and-publish.yml

@@ -12,6 +12,9 @@ env:
   CARGO_TERM_COLOR: always
   LIBVIRT_DEFAULT_URI: "qemu:///session"
   DEV_IMAGE: ghcr.io/bootc-dev/dev-bootc
+  # Retry parameters for `just build-fetch` (transient Koji/Copr/quay.io failures)
+  BOOTC_CI_RETRIES: "10"
+  BOOTC_CI_DELAY: "60"
 
 jobs:
   # Build and publish container images to ghcr.io
@@ -41,6 +44,9 @@ jobs:
             echo "BOOTC_buildroot_base=${BUILDROOTBASE}" >> $GITHUB_ENV
           fi
 
+      - name: Fetch external dependencies (with retry)
+        run: just build-fetch
+
       - name: Build container
         # TODO: Also consider building + publishing an image that is just "base + bootc"
         # as this implicitly changed to also publish our integration test images.

Dockerfile

@@ -134,8 +134,9 @@ ARG boot_type
 ARG seal_state
 ARG bootloader
 # All network-fetching operations: package installs from distro repos, Copr, Koji.
-# Separated so `just build-fetch --target=fetch` can be retried independently on
-# transient network failures without re-running the configuration phase.
+# Separated so `just build-fetch` can retry this (and the `tools` stage below)
+# independently on transient network failures without re-running the
+# configuration phase.
 RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
     --mount=type=bind,from=src,src=/src/hack,target=/run/hack <<-EOF
     set -ex

Justfile

@@ -96,38 +96,19 @@ build: package _keygen && _pull-lbi-images
 
 # Fetch all external dependencies with a retry loop.
 #
-# This runs `podman build --target=fetch` for both the main image and the
-# upgrade-source image, retrying on transient network failures (Koji 503s,
+# This pulls the base images and builds the network-fetching stages of
+# both the main image and the upgrade-source image, retrying on transient network failures (Koji 503s,
 # Copr outages, quay.io blips, etc.).  In CI this runs as its own step
 # before `just build` / `just test-upgrade` so that flakes don't require
 # re-queueing the entire PR.
 #
 # The retry parameters can be overridden via environment variables:
 #   BOOTC_CI_RETRIES=10 BOOTC_CI_DELAY=60 just build-fetch
 [group('core')]
-build-fetch: _keygen package
+build-fetch: _keygen _package-fetch package
     #!/bin/bash
     set -euo pipefail
-    retries=${BOOTC_CI_RETRIES:-3}
-    delay=${BOOTC_CI_DELAY:-30}
-    retry() {
-        local attempt
-        for attempt in $(seq 1 "$retries"); do
-            echo "--- Attempt ${attempt}/${retries}: $*"
-            if "$@"; then
-                return 0
-            fi
-            if [ "$attempt" -lt "$retries" ]; then
-                echo "--- Attempt ${attempt} failed, retrying in ${delay}s..."
-                sleep "$delay"
-            fi
-        done
-        echo "--- All ${retries} attempts failed: $*" >&2
-        return 1
-    }
-    # Pull the base images explicitly so failures are retried cleanly
-    # before we even start the container build.
-    retry podman pull -q {{base}}
+    {{_retry_fn}}
     retry podman pull -q {{buildroot_base}}
     # Pull LBI images (also fetched later by _pull-lbi-images, but doing it
     # here means a failure is retried rather than aborting the full build).
@@ -137,12 +118,13 @@ build-fetch: _keygen package
 
     pkg_path=$(realpath target/packages)
 
-    # Build the network-heavy fetch stage of the main image.  If this
-    # succeeds, `just build` will get a cache hit on the fetch layer and
-    # run entirely offline.
+    # Build the network-heavy stages of the main image: `tools` derives
+    # from `fetch` and additionally installs the sealing tools, and is
+    # always needed (via `sdboot-signed`).  If this succeeds, `just build`
+    # will get a cache hit on these layers and run entirely offline.
     # Note: buildargs (not base_buildargs) is needed here because the
     # target-base stage requires --cap-add/--security-opt for bwrap.
-    retry podman build {{_nocache_arg}} --build-context "packages=${pkg_path}" --target=fetch {{buildargs}} .
+    retry podman build {{_nocache_arg}} --build-context "packages=${pkg_path}" --target=tools {{buildargs}} .
     # Same for the upgrade-source image used by test-upgrade.
     retry podman build {{_nocache_arg}} --build-arg=base={{base}} \
         --target=fetch -f tmt/tests/Dockerfile.upgrade-source .
@@ -466,8 +448,49 @@ sysext:
 # Internal helpers (prefixed with _)
 # ============================================================================
 
+# Shell helper for the fetch recipes: `retry CMD...` reruns CMD on failure
+# to paper over transient network errors. Tunable via BOOTC_CI_RETRIES and
+# BOOTC_CI_DELAY (seconds).
+_retry_fn := '''
+retries=${BOOTC_CI_RETRIES:-3}
+delay=${BOOTC_CI_DELAY:-30}
+retry() {
+    local attempt
+    for attempt in $(seq 1 "$retries"); do
+        echo "--- Attempt ${attempt}/${retries}: $*"
+        if "$@"; then
+            return 0
+        fi
+        if [ "$attempt" -lt "$retries" ]; then
+            echo "--- Attempt ${attempt} failed, retrying in ${delay}s..."
+            sleep "$delay"
+        fi
+    done
+    echo "--- All ${retries} attempts failed: $*" >&2
+    return 1
+}
+'''
+
+# Fetch the network dependencies of `package` with retries: the base image
+# and the `buildroot` stage (distro packages via install-buildroot, crates
+# via cargo fetch). A subsequent `just package` then gets a cache hit on
+# the buildroot and only does offline work.
+_package-fetch:
+    #!/bin/bash
+    set -euo pipefail
+    {{_retry_fn}}
+    retry podman pull -q {{base}}
+    if test -n "${BOOTC_SKIP_PACKAGE:-}"; then
+        exit 0
+    fi
+    local_deps_args=$(just _local-deps-args)
+    retry podman build {{base_buildargs}} --target=buildroot $local_deps_args .
+
 _pull-lbi-images:
-    podman pull -q --retry 5 --retry-delay 5s {{lbi_images}}
+    #!/bin/bash
+    set -euo pipefail
+    {{_retry_fn}}
+    retry podman pull -q --retry 5 --retry-delay 5s {{lbi_images}}
 
 _git-build-vars:
     #!/bin/bash