diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index ac75eff266..06fd215795 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -12,6 +12,9 @@ env: CARGO_TERM_COLOR: always LIBVIRT_DEFAULT_URI: "qemu:///session" DEV_IMAGE: ghcr.io/bootc-dev/dev-bootc + # Retry parameters for `just build-fetch` (transient Koji/Copr/quay.io failures) + BOOTC_CI_RETRIES: "10" + BOOTC_CI_DELAY: "60" jobs: # Build and publish container images to ghcr.io @@ -41,6 +44,9 @@ jobs: echo "BOOTC_buildroot_base=${BUILDROOTBASE}" >> $GITHUB_ENV fi + - name: Fetch external dependencies (with retry) + run: just build-fetch + - name: Build container # TODO: Also consider building + publishing an image that is just "base + bootc" # as this implicitly changed to also publish our integration test images. diff --git a/Dockerfile b/Dockerfile index bd8c289a89..083c58bcf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,8 +134,9 @@ ARG boot_type ARG seal_state ARG bootloader # All network-fetching operations: package installs from distro repos, Copr, Koji. -# Separated so `just build-fetch --target=fetch` can be retried independently on -# transient network failures without re-running the configuration phase. +# Separated so `just build-fetch` can retry this (and the `tools` stage below) +# independently on transient network failures without re-running the +# configuration phase. RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \ --mount=type=bind,from=src,src=/src/hack,target=/run/hack <<-EOF set -ex diff --git a/Justfile b/Justfile index 86f14dd278..f12f902297 100644 --- a/Justfile +++ b/Justfile @@ -96,8 +96,8 @@ build: package _keygen && _pull-lbi-images # Fetch all external dependencies with a retry loop. # -# This runs `podman build --target=fetch` for both the main image and the -# upgrade-source image, retrying on transient network failures (Koji 503s, +# This pulls the base images and builds the network-fetching stages of +# both the main image and the upgrade-source image, retrying on transient network failures (Koji 503s, # Copr outages, quay.io blips, etc.). In CI this runs as its own step # before `just build` / `just test-upgrade` so that flakes don't require # re-queueing the entire PR. @@ -105,29 +105,10 @@ build: package _keygen && _pull-lbi-images # The retry parameters can be overridden via environment variables: # BOOTC_CI_RETRIES=10 BOOTC_CI_DELAY=60 just build-fetch [group('core')] -build-fetch: _keygen package +build-fetch: _keygen _package-fetch package #!/bin/bash set -euo pipefail - retries=${BOOTC_CI_RETRIES:-3} - delay=${BOOTC_CI_DELAY:-30} - retry() { - local attempt - for attempt in $(seq 1 "$retries"); do - echo "--- Attempt ${attempt}/${retries}: $*" - if "$@"; then - return 0 - fi - if [ "$attempt" -lt "$retries" ]; then - echo "--- Attempt ${attempt} failed, retrying in ${delay}s..." - sleep "$delay" - fi - done - echo "--- All ${retries} attempts failed: $*" >&2 - return 1 - } - # Pull the base images explicitly so failures are retried cleanly - # before we even start the container build. - retry podman pull -q {{base}} + {{_retry_fn}} retry podman pull -q {{buildroot_base}} # Pull LBI images (also fetched later by _pull-lbi-images, but doing it # here means a failure is retried rather than aborting the full build). @@ -137,12 +118,13 @@ build-fetch: _keygen package pkg_path=$(realpath target/packages) - # Build the network-heavy fetch stage of the main image. If this - # succeeds, `just build` will get a cache hit on the fetch layer and - # run entirely offline. + # Build the network-heavy stages of the main image: `tools` derives + # from `fetch` and additionally installs the sealing tools, and is + # always needed (via `sdboot-signed`). If this succeeds, `just build` + # will get a cache hit on these layers and run entirely offline. # Note: buildargs (not base_buildargs) is needed here because the # target-base stage requires --cap-add/--security-opt for bwrap. - retry podman build {{_nocache_arg}} --build-context "packages=${pkg_path}" --target=fetch {{buildargs}} . + retry podman build {{_nocache_arg}} --build-context "packages=${pkg_path}" --target=tools {{buildargs}} . # Same for the upgrade-source image used by test-upgrade. retry podman build {{_nocache_arg}} --build-arg=base={{base}} \ --target=fetch -f tmt/tests/Dockerfile.upgrade-source . @@ -466,8 +448,49 @@ sysext: # Internal helpers (prefixed with _) # ============================================================================ +# Shell helper for the fetch recipes: `retry CMD...` reruns CMD on failure +# to paper over transient network errors. Tunable via BOOTC_CI_RETRIES and +# BOOTC_CI_DELAY (seconds). +_retry_fn := ''' +retries=${BOOTC_CI_RETRIES:-3} +delay=${BOOTC_CI_DELAY:-30} +retry() { + local attempt + for attempt in $(seq 1 "$retries"); do + echo "--- Attempt ${attempt}/${retries}: $*" + if "$@"; then + return 0 + fi + if [ "$attempt" -lt "$retries" ]; then + echo "--- Attempt ${attempt} failed, retrying in ${delay}s..." + sleep "$delay" + fi + done + echo "--- All ${retries} attempts failed: $*" >&2 + return 1 +} +''' + +# Fetch the network dependencies of `package` with retries: the base image +# and the `buildroot` stage (distro packages via install-buildroot, crates +# via cargo fetch). A subsequent `just package` then gets a cache hit on +# the buildroot and only does offline work. +_package-fetch: + #!/bin/bash + set -euo pipefail + {{_retry_fn}} + retry podman pull -q {{base}} + if test -n "${BOOTC_SKIP_PACKAGE:-}"; then + exit 0 + fi + local_deps_args=$(just _local-deps-args) + retry podman build {{base_buildargs}} --target=buildroot $local_deps_args . + _pull-lbi-images: - podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} + #!/bin/bash + set -euo pipefail + {{_retry_fn}} + retry podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} _git-build-vars: #!/bin/bash