Wait for boot-time apt locks in self-hosted CI jobs

.github/workflows/ci.yml

@@ -443,8 +443,8 @@ jobs:
           shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
             iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted patch \
             podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
             libseccomp-dev nfs-kernel-server \
@@ -699,8 +699,8 @@ jobs:
           shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
             iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted patch \
             podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
             libseccomp-dev nfs-kernel-server \
@@ -769,7 +769,7 @@ jobs:
           # exit 125 before any test ran, 2026-08-13).
           ./fcvm/scripts/runner-podman-hygiene.sh
           # Install iperf3 for network benchmarks
-          sudo apt-get update && sudo apt-get install -y iperf3 || true
+          ./fcvm/scripts/ci-apt-get.sh update && ./fcvm/scripts/ci-apt-get.sh install -y iperf3 || true
       - name: Create test log directory
         run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
       - name: Clean test data

.github/workflows/kernels.yml

@@ -62,8 +62,8 @@ jobs:
 
       - name: Install build dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y flex bison bc libelf-dev libssl-dev libfuse3-dev libclang-dev clang musl-tools
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y flex bison bc libelf-dev libssl-dev libfuse3-dev libclang-dev clang musl-tools
 
       - name: Build fcvm and fc-agent
         working-directory: fcvm
@@ -127,8 +127,8 @@ jobs:
       - name: Install GitHub CLI
         run: |
           if ! command -v gh &> /dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y gh
+            ./fcvm/scripts/ci-apt-get.sh update
+            ./fcvm/scripts/ci-apt-get.sh install -y gh
           else
             echo "GitHub CLI already installed: $(gh --version)"
           fi
@@ -244,8 +244,8 @@ jobs:
 
       - name: Install build dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y flex bison bc libelf-dev libssl-dev libfuse3-dev libclang-dev clang musl-tools
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y flex bison bc libelf-dev libssl-dev libfuse3-dev libclang-dev clang musl-tools
 
       - name: Build fcvm and fc-agent
         working-directory: fcvm
@@ -286,8 +286,8 @@ jobs:
         run: |
           if ! command -v gh &> /dev/null; then
             echo "Installing GitHub CLI..."
-            sudo apt-get update
-            sudo apt-get install -y gh
+            ./fcvm/scripts/ci-apt-get.sh update
+            ./fcvm/scripts/ci-apt-get.sh install -y gh
           else
             echo "GitHub CLI already installed: $(gh --version)"
           fi
@@ -469,8 +469,8 @@ jobs:
 
       - name: Install build dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y flex bison bc libelf-dev libssl-dev libfuse3-dev libclang-dev clang musl-tools
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y flex bison bc libelf-dev libssl-dev libfuse3-dev libclang-dev clang musl-tools
 
       - name: Build fcvm and fc-agent
         working-directory: fcvm
@@ -511,8 +511,8 @@ jobs:
         run: |
           if ! command -v gh &> /dev/null; then
             echo "Installing GitHub CLI..."
-            sudo apt-get update
-            sudo apt-get install -y gh
+            ./fcvm/scripts/ci-apt-get.sh update
+            ./fcvm/scripts/ci-apt-get.sh install -y gh
           else
             echo "GitHub CLI already installed: $(gh --version)"
           fi

.github/workflows/teardown-evidence.yml

@@ -53,8 +53,8 @@ jobs:
           rustup show active-toolchain || true
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
             iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted patch \
             podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
             libseccomp-dev dmsetup

.github/workflows/weekly.yml

@@ -56,8 +56,8 @@ jobs:
           rustup show active-toolchain || true
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
             iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted patch \
             podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
             libseccomp-dev
@@ -154,8 +154,8 @@ jobs:
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
+          ./fcvm/scripts/ci-apt-get.sh update
+          ./fcvm/scripts/ci-apt-get.sh install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
             iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted patch \
             podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
             libseccomp-dev

scripts/ci-apt-get.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# apt-get for self-hosted CI jobs: wait for another apt process to release its lock instead
+# of failing on it.
+#
+# A self-hosted runner is a freshly booted instance, and its boot-time apt can still hold a
+# lock when the first job starts. On 2026-09-13 Host-arm64 on #921 was assigned 74 s after
+# its runner launched and failed in "Install dependencies" with
+#   E: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 3180 (apt)
+#
+# DPkg::Lock::Timeout makes `apt-get install` wait for the dpkg frontend lock. It does not
+# cover the lists lock that `apt-get update` takes. Measured with each lock held by another
+# process:
+#   install                              rc=100 in 0.0 s
+#   install -o DPkg::Lock::Timeout=30    waited, rc=0 in 11.4 s
+#   update                               rc=100 in 0.8 s
+#   update  -o DPkg::Lock::Timeout=30    rc=100 in 0.8 s
+# So the timeout is passed for the frontend lock, and a failure that reports a held lock is
+# retried until APT_LOCK_WAIT seconds have passed. Any other failure, or a lock still held at
+# the deadline, returns apt-get's own exit status. Each attempt is one whole apt-get run, so
+# there is no window between checking a lock and taking it.
+#
+# Usage: scripts/ci-apt-get.sh <apt-get arguments>
+# APT_GET, SUDO and APT_LOCK_RETRY_S exist for tests/test_ci_workflow_coverage.rs.
+set -uo pipefail
+
+wait_s=${APT_LOCK_WAIT:-600}
+retry_s=${APT_LOCK_RETRY_S:-5}
+apt_get=${APT_GET:-apt-get}
+sudo_cmd=${SUDO-sudo}
+
+out=$(mktemp) || exit 1
+trap 'rm -f "$out"' EXIT
+deadline=$((SECONDS + wait_s))
+attempt=0
+while :; do
+  attempt=$((attempt + 1))
+  left=$((deadline - SECONDS))
+  [ "$left" -gt 0 ] || left=1
+  $sudo_cmd "$apt_get" -o DPkg::Lock::Timeout="$left" "$@" 2>&1 | tee "$out"
+  rc=${PIPESTATUS[0]}
+  [ "$rc" -eq 0 ] && exit 0
+  if grep -qE '^E: Could not get lock ' "$out" && [ "$SECONDS" -lt "$deadline" ]; then
+    echo "ci-apt-get: attempt $attempt found an apt lock held by another process; retrying in ${retry_s}s ($((deadline - SECONDS))s left)" >&2
+    sleep "$retry_s"
+    [ "$SECONDS" -lt "$deadline" ] || exit "$rc"
+    continue
+  fi
+  exit "$rc"
+done

tests/test_ci_workflow_coverage.rs

@@ -1546,3 +1546,185 @@ fn the_changes_job_emits_every_output_the_matrix_gates_on() {
         );
     }
 }
+
+/// Every apt-get call in a self-hosted job goes through `scripts/ci-apt-get.sh`.
+///
+/// A self-hosted runner is a freshly booted instance, and its boot-time apt can still hold a
+/// lock when the job starts. On 2026-09-13 Host-arm64 on #921 was assigned 74 s after its
+/// runner launched and failed in "Install dependencies" with
+/// `E: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 3180 (apt)`.
+/// A bare `sudo apt-get` fails on a held lock; the script waits for it.
+#[test]
+fn self_hosted_apt_calls_wait_for_apt_locks() {
+    let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows");
+    let entries =
+        std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("cannot read {}: {e}", dir.display()));
+
+    let mut checked = 0usize;
+    for entry in entries {
+        let path = entry.expect("dir entry").path();
+        if path.extension().and_then(|e| e.to_str()) != Some("yml") {
+            continue;
+        }
+        let name = path.file_name().unwrap().to_string_lossy().to_string();
+        let wf: Value = serde_norway::from_str(&std::fs::read_to_string(&path).unwrap())
+            .unwrap_or_else(|e| panic!("{name} is not valid YAML: {e}"));
+        let Some(jobs) = wf.get("jobs").and_then(Value::as_mapping) else {
+            continue;
+        };
+        for (job_name, job) in jobs {
+            let runs_on = job
+                .get("runs-on")
+                .map(|v| format!("{v:?}"))
+                .unwrap_or_default();
+            if !runs_on.contains("self-hosted") {
+                continue;
+            }
+            let job_label = job_name.as_str().unwrap_or("<job>");
+            let Some(steps) = job.get("steps").and_then(Value::as_sequence) else {
+                continue;
+            };
+            for step in steps {
+                let Some(run) = step.get("run").and_then(Value::as_str) else {
+                    continue;
+                };
+                for line in run.lines().map(|l| l.split('#').next().unwrap_or("")) {
+                    if !line.contains("apt-get") {
+                        continue;
+                    }
+                    checked += 1;
+                    assert!(
+                        !line
+                            .replace("./fcvm/scripts/ci-apt-get.sh", "")
+                            .contains("apt-get"),
+                        "{name}: self-hosted job `{job_label}` calls apt-get directly: `{}`. \
+                         A boot-time apt on a fresh runner can hold the dpkg or lists lock, and \
+                         apt-get fails on it at once. Use ./fcvm/scripts/ci-apt-get.sh.",
+                        line.trim()
+                    );
+                    // The path is relative to the workspace root, where `path: fcvm` checks
+                    // the repo out.
+                    assert!(
+                        step.get("working-directory").is_none(),
+                        "{name}: job `{job_label}` calls ./fcvm/scripts/ci-apt-get.sh from a step \
+                         with a working-directory, where that path does not resolve"
+                    );
+                }
+            }
+        }
+    }
+    assert!(
+        checked > 0,
+        "found no apt-get calls in self-hosted jobs to inspect. The walk is broken, and a check \
+         that inspects nothing must not report success"
+    );
+}
+
+/// `scripts/ci-apt-get.sh` retries a held apt lock until its deadline, and nothing else.
+///
+/// Driven with a fake apt-get that records its arguments and plays one outcome per call:
+/// `lock` prints apt's held-lock error, `other` prints a different error (both exit 100),
+/// and anything else exits 0.
+#[test]
+fn ci_apt_get_retries_a_held_lock_and_nothing_else() {
+    use std::os::unix::fs::PermissionsExt;
+
+    let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scripts/ci-apt-get.sh");
+    let dir = std::env::temp_dir().join(format!("fcvm-ci-apt-get-{}", std::process::id()));
+    std::fs::create_dir_all(&dir).unwrap();
+    let fake = dir.join("apt-get");
+    std::fs::write(
+        &fake,
+        r#"#!/bin/bash
+d=$(dirname "$0")
+echo "$*" >> "$d/calls"
+step=$(sed -n "$(wc -l < "$d/calls")p" "$d/plan")
+case "$step" in
+  lock) echo 'E: Could not get lock /var/lib/apt/lists/lock. It is held by process 3180 (apt)'; exit 100 ;;
+  other) echo 'E: Unable to locate package no-such-package'; exit 100 ;;
+  *) exit 0 ;;
+esac
+"#,
+    )
+    .unwrap();
+    std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap();
+
+    let run = |plan: &[&str], wait_s: &str| -> (i32, Vec<String>) {
+        let _ = std::fs::remove_file(dir.join("calls"));
+        std::fs::write(dir.join("plan"), plan.join("\n") + "\n").unwrap();
+        let out = std::process::Command::new("bash")
+            .arg(&script)
+            .arg("update")
+            .env("APT_GET", &fake)
+            .env("SUDO", "")
+            .env("APT_LOCK_RETRY_S", "0")
+            .env("APT_LOCK_WAIT", wait_s)
+            .output()
+            .expect("bash must be runnable");
+        let calls = std::fs::read_to_string(dir.join("calls"))
+            .unwrap_or_default()
+            .lines()
+            .map(str::to_string)
+            .collect();
+        (out.status.code().unwrap_or(-1), calls)
+    };
+
+    let (code, calls) = run(&["lock", "lock", "ok"], "60");
+    assert_eq!(
+        code, 0,
+        "a lock released after two attempts must end in success: {calls:?}"
+    );
+    assert_eq!(calls.len(), 3, "{calls:?}");
+    assert!(
+        calls
+            .iter()
+            .all(|c| c.starts_with("-o DPkg::Lock::Timeout=") && c.ends_with(" update")),
+        "every attempt must pass the dpkg lock timeout and the caller's arguments: {calls:?}"
+    );
+
+    let (code, calls) = run(&["other", "ok"], "60");
+    assert_eq!(
+        code, 100,
+        "a failure that is not a held lock is apt-get's answer: {calls:?}"
+    );
+    assert_eq!(calls.len(), 1, "it must not be retried: {calls:?}");
+
+    let held = vec!["lock"; 20];
+    let (code, calls) = run(&held, "0");
+    assert_eq!(
+        code, 100,
+        "a lock still held at the deadline must fail with apt-get's status: {calls:?}"
+    );
+    assert_eq!(calls.len(), 1, "{calls:?}");
+
+    // A retry sleep that crosses the deadline ends the wait. Checking the deadline only before
+    // the sleep started one more apt-get after it had passed (CodeRabbit on #925).
+    let _ = std::fs::remove_file(dir.join("calls"));
+    std::fs::write(dir.join("plan"), held.join("\n") + "\n").unwrap();
+    let out = std::process::Command::new("bash")
+        .arg(&script)
+        .arg("update")
+        .env("APT_GET", &fake)
+        .env("SUDO", "")
+        .env("APT_LOCK_RETRY_S", "2")
+        .env("APT_LOCK_WAIT", "1")
+        .output()
+        .expect("bash must be runnable");
+    let calls: Vec<String> = std::fs::read_to_string(dir.join("calls"))
+        .unwrap_or_default()
+        .lines()
+        .map(str::to_string)
+        .collect();
+    assert_eq!(
+        out.status.code(),
+        Some(100),
+        "the last apt-get status is returned: {calls:?}"
+    );
+    assert_eq!(
+        calls.len(),
+        1,
+        "no apt-get attempt may start after the sleep crossed the deadline: {calls:?}"
+    );
+
+    let _ = std::fs::remove_dir_all(&dir);
+}