From dc6adae382b8c91f15ab9733c5f107ba9734e07f Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Wed, 19 Aug 2026 20:09:34 +0000 Subject: [PATCH 1/2] fix(ci): shrink per-attempt install timeout, add more retries PR #1267 surfaced a real gap in the apt reliability hardening: the apt.conf.d Acquire::http::Timeout/Acquire::Retries tuning does not reliably abort a connection that stalls completely mid-transfer after already receiving some bytes -- observed directly in a failing run: 19:06:45 Get:2 fonts-ipafont-gothic [3513 kB] -- completes 19:10:16 Get:3 fonts-freefont-ttf [5641 kB] -- request issued 19:15:49 (5.5 min of total silence, then our 10m outer timeout fires) On this runner's apt implementation, Acquire::http::Timeout appears to mostly govern connection setup rather than acting as an ongoing no-progress watchdog once a transfer is already in flight, so a connection that goes silent mid-download isn't aborted by that config regardless of the configured value. The one mechanism that has proven reliable every time this class of issue has come up this session is the outer bash `timeout` wrapper -- it fires and kills cleanly on schedule without exception. Shrink it from 10m to 3m per attempt and increase attempts from 3 to 5, so a stalled connection gets abandoned much faster and there are more chances at a working connection, while keeping worst-case total time per install (5 * 3m + 4 * 15s retry sleeps = ~16 min) well below the previous worst case (~30+ min for 3 * 10m). The apt.conf.d hardening from the prior commit is left in place -- it's still correct/harmless, just insufficient on its own for this specific stall shape. --- .github/workflows/e2e-tests-split.yml | 100 +++++++++++++------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index 485fd56dd..1acef5c8e 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -335,17 +335,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Chromium..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps chromium && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps chromium && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 + echo "ERROR: Playwright chromium install failed after 5 attempts" >&2 exit 1 fi done @@ -589,17 +589,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Chromium (required by security-tests dependency)..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps chromium && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps chromium && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 + echo "ERROR: Playwright chromium install failed after 5 attempts" >&2 exit 1 fi done @@ -608,17 +608,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Firefox..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps firefox && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps firefox && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright firefox install failed after 3 attempts" >&2 + echo "ERROR: Playwright firefox install failed after 5 attempts" >&2 exit 1 fi done @@ -865,17 +865,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Chromium (required by security-tests dependency)..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps chromium && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps chromium && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 + echo "ERROR: Playwright chromium install failed after 5 attempts" >&2 exit 1 fi done @@ -884,17 +884,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing WebKit..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps webkit && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps webkit && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright webkit install failed after 3 attempts" >&2 + echo "ERROR: Playwright webkit install failed after 5 attempts" >&2 exit 1 fi done @@ -1168,17 +1168,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Chromium..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps chromium && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps chromium && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 + echo "ERROR: Playwright chromium install failed after 5 attempts" >&2 exit 1 fi done @@ -1425,17 +1425,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Chromium (required by security-tests dependency)..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps chromium && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps chromium && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 + echo "ERROR: Playwright chromium install failed after 5 attempts" >&2 exit 1 fi done @@ -1444,17 +1444,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Firefox..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps firefox && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps firefox && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright firefox install failed after 3 attempts" >&2 + echo "ERROR: Playwright firefox install failed after 5 attempts" >&2 exit 1 fi done @@ -1701,17 +1701,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing Chromium (required by security-tests dependency)..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps chromium && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps chromium && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 + echo "ERROR: Playwright chromium install failed after 5 attempts" >&2 exit 1 fi done @@ -1720,17 +1720,17 @@ jobs: run: | set -uo pipefail echo "📦 Installing WebKit..." - for attempt in 1 2 3; do - timeout 10m npx playwright install --with-deps webkit && break - if [ "$attempt" -lt 3 ]; then - echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + for attempt in 1 2 3 4 5; do + timeout 3m npx playwright install --with-deps webkit && break + if [ "$attempt" -lt 5 ]; then + echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright webkit install failed after 3 attempts" >&2 + echo "ERROR: Playwright webkit install failed after 5 attempts" >&2 exit 1 fi done From cdc4737a72455c62f209bcddf45848dd8760bab9 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Wed, 19 Aug 2026 20:50:17 +0000 Subject: [PATCH 2/2] fix(ci): give WebKit's Playwright install a longer per-attempt budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WebKit's --with-deps pulls in the full GStreamer/media-codec stack (~170 packages, ~90MB) that Chromium/Firefox don't need. The 3m per-attempt timeout from the previous commit was killing healthy, non-stalled WebKit installs mid-download on a slow mirror, since a full download legitimately takes close to 10 minutes at reduced throughput. WebKit's two install steps go back to a 10m/3-attempt budget; Chromium/Firefox stay on the fail-fast 3m/5-attempt scheme. Also mask the per-run ephemeral CHARON_ENCRYPTION_KEY before writing it to GITHUB_ENV — it's generated via openssl rand rather than a registered secrets.* value, so GitHub's automatic log redaction never picked it up and it was appearing in plaintext in every job's env dump. --- .github/workflows/e2e-tests-split.yml | 60 ++++++++++++++++++++------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index 1acef5c8e..9e0d7f1ab 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -294,7 +294,10 @@ jobs: docker images | grep charon - name: Generate ephemeral encryption key - run: echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> "$GITHUB_ENV" + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::${key}" + echo "CHARON_ENCRYPTION_KEY=${key}" >> "$GITHUB_ENV" - name: Start test environment (Security Tests Profile) run: | @@ -548,7 +551,10 @@ jobs: docker images | grep charon - name: Generate ephemeral encryption key - run: echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> "$GITHUB_ENV" + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::${key}" + echo "CHARON_ENCRYPTION_KEY=${key}" >> "$GITHUB_ENV" - name: Start test environment (Security Tests Profile) run: | @@ -824,7 +830,10 @@ jobs: docker images | grep charon - name: Generate ephemeral encryption key - run: echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> "$GITHUB_ENV" + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::${key}" + echo "CHARON_ENCRYPTION_KEY=${key}" >> "$GITHUB_ENV" - name: Start test environment (Security Tests Profile) run: | @@ -884,17 +893,22 @@ jobs: run: | set -uo pipefail echo "📦 Installing WebKit..." - for attempt in 1 2 3 4 5; do - timeout 3m npx playwright install --with-deps webkit && break - if [ "$attempt" -lt 5 ]; then - echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 + # WebKit's --with-deps pulls the full GStreamer/media-codec stack + # (~170 packages, ~90MB) that Chromium/Firefox don't need, so it + # needs a longer per-attempt budget than the 3m used for the + # lighter browsers below - 3m was observed killing healthy, + # non-stalled WebKit installs mid-download on a slow mirror. + for attempt in 1 2 3; do + timeout 10m npx playwright install --with-deps webkit && break + if [ "$attempt" -lt 3 ]; then + echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright webkit install failed after 5 attempts" >&2 + echo "ERROR: Playwright webkit install failed after 3 attempts" >&2 exit 1 fi done @@ -1127,7 +1141,10 @@ jobs: docker images | grep charon - name: Generate ephemeral encryption key - run: echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> "$GITHUB_ENV" + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::${key}" + echo "CHARON_ENCRYPTION_KEY=${key}" >> "$GITHUB_ENV" - name: Start test environment (Non-Security Profile) run: | @@ -1384,7 +1401,10 @@ jobs: docker images | grep charon - name: Generate ephemeral encryption key - run: echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> "$GITHUB_ENV" + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::${key}" + echo "CHARON_ENCRYPTION_KEY=${key}" >> "$GITHUB_ENV" - name: Start test environment (Non-Security Profile) run: | @@ -1660,7 +1680,10 @@ jobs: docker images | grep charon - name: Generate ephemeral encryption key - run: echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> "$GITHUB_ENV" + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::${key}" + echo "CHARON_ENCRYPTION_KEY=${key}" >> "$GITHUB_ENV" - name: Start test environment (Non-Security Profile) run: | @@ -1720,17 +1743,22 @@ jobs: run: | set -uo pipefail echo "📦 Installing WebKit..." - for attempt in 1 2 3 4 5; do - timeout 3m npx playwright install --with-deps webkit && break - if [ "$attempt" -lt 5 ]; then - echo "Attempt ${attempt}/5 failed; retrying in 15s..." >&2 + # WebKit's --with-deps pulls the full GStreamer/media-codec stack + # (~170 packages, ~90MB) that Chromium/Firefox don't need, so it + # needs a longer per-attempt budget than the 3m used for the + # lighter browsers below - 3m was observed killing healthy, + # non-stalled WebKit installs mid-download on a slow mirror. + for attempt in 1 2 3; do + timeout 10m npx playwright install --with-deps webkit && break + if [ "$attempt" -lt 3 ]; then + echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else - echo "ERROR: Playwright webkit install failed after 5 attempts" >&2 + echo "ERROR: Playwright webkit install failed after 3 attempts" >&2 exit 1 fi done