From 5740b0225b37be082063ac7f65dc5a65c35711c9 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Mon, 16 Nov 2026 09:27:34 -0500 Subject: [PATCH] ci: raise normal tier timeout to 45 minutes Healthy lint and portable shard jobs that used to take 20-21 minutes now reach the 30 minute bound on slower runners and get cancelled while still making steady progress. Raise the shared normal tier to 45 minutes and update the policy doc and workflow test that pin it. --- .github/workflows/ci.yml | 13 ++++++++----- docs/fm-test-portable-shards.md | 4 ++-- tests/fm-ci-workflow.test.sh | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80d7d8b1fb8..484fd600843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,9 @@ concurrency: # Timeout policy: docs/fm-test-portable-shards.md "Timeouts" owns the three # tiers and their rationale; tests/fm-ci-workflow.test.sh guards this workflow. # Each job comment identifies the tier implemented by its executable value. +# The normal tier is 45 minutes: runner speed pushed healthy 20-21 minute lint +# and shard jobs up to the old 30-minute bound, so they were cancelled while +# still making steady progress rather than hung. # Fork opt-out: a fork sets the repository variable FM_CI_SKIP_PUSH to 1 to # skip the push run, and FM_CI_SKIP_MACOS to 1 to skip the macOS job, without @@ -59,7 +62,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -122,7 +125,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v6 with: @@ -167,7 +170,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v6 with: @@ -212,7 +215,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 strategy: # Every shard reports so one failure never hides another shard's result. fail-fast: false @@ -534,7 +537,7 @@ jobs: runs-on: macos-latest if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && vars.FM_CI_SKIP_MACOS != '1' && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v6 - name: Run snapshot consumers with stock Bash diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 2193aa8bcf0..cd6b532582f 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -161,9 +161,9 @@ A lane that reaches its tier bound needs investigation and a distribution or run | Tier | Jobs | Bound | Rationale | |---|---|---|---| | Fast | coverage guard, repo invariants, timing aggregate, Full suite verdict | 5 minutes | Seconds-long local work, so the tripwire only catches a hung runner. | -| Normal | lint partitions, portable parallel shards, portable serial shards, macOS stock Bash | 30 minutes, one value shared by every job in the tier | One shared hang tripwire keeps every ordinary test and lint lane on the same policy instead of allowing per-lane packing estimates or one-off caps to set the bound. | +| Normal | lint partitions, portable parallel shards, portable serial shards, macOS stock Bash | 45 minutes, one value shared by every job in the tier | One shared hang tripwire keeps every ordinary test and lint lane on the same policy instead of allowing per-lane packing estimates or one-off caps to set the bound. Runner speed moved healthy lint and shard jobs that used to take 20-21 minutes up to the old 30-minute bound with steady progress and no hang, so the bound rose to 45 minutes to keep the tripwire above healthy duration. | | Heavy | Herdr | family-run step 20 minutes under a 75-minute job-level last-resort backstop | Healthy runs finish in about 7-10 minutes, so the step tripwire fails a wedged suite while the `always()` cleanup and timing upload still run, and the job cap only catches a hang outside that step. | [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) holds the executable values and names each job's tier beside its `timeout-minutes`. -[`tests/fm-ci-workflow.test.sh`](../tests/fm-ci-workflow.test.sh) holds the policy against the parsed workflow: every job belongs to exactly one tier, the workflow carries exactly three distinct job-level values, the fast tier stays within 5-10 minutes, the normal jobs share one 30-minute budget, and the Herdr family-run step is the 20-minute tripwire below its job backstop with an `always()` teardown after it. +[`tests/fm-ci-workflow.test.sh`](../tests/fm-ci-workflow.test.sh) holds the policy against the parsed workflow: every job belongs to exactly one tier, the workflow carries exactly three distinct job-level values, the fast tier stays within 5-10 minutes, the normal jobs share one 45-minute budget, and the Herdr family-run step is the 20-minute tripwire below its job backstop with an `always()` teardown after it. A passing coverage guard does not establish a healthy job duration; refresh the healthy figures above from the lanes' uploaded timing artifacts. diff --git a/tests/fm-ci-workflow.test.sh b/tests/fm-ci-workflow.test.sh index dddb998fa5c..61353c8d65c 100755 --- a/tests/fm-ci-workflow.test.sh +++ b/tests/fm-ci-workflow.test.sh @@ -578,7 +578,7 @@ test_fast_tier_shares_one_short_tripwire() { pass "fast tier jobs share one $fast minute tripwire" } -# Normal tier: every test or lint lane shares ONE fixed 30-minute budget, +# Normal tier: every test or lint lane shares ONE fixed 45-minute budget, # above the fast tier. That budget is a hang tripwire, not a packing estimate. test_normal_tier_shares_one_budget() { local fast normal @@ -588,8 +588,8 @@ test_normal_tier_shares_one_budget() { normal=$(tier_timeout normal $NORMAL_TIER_JOBS) || exit 1 [ "$normal" -gt "$fast" ] \ || fail "normal tier ($normal) must exceed the fast tier ($fast)" - [ "$normal" = 30 ] \ - || fail "normal tier must be the single 30-minute shared budget, got $normal" + [ "$normal" = 45 ] \ + || fail "normal tier must be the single 45-minute shared budget, got $normal" pass "normal tier jobs share one $normal minute budget" }