diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80d7d8b1fb8..484fd600843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,9 @@ concurrency: # Timeout policy: docs/fm-test-portable-shards.md "Timeouts" owns the three # tiers and their rationale; tests/fm-ci-workflow.test.sh guards this workflow. # Each job comment identifies the tier implemented by its executable value. +# The normal tier is 45 minutes: runner speed pushed healthy 20-21 minute lint +# and shard jobs up to the old 30-minute bound, so they were cancelled while +# still making steady progress rather than hung. # Fork opt-out: a fork sets the repository variable FM_CI_SKIP_PUSH to 1 to # skip the push run, and FM_CI_SKIP_MACOS to 1 to skip the macOS job, without @@ -59,7 +62,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -122,7 +125,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v6 with: @@ -167,7 +170,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v6 with: @@ -212,7 +215,7 @@ jobs: runs-on: ubuntu-24.04 if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 strategy: # Every shard reports so one failure never hides another shard's result. fail-fast: false @@ -534,7 +537,7 @@ jobs: runs-on: macos-latest if: ${{ !(github.event_name == 'push' && vars.FM_CI_SKIP_PUSH == '1') && vars.FM_CI_SKIP_MACOS != '1' && !(github.event.action == 'labeled' && github.event.label.name != 'full-ci') && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.author_association == 'OWNER' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'full-ci')) }} # Normal tier (see the timeout policy above). - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v6 - name: Run snapshot consumers with stock Bash diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 2193aa8bcf0..cd6b532582f 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -161,9 +161,9 @@ A lane that reaches its tier bound needs investigation and a distribution or run | Tier | Jobs | Bound | Rationale | |---|---|---|---| | Fast | coverage guard, repo invariants, timing aggregate, Full suite verdict | 5 minutes | Seconds-long local work, so the tripwire only catches a hung runner. | -| Normal | lint partitions, portable parallel shards, portable serial shards, macOS stock Bash | 30 minutes, one value shared by every job in the tier | One shared hang tripwire keeps every ordinary test and lint lane on the same policy instead of allowing per-lane packing estimates or one-off caps to set the bound. | +| Normal | lint partitions, portable parallel shards, portable serial shards, macOS stock Bash | 45 minutes, one value shared by every job in the tier | One shared hang tripwire keeps every ordinary test and lint lane on the same policy instead of allowing per-lane packing estimates or one-off caps to set the bound. Runner speed moved healthy lint and shard jobs that used to take 20-21 minutes up to the old 30-minute bound with steady progress and no hang, so the bound rose to 45 minutes to keep the tripwire above healthy duration. | | Heavy | Herdr | family-run step 20 minutes under a 75-minute job-level last-resort backstop | Healthy runs finish in about 7-10 minutes, so the step tripwire fails a wedged suite while the `always()` cleanup and timing upload still run, and the job cap only catches a hang outside that step. | [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) holds the executable values and names each job's tier beside its `timeout-minutes`. -[`tests/fm-ci-workflow.test.sh`](../tests/fm-ci-workflow.test.sh) holds the policy against the parsed workflow: every job belongs to exactly one tier, the workflow carries exactly three distinct job-level values, the fast tier stays within 5-10 minutes, the normal jobs share one 30-minute budget, and the Herdr family-run step is the 20-minute tripwire below its job backstop with an `always()` teardown after it. +[`tests/fm-ci-workflow.test.sh`](../tests/fm-ci-workflow.test.sh) holds the policy against the parsed workflow: every job belongs to exactly one tier, the workflow carries exactly three distinct job-level values, the fast tier stays within 5-10 minutes, the normal jobs share one 45-minute budget, and the Herdr family-run step is the 20-minute tripwire below its job backstop with an `always()` teardown after it. A passing coverage guard does not establish a healthy job duration; refresh the healthy figures above from the lanes' uploaded timing artifacts. diff --git a/tests/fm-ci-workflow.test.sh b/tests/fm-ci-workflow.test.sh index dddb998fa5c..61353c8d65c 100755 --- a/tests/fm-ci-workflow.test.sh +++ b/tests/fm-ci-workflow.test.sh @@ -578,7 +578,7 @@ test_fast_tier_shares_one_short_tripwire() { pass "fast tier jobs share one $fast minute tripwire" } -# Normal tier: every test or lint lane shares ONE fixed 30-minute budget, +# Normal tier: every test or lint lane shares ONE fixed 45-minute budget, # above the fast tier. That budget is a hang tripwire, not a packing estimate. test_normal_tier_shares_one_budget() { local fast normal @@ -588,8 +588,8 @@ test_normal_tier_shares_one_budget() { normal=$(tier_timeout normal $NORMAL_TIER_JOBS) || exit 1 [ "$normal" -gt "$fast" ] \ || fail "normal tier ($normal) must exceed the fast tier ($fast)" - [ "$normal" = 30 ] \ - || fail "normal tier must be the single 30-minute shared budget, got $normal" + [ "$normal" = 45 ] \ + || fail "normal tier must be the single 45-minute shared budget, got $normal" pass "normal tier jobs share one $normal minute budget" }