From 049b5f1031c05bd93880a2c4fb0981648b4a716e Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Sat, 22 Aug 2026 14:29:48 +0700 Subject: [PATCH] ci(bootstrap): ratchet the test suite over failing names, with --no-fail-fast cargo test -p t27c --tests runs 1 of 73 targets: it stops after the first failing target, so every integration test under bootstrap/tests/ compiles and never runs. The regression guards added this week were unreachable, not broken. Measured on e53b9d048: 1 target / 13 failed without --no-fail-fast, 73 targets / 383 failed with it. 61 targets are clean; 358 of the failures are one target attributable to #2325; 12 across 10 targets are unexamined. Baseline is a set of names, not a count. Degenerate logs exit 2 rather than reading absence as a clean set. Closes #2382 --- .github/workflows/bootstrap-tests.yml | 99 +++++ docs/now/2026-08-22-bootstrap-test-ratchet.md | 61 +++ scripts/ci/test-baseline.txt | 387 ++++++++++++++++++ scripts/ci/test_ratchet.py | 164 ++++++++ 4 files changed, 711 insertions(+) create mode 100644 .github/workflows/bootstrap-tests.yml create mode 100644 docs/now/2026-08-22-bootstrap-test-ratchet.md create mode 100644 scripts/ci/test-baseline.txt create mode 100755 scripts/ci/test_ratchet.py diff --git a/.github/workflows/bootstrap-tests.yml b/.github/workflows/bootstrap-tests.yml new file mode 100644 index 000000000..6f1beb3e0 --- /dev/null +++ b/.github/workflows/bootstrap-tests.yml @@ -0,0 +1,99 @@ +# Ratchet the bootstrap test suite over a baseline of failing test NAMES. +# +# Why this exists (#2382): `cargo test -p t27c --tests` stops after the FIRST failing +# target. The unit-test target fails, so the other 72 targets compile and never run — +# every integration test under bootstrap/tests/ among them. That is why the regression +# guards added for #2363, #2003, #2006 and others were dead: not because CI lacked a +# test step, but because the step everyone had in mind would have run 1 target of 73. +# Hence `--no-fail-fast`, and hence the ratchet refuses a log with only one target. +# +# Why a ratchet and not a gate: master has 383 failing tests today. A plain gate would +# land red, and corpus-ratchet.yml already records what happens then — "a gate that +# lands red gets disabled rather than obeyed". +# +# Why names and not a count: 383 still passes when one failure is fixed and a different +# one appears. Only a set catches that. +# +# This job is NOT in the required-context set, so it does not block a merge (see #2376). +# That is the repository owner's decision to make, not this workflow's. + +name: Bootstrap Test Ratchet + +on: + push: + branches: [master] + paths: + - 'bootstrap/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'scripts/ci/test_ratchet.py' + - 'scripts/ci/test-baseline.txt' + - '.github/workflows/bootstrap-tests.yml' + pull_request: + # No `branches:` filter, deliberately — scripts/ci/check_pr_branch_filters.py + # enforces its absence so stacked PRs are not silently ungated. + paths: + - 'bootstrap/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'scripts/ci/test_ratchet.py' + - 'scripts/ci/test-baseline.txt' + - '.github/workflows/bootstrap-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test-ratchet: + name: test-ratchet + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-bootstrap-${{ hashFiles('Cargo.lock') }} + + # The suite has 383 known failures, so cargo exits non-zero on a healthy run. + # The exit code is recorded rather than obeyed, and the RATCHET is the verdict. + # This is not `|| true`: a build failure produces a log with zero targets, which + # the ratchet reports as "NOT evaluated" and exits 2 on. Absence never reads as a + # clean set. + - name: Run bootstrap tests (--no-fail-fast) + id: tests + run: | + set -o pipefail + cargo test -p t27c --tests --no-fail-fast 2>&1 | tee test-log.txt \ + || echo "cargo_exit=$?" >> "$GITHUB_OUTPUT" + echo "targets=$(grep -cE '^[[:space:]]+Running' test-log.txt || true)" >> "$GITHUB_OUTPUT" + + - name: Ratchet the failing set + if: always() + run: | + set -eux + python3 scripts/ci/test_ratchet.py \ + --log test-log.txt \ + --baseline scripts/ci/test-baseline.txt + + - name: Summary + if: always() + run: | + { + echo "### Bootstrap test ratchet" + echo "" + echo "- targets run: \`${{ steps.tests.outputs.targets }}\`" + echo "- cargo exit: \`${{ steps.tests.outputs.cargo_exit || '0' }}\` (recorded, not obeyed — see the workflow header)" + echo "- baseline: \`scripts/ci/test-baseline.txt\`" + echo "" + echo "A run reporting one target means \`--no-fail-fast\` was lost; the ratchet fails on it." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/now/2026-08-22-bootstrap-test-ratchet.md b/docs/now/2026-08-22-bootstrap-test-ratchet.md new file mode 100644 index 000000000..cd86f7ec6 --- /dev/null +++ b/docs/now/2026-08-22-bootstrap-test-ratchet.md @@ -0,0 +1,61 @@ +# NOW — cargo test ran 1 of 73 targets + +Last updated: 2026-08-22 + +## Ratchet the bootstrap suite over failing test names (Closes #2382) + +- Branch: `fix/2382-bootstrap-test-ratchet` +- Issue: #2382 + +### Что легло + +`scripts/ci/test_ratchet.py`, `scripts/ci/test-baseline.txt` (383 entries), and +`.github/workflows/bootstrap-tests.yml` (check name `test-ratchet`). + +The measurement that motivated it, on `e53b9d048`: + +| command | targets | passed | failed | +|---|---|---|---| +| `cargo test -p t27c --tests` | **1** | 1621 | 13 | +| same, `--no-fail-fast` | **73** | 2031 | **383** | + +`cargo test` stops after the first failing target. The unit target fails — those are the +13 named in #2292 — so the other 72 compile and never run. Every regression guard added +this week (#2363, #2003, #2006, #1977, #1985) lives in one of them. They were not broken +and not missing; they were unreachable. The guard for #2363 **passes** when actually run. + +Of the 383: **61 of 73 targets are completely clean**; 358 failures are one target +(`tests/icarus_lowerable.rs`, 0 passed of 358) attributable by shape to the emitter +defect #2325; 13 are the known unit failures; **12 across 10 targets have never been +examined**. + +Baseline is a set of `targettest` keys, not a count — 383 still passes when one +failure is fixed and another appears. A baselined test that starts passing is reported, +not failed on, so the set cannot rot unnoticed. + +### Границы честности (BINDING) + +- **The 12 unexamined failures were not diagnosed and their age was not established.** + Born-failing, long-failing and just-regressed need different responses. One of them, + `dma_local_addr_autoincrement_both_paths`, sits in the file #2345 changed — **that is + a lead, not a regression claim**. +- The 358 are attributed to #2325 by shape (whole-target, 0 passed, `iverilog` present), + not by reading each failure. +- Baseline measured on **macOS/arm64** at `e53b9d048` with a warm target dir; a Linux + runner may differ, and the first CI run will show whether it does. If it does, the + baseline must be regenerated from a runner log, not patched by hand. +- This job is **not a required context**, so it blocks no merge (#2376). Making it one is + the repository owner's decision. +- It does not fix anything. It stops the next regression from being invisible. + +### Evidence + +Three bars, measured. TRUE: unmodified log against its own baseline exits 0. BITING: with +`bootstrap/tests/on_clock_plain_assign.rs:85` altered to assert a string the emitter never +produces, a real rebuild + rerun gave 73 targets, 384 failures, and the ratchet exited 1 +naming `on_clock_emits_assignment_whose_rhs_does_not_reference_the_target`. Degenerate +inputs — a one-target (fail-fast) log, an empty log, a missing log — all exit 2 with +"NOT evaluated" rather than reading absence as a clean set. Exit codes were measured +without a pipeline, since `$?` after a pipe reports the last command's status. + +Cost: **1 min 49 s** wall on a warm cache. diff --git a/scripts/ci/test-baseline.txt b/scripts/ci/test-baseline.txt new file mode 100644 index 000000000..85171498a --- /dev/null +++ b/scripts/ci/test-baseline.txt @@ -0,0 +1,387 @@ +# Generated by scripts/ci/test_ratchet.py --emit-baseline. +# Do not hand-edit: regenerate from a real --no-fail-fast log. +# Each line is targettest for a test failing on master. +# 73 targets ran; 383 failing tests. +src/main.rs compiler::tests_compiler_rejects::lowers_only_first_of_two_modules_characterization +src/main.rs compiler::tests_phase40_coverage::test_for_range_loop_unroll +src/main.rs compiler::tests_phase40_coverage::test_parse_for_range +src/main.rs compiler::tests_phase40_coverage::test_parse_for_range_c +src/main.rs compiler::tests_phase40_coverage::test_parse_for_range_nested +src/main.rs compiler::tests_phase40_coverage::test_parse_for_range_rust +src/main.rs compiler::tests_phase40_coverage::test_parse_for_range_verilog +src/main.rs compiler::tests_phase40_coverage::test_parse_for_range_with_expr +src/main.rs compiler::tests_w458::local_array_named_after_a_verilog_keyword_is_escaped +src/main.rs compiler::tests_w458::tests_w459::array_param_bound_from_test_block +src/main.rs compiler::tests_w458::tests_w459::test_block_emits_real_function_call +src/main.rs lex_conform::tests::lexer_matches_its_conformance_table +src/main.rs parse_conform::tests::parser_matches_its_conformance_table +tests/bitnet_dma.rs dma_local_addr_autoincrement_both_paths +tests/bitnet_layer.rs spec_first_layer2_packs_two_neuron_trits +tests/bitnet_mlp.rs spec_first_mlp2_two_layer_inference_matches_reference +tests/bitnet_mlp3.rs spec_first_mlp3_three_layer_inference_matches_reference +tests/bitnet_neuron_nchunk.rs spec_first_neuron_n_accumulates_and_quantizes +tests/icarus_lowerable.rs accepts_known_lowerable_witnesses +tests/icarus_lowerable.rs accepts_w545_primitive_scalar_array_return +tests/icarus_lowerable.rs accepts_w547_signed_primitive_scalar_array_return +tests/icarus_lowerable.rs accepts_w548_multi_dimensional_primitive_scalar_array_return +tests/icarus_lowerable.rs accepts_w549_three_dimensional_primitive_scalar_array_return +tests/icarus_lowerable.rs accepts_w550_four_dimensional_primitive_scalar_array_return +tests/icarus_lowerable.rs accepts_w551_bench_block_cross_check +tests/icarus_lowerable.rs accepts_w552_bench_wide_cross_check +tests/icarus_lowerable.rs accepts_w553_bench_signed_cross_check +tests/icarus_lowerable.rs accepts_w554_bench_local_array_cross_check +tests/icarus_lowerable.rs accepts_w555_bench_whole_array_cross_check +tests/icarus_lowerable.rs accepts_w556_bench_multi_site_array_dedup +tests/icarus_lowerable.rs accepts_w557_bench_scalar_call_dedup +tests/icarus_lowerable.rs accepts_w558_bench_scalar_call_expected_side_dedup +tests/icarus_lowerable.rs accepts_w559_bench_whole_array_higher_rank_signed +tests/icarus_lowerable.rs accepts_w560_bench_scalar_struct_call_dedup +tests/icarus_lowerable.rs accepts_w562_bench_struct_array_field +tests/icarus_lowerable.rs accepts_w563_bench_array_of_struct_call_dedup +tests/icarus_lowerable.rs accepts_w564_bench_whole_aos_1d +tests/icarus_lowerable.rs accepts_w565_bench_multi_site_whole_aos +tests/icarus_lowerable.rs accepts_w566_bench_2d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w567_bench_3d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w568_bench_4d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w569_bench_4d_aos_call_dedup_nonp2 +tests/icarus_lowerable.rs accepts_w570_bench_5d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w571_bench_5d_aos_call_dedup_nonp2 +tests/icarus_lowerable.rs accepts_w572_bench_6d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w573_bench_7d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w574_bench_8d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w575_bench_9d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w576_bench_10d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w577_bench_11d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w578_bench_12d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w579_bench_13d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w580_bench_14d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w581_bench_15d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w582_bench_16d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w583_bench_module_3d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w584_bench_17d_aos_call_dedup +tests/icarus_lowerable.rs accepts_w585_bench_module_7d_aos_var_call_dedup +tests/icarus_lowerable.rs accepts_w586_bench_module_8d_aos_var_write +tests/icarus_lowerable.rs accepts_w587_bench_module_8d_aos_var_call_write +tests/icarus_lowerable.rs accepts_w588_bench_module_9d_aos_var_call_write +tests/icarus_lowerable.rs accepts_w589_bench_module_17d_aos_var_call_write +tests/icarus_lowerable.rs accepts_w590_bench_module_17d_aos_var_call_reassign +tests/icarus_lowerable.rs accepts_w591_bench_module_17d_aos_var_literal_reassign +tests/icarus_lowerable.rs accepts_w592_bench_module_3x2p15_aos_var_call_write +tests/icarus_lowerable.rs accepts_w593_bench_module_5x2p15_aos_var_call_write +tests/icarus_lowerable.rs accepts_w594_bench_module_7x2p14_aos_var_call_write +tests/icarus_lowerable.rs accepts_w595_bench_module_9x2p13_aos_var_call_write +tests/icarus_lowerable.rs accepts_w596_bench_module_11x2p12_aos_var_call_write +tests/icarus_lowerable.rs accepts_w597_bench_module_13x2p11_aos_var_call_write +tests/icarus_lowerable.rs accepts_w598_bench_module_15x2p10_aos_var_call_write +tests/icarus_lowerable.rs accepts_w599_bench_module_17x2p9_aos_var_call_write +tests/icarus_lowerable.rs accepts_w600_bench_module_19x2p8_aos_var_call_write +tests/icarus_lowerable.rs accepts_w601_bench_module_21x2p7_aos_var_call_write +tests/icarus_lowerable.rs accepts_w602_bench_module_23x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w603_bench_module_25x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w604_bench_module_27x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w605_bench_module_29x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w606_bench_module_31x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w607_bench_module_33x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w608_bench_module_35x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w609_bench_module_37x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w610_bench_module_39x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w611_bench_module_41x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w612_bench_module_43x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w613_bench_module_45x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w614_bench_module_47x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w615_bench_module_49x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w616_bench_module_51x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w617_bench_module_53x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w618_bench_module_55x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w619_bench_module_57x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w620_bench_module_59x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w621_bench_module_61x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w622_bench_module_63x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w623_bench_module_65x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w624_bench_module_67x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w625_bench_module_69x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w626_bench_module_71x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w627_bench_module_73x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w628_bench_module_75x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w629_bench_module_77x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w630_bench_module_79x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w631_bench_module_81x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w632_bench_module_83x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w633_bench_module_85x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w634_bench_module_87x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w635_bench_module_89x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w636_bench_module_91x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w637_bench_module_93x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w638_bench_module_95x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w639_bench_module_97x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w640_bench_module_99x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w641_bench_module_101x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w642_bench_module_103x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w643_bench_module_105x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w644_bench_module_107x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w645_bench_module_109x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w646_bench_module_111x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w647_bench_module_113x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w648_bench_module_115x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w649_bench_module_117x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w650_bench_module_119x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w651_bench_module_121x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w652_bench_module_123x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w653_bench_module_125x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w654_bench_module_127x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w655_bench_module_129x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w656_bench_module_131x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w657_bench_module_133x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w658_bench_module_135x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w659_bench_module_137x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w660_bench_module_139x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w661_bench_module_141x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w662_bench_module_143x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w663_bench_module_145x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w664_bench_module_147x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w665_bench_module_149x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w666_bench_module_151x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w667_bench_module_153x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w668_bench_module_155x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w669_bench_module_157x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w670_bench_module_159x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w671_bench_module_161x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w672_bench_module_163x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w673_bench_module_165x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w674_bench_module_167x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w675_bench_module_169x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w676_bench_module_171x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w677_bench_module_173x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w678_bench_module_175x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w679_bench_module_177x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w680_bench_module_179x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w681_bench_module_181x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w682_bench_module_183x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w683_bench_module_185x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w684_bench_module_187x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w685_bench_module_189x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w686_bench_module_191x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w687_bench_module_193x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w688_bench_module_195x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w689_bench_module_197x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w690_bench_module_199x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w691_bench_module_201x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w692_bench_module_203x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w693_bench_module_205x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w694_bench_module_207x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w695_bench_module_209x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w696_bench_module_211x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w697_bench_module_213x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w698_bench_module_215x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w699_bench_module_217x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w700_bench_module_219x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w701_bench_module_221x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w702_bench_module_223x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w703_bench_module_225x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w704_bench_module_227x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w705_bench_module_229x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w706_bench_module_231x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w707_bench_module_233x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w708_bench_module_235x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w709_bench_module_237x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w710_bench_module_239x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w711_bench_module_241x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w712_bench_module_243x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w713_bench_module_245x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w714_bench_module_247x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w715_bench_module_249x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w716_bench_module_251x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w717_bench_module_253x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w718_bench_module_255x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w719_bench_module_257x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w720_bench_module_259x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w721_bench_module_261x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w722_bench_module_263x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w723_bench_module_265x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w724_bench_module_267x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w725_bench_module_269x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w726_bench_module_271x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w727_bench_module_273x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w728_bench_module_275x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w729_bench_module_277x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w730_bench_module_279x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w731_bench_module_281x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w732_bench_module_283x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w733_bench_module_285x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w734_bench_module_287x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w735_bench_module_289x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w736_bench_module_291x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w737_bench_module_293x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w738_bench_module_295x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w739_bench_module_297x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w740_bench_module_299x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w741_bench_module_301x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w742_bench_module_303x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w743_bench_module_305x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w744_bench_module_307x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w745_bench_module_309x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w746_bench_module_311x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w747_bench_module_313x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w748_bench_module_315x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w749_bench_module_317x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w750_bench_module_319x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w751_bench_module_321x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w752_bench_module_323x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w753_bench_module_325x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w754_bench_module_327x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w755_bench_module_329x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w756_bench_module_331x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w757_bench_module_333x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w758_bench_module_335x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w759_bench_module_337x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w760_bench_module_339x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w761_bench_module_341x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w762_bench_module_343x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w763_bench_module_345x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w764_bench_module_347x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w765_bench_module_349x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w766_bench_module_351x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w767_bench_module_353x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w768_bench_module_355x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w769_bench_module_357x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w770_bench_module_359x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w771_bench_module_361x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w772_bench_module_363x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w773_bench_module_365x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w774_bench_module_367x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w775_bench_module_369x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w776_bench_module_371x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w777_bench_module_373x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w778_bench_module_375x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w779_bench_module_377x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w780_bench_module_379x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w781_bench_module_381x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w782_bench_module_383x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w783_bench_module_385x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w784_bench_module_387x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w785_bench_module_389x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w786_bench_module_391x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w787_bench_module_393x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w788_bench_module_395x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w789_bench_module_397x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w790_bench_module_399x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w791_bench_module_401x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w792_bench_module_403x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w793_bench_module_405x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w794_bench_module_407x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w795_bench_module_409x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w796_bench_module_411x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w797_bench_module_413x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w798_bench_module_415x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w799_bench_module_417x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w800_bench_module_419x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w801_bench_module_421x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w802_bench_module_423x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w803_bench_module_425x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w804_bench_module_427x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w805_bench_module_429x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w806_bench_module_431x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w807_bench_module_433x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w808_bench_module_435x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w809_bench_module_437x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w810_bench_module_439x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w811_bench_module_441x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w812_bench_module_443x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w813_bench_module_445x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w814_bench_module_447x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w815_bench_module_449x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w816_bench_module_451x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w817_bench_module_453x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w818_bench_module_455x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w819_bench_module_457x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w820_bench_module_459x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w821_bench_module_461x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w822_bench_module_463x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w823_bench_module_465x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w824_bench_module_467x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w825_bench_module_469x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w826_bench_module_471x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w827_bench_module_473x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w828_bench_module_475x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w829_bench_module_477x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w830_bench_module_479x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w831_bench_module_481x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w832_bench_module_483x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w833_bench_module_485x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w834_bench_module_487x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w835_bench_module_489x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w836_bench_module_491x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w837_bench_module_493x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w838_bench_module_495x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w839_bench_module_497x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w840_bench_module_499x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w841_bench_module_501x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w842_bench_module_503x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w843_bench_module_505x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w844_bench_module_507x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w845_bench_module_509x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w846_bench_module_511x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w847_bench_module_513x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w848_bench_module_515x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w849_bench_module_517x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w850_bench_module_519x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w851_bench_module_521x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w852_bench_module_523x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w853_bench_module_525x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w854_bench_module_527x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w855_bench_module_529x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w856_bench_module_531x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w857_bench_module_533x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w858_bench_module_535x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w859_bench_module_537x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w860_bench_module_539x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w861_bench_module_541x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w862_bench_module_543x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w863_bench_module_545x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w864_bench_module_547x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w865_bench_module_549x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w866_bench_module_551x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w867_bench_module_553x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w868_bench_module_555x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w869_bench_module_557x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w870_bench_module_559x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w871_bench_module_561x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w872_bench_module_563x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w873_bench_module_565x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w874_bench_module_567x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w875_bench_module_569x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w876_bench_module_571x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w877_bench_module_573x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w878_bench_module_575x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w879_bench_module_577x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w880_bench_module_579x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w881_bench_module_581x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w882_bench_module_583x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w883_bench_module_585x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w884_bench_module_587x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w885_bench_module_589x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w886_bench_module_591x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w887_bench_module_593x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w888_bench_module_595x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w889_bench_module_597x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w890_bench_module_599x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w891_bench_module_601x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w892_bench_module_603x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w893_bench_module_605x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w894_bench_module_607x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w895_bench_module_609x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w896_bench_module_611x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w897_bench_module_613x2p6_aos_var_call_write +tests/icarus_lowerable.rs accepts_w898_bench_module_615x2p6_aos_var_call_write +tests/icarus_lowerable.rs corpus_classifier_matches_lean_completeness +tests/icarus_lowerable.rs rejects_w534_negative_witnesses +tests/icarus_lowerable.rs rejects_w537_undefined_struct_witness +tests/icarus_lowerable.rs rejects_w543_nonlowerable_call_init_witness +tests/icarus_lowerable.rs rejects_w561_nonlowerable_struct_return_witnesses +tests/verilog_array_literal_expr.rs r_ca_2_synthetic_no_comment_only_call_argument +tests/verilog_array_param_index.rs array_param_index_is_element_part_select +tests/verilog_initial_decl.rs r_vd_1_real_uart_spec_no_integer_decl_inside_initial +tests/verilog_initial_decl.rs r_vd_1_synthetic_no_integer_decl_inside_initial +tests/verilog_r_si_1.rs r_si_1_emitter_injects_mul_noop_helper +tests/verilog_translate_off.rs r_tr_1_real_uart_spec_no_inline_translate_marker +tests/verilog_translate_off.rs r_tr_1_synthetic_no_inline_translate_marker diff --git a/scripts/ci/test_ratchet.py b/scripts/ci/test_ratchet.py new file mode 100755 index 000000000..17bb8c4ff --- /dev/null +++ b/scripts/ci/test_ratchet.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Ratchet the bootstrap test suite over a baseline of failing test NAMES. + +Why a ratchet and not a gate: `cargo test -p t27c --tests --no-fail-fast` has 383 +failures on master (2026-08-22, e53b9d048). A plain gate would land red, and a gate +that lands red gets disabled rather than obeyed. + +Why names and not a count: a count of 383 still passes when one failure is fixed and +a different one appears. Only a set catches that. + +Why --no-fail-fast matters, and this is the whole reason the suite was invisible: +without it `cargo test` stops after the FIRST failing target. The unit-test target +fails, so the other 72 targets compile and never run — every integration test under +bootstrap/tests/ among them. The recorded "13 failures" figure is that artefact. +See #2382. + +Usage: + cargo test -p t27c --tests --no-fail-fast 2>&1 | tee log.txt + scripts/ci/test_ratchet.py --log log.txt --emit-baseline > scripts/ci/test-baseline.txt + scripts/ci/test_ratchet.py --log log.txt --baseline scripts/ci/test-baseline.txt +""" + +import argparse +import os +import re +import sys + +RUNNING = re.compile(r"^\s+Running (?:unittests )?(\S+)") +FAILED = re.compile(r"^test (\S+) \.\.\. FAILED") +RESULT = re.compile(r"^test result: (?:ok|FAILED)\.") + + +def parse(path): + """Return (failures, targets_seen, result_lines). + + A failure is keyed "target\ttest" so the same test name in two targets stays + distinct. `target` is the source path cargo prints, not the hashed binary. + """ + failures = set() + targets = [] + results = 0 + current = None + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + m = RUNNING.match(line) + if m: + # "tests/foo.rs (…/deps/foo-1a2b)" -> "tests/foo.rs"; unittests -> "unittests" + raw = m.group(1) + current = "unittests" if raw.endswith(")") and "deps/" in raw and "tests/" not in raw else raw + if current not in targets: + targets.append(current) + continue + m = FAILED.match(line) + if m and current: + failures.add(f"{current}\t{m.group(1)}") + continue + if RESULT.match(line): + results += 1 + return failures, targets, results + + +def load_baseline(path): + out = set() + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if line and not line.startswith("#"): + out.add(line) + return out + + +def notice(msg): + if os.environ.get("GITHUB_ACTIONS"): + print(f"::notice title=test ratchet::{msg}") + print(msg) + + +def error(msg): + if os.environ.get("GITHUB_ACTIONS"): + print(f"::error title=test ratchet::{msg}") + print(msg, file=sys.stderr) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--log", required=True) + ap.add_argument("--baseline") + ap.add_argument("--emit-baseline", action="store_true") + args = ap.parse_args() + + if not os.path.exists(args.log): + error(f"test set NOT evaluated: no log at {args.log}") + return 2 + + failures, targets, results = parse(args.log) + + # Refuse rather than certify. An empty or truncated log has no failures in it, + # and reading that as a clean set is the fail-open this repo has closed ten times. + if not targets or results == 0: + error( + "test set NOT evaluated: the log has " + f"{len(targets)} target(s) and {results} 'test result:' line(s). " + "A run that did not happen is not a run that passed." + ) + return 2 + + # One target means --no-fail-fast was missing (or everything but one was filtered). + # That is the exact condition that hid 72 targets; do not let it read as success. + if len(targets) == 1: + error( + "test set NOT evaluated: only 1 target ran. cargo test stops after the " + "first failing target unless --no-fail-fast is passed, so this log " + "cannot speak for the suite. See #2382." + ) + return 2 + + if args.emit_baseline: + print("# Generated by scripts/ci/test_ratchet.py --emit-baseline.") + print("# Do not hand-edit: regenerate from a real --no-fail-fast log.") + print("# Each line is targettest for a test failing on master.") + print(f"# {len(targets)} targets ran; {len(failures)} failing tests.") + for key in sorted(failures): + print(key) + return 0 + + if not args.baseline: + error("--baseline is required unless --emit-baseline is given") + return 2 + + base = load_baseline(args.baseline) + new = sorted(failures - base) + recovered = sorted(base - failures) + + notice( + f"{len(targets)} targets ran; {len(failures)} failing test(s) " + f"(baseline {len(base)})" + ) + + # Recovery is reported, never failed on — but it must be visible, or the + # baseline rots into a list nobody can tell is stale. + if recovered: + print(f"\n{len(recovered)} baselined test(s) now PASS — prune them from the baseline:") + for key in recovered: + tgt, name = key.split("\t", 1) + print(f" - {name} ({tgt})") + + if new: + error(f"the failing set grew by {len(new)}") + print("\n**FAILED — these tests are newly failing:**") + for key in new: + tgt, name = key.split("\t", 1) + print(f" - `{name}` ({tgt})") + print( + "\nIf a failure is expected, regenerate the baseline from a master log " + "with --emit-baseline and say why in the PR." + ) + return 1 + + print("\nNo new failures. Baseline holds.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())