From 82a7831c0d7fb60d9237beae660da4bd7bea8210 Mon Sep 17 00:00:00 2001 From: quality Date: Thu, 8 Oct 2026 10:11:34 -0400 Subject: [PATCH] test(coverage): give the tests/tools/ harness a floor of its own `tests/tools/` is the third SOURCE_ROOTS entry, so --require-source-files insists every module in it be measured -- and then no ratio gate scored it. --check-source, --check-source-regions and --check-source-file-regions are all built from isSourceFile(), which drops everything under tests/, so the only floor over the harness was --check-regions, an all-files aggregate over ~8900 regions that the suite's own test files dominate. The measuring apparatus was required to be present and held to no standard, while tests/tools/e2e-coverage-report.mjs sat at 93.65% regions. Report `harness files` as its own row beside `src files`, built from the same SOURCE_ROOTS entry --require-source-files walks (isHarnessFile() is derived from SOURCE_ROOTS rather than restating the directory, so the row and the file-set floor cannot come to name different trees). Add --check-harness, --check-harness-regions and --check-harness-file-regions: the same three shapes as the source gates, with the same refusal to pass vacuously over a run that recorded no harness file, and the per-file failure naming the file, its ratio and its uncovered region start lines. It stays a separate row rather than widening `src files`, for the reason isSourceFile() already gives: the harness is not code this repository ships, and merging the two would let a harness gap be paid for out of src/'s slack -- the averaging --check-source-file-regions exists to stop. `src files` and `harness files` now share one accumulator, so a change to how one row is built cannot quietly stop applying to the other. Floors, measured on the suite this commit ships (harness files 100.00% lines / 97.95% regions, 3522/3522 lines, 1384/1413 regions; lowest harness file tests/tools/e2e-coverage-report.mjs at 93.65%): --check-harness 100, --check-harness-regions 97, --check-harness-file-regions 93. Recorded in tests/coverage-gate-thresholds.test.mjs, whose "every threshold the reporter supports is actually gated" test requires the gate script to pass all three. CONTRIBUTING.md and AGENTS.md state the new floors beside the existing ones. The four-number phrase tests/docs-threshold-parity.test.mjs pins is left exactly as it was; the harness floors are stated in a clause of their own. Closes #1192 Signed-off-by: quality --- AGENTS.md | 5 +- CONTRIBUTING.md | 8 +- package.json | 2 +- tests/coverage-gate-thresholds.test.mjs | 10 + tests/coverage-report-harness-floor.test.mjs | 300 +++++++++++++++++++ tests/tools/coverage-report.mjs | 205 +++++++++++-- 6 files changed, 495 insertions(+), 35 deletions(-) create mode 100644 tests/coverage-report-harness-floor.test.mjs diff --git a/AGENTS.md b/AGENTS.md index f65e6ddf..b3b8bbed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,8 +50,9 @@ or CI will fail the PR. - **Build**: `npm run build` - **Unit Tests**: `npm run test:unit:coverage:check` — required "Validate repository" CI check on every PR; enforces coverage thresholds (99% lines / - 100% source / 95% regions / 99% source regions). `npm run test:unit` alone - skips the coverage gate. + 100% source / 95% regions / 99% source regions), plus separate floors for the + `tests/tools/` harness (100% lines / 97% regions, 93% regions per harness + file). `npm run test:unit` alone skips the coverage gate. - **End-to-end Tests**: required "End-to-end tests" CI check on every PR. Needs a production build first, then a one-time browser install: `npm run build:production && npx playwright install --with-deps chromium && npm run test:e2e`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b24425db..4f5821bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,8 +178,12 @@ enough for a first contribution. 3. Run `npm run test:unit:coverage:check` — the required "Validate repository" check runs this on every PR, and it fails the build if unit-test coverage drops below its thresholds (99% lines / 100% source / 95% regions / 99% - source regions). `npm run test:unit` alone skips the coverage gate, so a - green `test:unit` locally does not guarantee a green PR check. + source regions). The harness under `tests/tools/` is floored separately (100% + lines / 97% regions, and 93% regions per harness file): it is measured like + source but reported in its own `harness files` row, so a gap in the test + apparatus cannot be paid for out of `src/`'s slack. `npm run test:unit` alone + skips the coverage gate, so a green `test:unit` locally does not guarantee a + green PR check. 4. Verify with `npm run build` before opening a PR. If your change touches pages, components, or navigation, also run the end-to-end suite — see [End-to-end tests](#end-to-end-tests) below. diff --git a/package.json b/package.json index 656fe65b..b0d7dda4 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ "pr-queue-hygiene": "node scripts/pr-queue-hygiene.mjs", "test:unit": "TZ=UTC node --test", "test:unit:coverage": "TZ=UTC node tests/tools/coverage-report.mjs", - "test:unit:coverage:check": "TZ=UTC node tests/tools/coverage-report.mjs --check 99 --check-source 100 --check-regions 95 --check-source-regions 99 --check-source-file-regions 97 --require-source-files", + "test:unit:coverage:check": "TZ=UTC node tests/tools/coverage-report.mjs --check 99 --check-source 100 --check-regions 95 --check-source-regions 99 --check-source-file-regions 97 --check-harness 100 --check-harness-regions 97 --check-harness-file-regions 93 --require-source-files", "test:e2e": "playwright test", "build:e2e:coverage": "npm run build:e2e:coverage:site && npm run build:e2e:coverage:variant", "build:e2e:coverage:site": "DOCUSAURUS_NO_PERSISTENT_CACHE=1 E2E_COVERAGE=1 npm run build:production", diff --git a/tests/coverage-gate-thresholds.test.mjs b/tests/coverage-gate-thresholds.test.mjs index c9b49cbb..ba749c0d 100644 --- a/tests/coverage-gate-thresholds.test.mjs +++ b/tests/coverage-gate-thresholds.test.mjs @@ -44,6 +44,16 @@ const FLOORS = { // the suite this commit ships, the lowest is // scripts/lib/architecture-content.mjs at 97.78%, floored to 97. '--check-source-file-regions': 97, + // The harness tree (tests/tools/) is walked by --require-source-files and + // entered none of the ratios above, which are all built from + // isSourceFile(). These are the same three shapes over its own row. + // Measured on the suite this commit ships: harness files 100.00% lines / + // 97.95% regions (3522/3522 lines, 1384/1413 regions), and the lowest + // harness file is tests/tools/e2e-coverage-report.mjs at 93.65%, floored + // to 93. + '--check-harness': 100, + '--check-harness-regions': 97, + '--check-harness-file-regions': 93, }; function parseGate(command) { diff --git a/tests/coverage-report-harness-floor.test.mjs b/tests/coverage-report-harness-floor.test.mjs new file mode 100644 index 00000000..13a9d671 --- /dev/null +++ b/tests/coverage-report-harness-floor.test.mjs @@ -0,0 +1,300 @@ +// `tests/tools/` is walked by `--require-source-files`, so every module in it +// is *required* to be measured -- and until the `--check-harness` family +// existed, nothing scored what the measurement said. +// +// The reason is structural rather than an oversight of degree. +// `--check-source`, `--check-source-regions` and `--check-source-file-regions` +// are all built from `isSourceFile()`, which classifies everything under +// `tests/` as non-source, so the harness enters none of their ratios. The only +// floor left over it was `--check-regions`, an all-files aggregate over ~8900 +// regions that the suite's own test files dominate: a harness module could +// lose every region it has and move that number by a fraction of a percent. +// +// That matters more here than the arithmetic suggests. `tests/tools/` is where +// the coverage reporters, the Playwright coverage fixture, the data-overlay +// loader and the JSX/DOM harness live, so a gap there is a gap in the +// instrument every other gate reads. +// +// These tests pin what the family is for: that the harness gets a row of its +// own rather than being folded into `src files`, that the row is built from +// the same `SOURCE_ROOTS` entry `--require-source-files` walks, that each of +// the three floors fails on a real run, that the per-file floor names the +// file and its uncovered regions, and that none of them passes vacuously over +// a run that recorded no harness file at all. +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + countsForScript, + isHarnessFile, + isSourceFile, + report, +} from './tools/coverage-report.mjs'; + +const REPO_ROOT = fileURLToPath(new URL('..', import.meta.url)); +const REPORTER = fileURLToPath( + new URL('./tools/coverage-report.mjs', import.meta.url), +); + +// Mirrors tests/coverage-report-file-region-floor.test.mjs: NODE_V8_COVERAGE +// stays set for the reporter itself (it overrides the variable for the `node +// --test` it spawns, so the grandchild writes elsewhere), and NODE_TEST_CONTEXT +// is removed because `node --test` refuses to run inside another runner. +function runReporter(args) { + const env = { ...process.env, TZ: 'UTC' }; + delete env.NODE_TEST_CONTEXT; + return spawnSync(process.execPath, [REPORTER, ...args], { + cwd: REPO_ROOT, + encoding: 'utf8', + env, + }); +} + +// A driver that imports a harness module without calling into it, so that +// module's function bodies are recorded as uncovered lines and regions. It is +// written inside the repository because the reporter maps coverage records +// onto repo-relative paths and drops everything outside, and it is +// dot-prefixed so `node --test`'s own discovery (which matches *.test.*) +// cannot pick it up while it exists; the reporter is handed the path +// explicitly. +function withImportOnlyDriver(body) { + const relative = `tests/.coverage-harness-floor-${randomUUID()}.mjs`; + writeFileSync( + join(REPO_ROOT, relative), + [ + "import test from 'node:test';", + "import * as staticReferences from './tools/static-references.mjs';", + "test('imports a harness module without exercising it', () => {", + " if (!staticReferences) throw new Error('module did not load');", + '});', + '', + ].join('\n'), + ); + try { + return body(relative); + } finally { + rmSync(join(REPO_ROOT, relative), { force: true }); + } +} + +function silently(body) { + const log = console.log; + console.log = () => {}; + try { + return body(); + } finally { + console.log = log; + } +} + +// One function whose body never ran, inside a file whose every line is +// otherwise executed: the sub-line gap shape the line gates cannot see. +const PARTIAL_SOURCE = 'export const pick = (value) => value ?? fallback();\n'; + +function partialCounts() { + return countsForScript( + { + functions: [ + { + ranges: [ + { startOffset: 0, endOffset: PARTIAL_SOURCE.length, count: 1 }, + { startOffset: 40, endOffset: 50, count: 0 }, + ], + }, + ], + }, + PARTIAL_SOURCE.length, + ); +} + +function mergedWith(files) { + return new Map( + files.map((file) => [ + file, + { source: PARTIAL_SOURCE, counts: partialCounts() }, + ]), + ); +} + +// The predicate is what keeps the row and the file-set floor naming the same +// tree. tests/tools/ is the one subtree of tests/ that --require-source-files +// enumerates, and it is deliberately not source: both halves of that have to +// hold at once or the harness falls through the gap between them again. +test('the harness tree is the part of SOURCE_ROOTS that is not source', () => { + assert.equal(isHarnessFile('tests/tools/coverage-report.mjs'), true); + assert.equal(isHarnessFile('tests/tools/docusaurus-stubs/Link.mjs'), true); + assert.equal(isSourceFile('tests/tools/coverage-report.mjs'), false); + + // Everything else under tests/ is the suite measuring itself: neither + // source nor harness, and in no gated row. + assert.equal(isHarnessFile('tests/static-assets.test.mjs'), false); + assert.equal(isHarnessFile('tests/tools-of-the-trade.mjs'), false); + // The shipped trees stay source; the harness row must not quietly annex + // them and score them against a laxer floor. + assert.equal(isHarnessFile('scripts/lib/validate-utils.mjs'), false); + assert.equal(isHarnessFile('src/components/Foo/index.js'), false); +}); + +test('report() summarises the harness separately from src files', () => { + const merged = mergedWith([ + 'src/lib/partial.mjs', + 'tests/tools/partial.mjs', + // Neither source nor harness: it must land in `all files` alone. + 'tests/partial.test.mjs', + ]); + + const totals = silently(() => report(merged)); + + assert.deepEqual( + totals.harnessFileRegions.map((entry) => entry.file), + ['tests/tools/partial.mjs'], + ); + assert.deepEqual( + totals.sourceFileRegions.map((entry) => entry.file), + ['src/lib/partial.mjs'], + ); + // The zero-count range splits the file into three regions: the covered run + // before it, the gap itself, and the covered run after. + const [entry] = totals.harnessFileRegions; + assert.equal(entry.regions, 3); + assert.equal(entry.covered, 2); + assert.equal(Number(entry.percent.toFixed(2)), 66.67); + assert.ok(entry.uncovered.length > 0); + assert.equal(Number(totals.harnessRegionPct.toFixed(2)), 66.67); + assert.equal(totals.harnessLinePct, 100); + assert.equal(totals.harnessExecutable, 1); + assert.equal(totals.harnessRegions, 3); + // Three rows, three denominators: folding the harness into `src files` + // would let its gap be paid for out of the shipped tree's slack. + assert.equal(totals.sourceRegions, 3); + assert.equal(Number(totals.totalRegionPct.toFixed(2)), 66.67); +}); + +test('the report prints a harness files row beside src files', () => { + const lines = []; + const log = console.log; + console.log = (line) => lines.push(String(line)); + try { + report(mergedWith(['src/lib/partial.mjs', 'tests/tools/partial.mjs'])); + } finally { + console.log = log; + } + const labels = lines + .map((line) => line.split('|')[0].trim()) + .filter((label) => + ['src files', 'harness files', 'all files'].includes(label), + ); + assert.deepEqual(labels, ['src files', 'harness files', 'all files']); + const harnessRow = lines.find((line) => line.startsWith('harness files')); + assert.match( + harnessRow, + /\| +100\.00 \| +66\.67 \| 1\/1 lines \| 2\/3 regions$/, + ); +}); + +test('--check-harness fails a run the source line gate passes', () => { + withImportOnlyDriver((driver) => { + // The driver loads a harness module and nothing under scripts/ or src/, + // so --check-source cannot speak to this run at all while the harness + // line ratio is plainly short of 100. + const result = runReporter(['--check-harness', '100', '--', driver]); + assert.equal(result.status, 1); + assert.match( + result.stderr, + /Harness line coverage \d+\.\d{2}% is below the required 100%\./, + ); + }); +}); + +test('--check-harness-regions fails on the harness alone', () => { + withImportOnlyDriver((driver) => { + const result = runReporter([ + '--check-harness-regions', + '100', + '--', + driver, + ]); + assert.equal(result.status, 1); + assert.match( + result.stderr, + /Harness region coverage \d+\.\d{2}% is below the required 100%\./, + ); + }); +}); + +test('--check-harness-file-regions names the file, its ratio and its uncovered regions', () => { + withImportOnlyDriver((driver) => { + const result = runReporter([ + '--check-harness-file-regions', + '100', + '--', + driver, + ]); + assert.equal(result.status, 1); + assert.match( + result.stderr, + /1 harness file\(s\) fall below the required 100% region coverage per file:/, + ); + assert.match( + result.stderr, + /^ {2}tests\/tools\/static-references\.mjs \d+\.\d{2}% \(\d+\/\d+ regions; uncovered at \d[\d\s-]*\)$/m, + ); + }); +}); + +test('the harness floors pass when the run clears them', () => { + withImportOnlyDriver((driver) => { + const result = runReporter([ + '--check-harness', + '1', + '--check-harness-regions', + '1', + '--check-harness-file-regions', + '1', + '--', + driver, + ]); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stderr, /Harness /); + assert.doesNotMatch(result.stderr, /fall below the required/); + }); +}); + +// A run that loads no harness module leaves every harness ratio a 0/0 +// fraction, which percent() reports as 100%. Without these guards the gates +// would be green on a run that verified nothing about the harness -- exactly +// the silence they were added to end. +const VACUOUS_CASES = [ + ['--check-harness', /--check-harness was requested, but no harness lines/], + [ + '--check-harness-regions', + /--check-harness-regions was requested, but no harness regions/, + ], + [ + '--check-harness-file-regions', + /--check-harness-file-regions was requested, but no harness file/, + ], +]; + +for (const [flag, expected] of VACUOUS_CASES) { + test(`${flag} refuses to pass when no harness file was measured`, () => { + // This test file touches no module under tests/tools/, so the run + // produces harness rows for nothing at all. + const result = runReporter([ + flag, + '97', + '--', + 'tests/markdown-link-check-config.test.mjs', + ]); + assert.equal(result.status, 1); + assert.match(result.stderr, expected); + // The directory is named so the reader is not left guessing which tree + // "harness" meant. + assert.match(result.stderr, /tests\/tools\//); + }); +} diff --git a/tests/tools/coverage-report.mjs b/tests/tools/coverage-report.mjs index fbc03ddd..08349ef3 100644 --- a/tests/tools/coverage-report.mjs +++ b/tests/tools/coverage-report.mjs @@ -23,6 +23,9 @@ // [--check-regions ] [--check-source ] // [--check-source-regions ] // [--check-source-file-regions ] [--require-source-files] +// [--check-harness ] +// [--check-harness-regions ] +// [--check-harness-file-regions ] // [-- ] // // The test files are themselves part of the recorded coverage, and they @@ -56,6 +59,17 @@ // its region percentage falls. --check-source-file-regions applies the floor // to each source file on its own, so a regression concentrated in one file // fails on that file's name instead of being averaged away. +// +// All five of those score `scripts/` and `src/` alone, because they are built +// from isSourceFile(), which drops everything under tests/. tests/tools/ is +// the third SOURCE_ROOTS entry, so --require-source-files insists every +// module in it be measured, and then no ratio gate scored it: the measuring +// apparatus was required to be present and held to no standard. The +// --check-harness family is the same three shapes over that tree, reported +// as a `harness files` row beside `src files`. It stays a separate row +// rather than widening `src files` for the reason isSourceFile() gives -- +// the harness is not code this repository ships, and merging the two would +// let a harness gap be paid for out of src/'s slack. import { spawnSync } from 'node:child_process'; import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; @@ -91,6 +105,25 @@ const SCRIPT_URL = /\.(js|jsx|mjs)$/; const SOURCE_ROOTS = ['scripts', 'src', join('tests', 'tools')]; const SOURCE_FILE = /\.(c|m)?jsx?$/; +// The SOURCE_ROOTS entries that --require-source-files walks but +// isSourceFile() classifies as non-source: the harness tree. Derived from +// SOURCE_ROOTS rather than restated so the row below and the file-set floor +// can never come to name different directories. +// +// Without a row of its own the harness is required to be measured and then +// scored by nothing: --check-source and its companions are built from +// isSourceFile(), which drops everything under tests/, and the only floor +// left over it is --check-regions, an all-files aggregate the suite's own +// test files dominate. The harness is where the coverage reporters, the +// Playwright coverage fixture, the data-overlay loader and the JSX/DOM +// harness live, so a gap there is a gap in the instrument every other gate +// reads. It is still summarised separately rather than folded into +// `src files`, for the reason isSourceFile() gives: it is not code this +// repository ships, and its ratios answer a different question. +const HARNESS_ROOTS = SOURCE_ROOTS.map((root) => + root.split(sep).join('/'), +).filter((root) => !isSourceFile(`${root}/`)); + // A flag takes one numeric percentage argument; the option key it sets on // the parsed options object. const PERCENT_FLAGS = { @@ -99,6 +132,9 @@ const PERCENT_FLAGS = { '--check-source': 'checkSource', '--check-source-regions': 'checkSourceRegions', '--check-source-file-regions': 'checkSourceFileRegions', + '--check-harness': 'checkHarness', + '--check-harness-regions': 'checkHarnessRegions', + '--check-harness-file-regions': 'checkHarnessFileRegions', }; function parseArgs(argv) { @@ -108,6 +144,9 @@ function parseArgs(argv) { checkSource: null, checkSourceRegions: null, checkSourceFileRegions: null, + checkHarness: null, + checkHarnessRegions: null, + checkHarnessFileRegions: null, requireSourceFiles: false, testArgs: [], }; @@ -445,6 +484,48 @@ function percent(covered, total) { return total === 0 ? 100 : (covered / total) * 100; } +// `src files` and `harness files` are the same arithmetic over two disjoint +// sets of rows, and the gates that read them are the same three shapes. They +// share an accumulator so a change to one row cannot quietly stop applying to +// the other. +function newTotals() { + return { + executable: 0, + covered: 0, + regions: 0, + regionsCovered: 0, + fileRegions: [], + }; +} + +function addRow(totals, row) { + totals.executable += row.lines.executable; + totals.covered += row.lines.covered; + totals.regions += row.regions.regions; + totals.regionsCovered += row.regions.covered; + // Files with no regions at all are left out: percent() reports 100% + // for a 0/0 fraction, so including them would add rows that can never + // fail the per-file floor and never say anything either. + if (row.regions.regions > 0) { + totals.fileRegions.push({ + file: row.file, + percent: percent(row.regions.covered, row.regions.regions), + covered: row.regions.covered, + regions: row.regions.regions, + uncovered: row.regions.uncovered, + }); + } +} + +function summaryRow(label, totals, width) { + const linePct = percent(totals.covered, totals.executable); + const regionPct = percent(totals.regionsCovered, totals.regions); + console.log( + `${label.padEnd(width)} | ${linePct.toFixed(2).padStart(6)} | ${regionPct.toFixed(2).padStart(8)} | ${totals.covered}/${totals.executable} lines | ${totals.regionsCovered}/${totals.regions} regions`, + ); + return { linePct, regionPct }; +} + // Coverage records exist for the test files too. They are not the code this // repository ships, so they are summarised separately rather than folded // into totals the suite's own near-complete self-coverage would dominate. @@ -452,6 +533,14 @@ export function isSourceFile(file) { return !file.startsWith('tests/'); } +// The harness modules under SOURCE_ROOTS: measured like source, summarised +// like neither. isSourceFile() keeps them out of the `src files` ratios, and +// this keeps them out of nothing -- it is what gives them the row and the +// floors of their own. +export function isHarnessFile(file) { + return HARNESS_ROOTS.some((root) => file.startsWith(`${root}/`)); +} + // Every module on disk under SOURCE_ROOTS, as repo-relative '/'-joined paths. // Walked rather than globbed so the traversal order is the sort order and a // directory that does not exist is simply absent instead of throwing -- the @@ -597,34 +686,15 @@ export function report(merged) { let covered = 0; let regions = 0; let regionsCovered = 0; - let sourceExecutable = 0; - let sourceCovered = 0; - let sourceRegions = 0; - let sourceRegionsCovered = 0; - const sourceFileRegions = []; + const source = newTotals(); + const harness = newTotals(); for (const row of rows) { executable += row.lines.executable; covered += row.lines.covered; regions += row.regions.regions; regionsCovered += row.regions.covered; - if (isSourceFile(row.file)) { - sourceExecutable += row.lines.executable; - sourceCovered += row.lines.covered; - sourceRegions += row.regions.regions; - sourceRegionsCovered += row.regions.covered; - // Files with no regions at all are left out: percent() reports 100% - // for a 0/0 fraction, so including them would add rows that can never - // fail the per-file floor and never say anything either. - if (row.regions.regions > 0) { - sourceFileRegions.push({ - file: row.file, - percent: percent(row.regions.covered, row.regions.regions), - covered: row.regions.covered, - regions: row.regions.regions, - uncovered: row.regions.uncovered, - }); - } - } + if (isSourceFile(row.file)) addRow(source, row); + else if (isHarnessFile(row.file)) addRow(harness, row); const linePct = percent(row.lines.covered, row.lines.executable) .toFixed(2) .padStart(6); @@ -636,10 +706,15 @@ export function report(merged) { ); } console.log('-'.repeat(header.length)); - const sourceLinePct = percent(sourceCovered, sourceExecutable); - const sourceRegionPct = percent(sourceRegionsCovered, sourceRegions); - console.log( - `${'src files'.padEnd(width)} | ${sourceLinePct.toFixed(2).padStart(6)} | ${sourceRegionPct.toFixed(2).padStart(8)} | ${sourceCovered}/${sourceExecutable} lines | ${sourceRegionsCovered}/${sourceRegions} regions`, + const { linePct: sourceLinePct, regionPct: sourceRegionPct } = summaryRow( + 'src files', + source, + width, + ); + const { linePct: harnessLinePct, regionPct: harnessRegionPct } = summaryRow( + 'harness files', + harness, + width, ); const totalLinePct = percent(covered, executable); const totalRegionPct = percent(regionsCovered, regions); @@ -651,9 +726,14 @@ export function report(merged) { totalRegionPct, sourceLinePct, sourceRegionPct, - sourceExecutable, - sourceRegions, - sourceFileRegions, + sourceExecutable: source.executable, + sourceRegions: source.regions, + sourceFileRegions: source.fileRegions, + harnessLinePct, + harnessRegionPct, + harnessExecutable: harness.executable, + harnessRegions: harness.regions, + harnessFileRegions: harness.fileRegions, }; } @@ -684,6 +764,11 @@ function main() { sourceExecutable, sourceRegions, sourceFileRegions, + harnessLinePct, + harnessRegionPct, + harnessExecutable, + harnessRegions, + harnessFileRegions, } = report(merged); const missing = options.requireSourceFiles ? missingSourceFiles([...merged.keys(), ...unmapped]) @@ -804,6 +889,66 @@ function main() { process.exit(1); } } + // The harness gates are the source gates applied to the other half of + // SOURCE_ROOTS. They are separate thresholds rather than a widened + // `src files` because the two trees answer different questions and sit + // at different percentages: folding tests/tools/ into the shipped-source + // ratios would let a harness gap be paid for out of src/'s slack, which + // is the averaging --check-source-file-regions already exists to stop. + if (options.checkHarness !== null && harnessExecutable === 0) { + console.error( + `\n--check-harness was requested, but no harness lines under ${HARNESS_ROOTS.join('/ or ')}/ were recorded.`, + ); + process.exit(1); + } + if ( + options.checkHarness !== null && + harnessLinePct + 1e-9 < options.checkHarness + ) { + console.error( + `\nHarness line coverage ${harnessLinePct.toFixed(2)}% is below the required ${options.checkHarness}%.`, + ); + process.exit(1); + } + if (options.checkHarnessRegions !== null && harnessRegions === 0) { + console.error( + `\n--check-harness-regions was requested, but no harness regions under ${HARNESS_ROOTS.join('/ or ')}/ were recorded.`, + ); + process.exit(1); + } + if ( + options.checkHarnessRegions !== null && + harnessRegionPct + 1e-9 < options.checkHarnessRegions + ) { + console.error( + `\nHarness region coverage ${harnessRegionPct.toFixed(2)}% is below the required ${options.checkHarnessRegions}%.`, + ); + process.exit(1); + } + if (options.checkHarnessFileRegions !== null) { + if (harnessFileRegions.length === 0) { + console.error( + `\n--check-harness-file-regions was requested, but no harness file under ${HARNESS_ROOTS.join('/ or ')}/ recorded any region.`, + ); + process.exit(1); + } + const below = harnessFileRegions.filter( + (entry) => entry.percent + 1e-9 < options.checkHarnessFileRegions, + ); + if (below.length > 0) { + console.error( + `\n${below.length} harness file(s) fall below the required ` + + `${options.checkHarnessFileRegions}% region coverage per file:`, + ); + for (const entry of below) { + console.error( + ` ${entry.file} ${entry.percent.toFixed(2)}% ` + + `(${entry.covered}/${entry.regions} regions; uncovered at ${formatRanges(entry.uncovered)})`, + ); + } + process.exit(1); + } + } } finally { rmSync(coverageDir, { recursive: true, force: true }); }