From cbd12b3068e1400bfef41ee74e01a8177b688dcd Mon Sep 17 00:00:00 2001 From: "kubestellar-hive[bot]" Date: Sun, 20 Sep 2026 11:11:54 -0400 Subject: [PATCH 1/2] test: resolve the test harness repo root with fileURLToPath tests/helpers.mjs and tests/validators-smoke.test.mjs both derived the repository root from URL.prototype.pathname, which stays percent-encoded. On a checkout whose path contains a space the root resolved to a directory that does not exist and all 55 tests failed in cpSync with ENOENT before any script under test was spawned. Resolve the root with fileURLToPath instead, and export the resolution so it can be asserted directly. Add tests/helpers.test.mjs covering the fixture-sandbox contract, which had no tests of its own even though every fixture-based validator test runs through it: percent-decoded root resolution, scripts/lib mirroring, nested fixture placement, isolation from the real repository, isolation between runs, temp-directory cleanup, and distinct reporting of status, stdout and stderr. Signed-off-by: kubestellar-hive[bot] --- tests/helpers.mjs | 12 ++- tests/helpers.test.mjs | 136 ++++++++++++++++++++++++++++++++ tests/validators-smoke.test.mjs | 3 +- 3 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 tests/helpers.test.mjs diff --git a/tests/helpers.mjs b/tests/helpers.mjs index 35b155d4..cefef189 100644 --- a/tests/helpers.mjs +++ b/tests/helpers.mjs @@ -2,8 +2,18 @@ import { spawnSync } from 'node:child_process'; import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; -const repoRoot = new URL('..', import.meta.url).pathname; +// `URL.prototype.pathname` stays percent-encoded, so a checkout whose path +// contains a space (or any other character the URL parser escapes) yields +// `/tmp/space%20dir/repo/` — a directory that does not exist. Every sandbox +// run then dies in cpSync with ENOENT. fileURLToPath performs the decoding +// that turns a file URL back into a filesystem path. +export function resolveRepoRoot(moduleUrl) { + return fileURLToPath(new URL('..', moduleUrl)); +} + +const repoRoot = resolveRepoRoot(import.meta.url); // Runs a script from scripts/ against fixture data by mirroring the repo // layout in a temp directory. The scripts resolve inputs relative to their diff --git a/tests/helpers.test.mjs b/tests/helpers.test.mjs new file mode 100644 index 00000000..396beca0 --- /dev/null +++ b/tests/helpers.test.mjs @@ -0,0 +1,136 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { existsSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { resolveRepoRoot, runScriptWithFixtures } from './helpers.mjs'; + +// tests/helpers.mjs is the harness every fixture-based validator test runs +// through, so a silent regression in it degrades those suites rather than +// failing them: a sandbox that quietly reads the real repository would make +// every "rejects X" assertion pass for the wrong reason. Nothing else in the +// suite exercises the harness itself. + +const repoRoot = resolveRepoRoot(import.meta.url); + +function sandboxCount() { + return readdirSync(tmpdir()).filter((entry) => + entry.startsWith('endusers-test-'), + ).length; +} + +test('resolveRepoRoot decodes percent-encoded path segments', () => { + // `new URL('..', url).pathname` returns '/tmp/space%20dir/repo/' here, which + // names no directory on disk. Every sandbox run under such a checkout dies + // in cpSync with ENOENT. + assert.equal( + resolveRepoRoot('file:///tmp/space dir/repo/tests/helpers.mjs'), + '/tmp/space dir/repo/', + ); +}); + +test('resolveRepoRoot resolves this checkout to the repository root', () => { + assert.ok( + existsSync(join(repoRoot, 'scripts')), + `resolveRepoRoot returned ${repoRoot}, which has no scripts/ directory`, + ); + assert.ok(existsSync(join(repoRoot, 'tests', 'helpers.mjs'))); +}); + +test('the sandbox mirrors scripts/lib so shared imports resolve', () => { + // validate-awards.mjs imports ./lib/validate-utils.mjs. If the mirror ever + // stopped copying scripts/lib, the script would die with + // ERR_MODULE_NOT_FOUND and a status of 1 — indistinguishable, to a test + // that only asserts a non-zero exit, from a rejected fixture. + const result = runScriptWithFixtures('validate-awards.mjs', { + 'data/awards.json': JSON.stringify({ + verifiedAt: '2026-08-08', + verifiedAgainst: 'https://contribute.cncf.io/community/awards/', + awards: [ + { + year: 2024, + slug: 'acme', + award: 'Top End User Award', + awardLabel: 'Winner', + organization: 'Acme Corp', + citation: 'For outstanding adoption of cloud native.', + event: 'KubeCon NA 2024', + announcementUrl: 'https://www.cncf.io/announcements/2024/example', + }, + ], + }), + }); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stderr, /ERR_MODULE_NOT_FOUND/); +}); + +test('fixtures are written at the repo-relative paths the script reads', () => { + // validate-button-contrast.mjs reads ../src/css/custom.css, a nested path + // that the harness has to create on the way. + const css = ['#005ea8', '#004f91', '#007f68', '#006f5b'] + .map((color, index) => + index % 2 === 0 + ? `--cncf-button-background: ${color};` + : `--cncf-button-background-hover: ${color};`, + ) + .join('\n'); + const result = runScriptWithFixtures('validate-button-contrast.mjs', { + 'src/css/custom.css': css, + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /4 theme backgrounds/); +}); + +test('a script reads the sandbox rather than the real repository', () => { + // Supplying no fixture must fail. If the sandbox leaked the real checkout, + // the script would find the repository's own custom.css and pass. + const result = runScriptWithFixtures('validate-button-contrast.mjs', {}); + assert.notEqual( + result.status, + 0, + 'the script succeeded with no fixture, so it read outside the sandbox', + ); +}); + +test('each run gets a sandbox of its own', () => { + const good = ['#005ea8', '#004f91', '#007f68', '#006f5b'] + .map((color, index) => + index % 2 === 0 + ? `--cncf-button-background: ${color};` + : `--cncf-button-background-hover: ${color};`, + ) + .join('\n'); + const first = runScriptWithFixtures('validate-button-contrast.mjs', { + 'src/css/custom.css': good, + }); + assert.equal(first.status, 0, first.stderr); + const second = runScriptWithFixtures('validate-button-contrast.mjs', {}); + assert.notEqual( + second.status, + 0, + 'the second run saw the first run\u2019s fixture', + ); +}); + +test('the sandbox directory is removed once the run returns', () => { + const before = sandboxCount(); + runScriptWithFixtures('validate-button-contrast.mjs', {}); + assert.equal( + sandboxCount(), + before, + 'a sandbox directory survived the run and leaked into the temp directory', + ); +}); + +test('stdout, stderr and status are reported separately', () => { + const result = runScriptWithFixtures('validate-awards.mjs', { + 'data/awards.json': JSON.stringify({ + verifiedAt: '2026-08-08', + verifiedAgainst: 'https://contribute.cncf.io/community/awards/', + awards: [], + }), + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /non-empty array/); + assert.equal(result.stdout, ''); +}); diff --git a/tests/validators-smoke.test.mjs b/tests/validators-smoke.test.mjs index adeb6c15..ef3cc55f 100644 --- a/tests/validators-smoke.test.mjs +++ b/tests/validators-smoke.test.mjs @@ -2,8 +2,9 @@ import { execFileSync } from 'node:child_process'; import assert from 'node:assert/strict'; import test from 'node:test'; import { join } from 'node:path'; +import { resolveRepoRoot } from './helpers.mjs'; -const repoRoot = new URL('..', import.meta.url).pathname; +const repoRoot = resolveRepoRoot(import.meta.url); // Read-only validators that must pass against the repository's current // generated data. Scripts that clone upstream repos or call network APIs From c96210b10eda9c83f65d3204dada81778d01ec66 Mon Sep 17 00:00:00 2001 From: "kubestellar-hive[bot]" Date: Sun, 20 Sep 2026 15:16:28 -0400 Subject: [PATCH 2/2] test: pin sandbox-cleanup assertion to the run's own directory The cleanup assertion counted endusers-test-* entries in the shared temp directory before and after a run. node --test executes test files in parallel processes, so a sandbox created and removed by another file changes that count mid-assertion; CI failed with 1 !== 2 against a harness that cleans up correctly. Recover the sandbox path from the ENOENT the fixture-less run emits and assert that exact directory is gone, which is independent of concurrent runs. Signed-off-by: kubestellar-hive[bot] --- tests/helpers.test.mjs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/helpers.test.mjs b/tests/helpers.test.mjs index 396beca0..83b1ed43 100644 --- a/tests/helpers.test.mjs +++ b/tests/helpers.test.mjs @@ -1,7 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { existsSync, readdirSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { resolveRepoRoot, runScriptWithFixtures } from './helpers.mjs'; @@ -13,12 +12,6 @@ import { resolveRepoRoot, runScriptWithFixtures } from './helpers.mjs'; const repoRoot = resolveRepoRoot(import.meta.url); -function sandboxCount() { - return readdirSync(tmpdir()).filter((entry) => - entry.startsWith('endusers-test-'), - ).length; -} - test('resolveRepoRoot decodes percent-encoded path segments', () => { // `new URL('..', url).pathname` returns '/tmp/space%20dir/repo/' here, which // names no directory on disk. Every sandbox run under such a checkout dies @@ -113,12 +106,21 @@ test('each run gets a sandbox of its own', () => { }); test('the sandbox directory is removed once the run returns', () => { - const before = sandboxCount(); - runScriptWithFixtures('validate-button-contrast.mjs', {}); - assert.equal( - sandboxCount(), - before, - 'a sandbox directory survived the run and leaked into the temp directory', + // A run with no fixture dies in readFileSync, and the ENOENT names the + // sandbox it was reading. Recovering the path that way pins the assertion to + // this run's own directory; counting `endusers-test-*` entries in the shared + // temp directory instead races the sandboxes the other test files create and + // delete concurrently, which is what made this assertion fail in CI against + // a harness that cleans up correctly. + const result = runScriptWithFixtures('validate-button-contrast.mjs', {}); + const match = result.stderr.match(/(\S*endusers-test-[^/\\]+)[/\\]src[/\\]/); + assert.ok( + match, + `could not recover the sandbox path from stderr: ${result.stderr}`, + ); + assert.ok( + !existsSync(match[1]), + `sandbox ${match[1]} survived the run and leaked into the temp directory`, ); });