From 920a4b67afae6a1ff652fd4f3e92c7f79d998ca5 Mon Sep 17 00:00:00 2001 From: Ludovico Fischer Date: Mon, 21 Sep 2026 22:58:50 +0200 Subject: [PATCH 1/3] chore: remove confusing comments --- src/lib/analyze.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/lib/analyze.js b/src/lib/analyze.js index cdf4d36..2f85a50 100644 --- a/src/lib/analyze.js +++ b/src/lib/analyze.js @@ -50,9 +50,6 @@ function analyzeType(node, depth = 0) { case 'Num': return resolved(numberType); case 'Dim': - // Percentages are contextual; their percent-ness is tracked so a - // `% / %` product cancels to a number. Unknown units are opaque, while - // known families can still reject px + seconds. return node.unit === '%' ? finish(percentageType, true, true) : resolved({ kind: 'dimension', base: baseOf(node.unit) }); @@ -83,9 +80,6 @@ function analyzeSum(node, depth) { if (isFailure(child.type)) { type = failureType; } else if (child.type.kind === 'unknown') { - // A pure percentage sum stays percentage-typed so a surrounding - // product can cancel `% / %`; any other opaque term must widen the - // sum back to unknown. if (isPercentage(child.type)) hasPercentage = true; else hasUnknown = true; } else if (type === null) { From 3a9193856ef659c1c16691cd377f9ef8fa5e50ac Mon Sep 17 00:00:00 2001 From: Ludovico Fischer Date: Tue, 22 Sep 2026 13:01:31 +0200 Subject: [PATCH 2/3] chore: update development deps --- package.json | 4 ++-- pnpm-lock.yaml | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index a7e4c98..7b56e20 100644 --- a/package.json +++ b/package.json @@ -57,8 +57,8 @@ }, "devDependencies": { "@csstools/css-calc": "^3.4.0", - "@types/node": "^26.6.1", - "fast-check": "^4.10.1", + "@types/node": "^26.6.2", + "fast-check": "^4.10.2", "oxfmt": "^0.68.0", "oxlint": "^1.83.0", "postcss": "^8.5.28", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 14f97f9..148cb45 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,11 +174,11 @@ importers: specifier: ^3.4.0 version: 3.4.0(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.1))(@csstools/css-tokenizer@4.0.1) '@types/node': - specifier: ^26.6.1 - version: 26.6.1 + specifier: ^26.6.2 + version: 26.6.2 fast-check: - specifier: ^4.10.1 - version: 4.10.1 + specifier: ^4.10.2 + version: 4.10.2 oxfmt: specifier: ^0.68.0 version: 0.68.0 @@ -455,8 +455,8 @@ packages: cpu: [x64] os: [win32] - '@types/node@26.6.1': - resolution: {integrity: sha512-VqGJBMCtdhqkBUCcBLvywI0NJ+KLuVzgNnlBUNFOQjqVxzo2lxLUNg1DSey8+u2u6ktswSAxg+s68QLzWHNOuA==} + '@types/node@26.6.2': + resolution: {integrity: sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g==} '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} @@ -578,8 +578,8 @@ packages: cpu: [x64] os: [win32] - fast-check@4.10.1: - resolution: {integrity: sha512-sB5Vghiu8MyCyToHoBVGsT0baZg3sZWNIY+a6Ct2EDrQJlT4YdH6MC1BSLNe3kX1k5i5g0q1O52XFgcKK/rGHg==} + fast-check@4.10.2: + resolution: {integrity: sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==} engines: {node: '>=12.17.0'} nanoid@3.3.19: @@ -766,7 +766,7 @@ snapshots: '@oxlint/binding-win32-x64-msvc@1.83.0': optional: true - '@types/node@26.6.1': + '@types/node@26.6.2': dependencies: undici-types: 8.9.0 @@ -830,7 +830,7 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true - fast-check@4.10.1: + fast-check@4.10.2: dependencies: pure-rand: 8.4.2 From d24f6fb23c223c2cad029d8a72a80915079b6b57 Mon Sep 17 00:00:00 2001 From: Ludovico Fischer Date: Tue, 22 Sep 2026 14:05:46 +0200 Subject: [PATCH 3/3] chore: remove support for old benchmark snapshot format --- scripts/compare-parser-benchmarks.js | 237 +------------- test/unit/compare-parser-benchmarks.test.js | 342 ++++++++++---------- 2 files changed, 186 insertions(+), 393 deletions(-) diff --git a/scripts/compare-parser-benchmarks.js b/scripts/compare-parser-benchmarks.js index 93a3c54..b203e75 100644 --- a/scripts/compare-parser-benchmarks.js +++ b/scripts/compare-parser-benchmarks.js @@ -1,6 +1,4 @@ -// Reanalyze one schema-v2 parser benchmark artifact. The six-transcript -// function below remains as a small compatibility API for older local tests; -// the command-line interface is intentionally artifact-based now. +// Reanalyze one schema-v2 parser benchmark artifact. import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { resolve } from 'node:path'; @@ -9,8 +7,7 @@ import { analyzeCorpus } from './lib/corpus-benchmark.js'; import { validateSchemaV2Artifact } from './lib/benchmark.js'; const usage = - 'Usage: node scripts/compare-parser-benchmarks.js \n' + - ' (legacy six-transcript arguments are accepted by the JS API only)'; + 'Usage: node scripts/compare-parser-benchmarks.js '; function readArtifact(path) { const artifact = JSON.parse(readFileSync(path, 'utf8')); @@ -27,210 +24,20 @@ export function reanalyzeParserBenchmark(path) { return { ...artifact, analysis }; } -/** @param {string} path @return {object} */ -function readResult(path) { - const line = readFileSync(path, 'utf8') - .split('\n') - .findLast((candidate) => candidate.startsWith('BENCHMARK_RESULT ')); - if (!line) throw new Error(`${path}: missing BENCHMARK_RESULT line`); - return JSON.parse(line.slice('BENCHMARK_RESULT '.length)); -} - -/** @param {number[]} values @return {number} */ -function median(values) { - const sorted = [...values].sort((a, b) => a - b); - return sorted[Math.floor(sorted.length / 2)]; -} - -/** - * @param {string[] | string} files - * @return {{benchmark: string, summaries: object[], failures: string[]} | object} - */ -function compareParserBenchmarks(files) { - if (typeof files === 'string') - return reanalyzeParserBenchmark(files).analysis; - if (files.length !== 6) throw new Error(usage); - - const results = files.map(readResult); - const benchmark = results[0].benchmark; - if ( - results.some( - (result) => result.schema !== 1 || result.benchmark !== benchmark - ) - ) { - throw new Error( - 'all benchmark transcripts must have the same schema and benchmark' - ); - } - - const baseline = results.slice(0, 3); - const candidate = results.slice(3); - const keyOf = (measurement) => - benchmark === 'arithmetic-chains' - ? `${measurement.kind}:${measurement.mode}:${measurement.size}` - : `${measurement.mode}:${measurement.depth}`; - - let expectedKeys; - if (benchmark === 'arithmetic-chains') { - expectedKeys = ['additive', 'multiplicative'].flatMap((kind) => - ['cold-index', 'hot-shared-index'].flatMap((mode) => - [1_000, 2_000, 4_000, 8_000].map((size) => `${kind}:${mode}:${size}`) - ) - ); - } else if (benchmark === 'nested-fallbacks') { - expectedKeys = ['cold-index', 'hot-shared-index'].flatMap((mode) => - [50, 100, 200, 400].map((depth) => `${mode}:${depth}`) - ); - } else { - throw new Error(`unsupported benchmark: ${benchmark}`); - } - - /** @param {object} result @param {string} path */ - function validateMeasurements(result, path) { - if (!Array.isArray(result.measurements)) { - throw new TypeError(`${path}: measurements must be an array`); - } - - const expected = new Set(expectedKeys); - const seen = new Set(); - for (const measurement of result.measurements) { - if (measurement === null || typeof measurement !== 'object') { - throw new TypeError(`${path}: invalid measurement`); - } - const key = keyOf(measurement); - if (!expected.has(key)) { - throw new Error(`${path}: unexpected measurement key ${key}`); - } - if (seen.has(key)) { - throw new Error(`${path}: duplicate measurement key ${key}`); - } - if ( - typeof measurement.medianMs !== 'number' || - !Number.isFinite(measurement.medianMs) || - measurement.medianMs <= 0 - ) { - throw new TypeError( - `${path}: invalid medianMs for ${key} (must be finite and > 0)` - ); - } - seen.add(key); - } - - const missing = expectedKeys.filter((key) => !seen.has(key)); - if (missing.length > 0) { - throw new Error( - `${path}: missing measurement keys ${missing.join(', ')}` - ); - } - } - - // Validate every transcript before aggregating any measurements. This keeps - // missing, duplicate, and empty runs from silently disappearing in a Map. - for (let i = 0; i < results.length; i++) { - validateMeasurements(results[i], files[i]); - } - - /** @param {object[]} runs @return {Map} */ - function valuesByKey(runs) { - /** @type {Map} */ - const values = new Map(); - for (const run of runs) { - for (const measurement of run.measurements) { - const key = keyOf(measurement); - const samples = values.get(key) ?? []; - samples.push(measurement.medianMs); - values.set(key, samples); - } - } - return values; - } - - const baseValues = valuesByKey(baseline); - const candidateValues = valuesByKey(candidate); - const failures = []; - const summaries = []; - - for (const [key, values] of candidateValues) { - const baselineSamples = baseValues.get(key); - if ( - !baselineSamples || - baselineSamples.length !== 3 || - values.length !== 3 - ) { - failures.push(`${key}: expected three baseline and candidate samples`); - continue; - } - const baseMedian = median(baselineSamples); - const candidateMedian = median(values); - const ratio = candidateMedian / baseMedian; - if (!Number.isFinite(ratio)) { - failures.push(`${key}: non-finite ratio`); - continue; - } - summaries.push({ key, baseMedian, candidateMedian, ratio }); - - const parts = key.split(':'); - const size = Number(parts.at(-1)); - const largest = - benchmark === 'arithmetic-chains' ? size === 8_000 : size === 400; - if (largest && ratio > 1.1) { - failures.push(`${key}: ${ratio.toFixed(2)}x baseline (limit 1.10x)`); - } +function compareParserBenchmarks(path) { + if (typeof path !== 'string') { + throw new TypeError(usage); } - - // Recompute growth from the three-run medians rather than trusting a single - // run's printed ratios. This makes the doubling gate auditable and resistant - // to a transient sample in one invocation. - const grouped = new Map(); - for (const summary of summaries) { - const parts = summary.key.split(':'); - const mode = benchmark === 'arithmetic-chains' ? parts[1] : parts[0]; - const family = benchmark === 'arithmetic-chains' ? parts[0] : ''; - const size = Number(parts.at(-1)); - const groupKey = - benchmark === 'arithmetic-chains' ? `${family}:${mode}` : mode; - const group = grouped.get(groupKey) ?? []; - group.push({ size, median: summary.candidateMedian }); - grouped.set(groupKey, group); - } - for (const [groupKey, points] of grouped) { - points.sort((a, b) => a.size - b.size); - for (let i = 1; i < points.length; i++) { - const growth = points[i].median / points[i - 1].median; - if (!Number.isFinite(growth)) { - failures.push( - `${groupKey} ${points[i - 1].size}->${points[i].size}: non-finite growth` - ); - continue; - } - if (growth > 2.5) { - failures.push( - `${groupKey} ${points[i - 1].size}->${points[i].size}: ${growth.toFixed(2)}x growth (limit 2.50x)` - ); - } - } - } - - return { benchmark, summaries, failures }; + return reanalyzeParserBenchmark(path).analysis; } -function printComparison(comparison) { - for (const summary of comparison.summaries) { - console.log( - `${summary.key.padEnd(38)} ${summary.baseMedian.toFixed(3).padStart(8)} ms -> ` + - `${summary.candidateMedian.toFixed(3).padStart(8)} ms ` + - `(${summary.ratio.toFixed(2)}x)` - ); - } - if (comparison.failures.length > 0) { - console.error('\nBenchmark gates failed:'); - for (const failure of comparison.failures) console.error(`- ${failure}`); - process.exitCode = 1; - } else { - console.log( - '\nBenchmark gates passed: largest medians <= 1.10x and every doubling <= 2.50x.' - ); - } +function exitCodeFor(status) { + if (status === 'pass') return 0; + if (status === 'regression') return 1; + if (status === 'postcss-calc faster' || status === 'postcss-calc slower') + return 0; + if (status === 'correctness-failure') return 3; + return 2; } const isMain = @@ -258,26 +65,10 @@ if (isMain) { console.error(error instanceof Error ? error.message : error); process.exitCode = 64; } - } else if (files.length !== 6) { + } else { console.error(usage); process.exitCode = 64; - } else { - try { - printComparison(compareParserBenchmarks(files)); - } catch (error) { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; - } } } -function exitCodeFor(status) { - if (status === 'pass') return 0; - if (status === 'regression') return 1; - if (status === 'postcss-calc faster' || status === 'postcss-calc slower') - return 0; - if (status === 'correctness-failure') return 3; - return 2; -} - export { compareParserBenchmarks, exitCodeFor }; diff --git a/test/unit/compare-parser-benchmarks.test.js b/test/unit/compare-parser-benchmarks.test.js index 8436f15..953f2f6 100644 --- a/test/unit/compare-parser-benchmarks.test.js +++ b/test/unit/compare-parser-benchmarks.test.js @@ -3,201 +3,203 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; import { compareParserBenchmarks, exitCodeFor, + reanalyzeParserBenchmark, } from '../../scripts/compare-parser-benchmarks.js'; +import { syntheticParserArtifact } from '../helpers/benchmark-artifact.js'; +import { + CORPUS_INTERVAL_METHOD, + DECISION_CONFIG_VERSION, + PRECISION_METHOD, +} from '../../scripts/lib/benchmark.js'; -const arithmeticKeys = ['additive', 'multiplicative'].flatMap((kind) => - ['cold-index', 'hot-shared-index'].flatMap((mode) => - [1_000, 2_000, 4_000, 8_000].map((size) => `${kind}:${mode}:${size}`) - ) -); -const nestedKeys = ['cold-index', 'hot-shared-index'].flatMap((mode) => - [50, 100, 200, 400].map((depth) => `${mode}:${depth}`) -); - -function makeResult(benchmark = 'arithmetic-chains', medianMs = 1) { - const keys = benchmark === 'arithmetic-chains' ? arithmeticKeys : nestedKeys; - return { - schema: 1, - benchmark, - measurements: keys.map((key) => { - const parts = key.split(':'); - if (benchmark === 'arithmetic-chains') { - const [kind, mode, size] = parts; - return { kind, mode, size: Number(size), medianMs }; - } - const [mode, depth] = parts; - return { mode, depth: Number(depth), medianMs }; - }), - }; -} +test('exitCodeFor maps benchmark analysis statuses to exit codes', () => { + assert.equal(exitCodeFor('pass'), 0); + assert.equal(exitCodeFor('regression'), 1); + assert.equal(exitCodeFor('postcss-calc faster'), 0); + assert.equal(exitCodeFor('postcss-calc slower'), 0); + assert.equal(exitCodeFor('correctness-failure'), 3); + assert.equal(exitCodeFor('inconclusive'), 2); + assert.equal(exitCodeFor('unknown'), 2); +}); -function runComparator(benchmark, edit = () => {}) { +test('reanalyzes schema-v2 parser benchmark artifacts', () => { const directory = mkdtempSync(join(tmpdir(), 'postcss-calc-benchmark-')); try { - const results = Array.from({ length: 6 }, () => makeResult(benchmark)); - edit(results); - const paths = results.map((value, index) => { - const path = join(directory, `run-${index}.json`); - writeFileSync(path, `BENCHMARK_RESULT ${JSON.stringify(value)}\n`); - return path; + const artifact = syntheticParserArtifact({ + rows: Array.from({ length: 20 }, () => ({ + baseline: [1, 2], + candidate: [1, 2], + })), + workloadKeys: ['additive:cold-index:1000', 'additive:cold-index:2000'], }); - return compareParserBenchmarks(paths); + const artifactPath = join(directory, 'parser-artifact.json'); + writeFileSync(artifactPath, JSON.stringify(artifact)); + + const result = reanalyzeParserBenchmark(artifactPath); + assert.equal(result.schema, 2); + assert.equal(result.benchmark, 'parser-simulation'); + assert.equal(result.analysis.status, 'pass'); + + const analysisOnly = compareParserBenchmarks(artifactPath); + assert.deepEqual(analysisOnly, result.analysis); } finally { rmSync(directory, { recursive: true, force: true }); } -} - -test('benchmark comparator rejects a run with a missing key', () => { - assert.throws( - () => - runComparator('arithmetic-chains', (runs) => { - runs[3].measurements.pop(); - }), - /missing measurement keys/ - ); }); -test('schema-v2 correctness failures use the correctness exit code', () => { - assert.equal(exitCodeFor('pass'), 0); - assert.equal(exitCodeFor('regression'), 1); - assert.equal(exitCodeFor('inconclusive'), 2); - assert.equal(exitCodeFor('correctness-failure'), 3); -}); - -test('benchmark comparator rejects an empty measurement run', () => { - assert.throws( - () => - runComparator('arithmetic-chains', (runs) => { - runs[0].measurements = []; - }), - /missing measurement keys/ - ); -}); - -test('benchmark comparator rejects duplicate keys', () => { - assert.throws( - () => - runComparator('arithmetic-chains', (runs) => { - runs[3].measurements.push(runs[3].measurements[0]); - }), - /duplicate measurement key/ - ); -}); - -test('benchmark comparator rejects schema mismatches', () => { - assert.throws( - () => - runComparator('arithmetic-chains', (runs) => { - runs[1].schema = 2; - }), - /same schema and benchmark/ - ); +test('reanalyzes schema-v2 corpus benchmark artifacts', () => { + const directory = mkdtempSync(join(tmpdir(), 'postcss-calc-benchmark-')); + try { + const groups = ['exact', 'sum']; + const artifact = { + schema: 2, + benchmark: 'corpus', + seed: 123, + config: { + decisionConfigVersion: DECISION_CONFIG_VERSION, + requestedBlocks: 20, + minimumBlocks: 20, + maxAttempts: 20, + targetBatchMs: 25, + warmupMinimum: 0, + warmupMaximum: 0, + measuredBatchCount: 6, + driftThreshold: 0.15, + bootstrapResamples: 1_000, + confidence: 0.95, + runtimeNonRegressionMargin: 1.1, + equivalenceMargin: 1.1, + precisionMargin: 1.1, + precisionMethod: PRECISION_METHOD, + growthThreshold: 2.5, + orderInteractionThreshold: Math.log(1.1), + intervalMethod: CORPUS_INTERVAL_METHOD, + replicates: 20, + batches: 6, + calibrationOrderBalanced: true, + }, + corpus: { lengthStrata: {}, rootShapeCounts: { sum: 2 } }, + correctness: { + accepted: 2, + counts: Object.fromEntries( + [ + 'accepted', + 'both-failed', + 'known-divergence', + 'malformed-input', + 'parser-rejected', + 'reference-rejected', + 'unexpected-divergence', + ].map((c) => [c, c === 'accepted' ? 2 : 0]) + ), + categoryHashes: Object.fromEntries( + [ + 'accepted', + 'both-failed', + 'known-divergence', + 'malformed-input', + 'parser-rejected', + 'reference-rejected', + 'unexpected-divergence', + ].map((c) => [c, c]) + ), + inputHash: 'inputs', + }, + replicates: Array.from({ length: 20 }, (_, replicate) => ({ + replicate, + calibrationOrder: replicate < 10 ? 'ours-first' : 'reference-first', + permutation: [0, 1], + batches: Array.from({ length: 6 }, (unusedBatch, batch) => ({ + order: batch < 3 ? 'ours-first' : 'reference-first', + measurements: groups.map((group) => ({ + group, + repetitions: 1, + calibrationOrder: replicate < 10 ? 'ours-first' : 'reference-first', + calibrationSamplesMs: [{ oursMs: 1, referenceMs: 1 }], + ours: { ms: 1, elapsedMs: 1, checksum: 1 }, + reference: { ms: 1, elapsedMs: 1, checksum: 1 }, + })), + })), + })), + }; + const artifactPath = join(directory, 'corpus-artifact.json'); + writeFileSync(artifactPath, JSON.stringify(artifact)); + + const result = reanalyzeParserBenchmark(artifactPath); + assert.equal(result.schema, 2); + assert.equal(result.benchmark, 'corpus'); + assert.ok(result.analysis); + assert.equal(typeof result.analysis.status, 'string'); + assert.ok(result.analysis.practical); + } finally { + rmSync(directory, { recursive: true, force: true }); + } }); -test('benchmark comparator rejects zero and negative timings', () => { - for (const medianMs of [0, -1]) { +test('rejects non-schema-v2 artifacts and non-string paths', () => { + const directory = mkdtempSync(join(tmpdir(), 'postcss-calc-benchmark-')); + try { + const invalidPath = join(directory, 'v1-artifact.json'); + writeFileSync(invalidPath, JSON.stringify({ schema: 1 })); + assert.throws( + () => reanalyzeParserBenchmark(invalidPath), + /artifact must use schema 2/ + ); assert.throws( - () => - runComparator('arithmetic-chains', (runs) => { - runs[0].measurements[0].medianMs = medianMs; - }), - /invalid medianMs.*finite and > 0/ + () => compareParserBenchmarks([invalidPath]), + /Usage: node scripts\/compare-parser-benchmarks\.js / ); + } finally { + rmSync(directory, { recursive: true, force: true }); } }); -test('benchmark comparator rejects non-finite computed ratios', () => { - const comparison = runComparator('arithmetic-chains', (runs) => { - for (const run of runs.slice(0, 3)) run.measurements[0].medianMs = 1e-308; - for (const run of runs.slice(3)) run.measurements[0].medianMs = 1e308; - }); - assert.ok( - comparison.failures.includes('additive:cold-index:1000: non-finite ratio') - ); - assert.equal( - comparison.summaries.some( - (summary) => summary.key === 'additive:cold-index:1000' - ), - false - ); -}); - -test('benchmark comparator rejects non-finite computed growth', () => { - const comparison = runComparator('nested-fallbacks', (runs) => { - for (const run of runs) { - for (const measurement of run.measurements) { - if (measurement.depth === 100) measurement.medianMs = 1e-308; - if (measurement.depth === 200) measurement.medianMs = 1e308; - } - } - }); - assert.deepEqual(comparison.failures, [ - 'cold-index 100->200: non-finite growth', - 'hot-shared-index 100->200: non-finite growth', - ]); -}); - -test('benchmark comparator reports largest-case regressions for every arithmetic kind and mode', () => { - for (const key of arithmeticKeys.filter((candidate) => - candidate.endsWith(':8000') - )) { - const comparison = runComparator('arithmetic-chains', (runs) => { - for (const run of runs.slice(3)) { - const measurement = run.measurements.find( - (candidate) => - `${candidate.kind}:${candidate.mode}:${candidate.size}` === key - ); - measurement.medianMs = 2; - } +test('CLI outputs analysis and exits with expected codes', () => { + const directory = mkdtempSync(join(tmpdir(), 'postcss-calc-benchmark-')); + try { + const artifact = syntheticParserArtifact({ + rows: Array.from({ length: 20 }, () => ({ + baseline: [1, 2], + candidate: [1, 2], + })), + workloadKeys: ['additive:cold-index:1000', 'additive:cold-index:2000'], }); - assert.deepEqual(comparison.failures, [ - `${key}: 2.00x baseline (limit 1.10x)`, - ]); - } -}); + const artifactPath = join(directory, 'parser-artifact.json'); + writeFileSync(artifactPath, JSON.stringify(artifact)); -test('benchmark comparator covers nested-fallbacks measurements and modes', () => { - const comparison = runComparator('nested-fallbacks'); - assert.deepEqual(comparison.failures, []); - assert.equal(comparison.summaries.length, nestedKeys.length); + const scriptPath = join( + process.cwd(), + 'scripts/compare-parser-benchmarks.js' + ); - for (const key of nestedKeys.filter((candidate) => - candidate.endsWith(':400') - )) { - const regression = runComparator('nested-fallbacks', (runs) => { - for (const run of runs.slice(3)) { - const measurement = run.measurements.find( - (candidate) => `${candidate.mode}:${candidate.depth}` === key - ); - measurement.medianMs = 2; - } + const validRun = spawnSync(process.execPath, [scriptPath, artifactPath], { + encoding: 'utf8', }); - assert.deepEqual(regression.failures, [ - `${key}: 2.00x baseline (limit 1.10x)`, - ]); - } -}); + assert.equal(validRun.status, 0); + assert.match(validRun.stdout, /Parser benchmark: pass/); -test('benchmark comparator fails a 2.5x doubling-growth regression', () => { - const comparison = runComparator('nested-fallbacks', (runs) => { - for (const run of runs.slice(3)) { - for (const measurement of run.measurements) { - if (measurement.depth === 200) measurement.medianMs = 0.1; - if (measurement.depth === 400) measurement.medianMs = 1; + const noArgs = spawnSync(process.execPath, [scriptPath], { + encoding: 'utf8', + }); + assert.equal(noArgs.status, 64); + assert.match(noArgs.stderr, /Usage:/); + + const invalidArtifactPath = join(directory, 'bad.json'); + writeFileSync(invalidArtifactPath, JSON.stringify({ schema: 1 })); + const invalidRun = spawnSync( + process.execPath, + [scriptPath, invalidArtifactPath], + { + encoding: 'utf8', } - } - }); - assert.deepEqual(comparison.failures, [ - 'cold-index 200->400: 10.00x growth (limit 2.50x)', - 'hot-shared-index 200->400: 10.00x growth (limit 2.50x)', - ]); -}); - -test('benchmark comparator accepts complete matching arithmetic runs', () => { - const comparison = runComparator('arithmetic-chains'); - assert.deepEqual(comparison.failures, []); - assert.equal(comparison.summaries.length, arithmeticKeys.length); + ); + assert.equal(invalidRun.status, 64); + assert.match(invalidRun.stderr, /artifact must use schema 2/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } });