Skip to content

chore: remove support for old benchmark format #329

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ludofischer merged 3 commits into from
Sep 22, 2026
Merged

chore: remove support for old benchmark format #329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
920a4b6
chore: remove confusing comments
ludofischer Sep 21, 2026
3a91938
chore: update development deps
ludofischer Sep 22, 2026
d24f6fb
chore: remove support for old benchmark snapshot format
ludofischer Sep 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@
},
"devDependencies": {
"@csstools/css-calc": "^3.4.0",
"@types/node": "^26.6.1",
"fast-check": "^4.10.1",
"@types/node": "^26.6.2",
"fast-check": "^4.10.2",
"oxfmt": "^0.68.0",
"oxlint": "^1.83.0",
"postcss": "^8.5.28",
Expand Down
20 changes: 10 additions & 10 deletions pnpm-lock.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

237 changes: 14 additions & 223 deletions scripts/compare-parser-benchmarks.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
// Reanalyze one schema-v2 parser benchmark artifact. The six-transcript
// function below remains as a small compatibility API for older local tests;
// the command-line interface is intentionally artifact-based now.
// Reanalyze one schema-v2 parser benchmark artifact.
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { resolve } from 'node:path';
Expand All @@ -9,8 +7,7 @@ import { analyzeCorpus } from './lib/corpus-benchmark.js';
import { validateSchemaV2Artifact } from './lib/benchmark.js';

const usage =
'Usage: node scripts/compare-parser-benchmarks.js <schema-v2-artifact>\n' +
' (legacy six-transcript arguments are accepted by the JS API only)';
'Usage: node scripts/compare-parser-benchmarks.js <schema-v2-artifact>';

function readArtifact(path) {
const artifact = JSON.parse(readFileSync(path, 'utf8'));
Expand All @@ -27,210 +24,20 @@ export function reanalyzeParserBenchmark(path) {
return { ...artifact, analysis };
}

/** @param {string} path @return {object} */
function readResult(path) {
const line = readFileSync(path, 'utf8')
.split('\n')
.findLast((candidate) => candidate.startsWith('BENCHMARK_RESULT '));
if (!line) throw new Error(`${path}: missing BENCHMARK_RESULT line`);
return JSON.parse(line.slice('BENCHMARK_RESULT '.length));
}

/** @param {number[]} values @return {number} */
function median(values) {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.floor(sorted.length / 2)];
}

/**
* @param {string[] | string} files
* @return {{benchmark: string, summaries: object[], failures: string[]} | object}
*/
function compareParserBenchmarks(files) {
if (typeof files === 'string')
return reanalyzeParserBenchmark(files).analysis;
if (files.length !== 6) throw new Error(usage);

const results = files.map(readResult);
const benchmark = results[0].benchmark;
if (
results.some(
(result) => result.schema !== 1 || result.benchmark !== benchmark
)
) {
throw new Error(
'all benchmark transcripts must have the same schema and benchmark'
);
}

const baseline = results.slice(0, 3);
const candidate = results.slice(3);
const keyOf = (measurement) =>
benchmark === 'arithmetic-chains'
? `${measurement.kind}:${measurement.mode}:${measurement.size}`
: `${measurement.mode}:${measurement.depth}`;

let expectedKeys;
if (benchmark === 'arithmetic-chains') {
expectedKeys = ['additive', 'multiplicative'].flatMap((kind) =>
['cold-index', 'hot-shared-index'].flatMap((mode) =>
[1_000, 2_000, 4_000, 8_000].map((size) => `${kind}:${mode}:${size}`)
)
);
} else if (benchmark === 'nested-fallbacks') {
expectedKeys = ['cold-index', 'hot-shared-index'].flatMap((mode) =>
[50, 100, 200, 400].map((depth) => `${mode}:${depth}`)
);
} else {
throw new Error(`unsupported benchmark: ${benchmark}`);
}

/** @param {object} result @param {string} path */
function validateMeasurements(result, path) {
if (!Array.isArray(result.measurements)) {
throw new TypeError(`${path}: measurements must be an array`);
}

const expected = new Set(expectedKeys);
const seen = new Set();
for (const measurement of result.measurements) {
if (measurement === null || typeof measurement !== 'object') {
throw new TypeError(`${path}: invalid measurement`);
}
const key = keyOf(measurement);
if (!expected.has(key)) {
throw new Error(`${path}: unexpected measurement key ${key}`);
}
if (seen.has(key)) {
throw new Error(`${path}: duplicate measurement key ${key}`);
}
if (
typeof measurement.medianMs !== 'number' ||
!Number.isFinite(measurement.medianMs) ||
measurement.medianMs <= 0
) {
throw new TypeError(
`${path}: invalid medianMs for ${key} (must be finite and > 0)`
);
}
seen.add(key);
}

const missing = expectedKeys.filter((key) => !seen.has(key));
if (missing.length > 0) {
throw new Error(
`${path}: missing measurement keys ${missing.join(', ')}`
);
}
}

// Validate every transcript before aggregating any measurements. This keeps
// missing, duplicate, and empty runs from silently disappearing in a Map.
for (let i = 0; i < results.length; i++) {
validateMeasurements(results[i], files[i]);
}

/** @param {object[]} runs @return {Map<string, number[]>} */
function valuesByKey(runs) {
/** @type {Map<string, number[]>} */
const values = new Map();
for (const run of runs) {
for (const measurement of run.measurements) {
const key = keyOf(measurement);
const samples = values.get(key) ?? [];
samples.push(measurement.medianMs);
values.set(key, samples);
}
}
return values;
}

const baseValues = valuesByKey(baseline);
const candidateValues = valuesByKey(candidate);
const failures = [];
const summaries = [];

for (const [key, values] of candidateValues) {
const baselineSamples = baseValues.get(key);
if (
!baselineSamples ||
baselineSamples.length !== 3 ||
values.length !== 3
) {
failures.push(`${key}: expected three baseline and candidate samples`);
continue;
}
const baseMedian = median(baselineSamples);
const candidateMedian = median(values);
const ratio = candidateMedian / baseMedian;
if (!Number.isFinite(ratio)) {
failures.push(`${key}: non-finite ratio`);
continue;
}
summaries.push({ key, baseMedian, candidateMedian, ratio });

const parts = key.split(':');
const size = Number(parts.at(-1));
const largest =
benchmark === 'arithmetic-chains' ? size === 8_000 : size === 400;
if (largest && ratio > 1.1) {
failures.push(`${key}: ${ratio.toFixed(2)}x baseline (limit 1.10x)`);
}
function compareParserBenchmarks(path) {
if (typeof path !== 'string') {
throw new TypeError(usage);
}

// Recompute growth from the three-run medians rather than trusting a single
// run's printed ratios. This makes the doubling gate auditable and resistant
// to a transient sample in one invocation.
const grouped = new Map();
for (const summary of summaries) {
const parts = summary.key.split(':');
const mode = benchmark === 'arithmetic-chains' ? parts[1] : parts[0];
const family = benchmark === 'arithmetic-chains' ? parts[0] : '';
const size = Number(parts.at(-1));
const groupKey =
benchmark === 'arithmetic-chains' ? `${family}:${mode}` : mode;
const group = grouped.get(groupKey) ?? [];
group.push({ size, median: summary.candidateMedian });
grouped.set(groupKey, group);
}
for (const [groupKey, points] of grouped) {
points.sort((a, b) => a.size - b.size);
for (let i = 1; i < points.length; i++) {
const growth = points[i].median / points[i - 1].median;
if (!Number.isFinite(growth)) {
failures.push(
`${groupKey} ${points[i - 1].size}->${points[i].size}: non-finite growth`
);
continue;
}
if (growth > 2.5) {
failures.push(
`${groupKey} ${points[i - 1].size}->${points[i].size}: ${growth.toFixed(2)}x growth (limit 2.50x)`
);
}
}
}

return { benchmark, summaries, failures };
return reanalyzeParserBenchmark(path).analysis;
}

function printComparison(comparison) {
for (const summary of comparison.summaries) {
console.log(
`${summary.key.padEnd(38)} ${summary.baseMedian.toFixed(3).padStart(8)} ms -> ` +
`${summary.candidateMedian.toFixed(3).padStart(8)} ms ` +
`(${summary.ratio.toFixed(2)}x)`
);
}
if (comparison.failures.length > 0) {
console.error('\nBenchmark gates failed:');
for (const failure of comparison.failures) console.error(`- ${failure}`);
process.exitCode = 1;
} else {
console.log(
'\nBenchmark gates passed: largest medians <= 1.10x and every doubling <= 2.50x.'
);
}
function exitCodeFor(status) {
if (status === 'pass') return 0;
if (status === 'regression') return 1;
if (status === 'postcss-calc faster' || status === 'postcss-calc slower')
return 0;
if (status === 'correctness-failure') return 3;
return 2;
}

const isMain =
Expand Down Expand Up @@ -258,26 +65,10 @@ if (isMain) {
console.error(error instanceof Error ? error.message : error);
process.exitCode = 64;
}
} else if (files.length !== 6) {
} else {
console.error(usage);
process.exitCode = 64;
} else {
try {
printComparison(compareParserBenchmarks(files));
} catch (error) {
console.error(error instanceof Error ? error.message : error);
process.exitCode = 1;
}
}
}

function exitCodeFor(status) {
if (status === 'pass') return 0;
if (status === 'regression') return 1;
if (status === 'postcss-calc faster' || status === 'postcss-calc slower')
return 0;
if (status === 'correctness-failure') return 3;
return 2;
}

export { compareParserBenchmarks, exitCodeFor };
6 changes: 0 additions & 6 deletions src/lib/analyze.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,6 @@ function analyzeType(node, depth = 0) {
case 'Num':
return resolved(numberType);
case 'Dim':
// Percentages are contextual; their percent-ness is tracked so a
// `% / %` product cancels to a number. Unknown units are opaque, while
// known families can still reject px + seconds.
return node.unit === '%'
? finish(percentageType, true, true)
: resolved({ kind: 'dimension', base: baseOf(node.unit) });
Expand Down Expand Up @@ -83,9 +80,6 @@ function analyzeSum(node, depth) {
if (isFailure(child.type)) {
type = failureType;
} else if (child.type.kind === 'unknown') {
// A pure percentage sum stays percentage-typed so a surrounding
// product can cancel `% / %`; any other opaque term must widen the
// sum back to unknown.
if (isPercentage(child.type)) hasPercentage = true;
else hasUnknown = true;
} else if (type === null) {
Expand Down
Loading
Loading