Add scoped retirement performance matrix

packages/perf-harness/README.md

@@ -0,0 +1,55 @@
+# Graphile scoped/retirement performance harness
+
+This package measures only the two changes in the preceding stacked PRs:
+
+| Arm | Catalog introspection | Introspection client | Build state |
+| --- | --- | --- | --- |
+| `stock` | stock | reuse | retained |
+| `scoped` | dependency closure | destroy | retained |
+| `retire` | stock | reuse | retired |
+| `scoped-retire` | dependency closure | destroy | retired |
+
+Every arm/repetition runs in a new Node process with `--expose-gc`. The harness
+uses the Graphile default presets directly, so CNC application plugins (including
+pg-many-to-many) are not part of the measurement. It verifies a simple query and
+requires every successful run to produce the same printed-schema hash.
+
+## Run
+
+Build the package, point it at a disposable PostgreSQL database, and use the
+same schema fixture for every arm:
+
+```sh
+pnpm --filter @constructive-io/perf-harness build
+export CPERF_DATABASE_URL=postgres:///cperf
+node packages/perf-harness/dist/index.js prepare \
+  --schema cperf_example \
+  --tables 64
+node packages/perf-harness/dist/index.js run \
+  --schemas cperf_example \
+  --repetitions 5 \
+  --seed 20260813 \
+  --output perf-results/scoped-retirement.json
+```
+
+`prepare` is intentionally conservative: it only accepts a previously absent
+schema whose name starts with `cperf_`; it never drops or replaces a schema.
+For an existing representative database, omit `prepare` and pass its exposed
+schemas to `run`. Add cross-schema dependencies explicitly with
+`--allowed-dependency-schemas`.
+
+By default each repetition gets a deterministic seeded shuffle. To pin an exact
+order, pass all four arms once, for example:
+
+```sh
+--order stock,scoped,retire,scoped-retire
+```
+
+The output JSON contains every raw sample, median summaries, all useful 2×2
+pairwise deltas, the complete schedule, process IDs, and validation results.
+Memory values are bytes and include both post-build snapshots and the process's
+cumulative peak RSS. Database credentials are passed to workers through the
+environment and are not written to the result.
+
+This is a focused regression/attribution harness. It is not a production
+capacity, concurrency, or tenant-density test.

packages/perf-harness/__tests__/fixture.test.ts

@@ -0,0 +1,20 @@
+import {
+  validateFixtureSchema,
+  validateFixtureTableCount,
+} from '../src/fixture';
+
+describe('fixture safety', () => {
+  test('only accepts a narrowly scoped benchmark schema name', () => {
+    expect(validateFixtureSchema('cperf_example_1')).toBe('cperf_example_1');
+    expect(() => validateFixtureSchema('public')).toThrow('must start with cperf_');
+    expect(() => validateFixtureSchema('cperf_example; drop schema public')).toThrow(
+      'must start with cperf_'
+    );
+  });
+
+  test('bounds generated fixture size', () => {
+    expect(validateFixtureTableCount(64)).toBe(64);
+    expect(() => validateFixtureTableCount(0)).toThrow('between 1 and 500');
+    expect(() => validateFixtureTableCount(501)).toThrow('between 1 and 500');
+  });
+});

packages/perf-harness/__tests__/fixtures/fake-worker.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const config = JSON.parse(
+  Buffer.from(process.env.CPERF_WORKER_CONFIG, 'base64url').toString('utf8')
+);
+const scoped = config.arm === 'scoped' || config.arm === 'scoped-retire';
+const retire = config.arm === 'retire' || config.arm === 'scoped-retire';
+const value = { stock: 40, scoped: 30, retire: 20, 'scoped-retire': 10 }[
+  config.arm
+];
+const memory = {
+  rss: value,
+  heapTotal: value,
+  heapUsed: value,
+  external: value,
+  arrayBuffers: value,
+};
+const result = {
+  status: 'ok',
+  pid: process.pid,
+  arm: config.arm,
+  definition: {
+    name: config.arm,
+    scopedIntrospection: scoped,
+    retireBuildState: retire,
+    introspectionMode: scoped ? 'scoped-required' : 'stock',
+    scopedCatalogTypes: scoped ? 'dependency-closure' : null,
+    introspectionClientReleaseMode: scoped ? 'destroy' : 'reuse',
+  },
+  buildMs: value,
+  schemaHash: 'fixture-schema-hash',
+  schemaTypeCount: 10,
+  queryVerified: true,
+  buildStateReleased: retire,
+  memory: {
+    baseline: memory,
+    afterBuild: memory,
+    delta: memory,
+    processPeakRss: value,
+  },
+};
+process.stdout.write(`CPERF_RESULT ${JSON.stringify(result)}\n`);

packages/perf-harness/__tests__/matrix.test.ts

@@ -0,0 +1,105 @@
+import { compareArms, makeSchedule, summarizeArm } from '../src/matrix';
+import { ARM_NAMES, type MatrixRun, type SuccessfulWorkerResult } from '../src/types';
+
+const successfulResult = (
+  arm: (typeof ARM_NAMES)[number],
+  value: number
+): SuccessfulWorkerResult => ({
+  status: 'ok',
+  pid: value,
+  arm,
+  definition: {
+    name: arm,
+    scopedIntrospection: arm === 'scoped' || arm === 'scoped-retire',
+    retireBuildState: arm === 'retire' || arm === 'scoped-retire',
+    introspectionMode:
+      arm === 'scoped' || arm === 'scoped-retire'
+        ? 'scoped-required'
+        : 'stock',
+    scopedCatalogTypes:
+      arm === 'scoped' || arm === 'scoped-retire'
+        ? 'dependency-closure'
+        : null,
+    introspectionClientReleaseMode:
+      arm === 'scoped' || arm === 'scoped-retire' ? 'destroy' : 'reuse',
+  },
+  buildMs: value,
+  schemaHash: 'same',
+  schemaTypeCount: 10,
+  queryVerified: true,
+  buildStateReleased: arm === 'retire' || arm === 'scoped-retire',
+  memory: {
+    baseline: {
+      rss: 10,
+      heapTotal: 10,
+      heapUsed: 10,
+      external: 10,
+      arrayBuffers: 10,
+    },
+    afterBuild: {
+      rss: value,
+      heapTotal: value,
+      heapUsed: value,
+      external: value,
+      arrayBuffers: value,
+    },
+    delta: {
+      rss: value - 10,
+      heapTotal: value - 10,
+      heapUsed: value - 10,
+      external: value - 10,
+      arrayBuffers: value - 10,
+    },
+    processPeakRss: value,
+  },
+});
+
+describe('benchmark matrix', () => {
+  test('seeded schedules are deterministic and cover every arm per repetition', () => {
+    const first = makeSchedule(5, 1234);
+    expect(makeSchedule(5, 1234)).toEqual(first);
+    expect(makeSchedule(5, 4321)).not.toEqual(first);
+    for (let repetition = 1; repetition <= 5; repetition += 1) {
+      expect(
+        first
+          .filter((coordinate) => coordinate.repetition === repetition)
+          .map((coordinate) => coordinate.arm)
+          .sort()
+      ).toEqual([...ARM_NAMES].sort());
+    }
+  });
+
+  test('exact order is repeated without changing its positions', () => {
+    const order = ['retire', 'stock', 'scoped-retire', 'scoped'] as const;
+    expect(makeSchedule(2, 1, order).map((coordinate) => coordinate.arm)).toEqual(
+      [...order, ...order]
+    );
+  });
+
+  test('summaries use medians and comparisons keep the factor direction', () => {
+    const runs: MatrixRun[] = [10, 30, 20].map((value, index) => ({
+      repetition: index + 1,
+      position: 1,
+      arm: 'stock',
+      result: successfulResult('stock', value),
+    }));
+    runs.push(
+      ...[5, 15, 10].map((value, index) => ({
+        repetition: index + 1,
+        position: 2,
+        arm: 'scoped' as const,
+        result: successfulResult('scoped', value),
+      }))
+    );
+    const stock = summarizeArm(runs, 'stock');
+    const scoped = summarizeArm(runs, 'scoped');
+    expect(stock?.buildMs.median).toBe(20);
+    expect(scoped?.buildMs.median).toBe(10);
+    expect(compareArms('stock', 'scoped', stock!, scoped!).buildMs).toEqual({
+      baseline: 20,
+      candidate: 10,
+      difference: -10,
+      percentChange: -50,
+    });
+  });
+});

packages/perf-harness/__tests__/process.test.ts

@@ -0,0 +1,29 @@
+import { resolve } from 'node:path';
+
+import { runWorkerProcess } from '../src/process';
+import type { WorkerConfig } from '../src/types';
+
+describe('fresh worker process', () => {
+  test('starts a distinct process for each measurement and forwards no database URL in output', async () => {
+    const worker = resolve(__dirname, 'fixtures/fake-worker.js');
+    const config: WorkerConfig = {
+      arm: 'stock' as const,
+      schemas: ['example'],
+      allowedDependencySchemas: [],
+    };
+    const first = await runWorkerProcess(
+      worker,
+      'postgres://secret@example.test/database',
+      config
+    );
+    const second = await runWorkerProcess(
+      worker,
+      'postgres://secret@example.test/database',
+      config
+    );
+    expect(first.pid).not.toBe(process.pid);
+    expect(second.pid).not.toBe(process.pid);
+    expect(first.pid).not.toBe(second.pid);
+    expect(JSON.stringify([first.result, second.result])).not.toContain('secret');
+  });
+});

packages/perf-harness/__tests__/run.test.ts

@@ -0,0 +1,64 @@
+import { resolve } from 'node:path';
+
+import { parseRunOptions, runMatrix } from '../src/run';
+
+describe('matrix runner', () => {
+  test('parses the exact four-arm order', () => {
+    const options = parseRunOptions([
+      '--database-url',
+      'postgres:///test',
+      '--schemas',
+      'app,private',
+      '--allowed-dependency-schemas',
+      'shared',
+      '--order',
+      'stock,scoped,retire,scoped-retire',
+      '--repetitions',
+      '2',
+    ]);
+    expect(options.schemas).toEqual(['app', 'private']);
+    expect(options.allowedDependencySchemas).toEqual(['shared']);
+    expect(options.order).toEqual([
+      'stock',
+      'scoped',
+      'retire',
+      'scoped-retire',
+    ]);
+  });
+
+  test('requires all four fresh workers to agree on schema identity', async () => {
+    const options = parseRunOptions([
+      '--database-url',
+      'postgres:///not-used-by-fake-worker',
+      '--schemas',
+      'example',
+      '--repetitions',
+      '1',
+      '--order',
+      'stock,scoped,retire,scoped-retire',
+    ]);
+    const report = await runMatrix(
+      options,
+      resolve(__dirname, 'fixtures/fake-worker.js')
+    );
+    expect(report.validation).toEqual(
+      expect.objectContaining({
+        allRunsSucceeded: true,
+        freshProcessPerRun: true,
+        schemaEquivalent: true,
+        schemaHash: 'fixture-schema-hash',
+        errors: [],
+      })
+    );
+    expect(new Set(report.runs.map((run) => run.result.pid)).size).toBe(4);
+    expect(report.comparisons).toEqual(
+      expect.objectContaining({
+        scopedVsStock: expect.any(Object),
+        retireVsStock: expect.any(Object),
+        combinedVsStock: expect.any(Object),
+        retireWithinScoped: expect.any(Object),
+        scopedWithinRetire: expect.any(Object),
+      })
+    );
+  });
+});

packages/perf-harness/jest.config.js

@@ -0,0 +1,12 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }]
+  },
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePathIgnorePatterns: ['dist/*'],
+  testPathIgnorePatterns: ['/__tests__/fixtures/']
+};

packages/perf-harness/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@constructive-io/perf-harness",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Focused fresh-process benchmark for Graphile catalog scoping and build-state retirement",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
+  "bin": {
+    "cperf": "index.js"
+  },
+  "scripts": {
+    "clean": "makage clean",
+    "build": "makage build",
+    "build:dev": "makage build --dev",
+    "lint": "eslint . --fix",
+    "test": "jest"
+  },
+  "dependencies": {
+    "graphile-build": "5.1.1",
+    "graphile-build-pg": "5.1.3",
+    "graphile-config": "1.1.0",
+    "graphile-settings": "workspace:*",
+    "graphql": "16.13.0",
+    "pg": "^8.21.0",
+    "postgraphile": "5.1.4"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.11",
+    "@types/pg": "^8.20.4",
+    "makage": "^0.3.0"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "license": "MIT"
+}