From c6d9b57c0c4d2bd885058b8e8aaf4bec3f9a10f9 Mon Sep 17 00:00:00 2001 From: Alex UA Date: Sun, 30 Aug 2026 18:07:00 -0400 Subject: [PATCH 1/2] fix: score immutable structured benchmark outputs --- .github/workflows/validate.yml | 1 + CLAUDE.md | 9 +- README.md | 29 +- .../benchmarks/fixtures/next/fixture-01.yaml | 4 +- .../benchmarks/fixtures/next/fixture-03.yaml | 2 +- .../benchmarks/fixtures/next/fixture-05.yaml | 2 +- .../benchmarks/fixtures/next/fixture-08.yaml | 2 +- .../fixtures/react-native/fixture-02.yaml | 2 +- .../fixtures/react-native/fixture-03.yaml | 4 +- .../fixtures/react-native/fixture-04.yaml | 3 + .../fixtures/react-native/fixture-08.yaml | 2 +- .../benchmarks/fixtures/react/fixture-01.yaml | 2 +- .../benchmarks/fixtures/react/fixture-02.yaml | 2 +- .../benchmarks/fixtures/react/fixture-05.yaml | 3 +- .../benchmarks/fixtures/react/fixture-06.yaml | 2 +- .../benchmarks/fixtures/react/fixture-08.yaml | 2 +- .../benchmarks/results/next-run-report.md | 61 +- .../results/react-native-run-report.md | 61 +- .../benchmarks/results/react-run-report.md | 61 +- .../benchmarks/results/stability-report.md | 45 +- research/benchmarks/rules/next.yaml | 10 + research/benchmarks/rules/react-native.yaml | 11 + research/benchmarks/rules/react.yaml | 8 + research/benchmarks/templates/baseline.txt | 9 + research/benchmarks/templates/candidate.txt | 12 + scripts/aggregate_stability.py | 159 +--- scripts/run_benchmark.py | 764 ++++++++++-------- tests/__init__.py | 1 + tests/test_run_benchmark.py | 400 +++++++++ 29 files changed, 956 insertions(+), 717 deletions(-) create mode 100644 research/benchmarks/rules/next.yaml create mode 100644 research/benchmarks/rules/react-native.yaml create mode 100644 research/benchmarks/rules/react.yaml create mode 100644 research/benchmarks/templates/baseline.txt create mode 100644 research/benchmarks/templates/candidate.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_run_benchmark.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9bbdc29..18470dc 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -17,3 +17,4 @@ jobs: - run: python3 scripts/verify_no_copied_skills.py # Note: add --verify-content for full content integrity checks; # omitted by default to avoid CI dependency on GitHub raw content availability. + - run: python3 -m unittest discover diff --git a/CLAUDE.md b/CLAUDE.md index 4358103..ed70274 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,8 +20,8 @@ python3 scripts/refresh_external_skills.py --approve # apply pin + hash upda python3 scripts/refresh_external_skills.py --check # CI: fail if updates needed python3 scripts/verify_no_copied_skills.py # validate manifest structure python3 scripts/verify_no_copied_skills.py --verify-content # fetch + hash verification -python3 scripts/run_benchmark.py --all -python3 scripts/aggregate_stability.py +python3 scripts/run_benchmark.py --critic react --capture-commit <40-hex-commit> --run-id +python3 -m unittest discover ``` ## Design Rules @@ -57,5 +57,6 @@ but should be verified before adding a new owner to `TRUSTED_OWNERS`. - Fixtures: `research/benchmarks/fixtures/{react,next,react-native}/` (8 per critic) - Results: `research/benchmarks/results/` -- Scoring: rubric-coverage evaluation (prompt checklist vs annotated fixture issues) -- Seeds: 3 jackknife windows per critic; aggregate stability in `stability-report.md` +- Model scoring: exact rule-and-line response matching from self-attested repository-native capture blobs at an exact Git commit. +- Windows: deterministic jackknife sensitivity references, never additional samples or provider/model seeds. +- Custody: provider/model/runtime values are self-attested labels, not independent authenticity proof. diff --git a/README.md b/README.md index 930e532..ec9102a 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,35 @@ python3 scripts/refresh_external_skills.py --approve # apply pin + hash upda python3 scripts/refresh_external_skills.py --check # CI: fail if updates needed python3 scripts/verify_no_copied_skills.py # validate manifest structure python3 scripts/verify_no_copied_skills.py --verify-content # fetch + hash verification -python3 scripts/run_benchmark.py --all -python3 scripts/aggregate_stability.py +python3 scripts/run_benchmark.py --critic react --capture-commit <40-hex-commit> --run-id +python3 -m unittest discover ``` +## Exact structured defect-detection benchmark + +The runner never calls a provider. It reads a run-specific, immutable +`captures/{critic}/{run-id}/run.json` and eight candidate plus eight baseline +capture blobs from an exact Git commit. Custody is `self-attested-repo-native`: +provider/model/runtime/tool labels in `run.json` are not independent proof of +execution authenticity. + +Prompts expose only neutral numbered source lines and a committed rule catalog. +Candidate prompts also include the committed critic skill; baseline prompts do +not. Gold issue IDs, severity, descriptions, remediation, and annotations stay +private. Each response is strict JSON containing only findings with a known +`rule_id`, sorted unique in-range `evidence_lines`, and `assertion: "present"`. + +The primary metric is micro exact-match precision, recall, and F1 across eight +unique captures per role. Jackknife windows provide sensitivity references only; +they do not increase sample size. The benchmark is not evidence of prose or +remediation quality, severity judgment, novel-finding ability, provider +authenticity, or statistical superiority. Historical reports remain rejected. + +Publication reserves the exact bundle directory before writing and adds a final +`COMPLETE` marker only after both report and manifest are in place. This is a +fail-closed local publication guard; it does not claim crash-durable or +independently witnessed publication. + ## License Apache 2.0 diff --git a/research/benchmarks/fixtures/next/fixture-01.yaml b/research/benchmarks/fixtures/next/fixture-01.yaml index d2c820f..173d349 100644 --- a/research/benchmarks/fixtures/next/fixture-01.yaml +++ b/research/benchmarks/fixtures/next/fixture-01.yaml @@ -23,10 +23,10 @@ issues: - severity: MAJOR category: rsc-boundary description: "'use client' on the page prevents server-side data fetching; getServerSession and db.query will fail or be called client-side, leaking db access to the browser bundle" - evidence: "'use client'; ... const session = await getServerSession();" + evidence: "'use client';" expected_checks: [rsc-boundary, server-actions-safety] - severity: MAJOR category: rsc-boundary description: "StatCard is a pure display component forced into the client bundle unnecessarily; only RevenueChart needs 'use client'" - evidence: "import StatCard from './StatCard'; // pure display" + evidence: "import StatCard from './StatCard';" expected_checks: [rsc-boundary, runtime-choice] diff --git a/research/benchmarks/fixtures/next/fixture-03.yaml b/research/benchmarks/fixtures/next/fixture-03.yaml index 2e085ee..9892c24 100644 --- a/research/benchmarks/fixtures/next/fixture-03.yaml +++ b/research/benchmarks/fixtures/next/fixture-03.yaml @@ -19,7 +19,7 @@ issues: - severity: CRITICAL category: server-actions-safety description: "role is written directly from form input with no authorization check; any user can elevate their own role to admin" - evidence: "const role = formData.get('role') as string; ... data: { bio, role }" + evidence: "const role = formData.get('role') as string;" expected_checks: [server-actions-safety, security] - severity: CRITICAL category: server-actions-safety diff --git a/research/benchmarks/fixtures/next/fixture-05.yaml b/research/benchmarks/fixtures/next/fixture-05.yaml index abffcc8..ed51a06 100644 --- a/research/benchmarks/fixtures/next/fixture-05.yaml +++ b/research/benchmarks/fixtures/next/fixture-05.yaml @@ -17,7 +17,7 @@ issues: - severity: CRITICAL category: runtime-choice description: "runtime='edge' is incompatible with fs.readFileSync and path.join; these are Node.js APIs unavailable in the Edge Runtime; route will fail at deploy time" - evidence: "export const runtime = 'edge'; import { readFileSync } from 'fs';" + evidence: "export const runtime = 'edge';" expected_checks: [runtime-choice] - severity: MINOR category: runtime-choice diff --git a/research/benchmarks/fixtures/next/fixture-08.yaml b/research/benchmarks/fixtures/next/fixture-08.yaml index cb73849..ec20be3 100644 --- a/research/benchmarks/fixtures/next/fixture-08.yaml +++ b/research/benchmarks/fixtures/next/fixture-08.yaml @@ -20,7 +20,7 @@ issues: - severity: MAJOR category: app-router-conventions description: "metadata exported as a plain object instead of generateMetadata(); post-specific title/og tags (title, description, og:image) cannot be set dynamically" - evidence: "export const metadata = { title: 'Blog Post' };" + evidence: "export const metadata = { // static, not dynamic" expected_checks: [app-router-conventions] - severity: MAJOR category: app-router-conventions diff --git a/research/benchmarks/fixtures/react-native/fixture-02.yaml b/research/benchmarks/fixtures/react-native/fixture-02.yaml index 71ece7c..46a9fc2 100644 --- a/research/benchmarks/fixtures/react-native/fixture-02.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-02.yaml @@ -21,7 +21,7 @@ issues: - severity: MAJOR category: list-performance description: "no removeClippedSubviews or windowSize config; on large lists all items stay mounted, causing memory pressure on low-end Android devices" - evidence: " { ... // no cleanup return" + evidence: "const id = setInterval(() => {" expected_checks: [memory-leak, hook-correctness] - severity: CRITICAL category: memory-leak description: "priceSocket subscription not unsubscribed on unmount; multiple subscriptions stack on each symbol change" - evidence: "const sub = priceSocket.subscribe(symbol, setPrice); // no unsubscribe" + evidence: "const sub = priceSocket.subscribe(symbol, setPrice);" expected_checks: [memory-leak, hook-correctness] diff --git a/research/benchmarks/fixtures/react-native/fixture-04.yaml b/research/benchmarks/fixtures/react-native/fixture-04.yaml index dd4d8be..182a66c 100644 --- a/research/benchmarks/fixtures/react-native/fixture-04.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-04.yaml @@ -23,9 +23,12 @@ issues: category: native-bridge description: "NativeModules synchronous call blocks the JS thread; on startup this can cause frame drops and ANR on Android if the native implementation is slow" evidence: "const deviceInfo = NativeModules.DeviceInfo.getInfo();" + evidence_lines: [6] expected_checks: [native-bridge, render-performance] - severity: MINOR category: native-bridge + rule_id: react-native.native-bridge.error-handling description: "no error handling around NativeModules call; if the native module is unavailable (web target, storybook), throws uncaught TypeError" evidence: "NativeModules.DeviceInfo.getInfo();" + evidence_lines: [6] expected_checks: [native-bridge, operability] diff --git a/research/benchmarks/fixtures/react-native/fixture-08.yaml b/research/benchmarks/fixtures/react-native/fixture-08.yaml index 6676cdf..8a28e11 100644 --- a/research/benchmarks/fixtures/react-native/fixture-08.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-08.yaml @@ -24,7 +24,7 @@ issues: - severity: MAJOR category: release-safety description: "no minimum app version check before applying OTA update; update might target a different native binary version and cause incompatibility crashes" - evidence: "const update = await Updates.checkForUpdateAsync();" + evidence: "await Updates.fetchUpdateAsync();" expected_checks: [release-safety, upgrade-safety] - severity: MINOR category: operability diff --git a/research/benchmarks/fixtures/react/fixture-01.yaml b/research/benchmarks/fixtures/react/fixture-01.yaml index 57032d5..07c11a5 100644 --- a/research/benchmarks/fixtures/react/fixture-01.yaml +++ b/research/benchmarks/fixtures/react/fixture-01.yaml @@ -23,7 +23,7 @@ issues: - severity: MAJOR category: async-correctness description: "no cancellation/teardown: concurrent fetches race and stale responses overwrite newer ones" - evidence: "fetch(`/api/search?...`).then(r => r.json()).then(data => setResults" + evidence: "fetch(`/api/search?q=${query}&page=${page}`)" expected_checks: [async-correctness] - severity: MINOR category: render-behavior diff --git a/research/benchmarks/fixtures/react/fixture-02.yaml b/research/benchmarks/fixtures/react/fixture-02.yaml index c1aa1e2..7ca7cd4 100644 --- a/research/benchmarks/fixtures/react/fixture-02.yaml +++ b/research/benchmarks/fixtures/react/fixture-02.yaml @@ -24,7 +24,7 @@ issues: - severity: MAJOR category: render-behavior description: "filter + nested traversal + regex compile runs synchronously on every render; O(rows × filters × depth) in the render path" - evidence: "const visibleRows = rows.filter(r => { return filters.every(f => {" + evidence: "const visibleRows = rows.filter(r => {" expected_checks: [render-behavior, hook-correctness] - severity: MAJOR category: render-behavior diff --git a/research/benchmarks/fixtures/react/fixture-05.yaml b/research/benchmarks/fixtures/react/fixture-05.yaml index ff2d709..e7ec078 100644 --- a/research/benchmarks/fixtures/react/fixture-05.yaml +++ b/research/benchmarks/fixtures/react/fixture-05.yaml @@ -29,10 +29,11 @@ issues: - severity: MAJOR category: async-correctness description: "no AbortController: if userId changes or component unmounts during fetch, stale setState calls fire on unmounted component" - evidence: ".then(r => r.json()).then(setPosts);" + evidence: ".then(setPosts);" expected_checks: [async-correctness, hook-correctness] - severity: MINOR category: operability description: "no error handling: fetch rejections are silently swallowed, user sees infinite spinner" evidence: ".then(r => r.json())" + evidence_lines: [12] expected_checks: [operability] diff --git a/research/benchmarks/fixtures/react/fixture-06.yaml b/research/benchmarks/fixtures/react/fixture-06.yaml index 73e23cf..f32ec38 100644 --- a/research/benchmarks/fixtures/react/fixture-06.yaml +++ b/research/benchmarks/fixtures/react/fixture-06.yaml @@ -25,5 +25,5 @@ issues: - severity: MINOR category: state-ownership description: "unrelated concerns (auth, theme, cart, notifications) colocated in one provider; should be split into separate contexts" - evidence: "const [user, setUser] = useState(null); const [theme..." + evidence: "const [user, setUser] = useState(null);" expected_checks: [state-ownership] diff --git a/research/benchmarks/fixtures/react/fixture-08.yaml b/research/benchmarks/fixtures/react/fixture-08.yaml index 895b922..1b07006 100644 --- a/research/benchmarks/fixtures/react/fixture-08.yaml +++ b/research/benchmarks/fixtures/react/fixture-08.yaml @@ -24,7 +24,7 @@ issues: - severity: MAJOR category: operability description: "no ErrorBoundary wrapping the Suspense; fetch errors propagate to root React error handler, crashing the full app instead of showing an inline error state" - evidence: "}>" + evidence: "}>" expected_checks: [operability] - severity: MINOR category: operability diff --git a/research/benchmarks/results/next-run-report.md b/research/benchmarks/results/next-run-report.md index b149251..63cf614 100644 --- a/research/benchmarks/results/next-run-report.md +++ b/research/benchmarks/results/next-run-report.md @@ -1,57 +1,6 @@ -# next Benchmark Run Report +# REJECTED-INTEGRITY — legacy Next.js report -## Summary (3 Seeds) - -| Metric | Value | -|--------|-------| -| Mean harsh score | 16.6 | -| Mean baseline score | 2.1 | -| Mean delta | +14.5 | -| Head-to-head (W-L-T) | 18-0-0 | - -## Per-Seed Results - -### Seed 1 - -- Fixtures: 6 -- Harsh mean: 15.8 | Baseline mean: 2.0 | Delta: +13.8 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| next-01 | 13.8 | 0.0 | +13.8 | W | -| next-02 | 10.8 | 0.0 | +10.8 | W | -| next-03 | 23.2 | 6.5 | +16.7 | W | -| next-04 | 21.2 | 5.5 | +15.7 | W | -| next-05 | 12.8 | 0.0 | +12.8 | W | -| next-06 | 12.8 | 0.0 | +12.8 | W | - -### Seed 2 - -- Fixtures: 6 -- Harsh mean: 16.4 | Baseline mean: 2.0 | Delta: +14.4 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| next-02 | 10.8 | 0.0 | +10.8 | W | -| next-03 | 23.2 | 6.5 | +16.7 | W | -| next-04 | 21.2 | 5.5 | +15.7 | W | -| next-05 | 12.8 | 0.0 | +12.8 | W | -| next-06 | 12.8 | 0.0 | +12.8 | W | -| next-07 | 17.8 | 0.0 | +17.8 | W | - -### Seed 3 - -- Fixtures: 6 -- Harsh mean: 17.7 | Baseline mean: 2.4 | Delta: +15.3 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| next-03 | 23.2 | 6.5 | +16.7 | W | -| next-04 | 21.2 | 5.5 | +15.7 | W | -| next-05 | 12.8 | 0.0 | +12.8 | W | -| next-06 | 12.8 | 0.0 | +12.8 | W | -| next-07 | 17.8 | 0.0 | +17.8 | W | -| next-08 | 18.2 | 2.5 | +15.7 | W | +This historical document is a static rubric-coverage projection. It contains no +captured candidate or baseline model outputs, provenance, or immutable artifact +custody. It is not model-behavior evidence and must not be used for comparison, +selection, ranking, stability, or recommendation claims. diff --git a/research/benchmarks/results/react-native-run-report.md b/research/benchmarks/results/react-native-run-report.md index d7e3e54..95f1f57 100644 --- a/research/benchmarks/results/react-native-run-report.md +++ b/research/benchmarks/results/react-native-run-report.md @@ -1,57 +1,6 @@ -# react-native Benchmark Run Report +# REJECTED-INTEGRITY — legacy React Native report -## Summary (3 Seeds) - -| Metric | Value | -|--------|-------| -| Mean harsh score | 14.7 | -| Mean baseline score | 1.0 | -| Mean delta | +13.7 | -| Head-to-head (W-L-T) | 18-0-0 | - -## Per-Seed Results - -### Seed 1 - -- Fixtures: 6 -- Harsh mean: 15.1 | Baseline mean: 0.8 | Delta: +14.3 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| rn-01 | 17.2 | 0.0 | +17.2 | W | -| rn-02 | 16.2 | 0.0 | +16.2 | W | -| rn-03 | 17.8 | 0.0 | +17.8 | W | -| rn-04 | 10.8 | 0.5 | +10.3 | W | -| rn-05 | 15.8 | 1.5 | +14.3 | W | -| rn-06 | 12.8 | 2.5 | +10.3 | W | - -### Seed 2 - -- Fixtures: 6 -- Harsh mean: 14.4 | Baseline mean: 0.9 | Delta: +13.5 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| rn-02 | 16.2 | 0.0 | +16.2 | W | -| rn-03 | 17.8 | 0.0 | +17.8 | W | -| rn-04 | 10.8 | 0.5 | +10.3 | W | -| rn-05 | 15.8 | 1.5 | +14.3 | W | -| rn-06 | 12.8 | 2.5 | +10.3 | W | -| rn-07 | 12.8 | 1.0 | +11.8 | W | - -### Seed 3 - -- Fixtures: 6 -- Harsh mean: 14.7 | Baseline mean: 1.3 | Delta: +13.4 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| rn-03 | 17.8 | 0.0 | +17.8 | W | -| rn-04 | 10.8 | 0.5 | +10.3 | W | -| rn-05 | 15.8 | 1.5 | +14.3 | W | -| rn-06 | 12.8 | 2.5 | +10.3 | W | -| rn-07 | 12.8 | 1.0 | +11.8 | W | -| rn-08 | 18.2 | 2.5 | +15.7 | W | +This historical document is a static rubric-coverage projection. It contains no +captured candidate or baseline model outputs, provenance, or immutable artifact +custody. It is not model-behavior evidence and must not be used for comparison, +selection, ranking, stability, or recommendation claims. diff --git a/research/benchmarks/results/react-run-report.md b/research/benchmarks/results/react-run-report.md index fa98b6b..9be0237 100644 --- a/research/benchmarks/results/react-run-report.md +++ b/research/benchmarks/results/react-run-report.md @@ -1,57 +1,6 @@ -# react Benchmark Run Report +# REJECTED-INTEGRITY — legacy React report -## Summary (3 Seeds) - -| Metric | Value | -|--------|-------| -| Mean harsh score | 15.6 | -| Mean baseline score | 1.7 | -| Mean delta | +13.9 | -| Head-to-head (W-L-T) | 18-0-0 | - -## Per-Seed Results - -### Seed 1 - -- Fixtures: 6 -- Harsh mean: 16.2 | Baseline mean: 1.5 | Delta: +14.7 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| react-01 | 20.2 | 0.0 | +20.2 | W | -| react-02 | 16.2 | 0.0 | +16.2 | W | -| react-03 | 15.8 | 0.0 | +15.8 | W | -| react-04 | 15.8 | 8.0 | +7.8 | W | -| react-05 | 18.2 | 1.0 | +17.2 | W | -| react-06 | 10.8 | 0.0 | +10.8 | W | - -### Seed 2 - -- Fixtures: 6 -- Harsh mean: 15.8 | Baseline mean: 1.5 | Delta: +14.3 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| react-02 | 16.2 | 0.0 | +16.2 | W | -| react-03 | 15.8 | 0.0 | +15.8 | W | -| react-04 | 15.8 | 8.0 | +7.8 | W | -| react-05 | 18.2 | 1.0 | +17.2 | W | -| react-06 | 10.8 | 0.0 | +10.8 | W | -| react-07 | 17.8 | 0.0 | +17.8 | W | - -### Seed 3 - -- Fixtures: 6 -- Harsh mean: 14.9 | Baseline mean: 2.1 | Delta: +12.8 -- Head-to-head: 6-0-0 - -| Fixture | Harsh | Baseline | Delta | H2H | -|---------|-------|----------|-------|-----| -| react-03 | 15.8 | 0.0 | +15.8 | W | -| react-04 | 15.8 | 8.0 | +7.8 | W | -| react-05 | 18.2 | 1.0 | +17.2 | W | -| react-06 | 10.8 | 0.0 | +10.8 | W | -| react-07 | 17.8 | 0.0 | +17.8 | W | -| react-08 | 10.8 | 3.5 | +7.3 | W | +This historical document is a static rubric-coverage projection. It contains no +captured candidate or baseline model outputs, provenance, or immutable artifact +custody. It is not model-behavior evidence and must not be used for comparison, +selection, ranking, stability, or recommendation claims. diff --git a/research/benchmarks/results/stability-report.md b/research/benchmarks/results/stability-report.md index 4ec6a2e..fc175ee 100644 --- a/research/benchmarks/results/stability-report.md +++ b/research/benchmarks/results/stability-report.md @@ -1,41 +1,6 @@ -# Benchmark Stability Report +# REJECTED-INTEGRITY — legacy stability report -Generated across 3 seeds × 3 critics (9 runs total). - -## Per-Critic Stability - -| Critic | Harsh Mean | Harsh StdDev | Baseline Mean | Delta Mean | Delta StdDev | W-L-T | -|--------|-----------|-------------|--------------|-----------|-------------|-------| -| next | 16.6 | ±0.97 | 2.1 | +14.5 | ±0.75 | 18-0-0 | -| react | 15.6 | ±0.67 | 1.7 | +13.9 | ±1.0 | 18-0-0 | -| react-native | 14.7 | ±0.35 | 1.0 | +13.7 | ±0.49 | 18-0-0 | - -## Per-Seed Harsh Scores - -| Critic | Seed 1 | Seed 2 | Seed 3 | StdDev | -|--------|--------|--------|--------|--------| -| next | 15.8 | 16.4 | 17.7 | ±0.97 | -| react | 16.2 | 15.8 | 14.9 | ±0.67 | -| react-native | 15.1 | 14.4 | 14.7 | ±0.35 | - -## Rank Stability - -- Seed 1 ranking: react > next > react-native -- Seed 2 ranking: next > react > react-native -- Seed 3 ranking: next > react > react-native -- Top critic stable across all seeds: NO -- Full ranking stable across all seeds: NO - -## Overall Composite (All 9 Runs) - -- Mean composite harsh: 15.6 -- Mean composite baseline: 1.6 -- Mean composite delta: +14.0 -- Aggregate W-L-T across all 9 runs: 54-0-0 - -## Recommendations - -- Best-performing critic: **next** (mean harsh 16.6, delta +14.5) -- Rank stability: PARTIAL - review seed variance before promoting to default -- Recommendation: keep specialized critics rather than collapsing to a single JS critic. - Delta advantage (+14.0 mean) confirms specialization carries measurable signal. \ No newline at end of file +This historical document is a static rubric-coverage projection. It contains no +captured candidate or baseline model outputs, provenance, or immutable artifact +custody. It is not model-behavior evidence and must not be used for comparison, +selection, ranking, stability, or recommendation claims. diff --git a/research/benchmarks/rules/next.yaml b/research/benchmarks/rules/next.yaml new file mode 100644 index 0000000..0938b37 --- /dev/null +++ b/research/benchmarks/rules/next.yaml @@ -0,0 +1,10 @@ +rules: + - {id: next.app-router-conventions, category: app-router-conventions} + - {id: next.cache-correctness, category: cache-correctness} + - {id: next.operability, category: operability} + - {id: next.route-handler-safety, category: route-handler-safety} + - {id: next.rsc-boundary, category: rsc-boundary} + - {id: next.runtime-choice, category: runtime-choice} + - {id: next.server-actions-safety, category: server-actions-safety} + - {id: next.suspense-boundary, category: suspense-boundary} + - {id: next.upgrade-safety, category: upgrade-safety} diff --git a/research/benchmarks/rules/react-native.yaml b/research/benchmarks/rules/react-native.yaml new file mode 100644 index 0000000..1be44d6 --- /dev/null +++ b/research/benchmarks/rules/react-native.yaml @@ -0,0 +1,11 @@ +rules: + - {id: react-native.expo-workflow, category: expo-workflow} + - {id: react-native.gesture-handling, category: gesture-handling} + - {id: react-native.hermes-compatibility, category: hermes-compatibility} + - {id: react-native.list-performance, category: list-performance} + - {id: react-native.memory-leak, category: memory-leak} + - {id: react-native.native-bridge, category: native-bridge} + - {id: react-native.native-bridge.error-handling, category: native-bridge} + - {id: react-native.operability, category: operability} + - {id: react-native.release-safety, category: release-safety} + - {id: react-native.render-performance, category: render-performance} diff --git a/research/benchmarks/rules/react.yaml b/research/benchmarks/rules/react.yaml new file mode 100644 index 0000000..76165d7 --- /dev/null +++ b/research/benchmarks/rules/react.yaml @@ -0,0 +1,8 @@ +rules: + - {id: react.async-correctness, category: async-correctness} + - {id: react.hook-correctness, category: hook-correctness} + - {id: react.operability, category: operability} + - {id: react.render-behavior, category: render-behavior} + - {id: react.security, category: security} + - {id: react.state-ownership, category: state-ownership} + - {id: react.upgrade-safety, category: upgrade-safety} diff --git a/research/benchmarks/templates/baseline.txt b/research/benchmarks/templates/baseline.txt new file mode 100644 index 0000000..a9a9931 --- /dev/null +++ b/research/benchmarks/templates/baseline.txt @@ -0,0 +1,9 @@ +Review the neutral numbered source lines using only the rule catalog. + +RULE_CATALOG +{{RULE_CATALOG}} + +SOURCE_LINES +{{SOURCE_LINES}} + +Return strict JSON only: {"findings":[{"rule_id":"catalog rule","evidence_lines":[1],"assertion":"present"}]}. diff --git a/research/benchmarks/templates/candidate.txt b/research/benchmarks/templates/candidate.txt new file mode 100644 index 0000000..a7f32b0 --- /dev/null +++ b/research/benchmarks/templates/candidate.txt @@ -0,0 +1,12 @@ +Review the neutral numbered source lines using the committed critic skill and the rule catalog. + +SKILL +{{SKILL}} + +RULE_CATALOG +{{RULE_CATALOG}} + +SOURCE_LINES +{{SOURCE_LINES}} + +Return strict JSON only: {"findings":[{"rule_id":"catalog rule","evidence_lines":[1],"assertion":"present"}]}. diff --git a/scripts/aggregate_stability.py b/scripts/aggregate_stability.py index f07fa56..425ab6b 100644 --- a/scripts/aggregate_stability.py +++ b/scripts/aggregate_stability.py @@ -1,156 +1,7 @@ #!/usr/bin/env python3 -""" -Aggregate stability report across all 3-rep benchmark runs. +"""Retired: exact structured scoring does not aggregate windows as samples.""" -Reads per-critic run reports (JSON sidecar from run_benchmark) and -produces a stability analysis: mean, stddev, and rank stability across seeds. - -Usage: - python3 scripts/aggregate_stability.py -""" - -import json -import math -from pathlib import Path - -REPO_ROOT = Path(__file__).parent.parent -RESULTS_DIR = REPO_ROOT / "research/benchmarks/results" -SKILL_DIR = REPO_ROOT / ".claude/skills" - -# Re-import scoring logic inline (avoid import dependency) -import sys -sys.path.insert(0, str(REPO_ROOT / "scripts")) -from run_benchmark import run_all_seeds, CRITIC_COVERAGE - - -def stdev(values: list[float]) -> float: - if len(values) < 2: - return 0.0 - mean = sum(values) / len(values) - variance = sum((v - mean) ** 2 for v in values) / (len(values) - 1) - return round(math.sqrt(variance), 2) - - -def main(): - critics = ["react", "next", "react-native"] - all_stats = {} - - for critic in critics: - seed_results = run_all_seeds(critic) - harsh_means = [r["mean_harsh"] for r in seed_results] - baseline_means = [r["mean_baseline"] for r in seed_results] - delta_means = [r["mean_delta"] for r in seed_results] - - all_stats[critic] = { - "harsh_mean": round(sum(harsh_means) / len(harsh_means), 1), - "harsh_stddev": stdev(harsh_means), - "baseline_mean": round(sum(baseline_means) / len(baseline_means), 1), - "baseline_stddev": stdev(baseline_means), - "delta_mean": round(sum(delta_means) / len(delta_means), 1), - "delta_stddev": stdev(delta_means), - "total_wins": sum(r["wins"] for r in seed_results), - "total_losses": sum(r["losses"] for r in seed_results), - "total_ties": sum(r["ties"] for r in seed_results), - "seed_harsh": harsh_means, - "seed_baseline": baseline_means, - } - - # Rank critics by mean harsh score - ranked = sorted(all_stats.items(), key=lambda x: x[1]["harsh_mean"], reverse=True) - - # Rank stability: are critics in the same order across all seeds? - seed_ranks = [] - for seed_idx in range(3): - seed_scores = {critic: run_all_seeds(critic)[seed_idx]["mean_harsh"] for critic in critics} - rank_order = sorted(seed_scores, key=lambda c: seed_scores[c], reverse=True) - seed_ranks.append(rank_order) - - top_critic_stable = all(sr[0] == seed_ranks[0][0] for sr in seed_ranks) - rank_fully_stable = all(sr == seed_ranks[0] for sr in seed_ranks) - - # Write stability report - lines = [ - "# Benchmark Stability Report", - "", - "Generated across 3 seeds × 3 critics (9 runs total).", - "", - "## Per-Critic Stability", - "", - "| Critic | Harsh Mean | Harsh StdDev | Baseline Mean | Delta Mean | Delta StdDev | W-L-T |", - "|--------|-----------|-------------|--------------|-----------|-------------|-------|", - ] - - for critic, stats in ranked: - wlt = f"{stats['total_wins']}-{stats['total_losses']}-{stats['total_ties']}" - lines.append( - f"| {critic} | {stats['harsh_mean']} | ±{stats['harsh_stddev']} | " - f"{stats['baseline_mean']} | +{stats['delta_mean']} | ±{stats['delta_stddev']} | {wlt} |" - ) - - lines += [ - "", - "## Per-Seed Harsh Scores", - "", - "| Critic | Seed 1 | Seed 2 | Seed 3 | StdDev |", - "|--------|--------|--------|--------|--------|", - ] - for critic, stats in ranked: - s = stats["seed_harsh"] - lines.append( - f"| {critic} | {s[0]} | {s[1]} | {s[2]} | ±{stats['harsh_stddev']} |" - ) - - lines += [ - "", - "## Rank Stability", - "", - f"- Seed 1 ranking: {' > '.join(seed_ranks[0])}", - f"- Seed 2 ranking: {' > '.join(seed_ranks[1])}", - f"- Seed 3 ranking: {' > '.join(seed_ranks[2])}", - f"- Top critic stable across all seeds: {'YES' if top_critic_stable else 'NO'}", - f"- Full ranking stable across all seeds: {'YES' if rank_fully_stable else 'NO'}", - "", - "## Overall Composite (All 9 Runs)", - "", - ] - - total_harsh = round(sum(s["harsh_mean"] for s in all_stats.values()) / len(all_stats), 1) - total_baseline = round(sum(s["baseline_mean"] for s in all_stats.values()) / len(all_stats), 1) - total_delta = round(total_harsh - total_baseline, 1) - total_wins = sum(s["total_wins"] for s in all_stats.values()) - total_losses = sum(s["total_losses"] for s in all_stats.values()) - total_ties = sum(s["total_ties"] for s in all_stats.values()) - - lines += [ - f"- Mean composite harsh: {total_harsh}", - f"- Mean composite baseline: {total_baseline}", - f"- Mean composite delta: +{total_delta}", - f"- Aggregate W-L-T across all 9 runs: {total_wins}-{total_losses}-{total_ties}", - "", - "## Recommendations", - "", - ] - - best_critic, best_stats = ranked[0] - lines += [ - f"- Best-performing critic: **{best_critic}** (mean harsh {best_stats['harsh_mean']}, " - f"delta +{best_stats['delta_mean']})", - f"- Rank stability: {'STABLE - rankings consistent across all 3 seeds' if rank_fully_stable else 'PARTIAL - review seed variance before promoting to default'}", - f"- Recommendation: keep specialized critics rather than collapsing to a single JS critic.", - f" Delta advantage (+{total_delta} mean) confirms specialization carries measurable signal.", - ] - - report_path = RESULTS_DIR / "stability-report.md" - report_path.write_text("\n".join(lines)) - print(f"Wrote {report_path}") - - # Also write per-critic run reports - from run_benchmark import write_run_report - for critic in critics: - seed_results = run_all_seeds(critic) - path = write_run_report(critic, seed_results) - print(f"Wrote {path}") - - -if __name__ == "__main__": - main() +raise SystemExit( + "Jackknife windows are sensitivity references, not independent benchmark samples. " + "Read the immutable exact-structured run bundle instead." +) diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index 71f5f99..fedb02d 100644 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -1,351 +1,445 @@ #!/usr/bin/env python3 -""" -Benchmark runner for react-critic suite. - -Scoring method: rubric-coverage evaluation. -For each fixture, we check whether the critic's SKILL.md must-check list -and rubric cover the categories needed to detect each annotated issue. -Harsh-critic bonus scoring rewards evidence requirements, multi-perspective -analysis, pre-commitment predictions, and Realist Check protocol. - -Usage: - python3 scripts/run_benchmark.py --critic react --seed 1 - python3 scripts/run_benchmark.py --critic next --seed 2 - python3 scripts/run_benchmark.py --all -""" +"""Exact structured defect-detection scoring from immutable Git artifacts.""" import argparse +import hashlib import json import os -import sys -from pathlib import Path +import re +import shutil +import subprocess +import tempfile +from pathlib import Path, PurePosixPath + import yaml + REPO_ROOT = Path(__file__).parent.parent -FIXTURE_DIR = REPO_ROOT / "research/benchmarks/fixtures" +FIXTURE_DIR = "research/benchmarks/fixtures" +RULE_DIR = "research/benchmarks/rules" +TEMPLATE_DIR = "research/benchmarks/templates" +CAPTURE_ROOT = "research/benchmarks/captures" RESULTS_DIR = REPO_ROOT / "research/benchmarks/results" -SKILL_DIR = REPO_ROOT / ".claude/skills" - -# --------------------------------------------------------------------------- -# Seed definitions: each seed is an ordered subset of 6 fixtures from the 8. -# Seed 1 = first 6 (fixtures 1-6), Seed 2 = fixtures 2-7, Seed 3 = fixtures 3-8. -# This jackknife windowing tests stability without full re-randomization. -# --------------------------------------------------------------------------- -SEED_WINDOWS = { - 1: [0, 1, 2, 3, 4, 5], - 2: [1, 2, 3, 4, 5, 6], - 3: [2, 3, 4, 5, 6, 7], -} - -# --------------------------------------------------------------------------- -# Critic rubric coverage maps. -# Maps fixture issue categories -> list of SKILL.md must-check terms that cover them. -# Harsh-critic coverage is dense; baseline is sparse. -# --------------------------------------------------------------------------- - -CRITIC_COVERAGE = { - "react": { - "must_check": [ - "hook-correctness", - "state-ownership", - "render-behavior", - "async-correctness", - "upgrade-safety", - "operability", - "security", - ], - "has_evidence_requirement": True, - "has_pre_commitment": True, - "has_realist_check": True, - "has_multi_perspective": True, - "perspective_count": 3, # security, new-hire, ops always-on - "has_explicit_gap_analysis": True, - }, - "next": { - "must_check": [ - "rsc-boundary", - "cache-correctness", - "server-actions-safety", - "route-handler-safety", - "runtime-choice", - "suspense-boundary", - "upgrade-safety", - "app-router-conventions", - "operability", - "security", - "ppr-correctness", - ], - "has_evidence_requirement": True, - "has_pre_commitment": True, - "has_realist_check": True, - "has_multi_perspective": True, - "perspective_count": 3, - "has_explicit_gap_analysis": True, - }, - "react-native": { - "must_check": [ - "render-performance", - "list-performance", - "memory-leak", - "native-bridge", - "expo-workflow", - "gesture-handling", - "hermes-compatibility", - "release-safety", - "upgrade-safety", - "operability", - "security", - "hook-correctness", - "native-styling", - ], - "has_evidence_requirement": True, - "has_pre_commitment": True, - "has_realist_check": True, - "has_multi_perspective": True, - "perspective_count": 3, - "has_explicit_gap_analysis": True, - }, -} - -# Baseline critic: a generic "find bugs" reviewer with no structure. -BASELINE_COVERAGE = { - "must_check": [ - "security", # most baselines catch obvious security - "operability", # and obvious crashes - ], - "has_evidence_requirement": False, - "has_pre_commitment": False, - "has_realist_check": False, - "has_multi_perspective": False, - "perspective_count": 0, - "has_explicit_gap_analysis": False, -} - -# --------------------------------------------------------------------------- -# Severity weights -# --------------------------------------------------------------------------- -SEVERITY_WEIGHT = { - "CRITICAL": 5.0, - "MAJOR": 3.0, - "MINOR": 1.0, -} - -# --------------------------------------------------------------------------- -# Scoring bonuses -# --------------------------------------------------------------------------- -EVIDENCE_BONUS_PER_FINDING = 0.5 -PRE_COMMITMENT_BONUS = 2.0 # per fixture where category was predicted -REALIST_CHECK_BONUS = 1.0 # per surviving CRITICAL/MAJOR -PERSPECTIVE_BONUS_PER = 0.3 # per active perspective × findings - - -def load_fixtures(critic: str) -> list[dict]: - """Load all YAML fixtures for a critic.""" - critic_fixture_dir = FIXTURE_DIR / critic - fixtures = [] - for p in sorted(critic_fixture_dir.glob("fixture-*.yaml")): - with open(p) as f: - fixtures.append(yaml.safe_load(f)) +SKILL_PATHS = {"react": ".claude/skills/react-critic/SKILL.md", "next": ".claude/skills/next-critic/SKILL.md", "react-native": ".claude/skills/react-native-critic/SKILL.md"} +ROLE_TEMPLATE_PATHS = {"candidate": f"{TEMPLATE_DIR}/candidate.txt", "baseline": f"{TEMPLATE_DIR}/baseline.txt"} +JACKKNIFE_WINDOWS = {1: [0, 1, 2, 3, 4, 5], 2: [1, 2, 3, 4, 5, 6], 3: [2, 3, 4, 5, 6, 7]} +SEVERITY_WEIGHT = {"CRITICAL": 5.0, "MAJOR": 3.0, "MINOR": 1.0} +SCORER_ID = "exact-rule-line-defect-detection" +SCORER_VERSION = "4" +CUSTODY_CLASSIFICATION = "self-attested-repo-native" +HASH_RE = re.compile(r"[0-9a-f]{64}\Z") +COMMIT_RE = re.compile(r"[0-9a-f]{40}\Z") +RUN_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z") + + +class IntegrityError(ValueError): + """Raised when exact scoring or artifact custody cannot be proven.""" + + +def require_hash(value: object, label: str) -> str: + if not isinstance(value, str) or not HASH_RE.fullmatch(value): + raise IntegrityError(f"{label} must be a full lowercase SHA-256") + return value + + +def canonical_hash(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def bytes_hash(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def text_hash(value: str) -> str: + return bytes_hash(value.encode()) + + +def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise IntegrityError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_nonfinite(value: str) -> None: + raise IntegrityError(f"Non-finite JSON number: {value}") + + +def strict_json_loads(value: str | bytes, label: str) -> object: + try: + return json.loads(value, object_pairs_hook=reject_duplicate_keys, parse_constant=reject_nonfinite) + except IntegrityError: + raise + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise IntegrityError(f"{label} is not strict UTF-8 JSON") from error + + +def git_bytes(*args: str) -> bytes: + result = subprocess.run(["git", *args], cwd=REPO_ROOT, capture_output=True) + if result.returncode: + raise IntegrityError(result.stderr.decode().strip() or "Git command failed") + return result.stdout + + +def git_text(*args: str) -> str: + return git_bytes(*args).decode() + + +def head_commit() -> str: + return git_text("rev-parse", "HEAD").strip() + + +def safe_repo_path(value: str) -> str: + path = PurePosixPath(value) + if not value or path.is_absolute() or any(part in ("", ".", "..") for part in path.parts): + raise IntegrityError(f"Unsafe repo-relative path: {value!r}") + return path.as_posix() + + +def verify_commit(value: str, flag: str = "--capture-commit") -> str: + if not COMMIT_RE.fullmatch(value): + raise IntegrityError(f"{flag} must be an exact 40-character lowercase SHA-1") + if git_text("cat-file", "-t", value).strip() != "commit": + raise IntegrityError(f"{flag} must identify a commit object, not a tag or another object") + return value + + +def tree_entry(commit: str, path: str) -> tuple[str, str, str]: + output = git_text("ls-tree", commit, "--", path).strip() + if not output or "\t" not in output: + raise IntegrityError(f"Missing committed path: {path}") + metadata, found = output.split("\t", 1) + parts = metadata.split() + if len(parts) != 3 or found != path: + raise IntegrityError(f"Ambiguous committed path: {path}") + return parts[0], parts[1], parts[2] + + +def read_regular_blob(commit: str, path: str) -> tuple[bytes, str]: + parts = safe_repo_path(path).split("/") + for index in range(1, len(parts)): + mode, kind, _ = tree_entry(commit, "/".join(parts[:index])) + if mode != "040000" or kind != "tree": + raise IntegrityError(f"Path has a non-tree component: {path}") + mode, kind, oid = tree_entry(commit, path) + if mode != "100644" or kind != "blob": + raise IntegrityError(f"Path must be a regular blob, not mode {mode}: {path}") + return git_bytes("show", f"{commit}:{path}"), oid + + +def assert_clean_inputs(critic: str, commit: str) -> None: + paths = ["scripts/run_benchmark.py", SKILL_PATHS[critic], f"{FIXTURE_DIR}/{critic}", f"{RULE_DIR}/{critic}.yaml", *ROLE_TEMPLATE_PATHS.values()] + dirty = subprocess.run(["git", "diff", "--quiet", commit, "--", *paths], cwd=REPO_ROOT).returncode + untracked = git_text("ls-files", "--others", "--exclude-standard", "--", *paths).strip() + if dirty or untracked: + raise IntegrityError("Scorer, skill, rules, templates, or fixtures are dirty") + + +def fixture_paths(commit: str, critic: str) -> list[str]: + prefix = safe_repo_path(f"{FIXTURE_DIR}/{critic}") + return sorted(path for path in git_text("ls-tree", "-r", "--name-only", commit, "--", prefix).splitlines() if path.endswith(".yaml")) + + +def evidence_lines(code: str, evidence: str, explicit: object = None) -> list[int]: + if explicit is not None: + if not isinstance(explicit, list) or not explicit or any(type(line) is not int or line < 1 or line > len(code.splitlines()) for line in explicit): + raise IntegrityError("Explicit evidence_lines are invalid") + if explicit != sorted(set(explicit)) or not any(evidence in code.splitlines()[line - 1] for line in explicit): + raise IntegrityError("Explicit evidence_lines do not ground the gold evidence") + return explicit + matches = [index for index, line in enumerate(code.splitlines(), 1) if evidence in line] + if len(matches) != 1: + raise IntegrityError("Gold evidence must occur on exactly one source line") + return matches + + +def issue_id(fixture: dict, issue: dict, index: int) -> str: + return f"issue-{canonical_hash({'fixture': fixture['id'], 'index': index, 'issue': issue})[:16]}" + + +def load_rule_catalog(commit: str, critic: str) -> tuple[dict, bytes, str]: + raw, oid = read_regular_blob(commit, f"{RULE_DIR}/{critic}.yaml") + data = yaml.safe_load(raw) + rules = data.get("rules") if isinstance(data, dict) else None + if not isinstance(rules, list) or not rules: + raise IntegrityError("Rule catalog must contain rules") + by_category, rule_ids = {}, set() + for rule in rules: + if not isinstance(rule, dict) or not isinstance(rule.get("category"), str) or not isinstance(rule.get("id"), str): + raise IntegrityError("Rule catalog has invalid rules") + by_category.setdefault(rule["category"], rule["id"]) + rule_ids.add(rule["id"]) + if len(rule_ids) != len(rules): + raise IntegrityError("Rule catalog has invalid or duplicate categories") + return {"by_category": by_category, "rule_ids": rule_ids}, raw, oid + + +def migrate_fixture(fixture: dict, catalog: dict) -> dict: + code = fixture.get("code") + if not isinstance(code, str): + raise IntegrityError("Fixture code must be text") + gold = [] + for index, issue in enumerate(fixture.get("issues", [])): + category = issue.get("category") + evidence = issue.get("evidence") + rule_id = issue.get("rule_id", catalog["by_category"].get(category)) + if rule_id not in catalog["rule_ids"] or not isinstance(evidence, str): + raise IntegrityError(f"Fixture {fixture.get('id')} has an unmapped issue") + gold.append({"issue_id": issue_id(fixture, issue, index), "rule_id": rule_id, "severity": issue.get("severity"), "category": category, "evidence_lines": evidence_lines(code, evidence, issue.get("evidence_lines"))}) + if len({(item['rule_id'], tuple(item['evidence_lines'])) for item in gold}) != len(gold): + raise IntegrityError(f"Fixture {fixture.get('id')} has duplicate gold defects") + return {"id": fixture.get("id"), "code": code, "gold": gold, "public_hash": canonical_hash(public_fixture(fixture)), "gold_hash": canonical_hash(gold), "full_hash": canonical_hash(fixture)} + + +def load_committed_fixtures(commit: str, critic: str, catalog: dict) -> list[dict]: + paths = fixture_paths(commit, critic) + fixtures = [migrate_fixture(yaml.safe_load(read_regular_blob(commit, path)[0]), catalog) for path in paths] + if len(fixtures) != 8: + raise IntegrityError(f"Expected exactly 8 fixtures, found {len(fixtures)}") return fixtures -def score_fixture(fixture: dict, coverage: dict, is_harsh: bool) -> dict: - """Score a single fixture against a coverage profile.""" - must_check = coverage["must_check"] - issues = fixture.get("issues", []) - - base_score = 0.0 - issue_details = [] - - for issue in issues: - severity = issue["severity"] - weight = SEVERITY_WEIGHT.get(severity, 1.0) - expected_checks = issue.get("expected_checks", []) - category = issue.get("category", "") - - # Coverage: at least one expected check covered by must_check list - covered_checks = [c for c in expected_checks if c in must_check] - # Also check primary category - if category in must_check and category not in covered_checks: - covered_checks.append(category) - - coverage_ratio = len(covered_checks) / max(len(expected_checks), 1) - issue_score = weight * coverage_ratio - - # Evidence bonus (harsh-critic only) - if coverage["has_evidence_requirement"] and coverage_ratio > 0: - issue_score += EVIDENCE_BONUS_PER_FINDING - - # Realist check bonus for CRITICAL/MAJOR (harsh-critic only) - if coverage["has_realist_check"] and severity in ("CRITICAL", "MAJOR") and coverage_ratio > 0: - issue_score += REALIST_CHECK_BONUS - - base_score += issue_score - issue_details.append({ - "severity": severity, - "category": category, - "covered_checks": covered_checks, - "coverage_ratio": round(coverage_ratio, 2), - "issue_score": round(issue_score, 2), - }) - - # Pre-commitment bonus: harsh-critic makes predictions, boosting coverage - # when the issue category is in must_check - if coverage["has_pre_commitment"]: - predicted_categories = set( - issue["category"] for issue in issues - if issue["category"] in must_check - ) - base_score += PRE_COMMITMENT_BONUS * len(predicted_categories) - - # Multi-perspective bonus - if coverage["has_multi_perspective"]: - active_perspectives = min(coverage["perspective_count"], 3) - base_score += PERSPECTIVE_BONUS_PER * active_perspectives * len(issues) - - # Explicit gap analysis bonus (harsh-critic finds "What's Missing") - if coverage["has_explicit_gap_analysis"]: - base_score += 1.0 # flat bonus per fixture for gap section - - return { - "fixture_id": fixture["id"], - "issue_count": len(issues), - "base_score": round(base_score, 2), - "issue_details": issue_details, - } - - -def run_seed(critic: str, seed: int, fixtures: list[dict], coverage: dict, is_harsh: bool) -> dict: - """Run one seed (subset of fixtures) and return aggregate scores.""" - indices = SEED_WINDOWS[seed] - subset = [fixtures[i] for i in indices if i < len(fixtures)] - - harsh_scores = [] - baseline_scores = [] - - results = [] - for fixture in subset: - harsh = score_fixture(fixture, coverage, is_harsh=True) - baseline = score_fixture(fixture, BASELINE_COVERAGE, is_harsh=False) - results.append({ - "fixture_id": fixture["id"], - "harsh_score": harsh["base_score"], - "baseline_score": baseline["base_score"], - "delta": round(harsh["base_score"] - baseline["base_score"], 2), - "head_to_head": "W" if harsh["base_score"] > baseline["base_score"] else - "T" if harsh["base_score"] == baseline["base_score"] else "L", - }) - harsh_scores.append(harsh["base_score"]) - baseline_scores.append(baseline["base_score"]) - - wins = sum(1 for r in results if r["head_to_head"] == "W") - ties = sum(1 for r in results if r["head_to_head"] == "T") - losses = sum(1 for r in results if r["head_to_head"] == "L") - - mean_harsh = round(sum(harsh_scores) / len(harsh_scores), 1) if harsh_scores else 0 - mean_baseline = round(sum(baseline_scores) / len(baseline_scores), 1) if baseline_scores else 0 - mean_delta = round(mean_harsh - mean_baseline, 1) - - return { - "critic": critic, - "seed": seed, - "fixture_count": len(subset), - "mean_harsh": mean_harsh, - "mean_baseline": mean_baseline, - "mean_delta": mean_delta, - "wins": wins, - "ties": ties, - "losses": losses, - "head_to_head_str": f"{wins}-{losses}-{ties}", - "fixture_results": results, - } - - -def run_all_seeds(critic: str) -> list[dict]: - fixtures = load_fixtures(critic) - coverage = CRITIC_COVERAGE[critic] - return [run_seed(critic, seed, fixtures, coverage, is_harsh=True) for seed in [1, 2, 3]] - - -def write_run_report(critic: str, seed_results: list[dict]) -> Path: - """Write per-critic run report.""" - mean_harsh = round(sum(r["mean_harsh"] for r in seed_results) / len(seed_results), 1) - mean_baseline = round(sum(r["mean_baseline"] for r in seed_results) / len(seed_results), 1) - mean_delta = round(mean_harsh - mean_baseline, 1) - total_wins = sum(r["wins"] for r in seed_results) - total_ties = sum(r["ties"] for r in seed_results) - total_losses = sum(r["losses"] for r in seed_results) - - lines = [ - f"# {critic} Benchmark Run Report", - f"", - f"## Summary (3 Seeds)", - f"", - f"| Metric | Value |", - f"|--------|-------|", - f"| Mean harsh score | {mean_harsh} |", - f"| Mean baseline score | {mean_baseline} |", - f"| Mean delta | +{mean_delta} |", - f"| Head-to-head (W-L-T) | {total_wins}-{total_losses}-{total_ties} |", - f"", - f"## Per-Seed Results", - f"", - ] - - for r in seed_results: - lines += [ - f"### Seed {r['seed']}", - f"", - f"- Fixtures: {r['fixture_count']}", - f"- Harsh mean: {r['mean_harsh']} | Baseline mean: {r['mean_baseline']} | Delta: +{r['mean_delta']}", - f"- Head-to-head: {r['head_to_head_str']}", - f"", - f"| Fixture | Harsh | Baseline | Delta | H2H |", - f"|---------|-------|----------|-------|-----|", - ] - for fr in r["fixture_results"]: - lines.append( - f"| {fr['fixture_id']} | {fr['harsh_score']} | {fr['baseline_score']} | +{fr['delta']} | {fr['head_to_head']} |" - ) - lines.append("") - - report_path = RESULTS_DIR / f"{critic}-run-report.md" - report_path.write_text("\n".join(lines)) - return report_path - - -def main(): - parser = argparse.ArgumentParser(description="Run harsh-critic benchmarks") - parser.add_argument("--critic", choices=["react", "next", "react-native"], help="Run a single critic") - parser.add_argument("--seed", type=int, choices=[1, 2, 3], help="Run a single seed") - parser.add_argument("--all", action="store_true", help="Run all critics × all seeds") +def strip_answer_comments(line: str) -> str: + line = re.sub(r"//.*$", "", line) + return re.sub(r"/\*.*?\*/", "", line).rstrip() + + +def public_fixture(fixture: dict) -> dict: + return {"source_lines": [{"line": index, "code": strip_answer_comments(line)} for index, line in enumerate(fixture["code"].splitlines(), 1)]} + + +def public_rule_catalog(catalog: dict) -> dict: + return {"rule_ids": sorted(catalog["rule_ids"])} + + +def load_role_templates(commit: str) -> tuple[dict, dict, dict]: + raw, oids, hashes = {}, {}, {} + for role, path in ROLE_TEMPLATE_PATHS.items(): + raw[role], oids[role] = read_regular_blob(commit, path) + hashes[role] = bytes_hash(raw[role]) + return raw, oids, hashes + + +def render_prompt(role: str, fixture: dict, catalog: dict, skill_text: str, templates: dict[str, bytes]) -> str: + template = templates[role].decode() + rendered = template.replace("{{RULE_CATALOG}}", json.dumps(public_rule_catalog(catalog), sort_keys=True, allow_nan=False)).replace("{{SOURCE_LINES}}", json.dumps(public_fixture(fixture), sort_keys=True, allow_nan=False)) + return rendered.replace("{{SKILL}}", skill_text if role == "candidate" else "") + + +def validate_response(response: str, fixture: dict, catalog: dict) -> set[tuple[str, tuple[int, ...]]]: + payload = strict_json_loads(response, "response") + if not isinstance(payload, dict) or set(payload) != {"findings"} or not isinstance(payload["findings"], list) or len(payload["findings"]) > 64: + raise IntegrityError("response must be exactly a findings object with at most 64 findings") + valid_rules = catalog["rule_ids"] + max_line = len(fixture["code"].splitlines()) + predictions = set() + for finding in payload["findings"]: + if not isinstance(finding, dict) or set(finding) != {"rule_id", "evidence_lines", "assertion"} or finding.get("assertion") != "present": + raise IntegrityError("Each finding must contain only rule_id, evidence_lines, assertion=present") + rule_id, lines = finding.get("rule_id"), finding.get("evidence_lines") + if rule_id not in valid_rules or not isinstance(lines, list) or not lines or any(type(line) is not int or line < 1 or line > max_line for line in lines): + raise IntegrityError("Finding rule or evidence lines are invalid") + if lines != sorted(set(lines)): + raise IntegrityError("Finding evidence_lines must be sorted and unique") + key = (rule_id, tuple(lines)) + if key in predictions: + raise IntegrityError("Duplicate structured finding") + predictions.add(key) + return predictions + + +def run_config_path(root: str, critic: str, run_id: str) -> str: + return safe_repo_path(f"{safe_repo_path(root)}/{critic}/{run_id}/run.json") + + +def capture_path(root: str, critic: str, run_id: str, fixture_id: str, role: str) -> str: + return safe_repo_path(f"{safe_repo_path(root)}/{critic}/{run_id}/{fixture_id}.{role}.json") + + +def load_run_config(commit: str, root: str, critic: str, run_id: str, inputs: dict) -> tuple[dict, bytes, str]: + raw, oid = read_regular_blob(commit, run_config_path(root, critic, run_id)) + config = strict_json_loads(raw, "run config") + fields = {"schema_version", "run_id", "critic", "custody_classification", "provider", "model", "runtime", "parameters", "tools", "input_commit", "fixture_ids", "skill_sha256", "rule_catalog_sha256", "scorer_id", "scorer_version", "template_hashes", "public_fixture_hashes", "gold_fixture_hashes"} + if not isinstance(config, dict) or set(config) != fields or config.get("schema_version") != 1: + raise IntegrityError("run.json schema is invalid") + if config["run_id"] != run_id or config["critic"] != critic or config["custody_classification"] != CUSTODY_CLASSIFICATION: + raise IntegrityError("run.json does not bind the requested run") + if not all(isinstance(config[field], str) and config[field] for field in ("provider", "model", "runtime")) or not isinstance(config["parameters"], dict) or not isinstance(config["tools"], list): + raise IntegrityError("run.json execution configuration is invalid") + require_hash(config["skill_sha256"], "run.json skill_sha256") + require_hash(config["rule_catalog_sha256"], "run.json rule_catalog_sha256") + if not isinstance(config["template_hashes"], dict) or not isinstance(config["public_fixture_hashes"], dict) or not isinstance(config["gold_fixture_hashes"], dict): + raise IntegrityError("run.json hash maps are invalid") + for mapping in (config["template_hashes"], config["public_fixture_hashes"], config["gold_fixture_hashes"]): + for value in mapping.values(): + require_hash(value, "run.json mapped hash") + verify_commit(config["input_commit"], "run.json input_commit") + if config["input_commit"] != inputs["commit"]: + raise IntegrityError("run.json input_commit is not the committed scoring-input authority") + if config["fixture_ids"] != inputs["fixture_ids"] or config["public_fixture_hashes"] != inputs["public_hashes"] or config["gold_fixture_hashes"] != inputs["gold_hashes"]: + raise IntegrityError("run.json fixture inputs do not match committed scoring inputs") + if config["skill_sha256"] != inputs["skill_sha"] or config["rule_catalog_sha256"] != inputs["rule_sha"] or config["scorer_id"] != SCORER_ID or config["scorer_version"] != SCORER_VERSION or config["template_hashes"] != inputs["template_hashes"]: + raise IntegrityError("run.json hashes or scorer identity do not match") + return config, raw, oid + + +def capture_input_commit(capture_commit: str, root: str, critic: str, run_id: str) -> str: + raw, _ = read_regular_blob(capture_commit, run_config_path(root, critic, run_id)) + config = strict_json_loads(raw, "run config") + if not isinstance(config, dict) or not isinstance(config.get("input_commit"), str): + raise IntegrityError("run.json must name an input_commit") + input_commit = verify_commit(config["input_commit"], "run.json input_commit") + require_ancestor(input_commit, capture_commit, "capture commit must equal or descend from run.json input_commit") + return input_commit + + +def require_ancestor(ancestor: str, descendant: str, message: str) -> None: + relation = subprocess.run( + ["git", "merge-base", "--is-ancestor", ancestor, descendant], + cwd=REPO_ROOT, + capture_output=True, + ) + if relation.returncode: + raise IntegrityError(message) + + +def validate_capture(record: object, fixture: dict, critic: str, role: str, run_id: str, config_sha: str, prompt: str, catalog: dict) -> set[tuple[str, tuple[int, ...]]]: + fields = {"schema_version", "run_id", "critic", "fixture_id", "role", "run_config_sha256", "rendered_prompt", "prompt_sha256", "response", "response_sha256"} + if not isinstance(record, dict) or set(record) != fields or record.get("schema_version") != 4: + raise IntegrityError("Capture schema must be exactly version 4") + if (record.get("run_id"), record.get("critic"), record.get("fixture_id"), record.get("role")) != (run_id, critic, fixture["id"], role): + raise IntegrityError("Capture identity does not bind the artifact") + if record.get("run_config_sha256") != config_sha or record.get("rendered_prompt") != prompt or not isinstance(record.get("response"), str): + raise IntegrityError("Capture run configuration or prompt binding is invalid") + if record.get("prompt_sha256") != text_hash(prompt) or record.get("response_sha256") != text_hash(record.get("response", "")): + raise IntegrityError("Capture prompt or response hash mismatch") + return validate_response(record.get("response"), fixture, catalog) + + +def score_artifact(raw: bytes, oid: str, fixture: dict, critic: str, role: str, run_id: str, config_sha: str, prompt: str, catalog: dict, path: str) -> dict: + record = strict_json_loads(raw, f"capture {path}") + predicted = validate_capture(record, fixture, critic, role, run_id, config_sha, prompt, catalog) + gold = {(item["rule_id"], tuple(item["evidence_lines"])) for item in fixture["gold"]} + return {"fixture_id": fixture["id"], "role": role, "predicted": sorted(predicted), "gold": sorted(gold), "capture_path": path, "capture_blob_oid": oid, "capture_raw_sha256": bytes_hash(raw), "rendered_prompt_sha256": record["prompt_sha256"], "response_sha256": record["response_sha256"]} + + +def metric_summary(artifacts: list[dict], fixtures: list[dict], role: str) -> dict: + by_id = {fixture["id"]: fixture for fixture in fixtures} + tp = fp = fn = 0 + severity = {name: [0, 0] for name in SEVERITY_WEIGHT} + per_fixture = [] + for artifact in [item for item in artifacts if item["role"] == role]: + predicted, gold = set(map(tuple, artifact["predicted"])), set(map(tuple, artifact["gold"])) + true = predicted & gold + tp += len(true); fp += len(predicted - gold); fn += len(gold - predicted) + per_fixture.append({"fixture_id": artifact["fixture_id"], "exact_set_f1": f1(len(true), len(predicted - gold), len(gold - predicted))}) + for issue in by_id[artifact["fixture_id"]]["gold"]: + bucket = severity[issue["severity"]] + bucket[1] += 1 + if (issue["rule_id"], tuple(issue["evidence_lines"])) in true: + bucket[0] += 1 + return {"tp": tp, "fp": fp, "fn": fn, "precision": ratio(tp, tp + fp), "recall": ratio(tp, tp + fn), "f1": f1(tp, fp, fn), "per_severity_recall": {name: ratio(hit, total) for name, (hit, total) in severity.items()}, "per_fixture_exact_set_f1": per_fixture, "clean_fixture_accuracy": "UNAVAILABLE: no adjudicated clean fixtures in the fixed eight-fixture corpus"} + + +def ratio(top: int, bottom: int) -> float: + return round(top / bottom, 4) if bottom else 0.0 + + +def f1(tp: int, fp: int, fn: int) -> float: + return ratio(2 * tp, 2 * tp + fp + fn) + + +def build_window_references(run_id: str, artifacts: list[dict], fixtures: list[dict]) -> list[dict]: + by_key = {(item["fixture_id"], item["role"]): item for item in artifacts} + refs = [] + for window, indices in JACKKNIFE_WINDOWS.items(): + for index in indices: + for role in ("candidate", "baseline"): + artifact = by_key[(fixtures[index]["id"], role)] + refs.append({"cell_id": f"{run_id}:window-{window}:{fixture['id'] if False else fixtures[index]['id']}:{role}", "window": window, "fixture_id": fixtures[index]["id"], "role": role, "capture_blob_oid": artifact["capture_blob_oid"]}) + if len(refs) != 36 or len({ref["cell_id"] for ref in refs}) != 36: + raise IntegrityError("Expected 36 unique window references") + return refs + + +def atomic_publish(bundle: Path, report: str, manifest: dict) -> tuple[Path, Path]: + if os.path.lexists(bundle): + raise IntegrityError(f"Refusing to overwrite existing bundle: {bundle}") + bundle.parent.mkdir(parents=True, exist_ok=True) + try: + bundle.mkdir() + except FileExistsError as error: + raise IntegrityError(f"Refusing to overwrite existing bundle: {bundle}") from error + temp = None + try: + temp = Path(tempfile.mkdtemp(prefix=".pending-model-benchmark-", dir=bundle.parent)) + (temp / "report.md").write_text(report + "\n") + (temp / "manifest.json").write_text(json.dumps(manifest, indent=2, allow_nan=False) + "\n") + (temp / "COMPLETE").write_text("complete\n") + for name in ("report.md", "manifest.json", "COMPLETE"): + os.replace(temp / name, bundle / name) + temp.rmdir() + except Exception: + if temp is not None: + shutil.rmtree(temp, ignore_errors=True) + shutil.rmtree(bundle, ignore_errors=True) + raise + return bundle / "report.md", bundle / "manifest.json" + + +def report_text(critic: str, run_id: str, candidate: dict, baseline: dict) -> str: + delta = round(candidate["f1"] - baseline["f1"], 4) + lines = [f"# {critic} Exact Structured Defect-Detection Benchmark", "", f"- Run ID: `{run_id}`", f"- Custody: `{CUSTODY_CLASSIFICATION}`; provider labels are self-attested, not independently verified.", "- Primary sample: 8 unique captures per role. Jackknife windows are sensitivity references only.", "- This is not proof of prose quality, remediation quality, severity judgment, novel-finding ability, provider authenticity, or statistical superiority.", "", "## Micro exact-match metrics", "", "| Role | TP | FP | FN | Precision | Recall | F1 |", "|---|---:|---:|---:|---:|---:|---:|", f"| Candidate | {candidate['tp']} | {candidate['fp']} | {candidate['fn']} | {candidate['precision']} | {candidate['recall']} | {candidate['f1']} |", f"| Baseline | {baseline['tp']} | {baseline['fp']} | {baseline['fn']} | {baseline['precision']} | {baseline['recall']} | {baseline['f1']} |", f"| Candidate minus baseline F1 | | | | | | {delta} |", "", "## Per-severity recall", "", "| Severity | Candidate | Baseline |", "|---|---:|---:|"] + lines += [f"| {severity} | {candidate['per_severity_recall'][severity]} | {baseline['per_severity_recall'][severity]} |" for severity in SEVERITY_WEIGHT] + lines += ["", "## Per-fixture exact-set F1", "", "| Fixture | Candidate | Baseline |", "|---|---:|---:|"] + baseline_f1 = {item["fixture_id"]: item["exact_set_f1"] for item in baseline["per_fixture_exact_set_f1"]} + lines += [f"| {item['fixture_id']} | {item['exact_set_f1']} | {baseline_f1[item['fixture_id']]} |" for item in candidate["per_fixture_exact_set_f1"]] + lines += ["", f"Clean-fixture accuracy: {candidate['clean_fixture_accuracy']}"] + return "\n".join(lines) + + +def run_model_benchmark(critic: str, capture_commit: str, run_id: str, capture_root: str, results_dir: Path) -> tuple[Path, Path]: + capture_commit = verify_commit(capture_commit) + if not RUN_ID_RE.fullmatch(run_id): + raise IntegrityError("--run-id must be an RFC4122 UUID") + input_commit = capture_input_commit(capture_commit, capture_root, critic, run_id) + current = head_commit() + require_ancestor(capture_commit, current, "current HEAD must equal or descend from capture commit") + assert_clean_inputs(critic, input_commit) + catalog, rule_raw, rule_oid = load_rule_catalog(input_commit, critic) + fixtures = load_committed_fixtures(input_commit, critic, catalog) + skill_raw, skill_oid = read_regular_blob(input_commit, SKILL_PATHS[critic]) + templates, template_oids, template_hashes = load_role_templates(input_commit) + inputs = {"commit": input_commit, "fixture_ids": [fixture["id"] for fixture in fixtures], "skill_sha": bytes_hash(skill_raw), "rule_sha": bytes_hash(rule_raw), "template_hashes": template_hashes, "public_hashes": {fixture["id"]: fixture["public_hash"] for fixture in fixtures}, "gold_hashes": {fixture["id"]: fixture["gold_hash"] for fixture in fixtures}} + _, config_raw, config_oid = load_run_config(capture_commit, capture_root, critic, run_id, inputs) + config_sha = bytes_hash(config_raw) + artifacts = [] + for fixture in fixtures: + for role in ("candidate", "baseline"): + prompt = render_prompt(role, fixture, catalog, skill_raw.decode(), templates) + path = capture_path(capture_root, critic, run_id, fixture["id"], role) + raw, oid = read_regular_blob(capture_commit, path) + artifacts.append(score_artifact(raw, oid, fixture, critic, role, run_id, config_sha, prompt, catalog, path)) + if len(artifacts) != 16 or len({(item['fixture_id'], item['role']) for item in artifacts}) != 16: + raise IntegrityError("Expected exactly 16 unique capture artifacts") + candidate, baseline = metric_summary(artifacts, fixtures, "candidate"), metric_summary(artifacts, fixtures, "baseline") + refs = build_window_references(run_id, artifacts, fixtures) + manifest = {"schema_version": 4, "benchmark": "exact-structured-defect-detection", "custody_classification": CUSTODY_CLASSIFICATION, "run_id": run_id, "critic": critic, "capture_commit": capture_commit, "input_commit": input_commit, "run_config": {"path": run_config_path(capture_root, critic, run_id), "blob_oid": config_oid, "sha256": config_sha}, "scoring_code": {"commit": current, "sha256": bytes_hash(read_regular_blob(current, "scripts/run_benchmark.py")[0])}, "skill": {"path": SKILL_PATHS[critic], "blob_oid": skill_oid, "sha256": inputs['skill_sha']}, "rule_catalog": {"path": f"{RULE_DIR}/{critic}.yaml", "blob_oid": rule_oid, "sha256": inputs['rule_sha']}, "templates": {role: {"path": ROLE_TEMPLATE_PATHS[role], "blob_oid": template_oids[role], "sha256": template_hashes[role]} for role in templates}, "fixtures": [{"id": item["id"], "public_sha256": item["public_hash"], "gold_sha256": item["gold_hash"], "full_sha256": item["full_hash"]} for item in fixtures], "capture_artifacts": artifacts, "capture_artifact_count": 16, "window_references": refs, "jackknife_window_reference_count": 36, "metrics": {"candidate": candidate, "baseline": baseline, "candidate_minus_baseline_f1": round(candidate['f1'] - baseline['f1'], 4)}} + return atomic_publish(results_dir / f"model-benchmark-{critic}-{run_id}", report_text(critic, run_id, candidate, baseline), manifest) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Score exact structured defect-detection captures") + parser.add_argument("--critic", choices=sorted(SKILL_PATHS), required=True) + parser.add_argument("--capture-commit", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--capture-root", default=CAPTURE_ROOT) + parser.add_argument("--results-dir", type=Path, default=RESULTS_DIR) args = parser.parse_args() - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - critics = ["react", "next", "react-native"] - - if args.all or (not args.critic and not args.seed): - all_results = {} - for critic in critics: - print(f"Running {critic}...") - seed_results = run_all_seeds(critic) - all_results[critic] = seed_results - path = write_run_report(critic, seed_results) - print(f" Wrote {path}") - return all_results - - elif args.critic: - critics_to_run = [args.critic] - seeds_to_run = [args.seed] if args.seed else [1, 2, 3] - for critic in critics_to_run: - fixtures = load_fixtures(critic) - coverage = CRITIC_COVERAGE[critic] - seed_results = [run_seed(critic, s, fixtures, coverage, True) for s in seeds_to_run] - path = write_run_report(critic, seed_results) - for r in seed_results: - print(f"{critic} seed {r['seed']}: harsh={r['mean_harsh']} baseline={r['mean_baseline']} delta=+{r['mean_delta']} h2h={r['head_to_head_str']}") - print(f"Wrote {path}") + report, manifest = run_model_benchmark(args.critic, args.capture_commit, args.run_id, args.capture_root, args.results_dir) + print(f"Wrote {report} and {manifest}") if __name__ == "__main__": diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3132581 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Focused integrity tests for benchmark custody and scoring.""" diff --git a/tests/test_run_benchmark.py b/tests/test_run_benchmark.py new file mode 100644 index 0000000..3b26609 --- /dev/null +++ b/tests/test_run_benchmark.py @@ -0,0 +1,400 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch + +import yaml + + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) +import run_benchmark + + +def git(repo, *args): + return subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, text=True).stdout.strip() + + +def catalog_from_rules(data): + rules = data["rules"] + return { + "by_category": {rule["category"]: next(item["id"] for item in rules if item["category"] == rule["category"]) for rule in rules}, + "rule_ids": {rule["id"] for rule in rules}, + } + + +class ExactBenchmarkRepo(unittest.TestCase): + critic = "react" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.repo = Path(self.temp.name) + self.run_id = str(uuid.uuid4()) + self.fixtures = self.write_inputs() + self.input_commit = self.commit("inputs") + self.write_run_and_captures() + self.capture_commit = self.commit("captures") + self.root_patch = patch.object(run_benchmark, "REPO_ROOT", self.repo) + self.root_patch.start() + + def tearDown(self): + self.root_patch.stop() + self.temp.cleanup() + + def write_inputs(self): + (self.repo / "scripts").mkdir() + (self.repo / "scripts/run_benchmark.py").write_text((SCRIPTS / "run_benchmark.py").read_text()) + skill = self.repo / run_benchmark.SKILL_PATHS[self.critic] + skill.parent.mkdir(parents=True) + skill.write_text("committed test skill\n") + for path in run_benchmark.ROLE_TEMPLATE_PATHS.values(): + target = self.repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text((Path(__file__).resolve().parents[1] / path).read_text()) + rules = self.repo / "research/benchmarks/rules/react.yaml" + rules.parent.mkdir(parents=True) + rules.write_text("rules:\n - {id: react.hook-correctness, category: hook-correctness}\n - {id: react.other, category: other}\n") + fixture_dir = self.repo / "research/benchmarks/fixtures/react" + fixture_dir.mkdir(parents=True) + fixtures = [] + for number in range(1, 9): + code = "\n".join([f"const bad{number} = true;"] + [f"const decoy{number}_{i} = false;" for i in range(1, 25)]) + fixture = {"id": f"react-{number:02d}", "description": "PRIVATE_GOLD_SENTINEL", "code": code, "issues": [{"severity": "MAJOR", "category": "hook-correctness", "description": "PRIVATE_GOLD_SENTINEL", "evidence": f"const bad{number} = true;", "expected_checks": ["hook-correctness"]}]} + fixtures.append(fixture) + (fixture_dir / f"fixture-{number:02d}.yaml").write_text(yaml.safe_dump(fixture)) + git(self.repo, "init") + git(self.repo, "config", "user.email", "test@example.com") + git(self.repo, "config", "user.name", "Test") + return fixtures + + def commit(self, message): + git(self.repo, "add", ".") + git(self.repo, "commit", "-m", message) + return git(self.repo, "rev-parse", "HEAD") + + def inputs(self): + catalog = {"by_category": {"hook-correctness": "react.hook-correctness", "other": "react.other"}, "rule_ids": {"react.hook-correctness", "react.other"}} + migrated = [run_benchmark.migrate_fixture(fixture, catalog) for fixture in self.fixtures] + skill = "committed test skill\n" + templates = {role: (self.repo / path).read_bytes() for role, path in run_benchmark.ROLE_TEMPLATE_PATHS.items()} + return catalog, migrated, skill, templates + + def run_config(self, migrated, templates): + config = {"schema_version": 1, "run_id": self.run_id, "critic": self.critic, "custody_classification": run_benchmark.CUSTODY_CLASSIFICATION, "provider": "self-attested", "model": "test-model", "runtime": "test-runtime", "parameters": {"temperature": 0}, "tools": [], "input_commit": self.input_commit, "fixture_ids": [item["id"] for item in migrated], "skill_sha256": run_benchmark.text_hash("committed test skill\n"), "rule_catalog_sha256": run_benchmark.bytes_hash((self.repo / "research/benchmarks/rules/react.yaml").read_bytes()), "scorer_id": run_benchmark.SCORER_ID, "scorer_version": run_benchmark.SCORER_VERSION, "template_hashes": {role: run_benchmark.bytes_hash(raw) for role, raw in templates.items()}, "public_fixture_hashes": {item["id"]: item["public_hash"] for item in migrated}, "gold_fixture_hashes": {item["id"]: item["gold_hash"] for item in migrated}} + return config + + def capture_record(self, fixture, role, config_sha, catalog, templates, findings=None, response=None): + prompt = run_benchmark.render_prompt(role, fixture, catalog, "committed test skill\n", templates) + findings = findings if findings is not None else [] + response = response if response is not None else json.dumps({"findings": findings}, allow_nan=False) + return {"schema_version": 4, "run_id": self.run_id, "critic": self.critic, "fixture_id": fixture["id"], "role": role, "run_config_sha256": config_sha, "rendered_prompt": prompt, "prompt_sha256": run_benchmark.text_hash(prompt), "response": response, "response_sha256": run_benchmark.text_hash(response)} + + def write_run_and_captures(self): + catalog, migrated, _, templates = self.inputs() + root = self.repo / "research/benchmarks/captures/react" / self.run_id + root.mkdir(parents=True) + config_raw = json.dumps(self.run_config(migrated, templates), sort_keys=True, allow_nan=False).encode() + (root / "run.json").write_bytes(config_raw) + config_sha = run_benchmark.bytes_hash(config_raw) + for fixture in migrated: + gold = fixture["gold"][0] + finding = {"rule_id": gold["rule_id"], "evidence_lines": gold["evidence_lines"], "assertion": "present"} + for role, findings in (("candidate", [finding]), ("baseline", [])): + record = self.capture_record(fixture, role, config_sha, catalog, templates, findings) + (root / f"{fixture['id']}.{role}.json").write_text(json.dumps(record, allow_nan=False)) + + def capture_file(self, fixture_id, role): + return self.repo / "research/benchmarks/captures/react" / self.run_id / f"{fixture_id}.{role}.json" + + def execute_benchmark(self, results=None, commit=None, run_id=None, capture_root="research/benchmarks/captures"): + return run_benchmark.run_model_benchmark(self.critic, commit or self.capture_commit, run_id or self.run_id, capture_root, results or self.repo / "results") + + +class ExactMetricTests(ExactBenchmarkRepo): + def test_correct_rule_and_lines_produce_tp_and_primary_metrics(self): + _, manifest_path = self.execute_benchmark() + metrics = json.loads(manifest_path.read_text())["metrics"] + self.assertEqual({"tp": 8, "fp": 0, "fn": 0}, {key: metrics["candidate"][key] for key in ("tp", "fp", "fn")}) + self.assertEqual(1.0, metrics["candidate"]["f1"]) + self.assertEqual(0.0, metrics["baseline"]["f1"]) + self.assertIn("UNAVAILABLE", metrics["candidate"]["clean_fixture_accuracy"]) + + def test_unrelated_rule_or_wrong_lines_are_fp_and_fn(self): + path = self.capture_file("react-01", "candidate") + record = json.loads(path.read_text()) + record["response"] = json.dumps({"findings": [{"rule_id": "react.hook-correctness", "evidence_lines": [2], "assertion": "present"}]}) + record["response_sha256"] = run_benchmark.text_hash(record["response"]) + path.write_text(json.dumps(record)) + commit = self.commit("wrong lines") + _, manifest_path = self.execute_benchmark(commit=commit) + metrics = json.loads(manifest_path.read_text())["metrics"]["candidate"] + self.assertEqual((7, 1, 1), (metrics["tp"], metrics["fp"], metrics["fn"])) + + def test_known_unrelated_rule_on_exact_lines_is_fp_and_fn(self): + path = self.capture_file("react-01", "candidate") + record = json.loads(path.read_text()) + record["response"] = json.dumps({"findings": [{"rule_id": "react.other", "evidence_lines": [1], "assertion": "present"}]}) + record["response_sha256"] = run_benchmark.text_hash(record["response"]) + path.write_text(json.dumps(record)) + commit = self.commit("unrelated rule") + _, manifest_path = self.execute_benchmark(commit=commit) + metrics = json.loads(manifest_path.read_text())["metrics"]["candidate"] + self.assertEqual((7, 1, 1), (metrics["tp"], metrics["fp"], metrics["fn"])) + + def test_twenty_false_positives_penalize_precision_and_f1(self): + fixture = self.fixtures[0] + catalog, migrated, _, _ = self.inputs() + gold = migrated[0]["gold"][0] + findings = [{"rule_id": gold["rule_id"], "evidence_lines": [line], "assertion": "present"} for line in range(1, 22)] + predicted = run_benchmark.validate_response(json.dumps({"findings": findings}), migrated[0], catalog) + artifact = {"fixture_id": fixture["id"], "role": "candidate", "predicted": sorted(predicted), "gold": [(gold["rule_id"], tuple(gold["evidence_lines"]))]} + metrics = run_benchmark.metric_summary([artifact], [migrated[0]], "candidate") + self.assertEqual((1, 20, 0), (metrics["tp"], metrics["fp"], metrics["fn"])) + self.assertLess(metrics["precision"], 0.1) + self.assertLess(metrics["f1"], 0.1) + + +class ContractAndCustodyTests(ExactBenchmarkRepo): + def test_not_present_and_extra_explanation_are_rejected(self): + fixture = self.fixtures[0] + catalog, migrated, _, _ = self.inputs() + bad = {"findings": [{"rule_id": migrated[0]["gold"][0]["rule_id"], "evidence_lines": [1], "assertion": "not_present"}]} + with self.assertRaises(run_benchmark.IntegrityError): + run_benchmark.validate_response(json.dumps(bad), migrated[0], catalog) + extra = {"findings": [{"rule_id": migrated[0]["gold"][0]["rule_id"], "evidence_lines": [1], "assertion": "present", "explanation": "forbidden"}]} + with self.assertRaises(run_benchmark.IntegrityError): + run_benchmark.validate_response(json.dumps(extra), migrated[0], catalog) + + def test_public_prompt_hides_private_sentinel_and_strips_comments(self): + catalog, migrated, skill, templates = self.inputs() + prompt = run_benchmark.render_prompt("candidate", migrated[0], catalog, skill, templates) + self.assertNotIn("PRIVATE_GOLD_SENTINEL", prompt) + self.assertNotIn("issues", prompt) + self.assertNotIn("expected_checks", prompt) + self.assertNotIn("//", run_benchmark.public_fixture(migrated[0])["source_lines"][0]["code"]) + + def test_template_hashes_and_all_custody_hashes_are_manifested(self): + _, manifest_path = self.execute_benchmark() + manifest = json.loads(manifest_path.read_text()) + self.assertEqual("complete\n", (manifest_path.parent / "COMPLETE").read_text()) + self.assertEqual(64, len(manifest["run_config"]["sha256"])) + self.assertEqual(64, len(manifest["skill"]["sha256"])) + self.assertEqual(64, len(manifest["rule_catalog"]["sha256"])) + self.assertTrue(all(len(value["sha256"]) == 64 for value in manifest["templates"].values())) + self.assertEqual(run_benchmark.bytes_hash((self.repo / run_benchmark.ROLE_TEMPLATE_PATHS["candidate"]).read_bytes()), manifest["templates"]["candidate"]["sha256"]) + self.assertTrue(all(len(item["public_sha256"]) == 64 and len(item["gold_sha256"]) == 64 for item in manifest["fixtures"])) + self.assertTrue(all(len(item["rendered_prompt_sha256"]) == 64 and len(item["response_sha256"]) == 64 for item in manifest["capture_artifacts"])) + + def test_mixed_config_is_rejected_and_artifacts_do_not_inflate_windows(self): + path = self.capture_file("react-01", "baseline") + record = json.loads(path.read_text()) + record["run_config_sha256"] = "0" * 64 + path.write_text(json.dumps(record)) + mixed = self.commit("mixed config") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=mixed) + git(self.repo, "checkout", self.capture_commit, "--", str(path.relative_to(self.repo))) + restored = self.commit("restore config") + _, manifest_path = self.execute_benchmark(commit=restored) + manifest = json.loads(manifest_path.read_text()) + self.assertEqual(16, manifest["capture_artifact_count"]) + self.assertEqual(36, manifest["jackknife_window_reference_count"]) + self.assertEqual(16, len({(item["fixture_id"], item["role"]) for item in manifest["capture_artifacts"]})) + + def test_commit_path_and_atomic_controls_remain_fail_closed(self): + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit="HEAD") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(run_id="bad") + results = self.repo / "results" + with patch.object(run_benchmark.os, "replace", side_effect=OSError("fault")): + with self.assertRaises(OSError): + self.execute_benchmark(results) + self.assertEqual([], list(results.glob("model-benchmark-*"))) + self.execute_benchmark(results) + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(results) + + def test_empty_directory_and_symlink_bundle_collisions_are_rejected(self): + results = self.repo / "collisions" + bundle = results / f"model-benchmark-{self.critic}-{self.run_id}" + bundle.mkdir(parents=True) + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(results) + bundle.rmdir() + os.symlink("missing-target", bundle) + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(results) + + def test_racing_bundle_creation_is_not_replaced(self): + results = self.repo / "race" + bundle = results / f"model-benchmark-{self.critic}-{self.run_id}" + original_mkdir = Path.mkdir + + def create_collision(path, *args, **kwargs): + if path == bundle: + original_mkdir(path, *args, **kwargs) + return original_mkdir(path, *args, **kwargs) + + with patch.object(Path, "mkdir", new=create_collision): + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(results) + self.assertTrue(bundle.is_dir()) + self.assertEqual([], list(bundle.iterdir())) + + def test_annotated_tag_object_id_is_not_a_commit(self): + git(self.repo, "tag", "-a", "capture-tag", "-m", "capture", self.capture_commit) + tag_oid = git(self.repo, "rev-parse", "capture-tag") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=tag_oid) + + def test_unrelated_capture_commit_is_rejected(self): + unrelated = git(self.repo, "commit-tree", f"{self.capture_commit}^{{tree}}", "-m", "unrelated") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=unrelated) + + def test_current_head_must_descend_from_capture_commit(self): + git(self.repo, "checkout", "--detach", self.input_commit) + self.assertEqual(self.input_commit, git(self.repo, "rev-parse", "HEAD")) + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=self.capture_commit) + + def test_mismatched_input_commit_hashes_are_rejected(self): + skill = self.repo / run_benchmark.SKILL_PATHS[self.critic] + skill.write_text("different committed skill\n") + changed_input = self.commit("different input") + config_path = self.repo / "research/benchmarks/captures/react" / self.run_id / "run.json" + config = json.loads(config_path.read_text()) + config["input_commit"] = changed_input + config_path.write_text(json.dumps(config, sort_keys=True, allow_nan=False)) + capture = self.commit("mismatched input authority") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=capture) + + def test_symlink_and_gitlink_artifacts_are_rejected(self): + path = self.capture_file("react-01", "candidate") + path.unlink() + os.symlink("react-01.baseline.json", path) + symlink_commit = self.commit("symlink artifact") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=symlink_commit) + relative = str(path.relative_to(self.repo)) + git(self.repo, "rm", "--cached", relative) + git(self.repo, "update-index", "--add", "--cacheinfo", f"160000,{self.capture_commit},{relative}") + git(self.repo, "commit", "-m", "gitlink artifact") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=git(self.repo, "rev-parse", "HEAD")) + + def test_dirty_scorer_and_dirty_fixture_fail_separately(self): + scorer = self.repo / "scripts/run_benchmark.py" + scorer.write_text("dirty\n") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark() + git(self.repo, "checkout", "--", "scripts/run_benchmark.py") + fixture = self.repo / "research/benchmarks/fixtures/react/fixture-01.yaml" + fixture.write_text(fixture.read_text() + "\n# dirty\n") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark() + + def test_dirty_committed_skill_fails(self): + skill = self.repo / run_benchmark.SKILL_PATHS[self.critic] + skill.write_text(skill.read_text() + "dirty\n") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark() + + def test_dirty_rule_catalog_fails(self): + rules = self.repo / "research/benchmarks/rules/react.yaml" + rules.write_text(rules.read_text() + "# dirty\n") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark() + + def test_dirty_prompt_template_fails(self): + template = self.repo / run_benchmark.ROLE_TEMPLATE_PATHS["candidate"] + template.write_text(template.read_text() + "dirty\n") + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark() + + def test_run_and_capture_json_reject_duplicate_and_nonfinite_values(self): + run_path = self.repo / "research/benchmarks/captures/react" / self.run_id / "run.json" + capture_path = self.capture_file("react-01", "candidate") + cases = [ + ("duplicate run key", run_path, lambda raw: raw[:-1] + ', "run_id": "duplicate"}'), + ("nonfinite run value", run_path, lambda raw: raw.replace('"temperature": 0', '"temperature": NaN')), + ("duplicate capture key", capture_path, lambda raw: raw[:-1] + ', "response": "{}"}'), + ("nonfinite capture value", capture_path, lambda raw: raw.replace('"schema_version": 4', '"schema_version": Infinity')), + ] + for label, path, mutate in cases: + raw = path.read_text() + changed = mutate(raw) + self.assertNotEqual(raw, changed, label) + path.write_text(changed) + commit = self.commit(label) + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(commit=commit) + git(self.repo, "checkout", self.capture_commit, "--", str(path.relative_to(self.repo))) + + def test_late_invalid_and_traversal_leave_no_partial_bundle(self): + path = self.capture_file("react-08", "baseline") + record = json.loads(path.read_text()) + record["response"] = "not-json" + record["response_sha256"] = run_benchmark.text_hash("not-json") + path.write_text(json.dumps(record)) + bad = self.commit("late invalid") + results = self.repo / "late-results" + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(results, bad) + self.assertFalse(results.exists()) + with self.assertRaises(run_benchmark.IntegrityError): + self.execute_benchmark(self.repo / "escape-results", self.capture_commit, capture_root="../captures") + + +class RealRepoIntegrationTests(unittest.TestCase): + def test_explicit_real_skill_paths_exist(self): + root = Path(__file__).resolve().parents[1] + expected = {"react": ".claude/skills/react-critic/SKILL.md", "next": ".claude/skills/next-critic/SKILL.md", "react-native": ".claude/skills/react-native-critic/SKILL.md"} + self.assertEqual(expected, run_benchmark.SKILL_PATHS) + self.assertTrue(all((root / path).is_file() for path in expected.values())) + + def test_real_catalogs_cover_real_fixture_categories(self): + root = Path(__file__).resolve().parents[1] + for critic in run_benchmark.SKILL_PATHS: + catalog = yaml.safe_load((root / f"research/benchmarks/rules/{critic}.yaml").read_text()) + categories = {rule["category"] for rule in catalog["rules"]} + fixture_categories = {issue["category"] for path in (root / f"research/benchmarks/fixtures/{critic}").glob("*.yaml") for issue in yaml.safe_load(path.read_text())["issues"]} + self.assertTrue(fixture_categories <= categories) + + def test_real_corpus_migrates_all_twenty_four_fixtures(self): + root = Path(__file__).resolve().parents[1] + total = 0 + for critic in run_benchmark.SKILL_PATHS: + catalog = catalog_from_rules(yaml.safe_load((root / f"research/benchmarks/rules/{critic}.yaml").read_text())) + paths = sorted((root / f"research/benchmarks/fixtures/{critic}").glob("*.yaml")) + self.assertEqual(8, len(paths)) + for path in paths: + migrated = run_benchmark.migrate_fixture(yaml.safe_load(path.read_text()), catalog) + keys = {(item["rule_id"], tuple(item["evidence_lines"])) for item in migrated["gold"]} + self.assertEqual(len(keys), len(migrated["gold"]), path.name) + total += 1 + self.assertEqual(24, total) + + def test_fixture_migration_binds_private_gold_and_strips_answer_comments(self): + fixture = {"id": "demo", "code": "const x = true; // answer\n", "issues": [{"severity": "MAJOR", "category": "hook-correctness", "description": "private", "evidence": "const x = true;", "expected_checks": []}]} + migrated = run_benchmark.migrate_fixture(fixture, {"by_category": {"hook-correctness": "react.hook-correctness"}, "rule_ids": {"react.hook-correctness"}}) + gold = migrated["gold"][0] + self.assertEqual(("react.hook-correctness", [1], "MAJOR", "hook-correctness"), (gold["rule_id"], gold["evidence_lines"], gold["severity"], gold["category"])) + self.assertTrue(gold["issue_id"].startswith("issue-")) + self.assertNotIn("answer", run_benchmark.public_fixture(migrated)["source_lines"][0]["code"]) + + def test_legacy_reports_remain_rejected(self): + root = Path(__file__).resolve().parents[1] + for name in ("react-run-report.md", "next-run-report.md", "react-native-run-report.md", "stability-report.md"): + self.assertTrue((root / "research/benchmarks/results" / name).read_text().startswith("# REJECTED-INTEGRITY")) + + +if __name__ == "__main__": + unittest.main() From af548cdd4d5666b78f77e4307efa9b248335db27 Mon Sep 17 00:00:00 2001 From: Alex UA Date: Sat, 19 Sep 2026 09:39:59 -0400 Subject: [PATCH 2/2] fix: redesign benchmark integrity foundation --- CLAUDE.md | 7 +- README.md | 44 +- research/benchmarks/DESIGN.md | 24 + .../benchmarks/fixtures/next/fixture-01.yaml | 78 ++- .../benchmarks/fixtures/next/fixture-02.yaml | 61 +- .../benchmarks/fixtures/next/fixture-03.yaml | 82 ++- .../benchmarks/fixtures/next/fixture-04.yaml | 76 +- .../benchmarks/fixtures/next/fixture-05.yaml | 65 +- .../benchmarks/fixtures/next/fixture-06.yaml | 78 ++- .../benchmarks/fixtures/next/fixture-07.yaml | 81 ++- .../benchmarks/fixtures/next/fixture-08.yaml | 73 +- .../fixtures/react-native/fixture-01.yaml | 76 +- .../fixtures/react-native/fixture-02.yaml | 71 +- .../fixtures/react-native/fixture-03.yaml | 82 ++- .../fixtures/react-native/fixture-04.yaml | 85 ++- .../fixtures/react-native/fixture-05.yaml | 71 +- .../fixtures/react-native/fixture-06.yaml | 77 ++- .../fixtures/react-native/fixture-07.yaml | 71 +- .../fixtures/react-native/fixture-08.yaml | 77 ++- .../benchmarks/fixtures/react/fixture-01.yaml | 84 ++- .../benchmarks/fixtures/react/fixture-02.yaml | 86 ++- .../benchmarks/fixtures/react/fixture-03.yaml | 84 ++- .../benchmarks/fixtures/react/fixture-04.yaml | 69 +- .../benchmarks/fixtures/react/fixture-05.yaml | 81 ++- .../benchmarks/fixtures/react/fixture-06.yaml | 64 +- .../benchmarks/fixtures/react/fixture-07.yaml | 79 ++- .../benchmarks/fixtures/react/fixture-08.yaml | 79 ++- research/benchmarks/rules/next.yaml | 64 +- research/benchmarks/rules/react-native.yaml | 71 +- research/benchmarks/rules/react.yaml | 50 +- research/benchmarks/templates/baseline.txt | 6 +- research/benchmarks/templates/candidate.txt | 8 +- scripts/benchmark_scorer.py | 647 ++++++++++++++++++ scripts/run_benchmark.py | 458 +------------ tests/test_run_benchmark.py | 535 +++++++++------ 35 files changed, 2411 insertions(+), 1333 deletions(-) create mode 100644 research/benchmarks/DESIGN.md create mode 100644 scripts/benchmark_scorer.py diff --git a/CLAUDE.md b/CLAUDE.md index ed70274..9fd5c82 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,6 +57,9 @@ but should be verified before adding a new owner to `TRUSTED_OWNERS`. - Fixtures: `research/benchmarks/fixtures/{react,next,react-native}/` (8 per critic) - Results: `research/benchmarks/results/` -- Model scoring: exact rule-and-line response matching from self-attested repository-native capture blobs at an exact Git commit. +- Corpus status: reviewed descriptive foundation only; no provider pilot or committed captures exist. +- Controls: `react-06`, `next-02`, `rn-01`, `rn-02`, and `rn-07` are intentional-clean restraint controls. +- Candidate treatment: only the committed inline `SKILL.md`; referenced rubrics, agents, router material, and external skills are excluded. +- Future model scoring: exact rule-and-line response matching from self-attested repository-native capture blobs at an exact Git commit. - Windows: deterministic jackknife sensitivity references, never additional samples or provider/model seeds. -- Custody: provider/model/runtime values are self-attested labels, not independent authenticity proof. +- Boundaries: this corpus supports no performance, authenticity, severity-quality, full-skill, or statistical claim. Historical reports remain rejected. diff --git a/README.md b/README.md index ec9102a..0a3510f 100644 --- a/README.md +++ b/README.md @@ -60,25 +60,31 @@ python3 scripts/run_benchmark.py --critic react --capture-commit <40-hex-commit> python3 -m unittest discover ``` -## Exact structured defect-detection benchmark - -The runner never calls a provider. It reads a run-specific, immutable -`captures/{critic}/{run-id}/run.json` and eight candidate plus eight baseline -capture blobs from an exact Git commit. Custody is `self-attested-repo-native`: -provider/model/runtime/tool labels in `run.json` are not independent proof of -execution authenticity. - -Prompts expose only neutral numbered source lines and a committed rule catalog. -Candidate prompts also include the committed critic skill; baseline prompts do -not. Gold issue IDs, severity, descriptions, remediation, and annotations stay -private. Each response is strict JSON containing only findings with a known -`rule_id`, sorted unique in-range `evidence_lines`, and `assertion: "present"`. - -The primary metric is micro exact-match precision, recall, and F1 across eight -unique captures per role. Jackknife windows provide sensitivity references only; -they do not increase sample size. The benchmark is not evidence of prose or -remediation quality, severity judgment, novel-finding ability, provider -authenticity, or statistical superiority. Historical reports remain rejected. +## Reviewed benchmark corpus and future exact scorer + +`research/benchmarks/` is a reviewed descriptive foundation for a future exact +structured defect-detection benchmark. It has eight synthetic fixtures per +critic, including five intentional-clean restraint controls: `react-06`, +`next-02`, `rn-01`, `rn-02`, and `rn-07`. Each fixture exposes the source +context, assumptions, requirements, rule, span-level evidence, and official +source used for adjudication. It does not represent provider execution or human +expert certification. + +Candidate and baseline prompts have identical substantive instructions, rule +definitions, source context, evidence-span contract, anti-speculation guard, +and permission to return no findings. Candidate treatment adds only the +committed inline `SKILL.md`; referenced rubrics, agents, router material, and +external skills are excluded from both treatments. Each response is strict JSON +containing only findings with a known `rule_id`, sorted unique in-range +`evidence_lines`, and `assertion: "present"`. + +The future runner never calls a provider. It requires run-specific immutable +captures before it can score any execution; provider/model/runtime/tool labels +would remain self-attested repository-native custody rather than independent +proof of authenticity. No provider pilot or capture set exists now. The corpus +therefore supports no performance, full-skill, prose/remediation, severity, +provider-authenticity, or statistical-superiority claim. Historical reports +remain rejected. Publication reserves the exact bundle directory before writing and adds a final `COMPLETE` marker only after both report and manifest are in place. This is a diff --git a/research/benchmarks/DESIGN.md b/research/benchmarks/DESIGN.md new file mode 100644 index 0000000..addb4a5 --- /dev/null +++ b/research/benchmarks/DESIGN.md @@ -0,0 +1,24 @@ +# Benchmark corpus design and adjudication + +This is a reviewed descriptive foundation for a future exact structured +defect-detection benchmark. It contains 24 synthetic, independently reviewable +fixtures: eight each for React, Next.js, and React Native. Every label names a +catalog rule, cites public source lines, records the public assumptions needed +to assess it, and cites an official primary source. This is corpus +adjudication, not provider execution or a human-expert certification. + +The corpus has five intentional-clean restraint controls: `react-06`, +`next-02`, `rn-01`, `rn-02`, and `rn-07`. Their `ADVERSARIAL` difficulty means +they are designed to attract unsupported advice; it does not mean they contain +a hidden defect. + +The candidate and baseline receive the same source context, rule catalog, +response schema, evidence-span contract, and permission to return no findings. +The candidate treatment adds only the committed inline `SKILL.md`. Referenced +rubrics, agents, router material, and external skills are excluded from both +prompts. + +No provider pilot, capture, model comparison, or aggregate run exists in this +repository. The corpus therefore supports no performance, provider-authenticity, +severity-quality, full-skill, or statistical claim. Historical benchmark +reports remain rejected rather than evidence of the redesigned corpus. diff --git a/research/benchmarks/fixtures/next/fixture-01.yaml b/research/benchmarks/fixtures/next/fixture-01.yaml index 173d349..38330e5 100644 --- a/research/benchmarks/fixtures/next/fixture-01.yaml +++ b/research/benchmarks/fixtures/next/fixture-01.yaml @@ -1,32 +1,56 @@ +schema_version: 2 id: next-01 category: rsc-boundary -description: use client placed too high, blocking RSC optimization -code: | - // app/dashboard/page.tsx - 'use client'; - import { getServerSession } from 'next-auth'; - import { db } from '@/lib/db'; - import RevenueChart from './RevenueChart'; // needs interactivity - import StatCard from './StatCard'; // pure display +description: A Client Component directive conflicts with server-only work in the same module. +intentional_clean: false +public: + file_path: app/dashboard/page.tsx + language: TypeScript JSX + frameworks: + next: '15' + react: '19' + runtime: nodejs + assumptions: + - db is a server-only database client. + - RevenueChart is an interactive Client Component. + requirements: + - The dashboard page must load its session and database data on the server. + code: | + 'use client'; - export default async function DashboardPage() { - const session = await getServerSession(); - const stats = await db.query.stats.findMany(); - return ( -
- - -
- ); - } + import { getServerSession } from 'next-auth'; + import { db } from '@/lib/db'; + import RevenueChart from './RevenueChart'; + import StatCard from './StatCard'; + + export default async function DashboardPage() { + const session = await getServerSession(); + const stats = await db.query.stats.findMany(); + return ; + } +metadata: + provenance: + kind: synthetic + origin: Next.js Server and Client Components documentation + revision_reason: Narrowed the label to the direct client-boundary conflict; no browser-leak claim is made. + difficulty: HAS-BUGS + domains: [server-components, boundaries] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/building-your-application/rendering/client-components + supports: The use client directive marks a Client Component entry point. + notes: The directive and server-only assumption make the incompatibility explicit. + excluded_claims: [Bundle contents and the placement of child component directives are not scored.] issues: - - severity: MAJOR - category: rsc-boundary - description: "'use client' on the page prevents server-side data fetching; getServerSession and db.query will fail or be called client-side, leaking db access to the browser bundle" - evidence: "'use client';" - expected_checks: [rsc-boundary, server-actions-safety] - - severity: MAJOR + - id: next-01-client-server-conflict + rule_id: next.rsc-boundary category: rsc-boundary - description: "StatCard is a pure display component forced into the client bundle unnecessarily; only RevenueChart needs 'use client'" - evidence: "import StatCard from './StatCard';" - expected_checks: [rsc-boundary, runtime-choice] + severity: MAJOR + description: The use client entry point conflicts with the stated server-only session and database work in the same module. + evidence_spans: + - lines: [1, 3, 4, 9, 10] + rationale: The module is marked as a Client Component while importing and awaiting server-only services. diff --git a/research/benchmarks/fixtures/next/fixture-02.yaml b/research/benchmarks/fixtures/next/fixture-02.yaml index 8433797..5b54895 100644 --- a/research/benchmarks/fixtures/next/fixture-02.yaml +++ b/research/benchmarks/fixtures/next/fixture-02.yaml @@ -1,24 +1,41 @@ +schema_version: 2 id: next-02 category: cache-correctness -description: overly aggressive no-store on every fetch disabling PPR -code: | - // app/products/page.tsx - export default async function ProductsPage() { - const [products, categories, featuredBanner] = await Promise.all([ - fetch('/api/products', { cache: 'no-store' }), - fetch('/api/categories', { cache: 'no-store' }), - fetch('/api/banner/featured', { cache: 'no-store' }), - ]); - // ...render - } -issues: - - severity: MAJOR - category: cache-correctness - description: "cache: 'no-store' on all fetches disables PPR; categories and banner are stable data and should use revalidate or force-cache; every page visit re-fetches all three even if data hasn't changed" - evidence: "fetch('/api/categories', { cache: 'no-store' })" - expected_checks: [cache-correctness, ppr-correctness] - - severity: MINOR - category: cache-correctness - description: "no granular revalidate tags; cache invalidation requires a full page re-build rather than targeted revalidatePath/revalidateTag" - evidence: "fetch('/api/products', { cache: 'no-store' })" - expected_checks: [cache-correctness] +description: Product data is intentionally uncached because it must reflect immediate writes. +intentional_clean: true +public: + file_path: app/products/page.tsx + language: TypeScript JSX + frameworks: + next: '15' + react: '19' + runtime: nodejs + assumptions: + - Inventory and prices can change immediately after a completed purchase. + - This route is the canonical confirmation view following a purchase. + requirements: + - The confirmation view must not serve a cached inventory or price response. + code: | + export default async function ProductsPage() { + const response = await fetch('https://catalog.example.test/products', { cache: 'no-store' }); + const products = await response.json(); + return ; + } +metadata: + provenance: + kind: synthetic + origin: Next.js data fetching documentation + revision_reason: Replaced a generic no-store performance preference with a clean freshness control. + difficulty: ADVERSARIAL + domains: [caching, freshness] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/building-your-application/data-fetching/fetching + supports: Next.js supports cache configuration for fetch requests. + notes: The stated requirement supports no-store for this route. + excluded_claims: [PPR, tags, and broader catalog caching policy are not scored.] +issues: [] diff --git a/research/benchmarks/fixtures/next/fixture-03.yaml b/research/benchmarks/fixtures/next/fixture-03.yaml index 9892c24..08dd549 100644 --- a/research/benchmarks/fixtures/next/fixture-03.yaml +++ b/research/benchmarks/fixtures/next/fixture-03.yaml @@ -1,33 +1,63 @@ +schema_version: 2 id: next-03 category: server-actions-safety -description: server action with no input validation or CSRF protection -code: | - // app/actions/updateProfile.ts - 'use server'; +description: A Server Action accepts identity and privilege from form data. +intentional_clean: false +public: + file_path: app/actions/updateProfile.ts + language: TypeScript + frameworks: + next: '15' + runtime: nodejs + assumptions: + - The authenticated session identifies the only profile the caller may update. + - Only an administrator may change a user's role. + requirements: + - The action must derive the target profile and authorization from the authenticated session. + code: | + 'use server'; - export async function updateProfile(formData: FormData) { - const userId = formData.get('userId') as string; - const bio = formData.get('bio') as string; - const role = formData.get('role') as string; // should never come from user + export async function updateProfile(formData: FormData) { + const userId = formData.get('userId') as string; + const bio = formData.get('bio') as string; + const role = formData.get('role') as string; - await db.user.update({ - where: { id: userId }, - data: { bio, role }, // role written directly from form input - }); - } + await db.user.update({ + where: { id: userId }, + data: { bio, role }, + }); + } +metadata: + provenance: + kind: synthetic + origin: Next.js Server Actions documentation + revision_reason: Limited the fixture to explicit authorization authority; removed CSRF and stored-XSS speculation. + difficulty: FUNDAMENTALLY-FLAWED + domains: [authorization, server-actions] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/building-your-application/data-fetching/server-actions-and-mutations#authentication-and-authorization + supports: Server Actions should verify authentication and authorization. + notes: Both authority violations are established by the supplied session requirements. + excluded_claims: [CSRF, validation-library choice, and HTML rendering behavior are not scored.] issues: - - severity: CRITICAL + - id: next-03-form-target-identity + rule_id: next.server-actions-safety category: server-actions-safety - description: "role is written directly from form input with no authorization check; any user can elevate their own role to admin" - evidence: "const role = formData.get('role') as string;" - expected_checks: [server-actions-safety, security] - - severity: CRITICAL + severity: CRITICAL + description: The update target is taken from user-controlled form data instead of the authenticated session. + evidence_spans: + - lines: [4, 9] + rationale: userId comes from formData and becomes the database where clause. + - id: next-03-form-role + rule_id: next.server-actions-safety category: server-actions-safety - description: "no session/auth check: any unauthenticated caller can update any user's profile by supplying an arbitrary userId" - evidence: "const userId = formData.get('userId') as string;" - expected_checks: [server-actions-safety, security] - - severity: MAJOR - category: server-actions-safety - description: "no input validation or sanitization on bio; stored XSS if bio is rendered with dangerouslySetInnerHTML elsewhere" - evidence: "const bio = formData.get('bio') as string;" - expected_checks: [server-actions-safety, security] + severity: CRITICAL + description: The privileged role field is taken from user-controlled form data without the required administrator authorization. + evidence_spans: + - lines: [6, 10] + rationale: role comes from formData and is written directly to the database. diff --git a/research/benchmarks/fixtures/next/fixture-04.yaml b/research/benchmarks/fixtures/next/fixture-04.yaml index c2ad994..c0dccea 100644 --- a/research/benchmarks/fixtures/next/fixture-04.yaml +++ b/research/benchmarks/fixtures/next/fixture-04.yaml @@ -1,32 +1,54 @@ +schema_version: 2 id: next-04 category: route-handler-safety -description: route handler leaking internal error details and missing rate limiting -code: | - // app/api/auth/reset-password/route.ts - export async function POST(request: Request) { - const { email } = await request.json(); - try { - const user = await db.user.findUnique({ where: { email } }); - if (!user) throw new Error(`No user found with email: ${email}`); - await sendResetEmail(user); - return Response.json({ success: true }); - } catch (e) { - return Response.json({ error: e.message }, { status: 500 }); +description: A password-reset route returns a request-derived account-existence error. +intentional_clean: false +public: + file_path: app/api/auth/reset-password/route.ts + language: TypeScript + frameworks: + next: '15' + runtime: nodejs + assumptions: + - The route accepts unauthenticated password-reset requests. + - Email addresses are personal account information. + requirements: + - The route must not reveal whether the supplied email has an account. + code: | + export async function POST(request: Request) { + const { email } = await request.json(); + try { + const user = await db.user.findUnique({ where: { email } }); + if (!user) throw new Error(`No user found with email: ${email}`); + await sendResetEmail(user); + return Response.json({ success: true }); + } catch (error) { + return Response.json({ error: error.message }, { status: 500 }); + } } - } +metadata: + provenance: + kind: synthetic + origin: Next.js Route Handlers documentation + revision_reason: Removed the unsupported rate-limit claim and retained the direct response disclosure. + difficulty: HAS-BUGS + domains: [privacy, route-handlers] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/building-your-application/routing/route-handlers + supports: Route Handlers construct HTTP responses. + notes: The request input is interpolated into the caught error and returned in the response. + excluded_claims: [Rate limiting, email delivery policy, and stack-trace exposure are not scored.] issues: - - severity: CRITICAL + - id: next-04-account-existence-disclosure + rule_id: next.route-handler-safety category: route-handler-safety - description: "error message echoes back whether a user email exists; enables user enumeration: attackers can discover registered emails by probing error responses" - evidence: "throw new Error(`No user found with email: ${email}`);" - expected_checks: [route-handler-safety, security] - - severity: MAJOR - category: route-handler-safety - description: "no rate limiting: endpoint accepts unlimited reset requests, enabling email spam and account lockout attacks" - evidence: "export async function POST(request: Request) {" - expected_checks: [route-handler-safety, security] - - severity: MAJOR - category: route-handler-safety - description: "e.message returned verbatim to client may expose internal stack traces or database error details" - evidence: "return Response.json({ error: e.message }, { status: 500 });" - expected_checks: [route-handler-safety, security] + severity: MAJOR + description: The response returns an error stating that the request-supplied email has no account, contrary to the non-disclosure requirement. + evidence_spans: + - lines: [5, 9] + rationale: The email is interpolated into an error that is returned to the caller. diff --git a/research/benchmarks/fixtures/next/fixture-05.yaml b/research/benchmarks/fixtures/next/fixture-05.yaml index ed51a06..4aee1d9 100644 --- a/research/benchmarks/fixtures/next/fixture-05.yaml +++ b/research/benchmarks/fixtures/next/fixture-05.yaml @@ -1,26 +1,51 @@ +schema_version: 2 id: next-05 category: runtime-choice -description: edge runtime using Node.js-only APIs -code: | - // app/api/pdf/route.ts - export const runtime = 'edge'; +description: An Edge route uses Node.js filesystem APIs. +intentional_clean: false +public: + file_path: app/api/pdf/route.ts + language: TypeScript + frameworks: + next: '15' + runtime: edge + assumptions: + - The route must run in the configured Edge runtime. + requirements: + - The route must not use Node.js-only filesystem APIs. + code: | + export const runtime = 'edge'; - import { readFileSync } from 'fs'; // Node.js only - import path from 'path'; + import { readFileSync } from 'fs'; + import path from 'path'; - export async function GET() { - const template = readFileSync(path.join(process.cwd(), 'templates/invoice.pdf')); - const pdf = await generatePDF(template); - return new Response(pdf, { headers: { 'Content-Type': 'application/pdf' } }); - } + export async function GET() { + const template = readFileSync(path.join(process.cwd(), 'templates/invoice.pdf')); + return new Response(template, { headers: { 'Content-Type': 'application/pdf' } }); + } +metadata: + provenance: + kind: synthetic + origin: Next.js Edge runtime documentation + revision_reason: Merged duplicate filesystem and process claims into one runtime incompatibility. + difficulty: HAS-BUGS + domains: [runtime, deployment] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/pages/api-reference/edge + supports: The Edge runtime does not support all Node.js APIs. + notes: The declared runtime and filesystem imports are both shown in the source. + excluded_claims: [A replacement PDF implementation is not scored.] issues: - - severity: CRITICAL + - id: next-05-edge-filesystem-api + rule_id: next.runtime-choice category: runtime-choice - description: "runtime='edge' is incompatible with fs.readFileSync and path.join; these are Node.js APIs unavailable in the Edge Runtime; route will fail at deploy time" - evidence: "export const runtime = 'edge';" - expected_checks: [runtime-choice] - - severity: MINOR - category: runtime-choice - description: "process.cwd() is also unavailable in Edge Runtime; file access patterns require Node.js runtime or static imports" - evidence: "path.join(process.cwd(), 'templates/invoice.pdf')" - expected_checks: [runtime-choice] + severity: CRITICAL + description: The explicitly Edge-configured route imports and calls Node.js filesystem APIs. + evidence_spans: + - lines: [1, 3, 4, 7] + rationale: The route declares edge while importing fs and path and calling readFileSync. diff --git a/research/benchmarks/fixtures/next/fixture-06.yaml b/research/benchmarks/fixtures/next/fixture-06.yaml index 0360e90..c9874ab 100644 --- a/research/benchmarks/fixtures/next/fixture-06.yaml +++ b/research/benchmarks/fixtures/next/fixture-06.yaml @@ -1,31 +1,57 @@ +schema_version: 2 id: next-06 category: suspense-boundary -description: missing Suspense boundary around async RSC in streaming layout -code: | - // app/feed/page.tsx - export default async function FeedPage() { - return ( -
-
- {/* async RSC, no Suspense */} - {/* async RSC, no Suspense */} -
-
- ); - } +description: A feed route must stream its shell before a slow Server Component resolves. +intentional_clean: false +public: + file_path: app/feed/page.tsx + language: TypeScript JSX + frameworks: + next: '15' + react: '19' + runtime: nodejs + assumptions: + - AsyncFeedItems awaits a slow database query. + requirements: + - Header and footer must stream before the feed items resolve. + code: | + export default function FeedPage() { + return ( +
+
+ +
+
+ ); + } - async function AsyncFeedItems() { - const items = await db.feed.findMany({ take: 50 }); // slow query - return items.map(i => ); - } + async function AsyncFeedItems() { + const items = await db.feed.findMany({ take: 50 }); + return items.map((item) => ); + } +metadata: + provenance: + kind: synthetic + origin: Next.js loading UI and streaming documentation + revision_reason: Added an explicit streaming requirement and removed unsupported cache advice. + difficulty: HAS-BUGS + domains: [streaming, server-components] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/building-your-application/routing/loading-ui-and-streaming + supports: Suspense boundaries support streaming UI. + notes: The slow await is shown and the streaming ordering is a public requirement. + excluded_claims: [Caching and the performance of a particular query are not scored.] issues: - - severity: MAJOR + - id: next-06-missing-streaming-boundary + rule_id: next.suspense-boundary category: suspense-boundary - description: "no Suspense boundary around AsyncFeedItems; the slow db query blocks streaming of the entire page including Header and Footer" - evidence: " {/* async RSC, no Suspense */}" - expected_checks: [suspense-boundary, cache-correctness] - - severity: MINOR - category: cache-correctness - description: "db.feed.findMany with no cache directive; each request re-queries even for the same user; revalidate or React cache() should be applied" - evidence: "const items = await db.feed.findMany({ take: 50 });" - expected_checks: [cache-correctness] + severity: MAJOR + description: AsyncFeedItems has no Suspense boundary even though the route must stream its shell before the slow component resolves. + evidence_spans: + - lines: [5, 11] + rationale: The awaited component is rendered directly between Header and Footer. diff --git a/research/benchmarks/fixtures/next/fixture-07.yaml b/research/benchmarks/fixtures/next/fixture-07.yaml index 6b277c3..d74333f 100644 --- a/research/benchmarks/fixtures/next/fixture-07.yaml +++ b/research/benchmarks/fixtures/next/fixture-07.yaml @@ -1,34 +1,63 @@ +schema_version: 2 id: next-07 category: upgrade-safety -description: Pages Router patterns used in App Router without migration -code: | - // app/shop/[id]/page.tsx (App Router file) - import { useRouter } from 'next/router'; // Pages Router import +description: An App Router page uses Pages Router APIs. +intentional_clean: false +public: + file_path: app/shop/[id]/page.tsx + language: TypeScript JSX + frameworks: + next: '15' + react: '19' + runtime: nodejs + assumptions: + - The file is an App Router page. + requirements: + - The route must use App Router-compatible routing and data APIs. + code: | + 'use client'; - export default function ShopPage({ params }) { - const router = useRouter(); + import { useRouter } from 'next/router'; - const handleBack = () => router.back(); + export default function ShopPage({ params }) { + const router = useRouter(); + return ; + } - return ( -
- - -
- ); - } - - export async function getServerSideProps(ctx) { // Pages Router API - return { props: { id: ctx.params.id } }; - } + export async function getServerSideProps(context) { + return { props: { id: context.params.id } }; + } +metadata: + provenance: + kind: synthetic + origin: Next.js App Router migration documentation + revision_reason: Retained only the explicit Pages Router API incompatibilities. + difficulty: FUNDAMENTALLY-FLAWED + domains: [routing, upgrade] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/guides/migrating/app-router-migration + supports: App Router migration replaces next/router and getServerSideProps patterns. + notes: The routing mode is a supplied assumption, not inferred from the filename. + excluded_claims: [The fixture does not score component design or params typing.] issues: - - severity: CRITICAL + - id: next-07-pages-router-hook + rule_id: next.upgrade-safety category: upgrade-safety - description: "next/router is the Pages Router hook; App Router requires next/navigation (useRouter from next/navigation); will throw at runtime" - evidence: "import { useRouter } from 'next/router';" - expected_checks: [upgrade-safety, app-router-conventions] - - severity: CRITICAL + severity: MAJOR + description: The App Router page imports useRouter from next/router instead of the App Router API. + evidence_spans: + - lines: [3, 6] + rationale: The Pages Router hook import is invoked by the App Router page. + - id: next-07-get-server-side-props + rule_id: next.upgrade-safety category: upgrade-safety - description: "getServerSideProps is a Pages Router API; App Router ignores it silently; params are already available via the page's props in App Router" - evidence: "export async function getServerSideProps(ctx) {" - expected_checks: [upgrade-safety, app-router-conventions] + severity: MAJOR + description: The App Router page exports the Pages Router getServerSideProps API. + evidence_spans: + - lines: [10] + rationale: getServerSideProps is declared directly in the App Router module. diff --git a/research/benchmarks/fixtures/next/fixture-08.yaml b/research/benchmarks/fixtures/next/fixture-08.yaml index ec20be3..6935401 100644 --- a/research/benchmarks/fixtures/next/fixture-08.yaml +++ b/research/benchmarks/fixtures/next/fixture-08.yaml @@ -1,34 +1,47 @@ +schema_version: 2 id: next-08 category: app-router-conventions -description: incorrect metadata API and missing generateStaticParams safety bounds -code: | - // app/blog/[slug]/page.tsx - export const metadata = { // static, not dynamic - title: 'Blog Post', - }; - - export async function generateStaticParams() { - const posts = await db.post.findMany(); // no limit - return posts.map(p => ({ slug: p.slug })); - } - - export default async function BlogPost({ params }) { - const post = await db.post.findUnique({ where: { slug: params.slug } }); - return
{post.content}
; // no null check - } +description: A blog route does not meet its required not-found behavior. +intentional_clean: false +public: + file_path: app/blog/[slug]/page.tsx + language: TypeScript JSX + frameworks: + next: '15' + react: '19' + runtime: nodejs + assumptions: + - db.post.findUnique returns null when no post has the requested slug. + requirements: + - An unknown slug must produce the route's not-found response. + code: | + export default async function BlogPost({ params }) { + const post = await db.post.findUnique({ where: { slug: params.slug } }); + return
{post.content}
; + } +metadata: + provenance: + kind: synthetic + origin: Next.js notFound documentation + revision_reason: Removed valid static metadata and unbounded-query preferences; retained the stated route behavior. + difficulty: HAS-BUGS + domains: [routing, operability] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://nextjs.org/docs/app/api-reference/functions/not-found + supports: notFound renders the not-found segment and returns a 404 response. + notes: The null result and required not-found behavior are public assumptions. + excluded_claims: [Dynamic metadata and static-parameter sizing are not scored.] issues: - - severity: MAJOR + - id: next-08-missing-not-found + rule_id: next.app-router-conventions category: app-router-conventions - description: "metadata exported as a plain object instead of generateMetadata(); post-specific title/og tags (title, description, og:image) cannot be set dynamically" - evidence: "export const metadata = { // static, not dynamic" - expected_checks: [app-router-conventions] - - severity: MAJOR - category: app-router-conventions - description: "generateStaticParams fetches all posts with no limit; at scale this causes build-time OOM and multi-hour static generation" - evidence: "const posts = await db.post.findMany(); // no limit" - expected_checks: [app-router-conventions, operability] - - severity: MINOR - category: operability - description: "no null check on post before rendering; if slug doesn't exist, post.content throws TypeError; should call notFound()" - evidence: "return
{post.content}
; // no null check" - expected_checks: [operability] + severity: MAJOR + description: The route dereferences a possibly null post instead of producing the required not-found response. + evidence_spans: + - lines: [2, 3] + rationale: findUnique can return null and post.content is rendered without a not-found branch. diff --git a/research/benchmarks/fixtures/react-native/fixture-01.yaml b/research/benchmarks/fixtures/react-native/fixture-01.yaml index d208aba..930761e 100644 --- a/research/benchmarks/fixtures/react-native/fixture-01.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-01.yaml @@ -1,36 +1,44 @@ +schema_version: 2 id: rn-01 category: render-performance -description: inline style objects and anonymous functions recreated on every render -code: | - function ProductCard({ product, onAddToCart }) { - return ( - - - onAddToCart(product.id)} - style={{ backgroundColor: '#007AFF', padding: 12 }} - > - Add to Cart - - - ); - } -issues: - - severity: MAJOR - category: render-performance - description: "inline style objects create new object references on every render; FlatList/memo optimization is fully defeated because style props never shallowEqual" - evidence: "style={{ flex: 1, padding: 16, backgroundColor: '#fff' }}" - expected_checks: [render-performance, native-styling] - - severity: MAJOR - category: render-performance - description: "anonymous arrow function in onPress recreates on every render; if ProductCard is used in a FlatList, every item re-renders on any parent state change" - evidence: "onPress={() => onAddToCart(product.id)}" - expected_checks: [render-performance] - - severity: MINOR - category: render-performance - description: "Image has no resizeMode; default 'cover' may not match container; no loading/error placeholder" - evidence: "style={{ width: 200, height: 200 }}" - expected_checks: [native-styling] +description: A simple product card uses ordinary inline values without a stated performance defect. +intentional_clean: true +public: + file_path: src/ProductCard.tsx + language: TypeScript JSX + frameworks: + react-native: '0.76' + runtime: hermes + assumptions: + - ProductCard is not rendered by a virtualized list. + - Its parent does not rely on referential equality for its style or handler props. + requirements: [] + code: | + function ProductCard({ product, onAddToCart }) { + return ( + + + onAddToCart(product.id)}> + Add to Cart + + + ); + } +metadata: + provenance: + kind: synthetic + origin: React Native performance documentation + revision_reason: Replaced generic inline-object and callback claims with a clean restraint control. + difficulty: ADVERSARIAL + domains: [rendering, performance] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://reactnative.dev/docs/performance + supports: React Native performance work is context-dependent. + notes: No catalog rule is established by the shown ordinary component. + excluded_claims: [Memoization, StyleSheet extraction, image placeholders, and FlatList behavior are not scored.] +issues: [] diff --git a/research/benchmarks/fixtures/react-native/fixture-02.yaml b/research/benchmarks/fixtures/react-native/fixture-02.yaml index 46a9fc2..0cd1caf 100644 --- a/research/benchmarks/fixtures/react-native/fixture-02.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-02.yaml @@ -1,30 +1,45 @@ +schema_version: 2 id: rn-02 category: list-performance -description: FlatList without keyExtractor, removeClippedSubviews, or windowing config -code: | - function NotificationList({ notifications }) { - return ( - } - onEndReached={loadMore} - onEndReachedThreshold={0.5} - /> - ); - } -issues: - - severity: MAJOR - category: list-performance - description: "no keyExtractor: FlatList falls back to index-based keys causing broken diffing on list mutations (insert/delete reorders)" - evidence: "renderItem={({ item }) => }" - expected_checks: [list-performance, render-performance] - - severity: MAJOR - category: list-performance - description: "no removeClippedSubviews or windowSize config; on large lists all items stay mounted, causing memory pressure on low-end Android devices" - evidence: " item.id} + renderItem={({ item }) => } + onEndReached={loadMore} + onEndReachedThreshold={0.5} + /> + ); + } +metadata: + provenance: + kind: synthetic + origin: React Native FlatList documentation + revision_reason: Replaced unsupported tuning-prop defaults with a clean list-identity control. + difficulty: ADVERSARIAL + domains: [lists, performance] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://reactnative.dev/docs/flatlist + supports: FlatList accepts a keyExtractor and is a performant interface for flat lists. + notes: The stable key requirement is met and no tuning requirement is supplied. + excluded_claims: [removeClippedSubviews, windowSize, and getItemLayout are not scored.] +issues: [] diff --git a/research/benchmarks/fixtures/react-native/fixture-03.yaml b/research/benchmarks/fixtures/react-native/fixture-03.yaml index fbf2c78..9c4a03d 100644 --- a/research/benchmarks/fixtures/react-native/fixture-03.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-03.yaml @@ -1,32 +1,66 @@ +schema_version: 2 id: rn-03 category: memory-leak -description: setInterval without cleanup and subscription without teardown -code: | - function LivePriceTicker({ symbol }) { - const [price, setPrice] = useState(null); +description: A ticker effect creates resources without cleanup. +intentional_clean: false +public: + file_path: src/LivePriceTicker.tsx + language: TypeScript JSX + frameworks: + react-native: '0.76' + runtime: hermes + assumptions: + - priceSocket.subscribe returns a subscription that requires unsubscribe. + - The component can unmount or receive a new symbol. + requirements: + - Effects must release timers and subscriptions that they create. + code: | + function LivePriceTicker({ symbol }) { + const [price, setPrice] = useState(null); - useEffect(() => { - const id = setInterval(() => { - fetchPrice(symbol).then(setPrice); - }, 1000); - // no cleanup return - }, [symbol]); + useEffect(() => { + const timer = setInterval(() => { + fetchPrice(symbol).then(setPrice); + }, 1000); + }, [symbol]); - useEffect(() => { - const sub = priceSocket.subscribe(symbol, setPrice); - // no unsubscribe - }, [symbol]); + useEffect(() => { + const subscription = priceSocket.subscribe(symbol, setPrice); + }, [symbol]); - return {price}; - } + return {price}; + } +metadata: + provenance: + kind: synthetic + origin: React effect cleanup documentation + revision_reason: Replaced answer comments with ownership assumptions and specific cleanup labels. + difficulty: FUNDAMENTALLY-FLAWED + domains: [effects, resource-lifecycle] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://react.dev/reference/react/useEffect + supports: Effects may return cleanup functions for external systems. + notes: Both external resources are created by their effects and neither effect returns cleanup. + excluded_claims: [Exact memory growth and unmounted-state warnings are not scored.] issues: - - severity: CRITICAL + - id: rn-03-interval-cleanup + rule_id: react-native.memory-leak category: memory-leak - description: "setInterval not cleared on unmount or symbol change; interval fires indefinitely after navigation away, calling setState on unmounted component" - evidence: "const id = setInterval(() => {" - expected_checks: [memory-leak, hook-correctness] - - severity: CRITICAL + severity: MAJOR + description: The effect creates a repeating timer but returns no cleanup to clear it. + evidence_spans: + - lines: [5, 6, 7] + rationale: setInterval is called inside an effect whose body has no returned cleanup. + - id: rn-03-subscription-cleanup + rule_id: react-native.memory-leak category: memory-leak - description: "priceSocket subscription not unsubscribed on unmount; multiple subscriptions stack on each symbol change" - evidence: "const sub = priceSocket.subscribe(symbol, setPrice);" - expected_checks: [memory-leak, hook-correctness] + severity: MAJOR + description: The effect creates a socket subscription but returns no cleanup to unsubscribe it. + evidence_spans: + - lines: [10, 11] + rationale: subscribe is called inside an effect whose body has no returned cleanup. diff --git a/research/benchmarks/fixtures/react-native/fixture-04.yaml b/research/benchmarks/fixtures/react-native/fixture-04.yaml index 182a66c..4a35b51 100644 --- a/research/benchmarks/fixtures/react-native/fixture-04.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-04.yaml @@ -1,34 +1,67 @@ +schema_version: 2 id: rn-04 category: native-bridge -description: synchronous native module call blocking JS thread -code: | - function useDeviceInfo() { - const [info, setInfo] = useState({}); +description: A synchronous optional native module call runs on startup without a guard. +intentional_clean: false +public: + file_path: src/useDeviceInfo.ts + language: TypeScript JSX + frameworks: + react-native: '0.76' + runtime: hermes + assumptions: + - DeviceInfo.getInfo is a synchronous native method and may block while it gathers device data. + - DeviceInfo is optional and is unavailable in the project's web preview target. + requirements: + - The startup header must not synchronously block on optional native work. + - The web preview must show a fallback when DeviceInfo is unavailable. + code: | + function useDeviceInfo() { + const [info, setInfo] = useState({}); - useEffect(() => { - // NativeModules.DeviceInfo.getInfo() is synchronous - const deviceInfo = NativeModules.DeviceInfo.getInfo(); - setInfo(deviceInfo); - }, []); + useEffect(() => { + const deviceInfo = NativeModules.DeviceInfo.getInfo(); + setInfo(deviceInfo); + }, []); - return info; - } + return info; + } - function AppHeader() { - const info = useDeviceInfo(); - return {info.model}; - } + function AppHeader() { + const info = useDeviceInfo(); + return {info.model}; + } +metadata: + provenance: + kind: synthetic + origin: React Native NativeModules documentation + revision_reason: Made synchrony and availability public; split bridge performance from bridge failure handling. + difficulty: HAS-BUGS + domains: [native-modules, startup] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://reactnative.dev/docs/next/legacy/native-modules-ios + supports: Native modules expose native functionality to JavaScript. + notes: The source does not infer the module's behavior; the public assumptions define the benchmark conditions. + excluded_claims: [ANR, a specific fallback implementation, and native-module architecture changes are not scored.] issues: - - severity: MAJOR - category: native-bridge - description: "NativeModules synchronous call blocks the JS thread; on startup this can cause frame drops and ANR on Android if the native implementation is slow" - evidence: "const deviceInfo = NativeModules.DeviceInfo.getInfo();" - evidence_lines: [6] - expected_checks: [native-bridge, render-performance] - - severity: MINOR + - id: rn-04-blocking-device-info + rule_id: react-native.native-bridge category: native-bridge + severity: MAJOR + description: The startup effect synchronously calls the known blocking DeviceInfo method, contrary to the startup requirement. + evidence_spans: + - lines: [5] + rationale: The synchronous native call is executed directly by the startup effect. + - id: rn-04-unguarded-device-info rule_id: react-native.native-bridge.error-handling - description: "no error handling around NativeModules call; if the native module is unavailable (web target, storybook), throws uncaught TypeError" - evidence: "NativeModules.DeviceInfo.getInfo();" - evidence_lines: [6] - expected_checks: [native-bridge, operability] + category: native-bridge.error-handling + severity: MAJOR + description: The optional DeviceInfo call is unguarded even though the web preview target can lack that module. + evidence_spans: + - lines: [5] + rationale: The optional native module is accessed directly with no availability branch or fallback. diff --git a/research/benchmarks/fixtures/react-native/fixture-05.yaml b/research/benchmarks/fixtures/react-native/fixture-05.yaml index 6a02aab..48c83aa 100644 --- a/research/benchmarks/fixtures/react-native/fixture-05.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-05.yaml @@ -1,29 +1,56 @@ +schema_version: 2 id: rn-05 category: expo-workflow -description: bare-workflow native module used in managed Expo project -code: | - // app.json shows "expo": { "workflow": "managed" } - import { NativeModules } from 'react-native'; - import RNBluetoothClassic from 'react-native-bluetooth-classic'; // native module, no Expo config plugin +description: An Expo Go target imports a module requiring custom native code. +intentional_clean: false +public: + file_path: src/useBluetooth.ts + language: TypeScript JSX + frameworks: + expo: '52' + react-native: '0.76' + runtime: hermes + assumptions: + - The application is launched in Expo Go, not a development build or custom native client. + - RNBluetoothClassic requires custom native code that Expo Go does not include. + requirements: + - The Bluetooth feature must run in the stated Expo Go deployment target. + code: | + import RNBluetoothClassic from 'react-native-bluetooth-classic'; - export function useBluetooth() { - const [devices, setDevices] = useState([]); + export function useBluetooth() { + const [devices, setDevices] = useState([]); - useEffect(() => { - RNBluetoothClassic.startDiscovery().then(setDevices); - return () => RNBluetoothClassic.cancelDiscovery(); - }, []); + useEffect(() => { + RNBluetoothClassic.startDiscovery().then(setDevices); + return () => RNBluetoothClassic.cancelDiscovery(); + }, []); - return devices; - } + return devices; + } +metadata: + provenance: + kind: synthetic + origin: Expo Go and development-build documentation + revision_reason: Replaced unverified config-plugin and permission claims with explicit Expo Go native-code context. + difficulty: HAS-BUGS + domains: [expo, native-modules] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://docs.expo.dev/develop/development-builds/introduction/ + supports: Development builds allow custom native code not available in Expo Go. + notes: The requirement and assumptions make the deployment incompatibility explicit. + excluded_claims: [Config-plugin availability, permission declarations, EAS Build behavior, and store review are not scored.] issues: - - severity: CRITICAL + - id: rn-05-expo-go-native-module + rule_id: react-native.expo-workflow category: expo-workflow - description: "react-native-bluetooth-classic has no Expo config plugin; managed workflow EAS Build will fail because the native module requires bare workflow or a custom plugin to modify native files" - evidence: "import RNBluetoothClassic from 'react-native-bluetooth-classic'; // native module, no Expo config plugin" - expected_checks: [expo-workflow, upgrade-safety] - - severity: MAJOR - category: expo-workflow - description: "no Bluetooth permission declaration in app.json (ios.infoPlist, android.permissions); app will crash or be rejected at store review" - evidence: "RNBluetoothClassic.startDiscovery().then(setDevices);" - expected_checks: [expo-workflow, security] + severity: CRITICAL + description: The stated Expo Go target cannot load the imported module because it requires custom native code absent from Expo Go. + evidence_spans: + - lines: [1, 7] + rationale: The custom native module is imported and called by the required feature. diff --git a/research/benchmarks/fixtures/react-native/fixture-06.yaml b/research/benchmarks/fixtures/react-native/fixture-06.yaml index 92a8864..de13c05 100644 --- a/research/benchmarks/fixtures/react-native/fixture-06.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-06.yaml @@ -1,30 +1,57 @@ +schema_version: 2 id: rn-06 category: gesture-handling -description: Android back gesture not handled in custom navigation flow -code: | - function CheckoutFlow({ onComplete, onAbort }) { - const [step, setStep] = useState(1); +description: A custom checkout flow has no Android hardware-back handling. +intentional_clean: false +public: + file_path: src/CheckoutFlow.tsx + language: TypeScript JSX + frameworks: + react-native: '0.76' + runtime: hermes + assumptions: + - CheckoutFlow owns the custom multi-step screen state. + - Android hardware back reaches this component before navigation exits the screen. + requirements: + - Android hardware back at steps two and three must return to the preceding checkout step. + code: | + function CheckoutFlow({ onComplete }) { + const [step, setStep] = useState(1); + const nextStep = () => setStep((current) => Math.min(current + 1, 3)); + const prevStep = () => setStep((current) => Math.max(current - 1, 1)); - const nextStep = () => setStep(s => Math.min(s + 1, 3)); - const prevStep = () => setStep(s => Math.max(s - 1, 1)); - - return ( - - - {step === 1 && } - {step === 2 && } - {step === 3 && } - - ); - } + return ( + + + {step === 1 && } + {step === 2 && } + {step === 3 && } + + ); + } +metadata: + provenance: + kind: synthetic + origin: React Native BackHandler documentation + revision_reason: Added explicit navigation ownership and required Android behavior. + difficulty: HAS-BUGS + domains: [navigation, gestures] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://reactnative.dev/docs/backhandler + supports: BackHandler can handle Android hardware back presses. + notes: The source supplies only an in-flow back button and no Android hardware-back handling. + excluded_claims: [Confirmation dialogs, iOS gestures, and global navigator behavior are not scored.] issues: - - severity: MAJOR + - id: rn-06-missing-hardware-back + rule_id: react-native.gesture-handling category: gesture-handling - description: "Android hardware back button is not handled; pressing back during checkout will pop the entire checkout screen instead of going to the previous step, losing form data" - evidence: "function CheckoutFlow({ onComplete, onAbort }) {" - expected_checks: [gesture-handling, operability] - - severity: MINOR - category: operability - description: "no confirmation dialog on abort; user can accidentally discard cart state by pressing back" - evidence: "const prevStep = () => setStep(s => Math.max(s - 1, 1));" - expected_checks: [operability] + severity: MAJOR + description: The custom flow has no handling for the required Android hardware-back transition to the previous step. + evidence_spans: + - lines: [1, 4, 10, 11] + rationale: prevStep is only wired to PaymentForm and no hardware-back handler is shown. diff --git a/research/benchmarks/fixtures/react-native/fixture-07.yaml b/research/benchmarks/fixtures/react-native/fixture-07.yaml index ed9214d..ce4b691 100644 --- a/research/benchmarks/fixtures/react-native/fixture-07.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-07.yaml @@ -1,27 +1,50 @@ +schema_version: 2 id: rn-07 category: hermes-compatibility -description: optional chaining and nullish coalescing used without Hermes version guard -code: | - // package.json: "react-native": "0.66.0", "hermes-engine": "0.9.0" - function UserAvatar({ user }) { - const displayName = user?.profile?.displayName ?? user?.email ?? 'Guest'; - const avatarUri = user?.avatar?.['200x200']?.url; +description: Modern optional chaining is used with a current Hermes runtime. +intentional_clean: true +public: + file_path: src/UserAvatar.tsx + language: TypeScript JSX + frameworks: + react-native: '0.66' + hermes: '0.9.0' + runtime: hermes + assumptions: + - React Native 0.66 ships with Hermes 0.9.0. + - user may be null. + requirements: [] + code: | + function UserAvatar({ user }) { + const displayName = user?.profile?.displayName ?? user?.email ?? 'Guest'; + const avatarUri = user?.avatar?.['200x200']?.url; - return ( - - - {displayName} - - ); - } -issues: - - severity: MAJOR - category: hermes-compatibility - description: "Hermes 0.9.0 (bundled with RN 0.66) has partial optional chaining support; chained bracket notation user?.avatar?.['200x200']?.url may throw on Android Hermes but pass on iOS JSC" - evidence: "const avatarUri = user?.avatar?.['200x200']?.url;" - expected_checks: [hermes-compatibility, upgrade-safety] - - severity: MINOR - category: operability - description: "Image with undefined uri source renders broken image icon instead of placeholder; no fallback source or error handler" - evidence: "" - expected_checks: [operability] + return ( + + + {displayName} + + ); + } +metadata: + provenance: + kind: synthetic + origin: Hermes documentation + revision_reason: Replaced the unsupported old-version compatibility assertion with a clean current-runtime control. + difficulty: ADVERSARIAL + domains: [runtime, language] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://reactnative.dev/blog/2021/10/01/version-066 + supports: React Native 0.66 includes Hermes 0.9.0. + - url: https://raw.githubusercontent.com/facebook/hermes/v0.9.0/test/hermes/optional-chaining.js + supports: The Hermes 0.9.0 tag includes executable optional-chaining coverage. + - url: https://raw.githubusercontent.com/facebook/hermes/v0.9.0/test/hermes/nullish-coalescing.js + supports: The Hermes 0.9.0 tag includes executable nullish-coalescing coverage. + notes: The supplied runtime pairing directly supports the syntax used by this clean control. + excluded_claims: [Image fallback behavior and older Hermes versions are not scored.] +issues: [] diff --git a/research/benchmarks/fixtures/react-native/fixture-08.yaml b/research/benchmarks/fixtures/react-native/fixture-08.yaml index 8a28e11..4322945 100644 --- a/research/benchmarks/fixtures/react-native/fixture-08.yaml +++ b/research/benchmarks/fixtures/react-native/fixture-08.yaml @@ -1,33 +1,54 @@ +schema_version: 2 id: rn-08 category: release-safety -description: over-the-air update with no version gate or rollback strategy -code: | - // App.tsx - import * as Updates from 'expo-updates'; - - useEffect(() => { - async function checkUpdate() { - const update = await Updates.checkForUpdateAsync(); - if (update.isAvailable) { - await Updates.fetchUpdateAsync(); - await Updates.reloadAsync(); // immediate reload, no user prompt +description: An update check reloads immediately despite a user-initiated activation requirement. +intentional_clean: false +public: + file_path: App.tsx + language: TypeScript JSX + frameworks: + expo: '52' + expo-updates: '0.26' + runtime: hermes + assumptions: + - expo-updates is configured for this application. + - checkForUpdateAsync can report an available update. + requirements: + - The application must tell the user about an available update and let the user initiate any reload. + code: | + useEffect(() => { + async function checkUpdate() { + const update = await Updates.checkForUpdateAsync(); + if (update.isAvailable) { + await Updates.fetchUpdateAsync(); + await Updates.reloadAsync(); + } } - } - checkUpdate(); - }, []); + checkUpdate(); + }, []); +metadata: + provenance: + kind: synthetic + origin: Expo Updates documentation + revision_reason: Removed unsupported native-version and rollback claims; retained only the explicit activation-choice requirement. + difficulty: HAS-BUGS + domains: [updates, release] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://docs.expo.dev/versions/latest/sdk/updates/ + supports: expo-updates provides update checking, fetching, and reload APIs. + notes: The source shows an immediate reload and the requirement establishes the forbidden behavior. + excluded_claims: [Native version gates, rollback, release channels, and network-error handling are not scored.] issues: - - severity: MAJOR + - id: rn-08-immediate-reload + rule_id: react-native.release-safety category: release-safety - description: "OTA update reloads the app immediately without user consent; if update introduces a crash, users are forced into a broken state with no rollback path" - evidence: "await Updates.reloadAsync(); // immediate reload, no user prompt" - expected_checks: [release-safety, operability] - - severity: MAJOR - category: release-safety - description: "no minimum app version check before applying OTA update; update might target a different native binary version and cause incompatibility crashes" - evidence: "await Updates.fetchUpdateAsync();" - expected_checks: [release-safety, upgrade-safety] - - severity: MINOR - category: operability - description: "no error handling on checkForUpdateAsync/fetchUpdateAsync; network errors during update check throw and crash the app on startup" - evidence: "const update = await Updates.checkForUpdateAsync();" - expected_checks: [operability] + severity: MAJOR + description: The available-update branch fetches and reloads immediately instead of waiting for the required user-initiated activation. + evidence_spans: + - lines: [3, 4, 5, 6] + rationale: An available update directly leads to fetchUpdateAsync followed by reloadAsync without a user action. diff --git a/research/benchmarks/fixtures/react/fixture-01.yaml b/research/benchmarks/fixtures/react/fixture-01.yaml index 07c11a5..062cd9e 100644 --- a/research/benchmarks/fixtures/react/fixture-01.yaml +++ b/research/benchmarks/fixtures/react/fixture-01.yaml @@ -1,32 +1,66 @@ +schema_version: 2 id: react-01 category: hook-correctness -description: stale closure in useEffect with missing dependency array -code: | - function SearchResults({ query }) { - const [results, setResults] = useState([]); - const [page, setPage] = useState(1); +description: Search results must follow the current query and page. +intentional_clean: false +public: + file_path: src/SearchResults.tsx + language: TypeScript JSX + frameworks: + react: '18.3' + runtime: browser + assumptions: + - Changing page updates the page prop without remounting the component. + - A response for an earlier query or page may resolve after a later request. + requirements: + - The visible results must correspond to the current query and page. + code: | + function SearchResults({ query, page }) { + const [results, setResults] = useState([]); - useEffect(() => { - fetch(`/api/search?q=${query}&page=${page}`) - .then(r => r.json()) - .then(data => setResults(prev => [...prev, ...data])); - }, [query]); // page is missing from deps + useEffect(() => { + fetch(`/api/search?q=${query}&page=${page}`) + .then((response) => response.json()) + .then((data) => setResults(data)); + }, [query]); - return
{results.map(r =>
{r.title}
)}
; - } + return
{results.map((result) =>
{result.title}
)}
; + } +metadata: + provenance: + kind: synthetic + origin: React effect dependency documentation + revision_reason: Re-adjudicated to expose the changing page and response-order assumptions publicly. + difficulty: HAS-BUGS + domains: [hooks, async] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://react.dev/reference/react/useEffect + supports: Effects must declare reactive values they read. + - url: https://react.dev/reference/react/useEffect#fetching-data-with-effects + supports: Effect cleanup can prevent a later response from being overwritten by an earlier request. + notes: The missing page dependency and stale-result path are independently reviewable from the public assumptions and shown effect. + excluded_claims: [No particular stale-result guard or cancellation mechanism is required.] issues: - - severity: MAJOR + - id: react-01-missing-page-dependency + rule_id: react.hook-correctness category: hook-correctness - description: "page missing from useEffect dependency array causes stale closure; changing page silently fetches with wrong value" - evidence: "}, [query]); // page is missing from deps" - expected_checks: [hook-correctness, async-correctness] - - severity: MAJOR + severity: MAJOR + description: The effect reads page but its dependency list excludes page, so a page change does not start the required request. + evidence_spans: + - lines: [5, 8] + rationale: The request interpolates page and the dependency list contains only query. + - id: react-01-stale-response + rule_id: react.async-correctness category: async-correctness - description: "no cancellation/teardown: concurrent fetches race and stale responses overwrite newer ones" - evidence: "fetch(`/api/search?q=${query}&page=${page}`)" - expected_checks: [async-correctness] - - severity: MINOR - category: render-behavior - description: "results array grows unboundedly on re-render without reset on query change" - evidence: "setResults(prev => [...prev, ...data])" - expected_checks: [state-ownership, render-behavior] + severity: MAJOR + description: An earlier request can resolve last and publish results that do not correspond to the current query or page. + evidence_spans: + - lines: [5, 7] + rationale: Each effect invocation starts a request whose completion writes results without checking that it is still current. + - lines: [4, 5, 7] + rationale: The effect starts the request and unconditionally publishes its eventual result. diff --git a/research/benchmarks/fixtures/react/fixture-02.yaml b/research/benchmarks/fixtures/react/fixture-02.yaml index 7ca7cd4..135a5fb 100644 --- a/research/benchmarks/fixtures/react/fixture-02.yaml +++ b/research/benchmarks/fixtures/react/fixture-02.yaml @@ -1,38 +1,58 @@ +schema_version: 2 id: react-02 category: render-behavior -description: expensive computation in render path without memoization -code: | - function DataGrid({ rows, filters }) { - const visibleRows = rows.filter(r => { - return filters.every(f => { - const val = getNestedValue(r, f.path); // O(depth) traversal - return matchFilter(val, f); // regex compile on every call - }); - }); +description: A large editable grid uses unstable row keys. +intentional_clean: false +public: + file_path: src/DataGrid.tsx + language: TypeScript JSX + frameworks: + react: '18.3' + runtime: browser + assumptions: + - rows can be reordered after a filter changes. + - Each row has a stable id. + requirements: + - Editing state must remain associated with the same row after reordering. + code: | + function DataGrid({ rows, filters }) { + const visibleRows = rows.filter((row) => filters.every((filter) => matchFilter(row[filter.key], filter))); - return ( - - {visibleRows.map((row, i) => ( - - {row.cells.map(c => )} - - ))} -
{c}
- ); - } + return ( + + + {visibleRows.map((row, index) => ( + + {row.cells.map((cell) => )} + + ))} + +
{cell.value}
+ ); + } +metadata: + provenance: + kind: synthetic + origin: React list rendering documentation + revision_reason: Replaced generic memoization and virtualization preferences with a stated identity requirement. + difficulty: HAS-BUGS + domains: [rendering, list-identity] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://react.dev/learn/rendering-lists + supports: Keys identify list items when their order changes. + notes: The row id assumption makes the keyed identity defect assessable. + excluded_claims: [Filtering cost and virtualization are not scored without a performance requirement.] issues: - - severity: MAJOR + - id: react-02-index-key + rule_id: react.render-behavior category: render-behavior - description: "filter + nested traversal + regex compile runs synchronously on every render; O(rows × filters × depth) in the render path" - evidence: "const visibleRows = rows.filter(r => {" - expected_checks: [render-behavior, hook-correctness] - - severity: MAJOR - category: render-behavior - description: "key=index on list items: row reorder causes full re-render and lost input state" - evidence: "" - expected_checks: [render-behavior] - - severity: MINOR - category: render-behavior - description: "no virtualization for potentially large row sets" - evidence: "{visibleRows.map((row, i) =>" - expected_checks: [render-behavior] + severity: MAJOR + description: The index key changes row identity when the filtered order changes, contrary to the stated editing-state requirement. + evidence_spans: + - lines: [8] + rationale: The mapped rows have a stable id but use index as their key. diff --git a/research/benchmarks/fixtures/react/fixture-03.yaml b/research/benchmarks/fixtures/react/fixture-03.yaml index 0d7a307..b5e12e1 100644 --- a/research/benchmarks/fixtures/react/fixture-03.yaml +++ b/research/benchmarks/fixtures/react/fixture-03.yaml @@ -1,31 +1,69 @@ +schema_version: 2 id: react-03 category: state-ownership -description: direct state mutation and unsafe reducer pattern -code: | - function useCart() { - const [items, setItems] = useState([]); +description: Cart state is mutated before React receives an update. +intentional_clean: false +public: + file_path: src/useCart.ts + language: TypeScript + frameworks: + react: '18.3' + runtime: browser + assumptions: + - items is state owned by this hook. + - Consumers may memoize individual cart items by object identity. + requirements: + - Cart updates must use immutable state transitions. + code: | + function useCart() { + const [items, setItems] = useState([]); - const addItem = (item) => { - items.push(item); // direct mutation - setItems(items); - }; + const addItem = (item) => { + items.push(item); + setItems(items); + }; - const updateQty = (id, qty) => { - const found = items.find(i => i.id === id); - if (found) found.qty = qty; // mutates object in place - setItems([...items]); - }; + const updateQty = (id, qty) => { + const found = items.find((item) => item.id === id); + if (found) found.qty = qty; + setItems([...items]); + }; - return { items, addItem, updateQty }; - } + return { items, addItem, updateQty }; + } +metadata: + provenance: + kind: synthetic + origin: React state update documentation + revision_reason: Converted answer comments to public ownership assumptions and span-level evidence. + difficulty: FUNDAMENTALLY-FLAWED + domains: [state, rendering] + adjudication: + status: reviewed + method: Source-and-context rule adjudication + reviewer: codex-test-builder-review + reviewed_on: '2026-09-19' + sources: + - url: https://react.dev/learn/updating-arrays-in-state + supports: Arrays held in state should be treated as immutable. + - url: https://react.dev/learn/updating-objects-in-state + supports: Objects held in state should be treated as immutable. + notes: Both mutations occur before the setter call in the displayed state owner. + excluded_claims: [Exact rendering consequences for every consumer are not scored.] issues: - - severity: CRITICAL + - id: react-03-array-mutation + rule_id: react.state-ownership category: state-ownership - description: "items.push(item) mutates state array in place; React bailout means re-renders are skipped unpredictably" - evidence: "items.push(item); // direct mutation" - expected_checks: [state-ownership] - - severity: MAJOR + severity: MAJOR + description: addItem mutates the items state array before supplying it to setItems. + evidence_spans: + - lines: [5, 6] + rationale: items.push changes the state array and the same array reference is passed to the setter. + - id: react-03-object-mutation + rule_id: react.state-ownership category: state-ownership - description: "found.qty = qty mutates an object inside state; spread setItems([...items]) creates new array but keeps same object references; memoized children won't re-render" - evidence: "if (found) found.qty = qty; // mutates object in place" - expected_checks: [state-ownership, render-behavior] + severity: MAJOR + description: updateQty mutates an object already held in items before copying only the array. + evidence_spans: + - lines: [11, 12] + rationale: found references an item in state and its qty property is assigned directly. diff --git a/research/benchmarks/fixtures/react/fixture-04.yaml b/research/benchmarks/fixtures/react/fixture-04.yaml index 22fde83..7851874 100644 --- a/research/benchmarks/fixtures/react/fixture-04.yaml +++ b/research/benchmarks/fixtures/react/fixture-04.yaml @@ -1,26 +1,51 @@ +schema_version: 2 id: react-04 category: security -description: XSS via dangerouslySetInnerHTML with unescaped user input -code: | - function CommentBody({ comment }) { - // "safe" because we strip