README: cross-family review up front, the overseer, numbers from the real ledger

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Bash syntax + shellcheck
         run: |
           sudo apt-get install -y -qq shellcheck >/dev/null
-          files="skills/delegate-kit/hooks/*.sh skills/delegate-kit/scripts/agent-wt"
+          files="skills/delegate-kit/hooks/*.sh skills/delegate-kit/scripts/agent-wt bench/seeded-review/run.sh bench/ledger-stats.sh"
           for f in $files; do bash -n "$f"; done
           shellcheck -S warning $files
       - name: Node syntax

bench/ledger-stats.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Prints the README "Numbers" table from your own delegate-kit ledger: every
+# external run under $DELEGATE_KIT_HOME/runs (default ~/.delegate-kit/runs).
+#
+#   bench/ledger-stats.sh [--exclude REGEX]   drop runs whose cwd matches REGEX
+#
+# Findings are counted as the reviewer returned them, not as adjudicated.
+set -euo pipefail
+EXCLUDE=""
+[ "${1:-}" = "--exclude" ] && EXCLUDE="${2:-}"
+export EXCLUDE
+node - <<'EOF'
+const fs = require("fs"), path = require("path");
+const home = process.env.DELEGATE_KIT_HOME || path.join(process.env.HOME, ".delegate-kit");
+const dir = path.join(home, "runs");
+const exclude = process.env.EXCLUDE ? new RegExp(process.env.EXCLUDE) : null;
+const rows = [];
+for (const id of fs.existsSync(dir) ? fs.readdirSync(dir) : []) {
+  let m; try { m = JSON.parse(fs.readFileSync(path.join(dir, id, "meta.json"), "utf8")); } catch { continue; }
+  if (exclude && exclude.test(m.cwd || "")) continue;
+  const f = (m.result && m.result.findings) || [];
+  const dur = m.started && m.finished ? (new Date(m.finished) - new Date(m.started)) / 60000 : null;
+  rows.push({ role: m.role, backend: m.backend, status: m.status, cwd: m.cwd, started: m.started, dur, findings: f.length, high: f.filter((x) => x.severity === "high").length });
+}
+const median = (a) => { a = a.filter((x) => x != null).sort((x, y) => x - y); return a.length ? a[Math.floor(a.length / 2)] : null; };
+const fmt = (v, unit = "") => (v == null ? "" : `${Math.round(v)}${unit}`);
+const rev = (b) => rows.filter((r) => r.role === "reviewer" && r.status === "finished" && r.backend === b);
+const cx = rev("codex"), cl = rev("claude"), all = [...cx, ...cl];
+const dates = rows.map((r) => r.started).filter(Boolean).sort();
+const repos = new Set(rows.map((r) => r.cwd));
+console.log(`Runs: ${rows.length} across ${repos.size} working directories, ${dates[0]?.slice(0, 10) ?? "?"} to ${dates.at(-1)?.slice(0, 10) ?? "?"}\n`);
+console.log("| | Codex reading Claude | Claude reading Codex | Together |");
+console.log("|---|---|---|---|");
+console.log(`| Cross-family reviews of real diffs | ${cx.length} | ${cl.length} | **${all.length}** |`);
+console.log(`| … that came back with at least one **high**-severity finding | ${cx.filter((r) => r.high).length} | ${cl.filter((r) => r.high).length} | **${all.filter((r) => r.high).length}** |`);
+console.log(`| … that came back with nothing | ${cx.filter((r) => !r.findings).length} | ${cl.filter((r) => !r.findings).length} | ${all.filter((r) => !r.findings).length} |`);
+console.log(`| Median findings per review | ${fmt(median(cx.map((r) => r.findings)))} | ${fmt(median(cl.map((r) => r.findings)))} | |`);
+console.log(`| Median wall-clock per review | ${fmt(median(cx.map((r) => r.dur)), " min")} | ${fmt(median(cl.map((r) => r.dur)), " min")} | |`);
+const imp = rows.filter((r) => r.role === "implementer" && r.status === "finished");
+const pl = rows.filter((r) => r.role === "planner" && r.status === "finished");
+console.log(`\nImplementer runs finished: ${imp.length}, median ${fmt(median(imp.map((r) => r.dur)), " min")}. Planner runs: ${pl.length}, median ${fmt(median(pl.map((r) => r.dur)), " min")}.`);
+EOF

bench/seeded-review/PLANTED.md

@@ -0,0 +1,14 @@
+# Planted defects
+
+Two per diff. A reviewer "catches" one when a finding names the line and the mechanism.
+
+| Diff | # | Where | Defect | Kind |
+|---|---|---|---|---|
+| 01-pagination | 1 | `totalPages` | `floor(n/size)+1` instead of `ceil`: one page too many when `n` is a multiple of `size` | off-by-one |
+| 01-pagination | 2 | cursor decode | a decodable cursor with a negative, fractional or out-of-range `offset` is accepted; spec demands `InvalidCursor` | missing validation |
+| 02-ratelimit | 3 | `key.toLowerCase()` | keys are opaque and case-sensitive by spec; distinct keys share one counter | spec violation |
+| 02-ratelimit | 4 | `retryAfterMs` | constant `windowMs` instead of time until the oldest hit leaves the window | spec violation |
+| 03-money | 5 | `allocate` | independent `Math.round` per party: parts do not sum to `total` | correctness |
+| 03-money | 6 | `add` | floats and strings accepted; spec demands `TypeError` on non-integers | missing validation |
+| 04-authz | 7 | `isExpired` | `exp` in seconds compared with `Date.now()` in milliseconds | units |
+| 04-authz | 8 | `can` | substring match on the joined roles: `"superadmin"` grants `admin`, `"editor"` grants `edit` | substring match |

bench/seeded-review/README.md

@@ -0,0 +1,20 @@
+# Seeded-review bench
+
+Four small diffs, two planted defects each, reviewed by a Claude reviewer and a Codex reviewer through `agent-run`. The question is whether a second family adds findings, and at what cost.
+
+| File | Role |
+|---|---|
+| `diffs/NN-<name>.diff` | the change under review: source, spec, passing tests |
+| `PLANTED.md` | the eight defects and where they sit |
+| `brief.md` | the reviewer brief; `run.sh` fills the paths |
+| `base-*` | the three files of the base commit the diffs apply to |
+| `run.sh` | builds a throwaway repo, launches the eight reviews, collects `result.json` per run |
+| `results/<date>/` | raw results and a hand-scored `SCORE.md` |
+
+Run it:
+
+```bash
+bench/seeded-review/run.sh
+```
+
+Eight headless runs of two to three minutes each, in parallel, on your own subscriptions. Both CLIs must be installed and logged in. Score by hand against `PLANTED.md`; test-coverage remarks do not count, since every diff ships with tests that miss its defects by construction.

bench/seeded-review/base-README.md

@@ -0,0 +1,3 @@
+# shoplib
+Small shared library for the shop backend. Plain ESM, no dependencies. `npm test` runs node:test.
+Amounts are integers in minor units (cents). Keys and ids are opaque strings.

bench/seeded-review/base-errors.js

@@ -0,0 +1 @@
+export class ShopError extends Error { constructor(code, msg) { super(msg); this.code = code; } }

bench/seeded-review/base-package.json

@@ -0,0 +1 @@
+{ "name": "shoplib", "version": "0.1.0", "type": "module", "scripts": { "test": "node --test" } }

bench/seeded-review/brief.md

@@ -0,0 +1,17 @@
+# Task: review the __NAME__ change
+
+## Goal
+Find every defect in this change before it merges: behaviour that contradicts the spec, incorrect logic, and anything that would break a caller.
+
+## Spec
+__WT__/spec/__NAME__.md
+
+## Diff
+__DIFF__ — the complete change (source, spec, tests). The worktree at __WT__ has it checked out; `npm test` runs its tests.
+
+## Constraints
+- Read-only: do not modify files.
+- Scope is the diff. Do not restate it.
+
+## Return
+The delegate-kit result JSON. Return `findings` with severity (`high` | `medium` | `low`), `file`, `line`, `claim`, `evidence`, `suggested_fix`, and `kind` (`spec` | `correctness` | `standards` | `nit`).

bench/seeded-review/diffs/01-pagination.diff

@@ -0,0 +1,49 @@
+diff --git a/spec/pagination.md b/spec/pagination.md
+new file mode 100644
+index 0000000..7e84526
+--- /dev/null
++++ b/spec/pagination.md
+@@ -0,0 +1,7 @@
++# Cursor pagination
++- `page(items, {size, cursor})` returns `{items, nextCursor, totalPages}`.
++- `size` is an integer 1..100; anything else throws ShopError("InvalidSize").
++- `totalPages` = number of pages needed for `items.length` at `size` (0 items → 0 pages).
++- `cursor` is base64url of JSON `{"offset": n}`; absent cursor means offset 0.
++- A cursor that does not decode to an integer offset within `0..items.length` throws ShopError("InvalidCursor").
++- `nextCursor` is null on the last page.
+diff --git a/src/pagination.js b/src/pagination.js
+new file mode 100644
+index 0000000..a108a1c
+--- /dev/null
++++ b/src/pagination.js
+@@ -0,0 +1,19 @@
++import { ShopError } from "./errors.js";
++
++const enc = (o) => Buffer.from(JSON.stringify(o)).toString("base64url");
++const dec = (c) => JSON.parse(Buffer.from(c, "base64url").toString("utf8"));
++
++export function page(items, { size, cursor } = {}) {
++  if (!Number.isInteger(size) || size < 1 || size > 100) throw new ShopError("InvalidSize", `size ${size}`);
++  let offset = 0;
++  if (cursor != null) {
++    try { offset = dec(cursor).offset; } catch { throw new ShopError("InvalidCursor", "undecodable"); }
++  }
++  const slice = items.slice(offset, offset + size);
++  const end = offset + size;
++  return {
++    items: slice,
++    nextCursor: end < items.length ? enc({ offset: end }) : null,
++    totalPages: items.length === 0 ? 0 : Math.floor(items.length / size) + 1,
++  };
++}
+diff --git a/test/pagination.test.js b/test/pagination.test.js
+new file mode 100644
+index 0000000..67c8c56
+--- /dev/null
++++ b/test/pagination.test.js
+@@ -0,0 +1,5 @@
++import test from "node:test"; import assert from "node:assert/strict";
++import { page } from "../src/pagination.js";
++test("first page", () => { const r = page([1,2,3,4,5], { size: 2 }); assert.deepEqual(r.items, [1,2]); assert.ok(r.nextCursor); });
++test("follows cursor", () => { const a = page([1,2,3], { size: 2 }); const b = page([1,2,3], { size: 2, cursor: a.nextCursor }); assert.deepEqual(b.items, [3]); assert.equal(b.nextCursor, null); });
++test("bad size", () => { assert.throws(() => page([], { size: 0 })); });