From 2902931b3ca14e2e0c2f7a9962a033197406cbf4 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:18:41 -0400 Subject: [PATCH 1/9] test(docs): the post-gate hooks and the npm publishers keep only the tests that guard a relation; test/root.ts's comment sits on its export --- test/docs/npm-publish-workflows.test.ts | 351 ++++------- test/docs/post-green-workflow.test.ts | 794 +++++++----------------- test/docs/workflow-loader.ts | 2 - test/root.ts | 3 +- 4 files changed, 361 insertions(+), 789 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index d41dafb5..d5ab2663 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -1,10 +1,12 @@ /** - * The two npm publish jobs (post-green.yml's publish-next, update-release.yml's - * publish-npm) share one contract, trusted publishing through the runner's OIDC - * token and nothing else, and one was copied from the other, so a fix to the - * shared steps must reach both. The probe, the floor guard, and the publish - * blocks also run under bash against stubs: the pin alone cannot show what a - * branch does. + * The two npm publishers (post-green.yml's publish-next, update-release.yml's publish-npm) share one contract, trusted publishing + * through the runner's OIDC token and nothing else, and one was copied from the other. The relations here: both are guarded to the + * repository package.json names; both take one lane; the stable one runs after every other job of its workflow; neither hands npm a + * token, as the library page promises; the steps they share are the same text; both publish to one registry. The probe, the floor + * guard, and the publish blocks also run under bash against stubs, since no pin shows what a branch does. + * + * The static guards catch ACCIDENTAL drift: a guard, lane, or env edited in plain YAML. Deliberately hiding a token or a second + * publisher behind other syntax is out of scope. */ import { describe, expect, test } from "bun:test"; @@ -12,6 +14,7 @@ import { execFileSync } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { ROOT } from "../root.js"; import { type Job, readWorkflow, type Step } from "./workflow-loader.js"; /** A job that runs steps (the publishers are never reusable-workflow calls). */ @@ -27,6 +30,13 @@ function runJob(job: Job | undefined, what: string): RunJob { return { ...found, steps: must(found.steps, `${what} steps`) }; } +/** A step condition as the runner evaluates it: with or without the `${{ }}` wrapper, whitespace aside. */ +const condition = (raw: unknown): string => + String(raw ?? "") + .trim() + .replace(/^\$\{\{\s*([\s\S]*?)\s*\}\}$/, "$1") + .trim(); + /** Every string a step hands the runner beyond its script: env values, with values, the gate. */ function stepInputs(step: Step): string[] { return [ @@ -47,219 +57,129 @@ const setupNode = (job: RunJob): Step => "setup-node step", ); -interface Contract { - /** The stable job's ceiling: read the source, mint the OIDC token, nothing else. */ - stablePermissions: Record | undefined; - /** publish-next declares no ceiling: it inherits the caller's id-token grant (a ceiling of its own would fail the call). */ - nextPermissions: Record | undefined; - /** Both jobs run only in this repository: a fork has no trusted publisher and must not try. */ - repositoryGuards: [string | undefined, string | undefined]; - /** Both jobs take one lane: the registry has no compare-and-set, so a verdict must still hold when its publish lands. */ - lanes: [unknown, unknown]; - stableNeeds: string[]; - /** Steps of publish-next after the probe that are not gated on it. */ - ungatedNextSteps: string[]; - /** Any env, with, or gate value in either job naming a secret or a registry token. */ - tokenInputs: string[]; - /** Whether the two floor guards and the two library builds are the same text. */ - sameFloor: boolean; - sameBuild: boolean; - /** Both setup-node steps' registry, and whether either carries an env of its own. */ - registries: unknown[]; - setupNodeEnvs: unknown[]; - /** The npm lines of each publish step: scripts.prepare leaves the manifest, then the one publish, the source sha - * reaching npm's provenance and the dist-tag the channel's. */ - publishCommands: [string[], string[]]; -} - -const OIDC_GATE = "steps.oidc.outputs.proceed == 'true'"; -/** The confirmation's gate: the publish step sets this output only on its OIDC-gated publish path. */ -const PUBLISHED_GATE = "steps.publish.outputs.published == 'true'"; -const EXPECTED: Contract = { - stablePermissions: { contents: "read", "id-token": "write" }, - nextPermissions: undefined, - repositoryGuards: [ - "github.repository == 'Vivswan/github-settings-as-code'", - "github.repository == 'Vivswan/github-settings-as-code'", - ], - lanes: [ - { group: "npm-publish", queue: "max", "cancel-in-progress": false }, - { group: "npm-publish", queue: "max", "cancel-in-progress": false }, - ], - stableNeeds: ["package-release", "verify-release"], - ungatedNextSteps: [], - tokenInputs: [], - sameFloor: true, - sameBuild: true, - registries: ["https://registry.npmjs.org", "https://registry.npmjs.org"], - setupNodeEnvs: [undefined, undefined], - publishCommands: [ - ["npm pkg delete scripts.prepare", 'GITHUB_SHA="$SOURCE_SHA" npm publish --tag next'], - ["npm pkg delete scripts.prepare", 'GITHUB_SHA="$SOURCE_SHA" npm publish ;;'], - ], -}; +const STABLE_FILE = "update-release.yml"; +const publishers = () => ({ + next: runJob(readWorkflow("post-green.yml").jobs["publish-next"], "publish-next job"), + stableWorkflow: readWorkflow(STABLE_FILE), +}); -function contractOf(next: RunJob, stable: RunJob): Contract { - const probe = next.steps.findIndex((step) => step.id === "oidc"); - if (probe < 0) throw new Error("publish-next has no oidc probe"); - const publishLines = (step: Step): string[] => - (step.run ?? "") - .split("\n") - .map((line) => line.trim()) - .filter((line) => /\bnpm (?:publish|pkg delete)\b/.test(line)); - return { - stablePermissions: stable.permissions, - nextPermissions: next.permissions, - repositoryGuards: [next.if, stable.if], - lanes: [next.concurrency, stable.concurrency], - stableNeeds: [stable.needs ?? []].flat().sort(), - ungatedNextSteps: next.steps - .slice(probe + 1) - .filter((step) => step.if !== OIDC_GATE && step.if !== PUBLISHED_GATE) - .map((step) => step.name ?? step.uses ?? ""), - tokenInputs: [...next.steps, ...stable.steps] - .flatMap(stepInputs) - .filter((value) => /secrets\.|NODE_AUTH_TOKEN|NPM_TOKEN/.test(value)), - sameFloor: - stepNamed(next, "Require an npm that publishes through OIDC").run === - stepNamed(stable, "Require an npm that publishes through OIDC").run, - sameBuild: - stepNamed(next, "Build the library").run === stepNamed(stable, "Build the library").run, - registries: [setupNode(next).with?.["registry-url"], setupNode(stable).with?.["registry-url"]], - setupNodeEnvs: [setupNode(next).env, setupNode(stable).env], - publishCommands: [ - publishLines(stepNamed(next, "Publish the pre-release under the next dist-tag")), - publishLines(stepNamed(stable, "Publish the release to npm")), - ], +/** The `owner/name` the manifest's repository URL names: the one repository whose CI is the trusted publisher. */ +function manifestSlug(): string { + const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8")) as { + repository: { url: string }; }; + return must( + pkg.repository.url.match(/^git\+https:\/\/github\.com\/([^/]+\/[^/]+)\.git$/)?.[1], + "owner/name in package.json's repository.url", + ); +} + +/** + * The steps the two publishers share, by name or by action, as [label, next's, stable's]. The checkout is not one: each publisher + * checks out the source its own workflow resolved. + */ +function sharedSteps(next: RunJob, stable: RunJob): Array<[string, Step, Step]> { + return next.steps.flatMap((step) => { + if ((step.uses ?? "").startsWith("actions/checkout@")) return []; + const twin = stable.steps.find( + (candidate) => + (step.name !== undefined && candidate.name === step.name) || + (step.uses !== undefined && candidate.uses === step.uses), + ); + return twin + ? [[step.name ?? step.uses ?? "unnamed step", step, twin] as [string, Step, Step]] + : []; + }); } +/** A step stripped of what legitimately differs between the two publishers: its gate and its id. */ +const body = ({ if: _gate, id: _id, ...rest }: Step): Omit => rest; + +/** The guard both publishers must carry. */ +const forkGuard = () => `github.repository == '${manifestSlug()}'`; + describe("the npm publish jobs", () => { - const next = runJob(readWorkflow("post-green.yml").jobs["publish-next"], "publish-next job"); - const stable = runJob(readWorkflow("update-release.yml").jobs["publish-npm"], "publish-npm job"); + const { next, stableWorkflow } = publishers(); + const stable = runJob(stableWorkflow.jobs["publish-npm"], "publish-npm job"); - test("both publish through OIDC alone, with the same guard and build, and publish-next skips whole without a token", () => { - expect(contractOf(next, stable)).toEqual(EXPECTED); + test("both run only in the repository the manifest names: a fork has no trusted publisher", () => { + expect(condition(next.if)).toBe(forkGuard()); + expect(condition(stable.if)).toBe(forkGuard()); }); - const REGRESSIONS: Array<[string, (next: RunJob, stable: RunJob) => void, keyof Contract]> = [ - [ - "a stable publish without the OIDC grant", - (_next, stable) => (stable.permissions = { contents: "read" }), - "stablePermissions", - ], - [ - "a stable publish with a ceiling beyond its two grants", - (_next, stable) => (stable.permissions = { ...stable.permissions, contents: "write" }), - "stablePermissions", - ], - [ - "a pre-release publish with a ceiling of its own, which the caller's rejects as a whole", - (next) => (next.permissions = { contents: "read", "id-token": "write" }), - "nextPermissions", - ], - ["a stable publish open to forks", (_next, stable) => delete stable.if, "repositoryGuards"], - ["a pre-release publish open to forks", (next) => delete next.if, "repositoryGuards"], - [ - "a stable publish outside the shared lane, free to overlap a pre-release publish", - (_next, stable) => delete stable.concurrency, - "lanes", - ], - [ - "a pre-release publish on a lane of its own", - (next) => (next.concurrency = { ...next.concurrency, group: "publish-next" }), - "lanes", - ], - [ - "a lane that keeps one pending publish and drops the rest", - (_next, stable) => delete stable.concurrency?.queue, - "lanes", - ], - [ - "a lane pairing queue: max with cancel-in-progress, which GitHub rejects at validation", - (next) => (next.concurrency = { ...next.concurrency, "cancel-in-progress": true }), - "lanes", - ], - [ - "a stable publish ahead of the ref verification", - (_next, stable) => (stable.needs = ["package-release"]), - "stableNeeds", - ], - [ - "a pre-release step that runs without the OIDC gate", - (next) => (must(next.steps.at(-2), "publish step").if = undefined), - "ungatedNextSteps", - ], - [ - "a confirmation step that runs whether or not this job published", - (next) => (must(next.steps.at(-1), "confirm step").if = undefined), - "ungatedNextSteps", - ], - [ - "a registry token handed to npm through the environment", - (_next, stable) => - (must(stable.steps.at(-1), "publish step").env = { - NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}`, - }), - "tokenInputs", - ], - [ - "a registry token handed to setup-node", - (next) => (setupNode(next).env = { NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}` }), - "tokenInputs", - ], - [ - "a floor guard fixed in one job only", - (next) => { - const guard = stepNamed(next, "Require an npm that publishes through OIDC"); - guard.run = guard.run?.replace("11.5.1", "11.6.0"); - }, - "sameFloor", - ], + test("both take the one lane: the registry has no compare-and-set, so a verdict must still hold when its publish lands", () => { + expect(next.concurrency).toEqual(stable.concurrency); + expect(typeof next.concurrency?.group).toBe("string"); + expect(next.concurrency?.group).not.toContain("${{"); + }); + + test("the stable publish runs after every other job of its workflow, so nothing reaches the registry before the draft's assets check out", () => { + const others = Object.keys(stableWorkflow.jobs) + .filter((id) => id !== "publish-npm") + .sort(); + expect(others.length).toBeGreaterThan(1); + expect([stable.needs ?? []].flat().sort()).toEqual(others); + }); + + test("neither hands npm a registry token, as the library page promises", () => { + const page = readFileSync(join(ROOT, "docs", "reference", "library.md"), "utf8"); + expect(page).toContain("no registry token exists anywhere"); + const tokens = [...next.steps, ...stable.steps] + .flatMap(stepInputs) + .filter((value) => /secrets\.|NODE_AUTH_TOKEN|NPM_TOKEN/.test(value)); + expect(tokens).toEqual([]); + }); + + test("the steps the two publishers share are the same text, so a fix reaches both", () => { + const shared = sharedSteps(next, stable); + // The setup composite, setup-node, the floor guard, and the library build are shared today. + expect(shared.length).toBeGreaterThan(3); + for (const [label, a, b] of shared) { + expect(body(a), `"${label}" diverged between the publishers`).toEqual(body(b)); + } + }); + + test("both publish to one registry, the one setup-node writes into .npmrc", () => { + const registry = setupNode(next).with?.["registry-url"]; + expect(typeof registry).toBe("string"); + expect(setupNode(stable).with?.["registry-url"]).toBe(registry); + }); + + /** Each mutation, then whether the relation it targets still holds; the control passes when it does not. */ + test.each<[string, (next: RunJob, stable: RunJob) => boolean]>([ [ - "a library build that diverged between the jobs", - (_next, stable) => { - const build = stepNamed(stable, "Build the library"); - build.run = build.run?.replace("build:lib", "build"); + "a stable publish open to forks", + (_n, s) => { + delete s.if; + return condition(s.if) === forkGuard(); }, - "sameBuild", - ], - [ - "a setup-node without the registry the trusted publisher is configured on", - (_next, stable) => delete setupNode(stable).with?.["registry-url"], - "registries", ], [ - "a pre-release published under the default dist-tag", - (next) => { - const step = must(next.steps.at(-2), "publish step"); - step.run = step.run?.replace("npm publish --tag next", "npm publish"); + "a pre-release publish on a lane of its own", + (n, s) => { + n.concurrency = { ...n.concurrency, group: "publish-next" }; + return JSON.stringify(n.concurrency) === JSON.stringify(s.concurrency); }, - "publishCommands", ], [ - "a stable publish that ships scripts.prepare (lefthook is not in the tarball, and npm blocks install scripts)", - (_next, stable) => { - const step = must(stable.steps.at(-1), "publish step"); - step.run = step.run?.replace("npm pkg delete scripts.prepare\n", ""); + "a token handed to setup-node", + (n) => { + setupNode(n).env = { NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}` }; + return n.steps.flatMap(stepInputs).every((value) => !/secrets\./.test(value)); }, - "publishCommands", ], [ - "a publish whose provenance names this run's head instead of the source", - (_next, stable) => { - const step = must(stable.steps.at(-1), "publish step"); - step.run = step.run?.replace('GITHUB_SHA="$SOURCE_SHA" npm publish ;;', "npm publish ;;"); + "a floor guard fixed in one job only", + (n, s) => { + const guard = stepNamed(n, "Require an npm that publishes through OIDC"); + guard.run = guard.run?.replace("11.5.1", "11.6.0"); + return sharedSteps(n, s).every( + ([, a, b]) => JSON.stringify(body(a)) === JSON.stringify(body(b)), + ); }, - "publishCommands", ], - ]; - test.each(REGRESSIONS)("catches %s (negative control)", (_label, mutate, key) => { - const driftedNext = structuredClone(next); - const driftedStable = structuredClone(stable); - mutate(driftedNext, driftedStable); - expect(contractOf(driftedNext, driftedStable)[key]).not.toEqual(EXPECTED[key]); - expect(() => expect(contractOf(driftedNext, driftedStable)).toEqual(EXPECTED)).toThrow(); + ])("%s fails its relation (negative control)", (_case, mutateAndHolds) => { + expect(mutateAndHolds(structuredClone(next), structuredClone(stable))).toBe(false); }); }); @@ -301,7 +221,7 @@ function runStep( } describe("the OIDC probe under bash", () => { - const next = runJob(readWorkflow("post-green.yml").jobs["publish-next"], "publish-next job"); + const { next } = publishers(); const run = must(must(next.steps[0], "probe step").run, "probe run"); test("a runner that minted a token URL proceeds silently", () => { @@ -309,24 +229,23 @@ describe("the OIDC probe under bash", () => { expect(probe).toEqual({ lines: [], status: 0, output: "proceed=true\n" }); }); - test("a runner without one warns naming the caller's missing grant and skips (control)", () => { + test("a runner without one warns naming the missing grant and skips (control)", () => { const probe = runStep(run, {}); expect(probe.status).toBe(0); expect(probe.output).toBe("proceed=false\n"); - expect(probe.lines).toEqual([ - "::warning::this run has no OIDC token (the post-green call in the managed ci.yml grants no " + - "id-token: write); the library pre-release was not published to npm. Add id-token: write " + - "to that call's permissions in Vivswan/repo-platform to publish every green push to @next.", - ]); + expect(probe.lines).toHaveLength(1); + expect(probe.lines[0]).toMatch(/^::warning::/); + expect(probe.lines[0]).toContain("id-token: write"); }); }); describe("the npm floor guard under bash", () => { - const stable = runJob(readWorkflow("update-release.yml").jobs["publish-npm"], "publish-npm job"); + const stable = runJob(readWorkflow(STABLE_FILE).jobs["publish-npm"], "publish-npm job"); const run = must( stepNamed(stable, "Require an npm that publishes through OIDC").run, "guard run", ); + const floor = must(run.match(/^floor=(\S+)$/m)?.[1], "floor= line in the guard"); /** An npm stub reporting `before` until `npm install -g` runs, then `after`. */ const stubNpm = @@ -350,22 +269,22 @@ describe("the npm floor guard under bash", () => { }; const floors: [string, string, string, { lines: string[]; status: number }][] = [ - ["at the floor", "11.5.1", "11.5.1", { lines: [], status: 0 }], - ["above it", "11.9.0", "11.9.0", { lines: [], status: 0 }], + ["at the floor", floor, floor, { lines: [], status: 0 }], + ["above it", "99.0.0", "99.0.0", { lines: [], status: 0 }], [ "below it, upgraded past it", - "11.4.2", - "11.9.0", + "1.0.0", + "99.0.0", { lines: ["installed install -g npm@latest"], status: 0 }, ], [ "below it and still below after the upgrade", - "10.9.4", - "10.9.4", + "1.0.0", + "1.0.0", { lines: [ "installed install -g npm@latest", - "::error::npm 10.9.4 cannot publish through OIDC; trusted publishing needs npm 11.5.1 or newer.", + `::error::npm 1.0.0 cannot publish through OIDC; trusted publishing needs npm ${floor} or newer.`, ], status: 1, }, @@ -378,8 +297,8 @@ describe("the npm floor guard under bash", () => { }); describe("the publish blocks under bash", () => { - const next = runJob(readWorkflow("post-green.yml").jobs["publish-next"], "publish-next job"); - const stable = runJob(readWorkflow("update-release.yml").jobs["publish-npm"], "publish-npm job"); + const { next } = publishers(); + const stable = runJob(readWorkflow(STABLE_FILE).jobs["publish-npm"], "publish-npm job"); const nextRun = must( stepNamed(next, "Publish the pre-release under the next dist-tag").run, "next publish run", diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 5332d939..39413668 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -1,592 +1,243 @@ -/** post-green.yml runs only from ci.yml's post-green slot, so neither a build tag, latest, nor npm is written from a - * commit the all-green gate has not judged. */ +/** + * The hooks ci.yml calls after the all-green gate (post-green.yml, update-release.yml, update-release-pr.yml) push refs and publish + * packages, so what they can do follows from where they are reachable and what each job is granted. The relations here: a hook is + * reachable through workflow_call alone and every ci.yml job that calls one sits downstream of all-green; a job's effective grant covers + * what its steps consume; post-green's judged sha reaches every checkout and packaging step; every step after a probe runs on a verdict + * an earlier step wrote. The push probe also runs under bash against a stubbed git, since no pin shows what a branch does. + * + * The static guards catch ACCIDENTAL drift: a trigger, grant, or gate added or dropped in plain YAML. Deliberately hiding one behind + * other syntax is out of scope. + */ import { describe, expect, test } from "bun:test"; import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { readWorkflow, SETUP_USES, type Workflow } from "./workflow-loader.js"; +import { type Job, readWorkflow, type Step, type Workflow } from "./workflow-loader.js"; + +const CI = readWorkflow("ci.yml"); +const CALLER_PREFIX = "./.github/workflows/"; function must(value: T | undefined, what: string): T { - if (value === undefined) throw new Error(`post-green.yml has no ${what}`); + if (value === undefined) throw new Error(`no ${what}`); return value; } -interface PinnedStep { - name: string | undefined; - id: string | undefined; - uses: string | undefined; - if: string | undefined; - run: string | undefined; - env: Record | undefined; - with: Record | undefined; +/** A step condition as the runner evaluates it: with or without the `${{ }}` wrapper, whitespace aside. */ +const condition = (raw: unknown): string => + String(raw ?? "") + .trim() + .replace(/^\$\{\{\s*([\s\S]*?)\s*\}\}$/, "$1") + .trim(); + +const needsOf = (job: Job | undefined): string[] => [job?.needs ?? []].flat(); + +/** Every job reachable from `gate` by following `needs` edges away from it, however many hops. */ +function downstreamOf(jobs: Record, gate: string): Set { + const downstream = new Set(); + for (let grew = true; grew; ) { + grew = false; + for (const [name, job] of Object.entries(jobs)) { + if (downstream.has(name)) continue; + if (needsOf(job).some((dep) => dep === gate || downstream.has(dep))) { + downstream.add(name); + grew = true; + } + } + } + return downstream; } -interface CallerContract { - /** The workflow's top-level keys: no lane, permissions, or env above the jobs. */ - topLevel: string[]; - triggers: string[]; - /** The whole workflow_call interface ci.yml must satisfy. */ - inputs: Record; - secrets: string[]; - /** Pinned to the exact key set so nothing gates or extends the two jobs; no permissions ceiling of their own, so - * they inherit the caller's grant. */ - jobs: Array<{ - id: string; - keys: string[]; - uses: unknown; - with: unknown; - secrets: unknown; - permissions: unknown; - steps: PinnedStep[] | undefined; - }>; +/** ci.yml's calls of this repository's own workflows: the calling job and the callee file. */ +const localCalls = (ci: Workflow): Array<{ caller: string; job: Job; file: string }> => + Object.entries(ci.jobs).flatMap(([caller, job]) => + (job.uses ?? "").startsWith(CALLER_PREFIX) + ? [{ caller, job, file: (job.uses ?? "").slice(CALLER_PREFIX.length) }] + : [], + ); + +/** The hooks: callees of ci.yml jobs downstream of all-green, with the caller that reaches each. */ +const postGateHooks = (ci: Workflow) => { + const gated = downstreamOf(ci.jobs, "all-green"); + return localCalls(ci).filter(({ caller }) => gated.has(caller)); +}; + +describe("the hooks ci.yml calls after the gate", () => { + const hooks = postGateHooks(CI); + + test("every post-gate hook is reachable through workflow_call alone, and its caller sits downstream of all-green", () => { + // Non-vacuity: the packaging and release hooks are called this way today. + expect(hooks.map((hook) => hook.file).sort()).toContain("post-green.yml"); + expect(hooks.length).toBeGreaterThan(1); + for (const { file } of hooks) { + expect( + Object.keys(readWorkflow(file).on), + `${file} is reachable outside ci.yml's gate`, + ).toEqual(["workflow_call"]); + } + // The pre-gate call (checks.yml) is not a hook: it is what the gate judges. + expect(localCalls(CI).map((call) => call.file)).toContain("checks.yml"); + expect(hooks.map((hook) => hook.file)).not.toContain("checks.yml"); + }); + + test.each(hooks.map((hook) => [hook.file, hook.job] as const))( + "%s: ci.yml passes exactly the inputs it declares", + (file, job) => { + const declared = Object.keys(readWorkflow(file).on.workflow_call?.inputs ?? {}).sort(); + expect(Object.keys(job.with ?? {}).sort()).toEqual(declared); + }, + ); + + test("a hook that grew a dispatch trigger fails, and a caller moved ahead of the gate stops counting as a hook (negative controls)", () => { + const dispatchable = structuredClone(readWorkflow("post-green.yml")); + dispatchable.on.workflow_dispatch = null; + expect(Object.keys(dispatchable.on)).not.toEqual(["workflow_call"]); + const ci = structuredClone(CI); + must(ci.jobs["post-green"], "post-green caller").needs = ["ci"]; + expect(postGateHooks(ci).map((hook) => hook.file)).not.toContain("post-green.yml"); + }); +}); + +/** The grant a step's commands consume: a push or a release write needs contents: write, an OIDC-authenticated publish id-token: write. */ +function consumedGrants(step: Step): Array<[scope: string, why: string]> { + const run = step.run ?? ""; + const label = step.name ?? "unnamed step"; + const grants: Array<[string, string]> = []; + if (/\bgit\s+push\b|\bgh\s+release\s+(?:upload|edit|create)\b/.test(run)) { + grants.push(["contents", `"${label}" pushes or writes a release`]); + } + if (/\bnpm\s+publish\b|ACTIONS_ID_TOKEN_REQUEST_URL/.test(run)) { + grants.push(["id-token", `"${label}" publishes through OIDC`]); + } + return grants; } -/** The build job's gate: every step after the push probe runs only when the token can push. */ -const PROCEED = "steps.token.outputs.proceed == 'true'"; +/** A called job's grant: its own permissions block, else the caller's (GitHub inherits the caller's when the job declares none). */ +const effectiveGrant = (job: Job, caller: Job): Record => + job.permissions ?? caller.permissions ?? {}; + +describe("the hooks' grants", () => { + test("every job's effective grant covers what its steps consume", () => { + // A missing grant is quiet here: the push probe warns and skips, the OIDC probe warns and skips, and the job stays green. + const consumers = postGateHooks(CI).flatMap(({ file, job: caller }) => + Object.entries(readWorkflow(file).jobs).flatMap(([id, job]) => + (job.steps ?? []).flatMap((step) => + consumedGrants(step).map(([scope, why]) => ({ + where: `${file}#${id}`, + granted: effectiveGrant(job, caller)[scope], + scope, + why, + })), + ), + ), + ); + // The packaging job pushes and both publishers mint OIDC tokens today. + expect(consumers.length).toBeGreaterThan(2); + for (const { where, granted, scope, why } of consumers) { + expect(granted, `${where}: ${why}, so it needs ${scope}: write`).toBe("write"); + } + }); + + test("a publisher whose own ceiling drops the OIDC grant fails, whatever the caller grants (negative control)", () => { + const caller: Job = { permissions: { contents: "write", "id-token": "write" } }; + const job: Job = { permissions: { contents: "read" } }; + expect(effectiveGrant(job, caller)["id-token"]).toBeUndefined(); + expect(consumedGrants({ run: "npm publish --tag next" })).toEqual([ + ["id-token", '"unnamed step" publishes through OIDC'], + ]); + }); +}); + +/** The probe steps: each writes a `proceed=` verdict to GITHUB_OUTPUT for the steps after it to read. */ +const isProbe = (step: Step) => + /echo "proceed=(?:true|false)" >> "\$GITHUB_OUTPUT"/.test(step.run ?? ""); + /** - * git's stderr prints inside a stop-commands fence keyed by a token minted for the run, so remote-supplied text can neither forge a workflow command - * nor swallow the static error that follows. - * % and encoded line breaks in a message -> the runner decodes them, so one line can carry a second command - * a line whose first non-blank text is "::" -> the runner acts on it, wherever it came from - * the fence's own token -> the only resume; awk terminates probe.err's last line, so the closing fence is a line of its own + * Whether the step at `index` runs on a verdict: its condition is `steps..outputs. == 'true'` where `` names an + * earlier step that is the probe itself or is gated the same way (a chain back to the probe). */ -const FENCED_STDERR = [ - " fence=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \\n')", - ' echo "::stop-commands::$fence"', - ' echo "probe stderr:"', - " awk '{ print \" \" $0 }' probe.err", - ' echo "::$fence::"', -].join("\n"); -const PUSH_PROBE = [ - "if git push --dry-run --quiet origin HEAD:refs/dry-run/token-probe 2>probe.err; then", - ' echo "proceed=true" >> "$GITHUB_OUTPUT"', - 'elif [ "$PAT_SET" = "true" ]; then', - FENCED_STDERR, - ' echo "::error::REPO_PLATFORM_TOKEN cannot push to this repository; git\'s refusal is in the probe stderr lines above."', - " rm -f probe.err", - " exit 1", - "else", - ' echo "::warning::this run\'s token cannot push (the caller grants contents: read);" \\', - ' "this commit was not packaged and the latest tag was not moved here (the release hook packages each release and moves latest itself)." \\', - ' "Raise the caller\'s ceiling to contents: write, or add a REPO_PLATFORM_TOKEN PAT secret" \\', - ' "with Contents (read and write) on this repository, to publish every green push to @latest."', - ' echo "proceed=false" >> "$GITHUB_OUTPUT"', - "fi", - "rm -f probe.err", - "", -].join("\n"); - -/** The publish job's gate: every step after the OIDC probe runs only when the runner minted a token URL. */ -const OIDC_PROCEED = "steps.oidc.outputs.proceed == 'true'"; -/** The OIDC probe: the runner sets ACTIONS_ID_TOKEN_REQUEST_URL only under an - * id-token: write grant (the managed caller gives post-green one; a fork or a - * ceiling change may not); without it the job warns, naming the caller's - * ceiling, and skips. */ -const OIDC_PROBE = [ - 'if [ -n "$ACTIONS_ID_TOKEN_REQUEST_URL" ]; then', - ' echo "proceed=true" >> "$GITHUB_OUTPUT"', - "else", - ' echo "::warning::this run has no OIDC token (the post-green call in the managed ci.yml grants no id-token: write);" \\', - ' "the library pre-release was not published to npm." \\', - ' "Add id-token: write to that call\'s permissions in Vivswan/repo-platform to publish every green push to @next."', - ' echo "proceed=false" >> "$GITHUB_OUTPUT"', - "fi", - "", -].join("\n"); -/** The npm floor: trusted publishing needs 11.5.1; an older npm is upgraded once, then held to it. */ -const NPM_FLOOR = [ - "floor=11.5.1", - 'below_floor() { [ "$(printf \'%s\\n\' "$floor" "$(npm --version)" | sort -V | head -n1)" != "$floor" ]; }', - "if below_floor; then", - " npm install -g npm@latest", - "fi", - "if below_floor; then", - ' echo "::error::npm $(npm --version) cannot publish through OIDC; trusted publishing needs npm $floor or newer."', - " exit 1", - "fi", - "", -].join("\n"); -/** The publish: the pipeline's verdict names the version or the reason to skip; the source sha reaches npm's provenance; no token is passed. */ -const PUBLISH_NEXT = [ - `verdict="$(GITHUB_SHA="$SOURCE_SHA" bun .github/scripts/release-pipeline.ts npm-verdict next)"`, - 'case "$verdict" in', - " publish\\ *)", - ` npm version "\${verdict#publish }" --no-git-tag-version`, - " npm pkg delete scripts.prepare", - ` GITHUB_SHA="$SOURCE_SHA" npm publish --tag next`, - ' echo "published=true" >> "$GITHUB_OUTPUT" ;;', - ` skip\\ *) echo "::notice::\${verdict#skip }" ;;`, - " *)", - ' echo "unexpected npm-verdict output: $verdict"', - ' echo "::error::npm-verdict printed neither publish nor skip; see the line above."', - " exit 1 ;;", - "esac", - "", -].join("\n"); - -/** The confirmation's gate: it runs only after this job's own publish, so the lane is held until the registry shows it. */ -const PUBLISHED = "steps.publish.outputs.published == 'true'"; -/** After the publish: the registry is read until it shows the version; next behind a newer pre-release fails the job - * (trusted publishing cannot move a dist-tag), and a version the registry never shows warns. */ -const CONFIRM_NEXT = [ - `confirmed="$(GITHUB_SHA="$SOURCE_SHA" bun .github/scripts/release-pipeline.ts npm-confirm next)"`, - 'case "$confirmed" in', - ` settled\\ *) echo "::notice::\${confirmed#settled }" ;;`, - ` unsettled\\ *) echo "::warning::\${confirmed#unsettled }" ;;`, - " behind\\ *)", - ` echo "::error::\${confirmed#behind }"`, - " exit 1 ;;", - " *)", - ' echo "unexpected npm-confirm output: $confirmed"', - ' echo "::error::npm-confirm printed neither settled, unsettled, nor behind; see the line above."', - " exit 1 ;;", - "esac", - "", -].join("\n"); - -const CALLER_EXPECTED: CallerContract = { - topLevel: ["jobs", "name", "on"], - triggers: ["workflow_call"], - inputs: { sha: { required: true, type: "string", hasDefault: false } }, - secrets: [], - jobs: [ - { - id: "build", - keys: ["runs-on", "steps", "timeout-minutes"], - uses: undefined, - with: undefined, - secrets: undefined, - permissions: undefined, - steps: [ - { - name: undefined, - id: undefined, - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", - if: undefined, - run: undefined, - env: undefined, - with: { - ref: `\${{ inputs.sha }}`, - "fetch-depth": 0, - token: `\${{ secrets.REPO_PLATFORM_TOKEN || github.token }}`, - }, - }, - { - name: "Check the token can push", - id: "token", - uses: undefined, - if: undefined, - run: PUSH_PROBE, - env: { PAT_SET: `\${{ secrets.REPO_PLATFORM_TOKEN != '' }}` }, - with: undefined, - }, - { - name: undefined, - id: undefined, - uses: SETUP_USES, - if: PROCEED, - run: undefined, - env: undefined, - with: undefined, - }, - { - name: "Build the bundle and the library", - id: undefined, - uses: undefined, - if: PROCEED, - run: "bun run build:bundle\nbun run build:lib\n", - env: undefined, - with: undefined, - }, - { - name: "Package this commit under its build tag, prune the window, and point latest at the newest main source", - id: undefined, - uses: undefined, - if: PROCEED, - run: 'GITHUB_SHA="$SOURCE_SHA" bun .github/scripts/release-pipeline.ts package-commit', - env: { - SOURCE_SHA: `\${{ inputs.sha }}`, - RUN_URL: `\${{ github.server_url }}/\${{ github.repository }}/actions/runs/\${{ github.run_id }}`, - }, - with: undefined, - }, - ], - }, - { - id: "publish-next", - keys: ["concurrency", "if", "runs-on", "steps", "timeout-minutes"], - uses: undefined, - with: undefined, - secrets: undefined, - permissions: undefined, - steps: [ - { - name: "Check the caller grants an OIDC token", - id: "oidc", - uses: undefined, - if: undefined, - run: OIDC_PROBE, - env: undefined, - with: undefined, - }, - { - name: undefined, - id: undefined, - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", - if: OIDC_PROCEED, - run: undefined, - env: undefined, - with: { ref: `\${{ inputs.sha }}`, "fetch-depth": 0, "persist-credentials": false }, - }, - { - name: undefined, - id: undefined, - uses: SETUP_USES, - if: OIDC_PROCEED, - run: undefined, - env: undefined, - with: undefined, - }, - { - name: undefined, - id: undefined, - uses: "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", - if: OIDC_PROCEED, - run: undefined, - env: undefined, - with: { "node-version": 24, "registry-url": "https://registry.npmjs.org" }, - }, - { - name: "Require an npm that publishes through OIDC", - id: undefined, - uses: undefined, - if: OIDC_PROCEED, - run: NPM_FLOOR, - env: undefined, - with: undefined, - }, - { - name: "Build the library", - id: undefined, - uses: undefined, - if: OIDC_PROCEED, - run: "bun run build:lib", - env: undefined, - with: undefined, - }, - { - name: "Publish the pre-release under the next dist-tag", - id: "publish", - uses: undefined, - if: OIDC_PROCEED, - run: PUBLISH_NEXT, - env: { SOURCE_SHA: `\${{ inputs.sha }}` }, - with: undefined, - }, - { - name: "Confirm the registry shows the publish and next moved forward", - id: undefined, - uses: undefined, - if: PUBLISHED, - run: CONFIRM_NEXT, - env: { SOURCE_SHA: `\${{ inputs.sha }}` }, - with: undefined, - }, - ], - }, - ], -}; - -function callerContractOf(wf: Workflow): CallerContract { - const call = wf.on.workflow_call; - return { - topLevel: Object.keys(wf).sort(), - triggers: Object.keys(wf.on).sort(), - inputs: Object.fromEntries( - Object.entries(call?.inputs ?? {}).map(([name, input]) => [ - name, - { required: input.required === true, type: input.type, hasDefault: "default" in input }, - ]), - ), - secrets: Object.keys(call?.secrets ?? {}).sort(), - jobs: Object.entries(wf.jobs).map(([id, job]) => ({ - id, - keys: Object.keys(job).sort(), - uses: job.uses, - with: job.with, - secrets: job.secrets, - permissions: job.permissions, - steps: job.steps?.map((step) => ({ - name: step.name, - id: step.id, - uses: step.uses, - if: step.if, - run: step.run, - env: step.env, - with: step.with, - })), - })), - }; +function gatedOnProbe(steps: Step[], index: number, probe: number): boolean { + const match = condition(steps[index]?.if).match(/^steps\.([\w-]+)\.outputs\.[\w-]+ == 'true'$/); + if (!match) return false; + const source = steps.findIndex((step, at) => at < index && step.id === match[1]); + if (source < 0) return false; + return source === probe || gatedOnProbe(steps, source, probe); } -function expectCallerContract(wf: Workflow): void { - expect(callerContractOf(wf)).toEqual(CALLER_EXPECTED); +/** The job ids whose steps after a probe are not gated on it, with the offending step names. */ +function ungatedAfterProbe(workflow: Workflow): string[] { + return Object.entries(workflow.jobs).flatMap(([id, job]) => { + const steps = job.steps ?? []; + const probe = steps.findIndex(isProbe); + if (probe < 0) return []; + return steps.flatMap((step, index) => + index > probe && !gatedOnProbe(steps, index, probe) + ? [`${id}: "${step.name ?? step.uses ?? "unnamed step"}" runs whatever the probe found`] + : [], + ); + }); } -describe("post-green.yml packages the judged commit", () => { - const wf = readWorkflow("post-green.yml"); +describe("post-green.yml", () => { + const workflow = readWorkflow("post-green.yml"); + const caller = must( + localCalls(CI).find((call) => call.file === "post-green.yml"), + "post-green caller", + ).job; - test("two self-contained jobs, each gated on its probe, with the judged sha as the only input", () => { - expectCallerContract(wf); + test("every step after a probe runs on the probe's verdict", () => { + const probes = Object.values(workflow.jobs).filter((job) => (job.steps ?? []).some(isProbe)); + // Both jobs open with a probe today; a job without one is not judged here. + expect(probes.length).toBe(Object.keys(workflow.jobs).length); + expect(ungatedAfterProbe(workflow)).toEqual([]); }); - const REGRESSIONS: Array<[string, (w: Workflow) => void, keyof CallerContract]> = [ - [ - "an undeclared job beside the publisher", - (w) => (w.jobs.extra = { "runs-on": "ubuntu-latest", steps: [{ run: "echo" }] }), - "jobs", - ], - [ - "a job that calls another workflow with inherited secrets", - (w) => { - w.jobs.call = { - uses: "./.github/workflows/checks.yml", - with: { sha: `\${{ inputs.sha }}` }, - secrets: "inherit", - }; - }, - "jobs", - ], - [ - "a build step that runs the packaging without the token gate", - (w) => { - const step = must(w.jobs.build, "build job").steps?.at(-1); - must(step, "packaging step").if = undefined; - }, - "jobs", - ], - [ - "a build job whose packaging runs another subcommand", - (w) => { - const step = must(w.jobs.build, "build job").steps?.at(-1); - must(step, "packaging step").run = - 'GITHUB_SHA="$SOURCE_SHA" bun .github/scripts/release-pipeline.ts package'; - }, - "jobs", - ], - [ - "a build job with a ceiling of its own instead of the caller's grant", - (w) => (must(w.jobs.build, "build job").permissions = { contents: "read" }), - "jobs", - ], - [ - "a publish job with a ceiling of its own, which the managed caller's rejects as a whole", - (w) => - (must(w.jobs["publish-next"], "publish-next job").permissions = { - contents: "read", - "id-token": "write", - }), - "jobs", - ], - [ - "a publish step that runs without the OIDC gate", - (w) => { - const step = must(w.jobs["publish-next"], "publish-next job").steps?.at(-2); - must(step, "publish step").if = undefined; - }, - "jobs", - ], - [ - "a confirmation that runs without the publish gate, holding the lane for a publish that never happened", - (w) => { - const step = must(w.jobs["publish-next"], "publish-next job").steps?.at(-1); - must(step, "confirm step").if = OIDC_PROCEED; - }, - "jobs", - ], + test.each<[string, (w: Workflow) => void]>([ [ - "a confirmation that treats next behind a newer pre-release as a warning", - (w) => { - const step = must(w.jobs["publish-next"], "publish-next job").steps?.at(-1); - must(step, "confirm step").run = CONFIRM_NEXT.replace( - `echo "::error::\${confirmed#behind }"\n exit 1 ;;`, - `echo "::warning::\${confirmed#behind }" ;;`, - ); - }, - "jobs", - ], - [ - "a publish job that hands npm a registry token", - (w) => { - const step = must(w.jobs["publish-next"], "publish-next job").steps?.at(-2); - must(step, "publish step").env = { - ...must(step, "publish step").env, - NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}`, - }; - }, - "jobs", - ], - [ - "a publish job that publishes under the default dist-tag", - (w) => { - const step = must(w.jobs["publish-next"], "publish-next job").steps?.at(-2); - must(step, "publish step").run = PUBLISH_NEXT.replace( - "npm publish --tag next", - "npm publish", - ); - }, - "jobs", - ], - [ - "a publish job open to forks", - (w) => delete must(w.jobs["publish-next"], "publish-next job").if, - "jobs", - ], - [ - "a push probe whose warning names only one of the two remedies", - (w) => { - const step = must(w.jobs.build, "build job").steps?.[1]; - must(step, "probe step").run = PUSH_PROBE.replace( - "Raise the caller's ceiling to contents: write, or add", - "Add", - ); - }, - "jobs", - ], - [ - "a push probe that interpolates git's stderr into the workflow command", - (w) => { - const step = must(w.jobs.build, "build job").steps?.[1]; - must(step, "probe step").run = PUSH_PROBE.replace( - `${FENCED_STDERR}\n echo "::error::REPO_PLATFORM_TOKEN cannot push to this repository; git's refusal is in the probe stderr lines above."`, - " echo \"::error::REPO_PLATFORM_TOKEN cannot push to this repository: $(tr '\\n' ' ' { - const step = must(w.jobs.build, "build job").steps?.[1]; - must(step, "probe step").run = PUSH_PROBE.replace( - FENCED_STDERR, - ' echo "probe stderr:"\n awk \'{ print " " $0 }\' probe.err', - ); - }, - "jobs", - ], - [ - "a push probe whose fence token is fixed, so remote text could name it and resume commands", - (w) => { - const step = must(w.jobs.build, "build job").steps?.[1]; - must(step, "probe step").run = PUSH_PROBE.replace( - " fence=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \\n')", - " fence=probe-marker", - ); - }, - "jobs", - ], - [ - "a probe whose output the gates cannot read (its id gone)", - (w) => { - const step = must(w.jobs.build, "build job").steps?.[1]; - delete must(step, "probe step").id; - }, - "jobs", + "the packaging step without its gate", + (w) => delete must(must(w.jobs.build, "build").steps?.at(-1), "step").if, ], [ - "a probe that cannot tell a rejected PAT from a read ceiling (PAT_SET gone)", + "the confirmation gated on a step that is not gated itself", (w) => { - const step = must(w.jobs.build, "build job").steps?.[1]; - delete must(step, "probe step").env; + const steps = must(must(w.jobs["publish-next"], "publish-next").steps, "steps"); + must(steps.at(-1), "confirm").if = "steps.oidc-copy.outputs.proceed == 'true'"; + steps.splice(1, 0, { id: "oidc-copy", run: "echo" }); }, - "jobs", ], [ - "a shallow checkout, which package-commit refuses (fetch-depth gone)", + "a gate that is not an equality on true", (w) => { - const step = must(w.jobs.build, "build job").steps?.[0]; - delete must(must(step, "checkout step").with, "checkout with")["fetch-depth"]; + must(must(w.jobs.build, "build").steps?.at(-1), "step").if = + "always() || steps.token.outputs.proceed == 'true'"; }, - "jobs", ], - [ - "a shallow publish-next checkout, which prerelease-version refuses (fetch-depth gone)", - (w) => { - const step = must(w.jobs["publish-next"], "publish-next job").steps?.[1]; - delete must(must(step, "checkout step").with, "checkout with")["fetch-depth"]; - }, - "jobs", - ], - [ - "a checkout that never tries the caller's token", - (w) => { - const step = must(w.jobs.build, "build job").steps?.[0]; - must(must(step, "checkout step").with, "checkout with").token = - `\${{ secrets.REPO_PLATFORM_TOKEN }}`; - }, - "jobs", - ], - [ - "a checkout of the live default branch instead of the judged sha", - (w) => { - const step = must(w.jobs.build, "build job").steps?.[0]; - must(must(step, "checkout step").with, "checkout with").ref = - `\${{ github.event.repository.default_branch }}`; - }, - "jobs", - ], - [ - "a workflow-level lane above the jobs", - (w) => (w.concurrency = { group: `post-green-\${{ github.repository }}` }), - "topLevel", - ], - [ - "an optional judged sha", - (w) => (must(w.on.workflow_call?.inputs?.sha, "sha input").required = false), - "inputs", - ], - [ - "a judged sha with a fallback default", - (w) => (must(w.on.workflow_call?.inputs?.sha, "sha input").default = ""), - "inputs", - ], - [ - "a judged sha that is not a string", - (w) => (must(w.on.workflow_call?.inputs?.sha, "sha input").type = "boolean"), - "inputs", - ], - [ - "a second required input ci.yml does not pass", - (w) => - (must(must(w.on.workflow_call ?? undefined, "workflow_call").inputs, "inputs").mode = { - required: true, - type: "string", - }), - "inputs", - ], - [ - "a declared secret ci.yml does not pass by name", - (w) => - (must(w.on.workflow_call ?? undefined, "workflow_call").secrets = { - REPO_PLATFORM_TOKEN: { required: true }, - }), - "secrets", - ], - ["a push trigger of its own", (w) => (w.on.push = { branches: ["main"] }), "triggers"], - [ - "a dispatch that could reach the publisher outside the gate", - (w) => (w.on.workflow_dispatch = null), - "triggers", - ], - ]; - test.each(REGRESSIONS)("catches %s (negative control)", (_label, mutate, key) => { - const drifted = structuredClone(wf); + ])("%s fails the gate relation (negative control)", (_case, mutate) => { + const drifted = structuredClone(workflow); mutate(drifted); - expect(callerContractOf(drifted)[key]).not.toEqual(CALLER_EXPECTED[key]); - expect(() => expectCallerContract(drifted)).toThrow(); + expect(ungatedAfterProbe(drifted)).not.toEqual([]); + }); + + test("the judged sha the caller passes is the ref every checkout takes and the source every packaging step names", () => { + const [input, ...rest] = Object.keys(workflow.on.workflow_call?.inputs ?? {}); + expect(rest, "post-green.yml takes more than the one judged sha").toEqual([]); + expect(caller.with?.[input ?? ""]).toBe(`\${{ github.sha }}`); + const judged = `\${{ inputs.${input} }}`; + const steps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); + const checkouts = steps.filter((step) => (step.uses ?? "").startsWith("actions/checkout@")); + const sources = steps.filter((step) => step.env?.SOURCE_SHA !== undefined); + expect(checkouts.length).toBeGreaterThan(1); + expect(sources.length).toBeGreaterThan(1); + for (const step of checkouts) { + expect(step.with?.ref, "a checkout of something other than the judged sha").toBe(judged); + } + for (const step of sources) { + expect( + step.env?.SOURCE_SHA, + `"${step.name}" packages something other than the judged sha`, + ).toBe(judged); + } }); }); @@ -600,7 +251,7 @@ interface ProbeRun { } /** - * Run the pinned probe under `bash -e` (what a `run:` step gets on a Linux runner) with git stubbed to write `stderr` and exit `gitStatus`; the + * Run the probe under `bash -e` (what a `run:` step gets on a Linux runner) with git stubbed to write `stderr` and exit `gitStatus`; the * scratch directory is removed on every path. */ function runProbe(run: string, stderr: string, gitStatus: number, patSet: boolean): ProbeRun { @@ -646,22 +297,27 @@ function runProbe(run: string, stderr: string, gitStatus: number, patSet: boolea } } -const STATIC_ERROR = - "::error::REPO_PLATFORM_TOKEN cannot push to this repository; git's refusal is in the probe stderr lines above."; const FENCE_OPEN = /^::stop-commands::([0-9a-f]{32})$/; describe("the push probe under bash", () => { - const wf = readWorkflow("post-green.yml"); - const run = must(must(must(wf.jobs.build, "build job").steps?.[1], "probe step").run, "run"); + const steps = must(readWorkflow("post-green.yml").jobs.build, "build job").steps ?? []; + const run = must(must(steps.find(isProbe), "probe step").run, "probe run"); + /** + * The fenced block: git's stderr, indented, between a stop-commands line keyed by a fresh 32-hex token and its own resume line, then one + * static error. The runner trims, so an indented "::error::forged" still reads as a command outside a fence; the indent only marks the + * lines as quoted text in the log. + */ function expectFenced(lines: string[], inner: string[]): void { const [open, header, ...rest] = lines; const token = open?.match(FENCE_OPEN)?.[1]; expect(token).toBeDefined(); expect(header).toBe("probe stderr:"); - expect(rest).toEqual([...inner, `::${token}::`, STATIC_ERROR, ""]); - // The runner trims, so an indented "::error::forged" still reads as a command outside a fence; the indent only marks the lines as quoted text in - // the log. + expect(rest.slice(0, inner.length)).toEqual(inner); + expect(rest[inner.length]).toBe(`::${token}::`); + const after = rest.slice(inner.length + 1).filter(Boolean); + expect(after).toHaveLength(1); + expect(after[0]).toMatch(/^::error::/); for (const line of inner) { expect(line.startsWith(" ")).toBe(true); } @@ -694,15 +350,15 @@ describe("the push probe under bash", () => { expect(probe.probeErrLeft).toBe(false); }); - test("without a PAT a refused probe warns, skips, and prints no stderr (control)", () => { + test("without a PAT a refused probe warns naming both remedies, skips, and prints no stderr (control)", () => { const probe = runProbe(run, "refused\n", 1, false); expect(probe.status).toBe(0); - expect(probe.lines.filter((line) => line.startsWith("::"))).toEqual([ - "::warning::this run's token cannot push (the caller grants contents: read); this commit was not " + - "packaged and the latest tag was not moved here (the release hook packages each release and moves latest itself). " + - "Raise the caller's ceiling to contents: write, or add a REPO_PLATFORM_TOKEN PAT secret with " + - "Contents (read and write) on this repository, to publish every green push to @latest.", - ]); + const commands = probe.lines.filter((line) => line.startsWith("::")); + expect(commands).toHaveLength(1); + expect(commands[0]).toMatch(/^::warning::/); + // The two ways to let the hook push: a wider caller ceiling, or the PAT the checkout falls back from. + expect(commands[0]).toContain("contents: write"); + expect(commands[0]).toContain("REPO_PLATFORM_TOKEN"); expect(probe.lines).not.toContain(" refused"); expect(probe.output).toBe("proceed=false\n"); expect(probe.probeErrLeft).toBe(false); diff --git a/test/docs/workflow-loader.ts b/test/docs/workflow-loader.ts index 316c642f..acd2a4be 100644 --- a/test/docs/workflow-loader.ts +++ b/test/docs/workflow-loader.ts @@ -9,8 +9,6 @@ import { parse as parseYaml } from "yaml"; import { ROOT } from "../root.js"; const WORKFLOWS_DIR = join(ROOT, ".github", "workflows"); -/** The setup composite (bun, the locked install, yamllint on request), as a job's `uses:` spells it. */ -export const SETUP_USES = "./.github/actions/setup"; export interface Step { name?: string; diff --git a/test/root.ts b/test/root.ts index aaa130cd..54ca7963 100644 --- a/test/root.ts +++ b/test/root.ts @@ -1,5 +1,4 @@ -/** The repository root, for tests that read committed files or spawn the scripts by path. */ - import { join } from "node:path"; +/** The repository root, for tests that read committed files or spawn the scripts by path. */ export const ROOT = join(import.meta.dir, ".."); From 5692f89e9c78046cdeb153a94597854f4d36f0d1 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:31:56 -0400 Subject: [PATCH 2/9] test(docs): the hooks' grants, gates, outputs, and the library page's publishing claims are relations; a fixed fence token and a token in a job's env fail --- test/docs/npm-publish-workflows.test.ts | 188 ++++++----- test/docs/post-green-workflow.test.ts | 406 +++++++++++++++++++----- test/docs/workflow-loader.ts | 1 + 3 files changed, 440 insertions(+), 155 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index d5ab2663..038169f4 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -15,7 +15,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import { tmpdir } from "node:os"; import { join } from "node:path"; import { ROOT } from "../root.js"; -import { type Job, readWorkflow, type Step } from "./workflow-loader.js"; +import { type Job, readWorkflow, type Step, type Workflow } from "./workflow-loader.js"; /** A job that runs steps (the publishers are never reusable-workflow calls). */ type RunJob = Job & { steps: Step[] }; @@ -37,12 +37,16 @@ const condition = (raw: unknown): string => .replace(/^\$\{\{\s*([\s\S]*?)\s*\}\}$/, "$1") .trim(); -/** Every string a step hands the runner beyond its script: env values, with values, the gate. */ -function stepInputs(step: Step): string[] { +/** Every name and value a job hands the runner beyond its scripts: its env, each step's env, with, and gate. */ +function runnerInputs(job: RunJob): string[] { return [ - ...Object.values(step.env ?? {}), - ...Object.values(step.with ?? {}).map(String), - step.if ?? "", + ...Object.entries(job.env ?? {}).flat(), + ...job.steps.flatMap((step) => [ + ...Object.entries(step.env ?? {}).flat(), + ...Object.keys(step.with ?? {}), + ...Object.values(step.with ?? {}).map(String), + step.if ?? "", + ]), ]; } @@ -58,6 +62,7 @@ const setupNode = (job: RunJob): Step => ); const STABLE_FILE = "update-release.yml"; +const STABLE_JOB = "publish-npm"; const publishers = () => ({ next: runJob(readWorkflow("post-green.yml").jobs["publish-next"], "publish-next job"), stableWorkflow: readWorkflow(STABLE_FILE), @@ -95,91 +100,113 @@ function sharedSteps(next: RunJob, stable: RunJob): Array<[string, Step, Step]> /** A step stripped of what legitimately differs between the two publishers: its gate and its id. */ const body = ({ if: _gate, id: _id, ...rest }: Step): Omit => rest; -/** The guard both publishers must carry. */ -const forkGuard = () => `github.repository == '${manifestSlug()}'`; +/** Every way the two publishers break their shared contract; the assertions and the negative controls read this one list. */ +function publisherProblems(next: RunJob, stableWorkflow: Workflow): string[] { + const stable = runJob(stableWorkflow.jobs[STABLE_JOB], `${STABLE_JOB} job`); + const problems: string[] = []; + const guard = `github.repository == '${manifestSlug()}'`; + for (const [label, job] of [ + ["publish-next", next], + [STABLE_JOB, stable], + ] as const) { + if (condition(job.if) !== guard) problems.push(`${label} is not guarded to ${guard}`); + } + if (JSON.stringify(next.concurrency) !== JSON.stringify(stable.concurrency)) { + problems.push("the publishers take different lanes"); + } + if (typeof next.concurrency?.group !== "string" || next.concurrency.group.includes("${{")) { + problems.push("the lane's group is not one literal"); + } + const others = Object.keys(stableWorkflow.jobs) + .filter((id) => id !== STABLE_JOB) + .sort(); + if (JSON.stringify([stable.needs ?? []].flat().sort()) !== JSON.stringify(others)) { + problems.push(`${STABLE_JOB} does not run after every other job (${others.join(", ")})`); + } + for (const value of [...runnerInputs(next), ...runnerInputs(stable)]) { + if (/secrets\.|NODE_AUTH_TOKEN|NPM_TOKEN/.test(value)) + problems.push(`a token reaches npm: ${value}`); + } + for (const [label, a, b] of sharedSteps(next, stable)) { + if (JSON.stringify(body(a)) !== JSON.stringify(body(b))) + problems.push(`"${label}" diverged between the publishers`); + } + const registry = setupNode(next).with?.["registry-url"]; + if (typeof registry !== "string" || setupNode(stable).with?.["registry-url"] !== registry) { + problems.push("the publishers name different registries"); + } + return problems; +} describe("the npm publish jobs", () => { const { next, stableWorkflow } = publishers(); - const stable = runJob(stableWorkflow.jobs["publish-npm"], "publish-npm job"); - - test("both run only in the repository the manifest names: a fork has no trusted publisher", () => { - expect(condition(next.if)).toBe(forkGuard()); - expect(condition(stable.if)).toBe(forkGuard()); - }); - - test("both take the one lane: the registry has no compare-and-set, so a verdict must still hold when its publish lands", () => { - expect(next.concurrency).toEqual(stable.concurrency); - expect(typeof next.concurrency?.group).toBe("string"); - expect(next.concurrency?.group).not.toContain("${{"); - }); - - test("the stable publish runs after every other job of its workflow, so nothing reaches the registry before the draft's assets check out", () => { - const others = Object.keys(stableWorkflow.jobs) - .filter((id) => id !== "publish-npm") - .sort(); - expect(others.length).toBeGreaterThan(1); - expect([stable.needs ?? []].flat().sort()).toEqual(others); - }); + const stable = runJob(stableWorkflow.jobs[STABLE_JOB], `${STABLE_JOB} job`); - test("neither hands npm a registry token, as the library page promises", () => { + test("both are guarded to the manifest's repository, take one lane, hand npm no token, share their steps' text, and name one registry; the stable one runs last", () => { + // The library page's promise is the second artifact of the token relation. const page = readFileSync(join(ROOT, "docs", "reference", "library.md"), "utf8"); expect(page).toContain("no registry token exists anywhere"); - const tokens = [...next.steps, ...stable.steps] - .flatMap(stepInputs) - .filter((value) => /secrets\.|NODE_AUTH_TOKEN|NPM_TOKEN/.test(value)); - expect(tokens).toEqual([]); - }); - - test("the steps the two publishers share are the same text, so a fix reaches both", () => { - const shared = sharedSteps(next, stable); - // The setup composite, setup-node, the floor guard, and the library build are shared today. - expect(shared.length).toBeGreaterThan(3); - for (const [label, a, b] of shared) { - expect(body(a), `"${label}" diverged between the publishers`).toEqual(body(b)); - } - }); - - test("both publish to one registry, the one setup-node writes into .npmrc", () => { - const registry = setupNode(next).with?.["registry-url"]; - expect(typeof registry).toBe("string"); - expect(setupNode(stable).with?.["registry-url"]).toBe(registry); + // Non-vacuity: the setup composite, setup-node, the floor guard, and the library build are shared today, and the stable + // workflow has jobs for the publish to wait on. + expect(sharedSteps(next, stable).length).toBeGreaterThan(3); + expect(Object.keys(stableWorkflow.jobs).length).toBeGreaterThan(2); + expect(publisherProblems(next, stableWorkflow)).toEqual([]); }); - /** Each mutation, then whether the relation it targets still holds; the control passes when it does not. */ - test.each<[string, (next: RunJob, stable: RunJob) => boolean]>([ + test.each<[string, (next: RunJob, stableWorkflow: Workflow) => void, RegExp]>([ [ "a stable publish open to forks", - (_n, s) => { - delete s.if; - return condition(s.if) === forkGuard(); - }, + (_n, w) => delete must(w.jobs[STABLE_JOB], "stable").if, + /publish-npm is not guarded/, ], [ "a pre-release publish on a lane of its own", - (n, s) => { + (n) => { n.concurrency = { ...n.concurrency, group: "publish-next" }; - return JSON.stringify(n.concurrency) === JSON.stringify(s.concurrency); }, + /different lanes/, + ], + [ + "a stable publish ahead of the asset verification", + (_n, w) => { + must(w.jobs[STABLE_JOB], "stable").needs = ["package-release"]; + }, + /does not run after every other job/, ], [ "a token handed to setup-node", (n) => { setupNode(n).env = { NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}` }; - return n.steps.flatMap(stepInputs).every((value) => !/secrets\./.test(value)); }, + /a token reaches npm: NODE_AUTH_TOKEN/, + ], + [ + "a token in the job's own env under an innocent value", + (_n, w) => { + must(w.jobs[STABLE_JOB], "stable").env = { NODE_AUTH_TOKEN: `\${{ github.token }}` }; + }, + /a token reaches npm: NODE_AUTH_TOKEN/, ], [ "a floor guard fixed in one job only", - (n, s) => { + (n) => { const guard = stepNamed(n, "Require an npm that publishes through OIDC"); guard.run = guard.run?.replace("11.5.1", "11.6.0"); - return sharedSteps(n, s).every( - ([, a, b]) => JSON.stringify(body(a)) === JSON.stringify(body(b)), - ); }, + /"Require an npm that publishes through OIDC" diverged/, ], - ])("%s fails its relation (negative control)", (_case, mutateAndHolds) => { - expect(mutateAndHolds(structuredClone(next), structuredClone(stable))).toBe(false); + [ + "a setup-node without the registry the trusted publisher is configured on", + (_n, w) => { + delete setupNode(runJob(w.jobs[STABLE_JOB], "stable")).with?.["registry-url"]; + }, + /different registries|diverged/, + ], + ])("%s fails its relation (negative control)", (_case, mutate, message) => { + const driftedNext = structuredClone(next); + const driftedWorkflow = structuredClone(stableWorkflow); + mutate(driftedNext, driftedWorkflow); + expect(publisherProblems(driftedNext, driftedWorkflow).join("\n")).toMatch(message); }); }); @@ -240,13 +267,24 @@ describe("the OIDC probe under bash", () => { }); describe("the npm floor guard under bash", () => { - const stable = runJob(readWorkflow(STABLE_FILE).jobs["publish-npm"], "publish-npm job"); + const stable = runJob(readWorkflow(STABLE_FILE).jobs[STABLE_JOB], `${STABLE_JOB} job`); const run = must( stepNamed(stable, "Require an npm that publishes through OIDC").run, "guard run", ); + /** Trusted publishing exists from this npm on (npm's changelog for 11.5.1); the script must hold that floor or a newer one. */ + const OIDC_NPM = "11.5.1"; const floor = must(run.match(/^floor=(\S+)$/m)?.[1], "floor= line in the guard"); + test("the script's floor is not below the npm that introduced trusted publishing", () => { + const [scriptFloor] = [floor, OIDC_NPM].sort((a, b) => + a.localeCompare(b, undefined, { numeric: true }), + ); + expect(scriptFloor, `floor=${floor} admits an npm that cannot publish through OIDC`).toBe( + OIDC_NPM, + ); + }); + /** An npm stub reporting `before` until `npm install -g` runs, then `after`. */ const stubNpm = (before: string, after: string) => @@ -272,8 +310,8 @@ describe("the npm floor guard under bash", () => { ["at the floor", floor, floor, { lines: [], status: 0 }], ["above it", "99.0.0", "99.0.0", { lines: [], status: 0 }], [ - "below it, upgraded past it", - "1.0.0", + "just below the OIDC floor, upgraded past it", + "11.4.2", "99.0.0", { lines: ["installed install -g npm@latest"], status: 0 }, ], @@ -298,7 +336,7 @@ describe("the npm floor guard under bash", () => { describe("the publish blocks under bash", () => { const { next } = publishers(); - const stable = runJob(readWorkflow(STABLE_FILE).jobs["publish-npm"], "publish-npm job"); + const stable = runJob(readWorkflow(STABLE_FILE).jobs[STABLE_JOB], `${STABLE_JOB} job`); const nextRun = must( stepNamed(next, "Publish the pre-release under the next dist-tag").run, "next publish run", @@ -316,9 +354,9 @@ describe("the publish blocks under bash", () => { }; const source = "b8df084c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a"; - const cases: [string, string, string, { lines: string[]; status: number }][] = [ + const cases: [string, string, string, { lines: string[]; status: number; output: string }][] = [ [ - "next: a publish verdict sets the version and publishes under next with the source as GITHUB_SHA", + "next: a publish verdict sets the version, publishes under next with the source as GITHUB_SHA, and reports the publish for the confirmation", nextRun, "publish 2.0.1-main.446.20260913.gb8df084", { @@ -328,13 +366,14 @@ describe("the publish blocks under bash", () => { "npm publish --tag next (GITHUB_SHA=b8df084c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a)", ], status: 0, + output: "published=true\n", }, ], [ - "next: a skip verdict calls npm not at all and reports why", + "next: a skip verdict calls npm not at all, reports why, and leaves the confirmation ungated", nextRun, "skip the registry's next is newer", - { lines: ["::notice::the registry's next is newer"], status: 0 }, + { lines: ["::notice::the registry's next is newer"], status: 0, output: "" }, ], [ "next: anything else fails the step", @@ -346,6 +385,7 @@ describe("the publish blocks under bash", () => { "::error::npm-verdict printed neither publish nor skip; see the line above.", ], status: 1, + output: "", }, ], [ @@ -358,17 +398,17 @@ describe("the publish blocks under bash", () => { "npm publish (GITHUB_SHA=b8df084c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a)", ], status: 0, + output: "", }, ], [ "stable: a skip verdict calls npm not at all and warns", stableRun, "skip 2.1.0 is already on the registry", - { lines: ["::warning::2.1.0 is already on the registry"], status: 0 }, + { lines: ["::warning::2.1.0 is already on the registry"], status: 0, output: "" }, ], ]; test.each(cases)("%s", (_name, run, verdict, expected) => { - const step = runStep(run, { SOURCE_SHA: source, TAG: "v2.1.0" }, stubs(verdict)); - expect({ lines: step.lines, status: step.status }).toEqual(expected); + expect(runStep(run, { SOURCE_SHA: source, TAG: "v2.1.0" }, stubs(verdict))).toEqual(expected); }); }); diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 39413668..9ef6f9ac 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -1,12 +1,14 @@ /** * The hooks ci.yml calls after the all-green gate (post-green.yml, update-release.yml, update-release-pr.yml) push refs and publish - * packages, so what they can do follows from where they are reachable and what each job is granted. The relations here: a hook is - * reachable through workflow_call alone and every ci.yml job that calls one sits downstream of all-green; a job's effective grant covers - * what its steps consume; post-green's judged sha reaches every checkout and packaging step; every step after a probe runs on a verdict - * an earlier step wrote. The push probe also runs under bash against a stubbed git, since no pin shows what a branch does. + * packages, so what they can do follows from where they are reachable and what each job is granted. The relations here: every hook is + * reachable through workflow_call alone and its ci.yml caller sits downstream of all-green; a job's effective grant covers what its + * steps consume and nothing granted goes unconsumed; a hook job's own condition is the fork guard or nothing; post-green's judged sha + * reaches every checkout and packaging step; every output a step writes is read by a later step, and every gate reads an output an + * earlier step writes, back to the probe. The push probe also runs under bash against a stubbed git, since no pin shows what a branch + * does. * - * The static guards catch ACCIDENTAL drift: a trigger, grant, or gate added or dropped in plain YAML. Deliberately hiding one behind - * other syntax is out of scope. + * The static guards catch ACCIDENTAL drift: a trigger, grant, gate, or step added or dropped in plain YAML. Deliberately hiding one + * behind other syntax is out of scope. */ import { describe, expect, test } from "bun:test"; @@ -14,6 +16,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { ROOT } from "../root.js"; import { type Job, readWorkflow, type Step, type Workflow } from "./workflow-loader.js"; const CI = readWorkflow("ci.yml"); @@ -49,39 +52,66 @@ function downstreamOf(jobs: Record, gate: string): Set { return downstream; } +interface LocalCall { + caller: string; + job: Job; + file: string; +} + /** ci.yml's calls of this repository's own workflows: the calling job and the callee file. */ -const localCalls = (ci: Workflow): Array<{ caller: string; job: Job; file: string }> => +const localCalls = (ci: Workflow): LocalCall[] => Object.entries(ci.jobs).flatMap(([caller, job]) => (job.uses ?? "").startsWith(CALLER_PREFIX) ? [{ caller, job, file: (job.uses ?? "").slice(CALLER_PREFIX.length) }] : [], ); -/** The hooks: callees of ci.yml jobs downstream of all-green, with the caller that reaches each. */ -const postGateHooks = (ci: Workflow) => { - const gated = downstreamOf(ci.jobs, "all-green"); - return localCalls(ci).filter(({ caller }) => gated.has(caller)); -}; +/** + * The hooks: every local call whose caller the gate does not itself judge (the judged call is what all-green needs). Each must sit + * downstream of the gate; `misplaced` names the ones that do not. + */ +function postGateHooks(ci: Workflow): { hooks: LocalCall[]; misplaced: string[] } { + const judged = new Set(needsOf(ci.jobs["all-green"])); + const downstream = downstreamOf(ci.jobs, "all-green"); + const hooks = localCalls(ci).filter(({ caller }) => !judged.has(caller)); + return { + hooks, + misplaced: hooks.filter(({ caller }) => !downstream.has(caller)).map(({ caller }) => caller), + }; +} -describe("the hooks ci.yml calls after the gate", () => { - const hooks = postGateHooks(CI); +const { hooks: HOOKS } = postGateHooks(CI); + +/** The `owner/name` the manifest's repository URL names: the one repository whose CI is the publisher. */ +function manifestSlug(): string { + const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8")) as { + repository: { url: string }; + }; + return must( + pkg.repository.url.match(/^git\+https:\/\/github\.com\/([^/]+\/[^/]+)\.git$/)?.[1], + "owner/name in package.json's repository.url", + ); +} - test("every post-gate hook is reachable through workflow_call alone, and its caller sits downstream of all-green", () => { - // Non-vacuity: the packaging and release hooks are called this way today. - expect(hooks.map((hook) => hook.file).sort()).toContain("post-green.yml"); - expect(hooks.length).toBeGreaterThan(1); - for (const { file } of hooks) { +describe("the hooks ci.yml calls after the gate", () => { + test("every hook is reachable through workflow_call alone, and its caller sits downstream of all-green", () => { + const { misplaced } = postGateHooks(CI); + // Non-vacuity: the packaging hook and the two release hooks are called this way today. + expect(HOOKS.map((hook) => hook.file)).toContain("post-green.yml"); + expect(HOOKS.length).toBeGreaterThan(2); + expect(misplaced, "hook callers that all-green does not gate").toEqual([]); + for (const { file } of HOOKS) { expect( Object.keys(readWorkflow(file).on), `${file} is reachable outside ci.yml's gate`, ).toEqual(["workflow_call"]); } - // The pre-gate call (checks.yml) is not a hook: it is what the gate judges. + // The judged call (checks.yml) is not a hook: it is what the gate waits for. expect(localCalls(CI).map((call) => call.file)).toContain("checks.yml"); - expect(hooks.map((hook) => hook.file)).not.toContain("checks.yml"); + expect(HOOKS.map((hook) => hook.file)).not.toContain("checks.yml"); }); - test.each(hooks.map((hook) => [hook.file, hook.job] as const))( + test.each(HOOKS.map((hook) => [hook.file, hook.job] as const))( "%s: ci.yml passes exactly the inputs it declares", (file, job) => { const declared = Object.keys(readWorkflow(file).on.workflow_call?.inputs ?? {}).sort(); @@ -89,23 +119,42 @@ describe("the hooks ci.yml calls after the gate", () => { }, ); - test("a hook that grew a dispatch trigger fails, and a caller moved ahead of the gate stops counting as a hook (negative controls)", () => { + test.each(HOOKS.map((hook) => [hook.file] as const))( + "%s: a job's own condition is the fork guard or nothing, so no job skips a green push quietly", + (file) => { + const guard = `github.repository == '${manifestSlug()}'`; + for (const [id, job] of Object.entries(readWorkflow(file).jobs)) { + if (job.if !== undefined) { + expect(condition(job.if), `${file}#${id} carries a condition of its own`).toBe(guard); + } + } + }, + ); + + test("a hook that grew a dispatch trigger, and a release hook moved ahead of the gate, both fail (negative controls)", () => { const dispatchable = structuredClone(readWorkflow("post-green.yml")); dispatchable.on.workflow_dispatch = null; expect(Object.keys(dispatchable.on)).not.toEqual(["workflow_call"]); const ci = structuredClone(CI); - must(ci.jobs["post-green"], "post-green caller").needs = ["ci"]; - expect(postGateHooks(ci).map((hook) => hook.file)).not.toContain("post-green.yml"); + must(ci.jobs["update-release"], "update-release caller").needs = ["ci"]; + expect(postGateHooks(ci).misplaced).toEqual(["update-release"]); }); }); -/** The grant a step's commands consume: a push or a release write needs contents: write, an OIDC-authenticated publish id-token: write. */ +/** + * The grant a step's commands consume: a push, a release read or write (drafts are visible to a write grant alone), or a pipeline + * subcommand that pushes needs contents: write; an OIDC-authenticated publish needs id-token: write. + */ function consumedGrants(step: Step): Array<[scope: string, why: string]> { const run = step.run ?? ""; const label = step.name ?? "unnamed step"; const grants: Array<[string, string]> = []; - if (/\bgit\s+push\b|\bgh\s+release\s+(?:upload|edit|create)\b/.test(run)) { - grants.push(["contents", `"${label}" pushes or writes a release`]); + if ( + /\bgit\s+push\b|\bgh\s+release\s+(?:view|upload|edit|create)\b|release-pipeline\.ts (?:package|package-commit|retag-major|anchor)\b/.test( + run, + ) + ) { + grants.push(["contents", `"${label}" pushes or touches a release`]); } if (/\bnpm\s+publish\b|ACTIONS_ID_TOKEN_REQUEST_URL/.test(run)) { grants.push(["id-token", `"${label}" publishes through OIDC`]); @@ -113,98 +162,247 @@ function consumedGrants(step: Step): Array<[scope: string, why: string]> { return grants; } -/** A called job's grant: its own permissions block, else the caller's (GitHub inherits the caller's when the job declares none). */ -const effectiveGrant = (job: Job, caller: Job): Record => - job.permissions ?? caller.permissions ?? {}; +/** + * A called job's grant: its own permissions block, else its workflow's, else the caller's (GitHub applies the nearest block; a called + * job with none inherits the caller's). + */ +const effectiveGrant = (job: Job, workflow: Workflow, caller: Job): Record => + job.permissions ?? workflow.permissions ?? caller.permissions ?? {}; + +/** Every hook job whose steps consume a grant its effective permissions do not hold. */ +function grantProblems(hooks: LocalCall[]): string[] { + return hooks.flatMap(({ file, job: caller }) => { + const workflow = readWorkflow(file); + return Object.entries(workflow.jobs).flatMap(([id, job]) => { + const granted = effectiveGrant(job, workflow, caller); + return (job.steps ?? []) + .flatMap(consumedGrants) + .filter(([scope]) => granted[scope] !== "write") + .map(([scope, why]) => `${file}#${id}: ${why}, so it needs ${scope}: write`); + }); + }); +} describe("the hooks' grants", () => { test("every job's effective grant covers what its steps consume", () => { // A missing grant is quiet here: the push probe warns and skips, the OIDC probe warns and skips, and the job stays green. - const consumers = postGateHooks(CI).flatMap(({ file, job: caller }) => - Object.entries(readWorkflow(file).jobs).flatMap(([id, job]) => - (job.steps ?? []).flatMap((step) => - consumedGrants(step).map(([scope, why]) => ({ - where: `${file}#${id}`, - granted: effectiveGrant(job, caller)[scope], - scope, - why, - })), - ), + const consumers = HOOKS.flatMap(({ file }) => + Object.values(readWorkflow(file).jobs).flatMap((job) => + (job.steps ?? []).flatMap(consumedGrants), ), ); - // The packaging job pushes and both publishers mint OIDC tokens today. - expect(consumers.length).toBeGreaterThan(2); - for (const { where, granted, scope, why } of consumers) { - expect(granted, `${where}: ${why}, so it needs ${scope}: write`).toBe("write"); + expect(consumers.length).toBeGreaterThan(4); + expect(grantProblems(HOOKS)).toEqual([]); + }); + + test.each<[string, (hooks: LocalCall[]) => LocalCall[], RegExp]>([ + [ + "a caller ceiling below what the build job pushes with", + (hooks) => + hooks.map((hook) => + hook.file === "post-green.yml" + ? { + ...hook, + job: { ...hook.job, permissions: { contents: "read", "id-token": "write" } }, + } + : hook, + ), + /post-green\.yml#build: .* needs contents: write/, + ], + [ + "a caller that grants no OIDC token", + (hooks) => + hooks.map((hook) => + hook.file === "post-green.yml" + ? { ...hook, job: { ...hook.job, permissions: { contents: "write" } } } + : hook, + ), + /post-green\.yml#publish-next: .* needs id-token: write/, + ], + ])("%s fails the grant relation (negative control)", (_case, mutate, message) => { + expect(grantProblems(mutate(structuredClone(HOOKS))).join("\n")).toMatch(message); + }); + + test("an anchor job narrowed to read while its subcommand pushes fails (negative control)", () => { + const workflow = readWorkflow("update-release-pr.yml"); + const anchor = must(workflow.jobs.anchor, "anchor job"); + anchor.permissions = { contents: "read" }; + const caller = must( + HOOKS.find((hook) => hook.file === "update-release-pr.yml"), + "caller", + ).job; + const unmet = (anchor.steps ?? []) + .flatMap(consumedGrants) + .filter(([scope]) => effectiveGrant(anchor, workflow, caller)[scope] !== "write"); + expect(unmet.map(([scope]) => scope)).toEqual(["contents"]); + }); +}); + +/** The hook a library-page claim about "every green push" or "every release cut" points at, by its ci.yml caller's condition. */ +function hookFor(claim: string): { file: string; workflow: Workflow } { + const release = /release cut/i.test(claim); + const hook = must( + HOOKS.find(({ job }) => /release_created == 'true'/.test(condition(job.if)) === release), + `a hook whose caller runs on ${release ? "a release cut" : "a green push"}`, + ); + return { file: hook.file, workflow: readWorkflow(hook.file) }; +} + +/** Whether some step of the workflow runs the pipeline subcommand (with its argument, when one is named). */ +const runsSubcommand = (workflow: Workflow, subcommand: string): boolean => + Object.values(workflow.jobs).some((job) => + (job.steps ?? []).some((step) => + new RegExp(`release-pipeline\\.ts ${subcommand}(?![\\w-])`).test(step.run ?? ""), + ), + ); + +/** The library page's claims about what publishes when, against the steps the hooks run. */ +function claimProblems(page: string): string[] { + const problems: string[] = []; + const rows = [...page.matchAll(/^\| `(next|latest)` \| ([^|]+) \|/gm)]; + if (rows.length !== 2) + problems.push("the dist-tag table no longer has a next row and a latest row"); + for (const [, tag = "", publishesOn = ""] of rows) { + const channel = tag === "latest" ? "stable" : tag; + const { file, workflow } = hookFor(publishesOn); + if (!runsSubcommand(workflow, `npm-verdict ${channel}`)) { + problems.push( + `the page says ${tag} publishes on "${publishesOn.trim()}", but ${file} runs no npm-verdict ${channel}`, + ); } + } + const build = page.match(/^- Every green push to `main` mints one under the tag `build\//m); + if (!build) problems.push("the page no longer claims every green push mints a build tag"); + else if (!runsSubcommand(hookFor("green push").workflow, "package-commit")) { + problems.push( + "the page says every green push mints a build tag, but the green-push hook runs no package-commit", + ); + } + return problems; +} + +describe("the library page's publishing claims", () => { + const page = readFileSync(join(ROOT, "docs", "reference", "library.md"), "utf8"); + + test("each channel the page says publishes on a green push or a release cut is published by that hook", () => { + expect(claimProblems(page)).toEqual([]); }); - test("a publisher whose own ceiling drops the OIDC grant fails, whatever the caller grants (negative control)", () => { - const caller: Job = { permissions: { contents: "write", "id-token": "write" } }; - const job: Job = { permissions: { contents: "read" } }; - expect(effectiveGrant(job, caller)["id-token"]).toBeUndefined(); - expect(consumedGrants({ run: "npm publish --tag next" })).toEqual([ - ["id-token", '"unnamed step" publishes through OIDC'], + test("a packaging step gone, or a channel's verdict gone, fails the claim (negative control)", () => { + const { workflow } = hookFor("green push"); + const build = must(workflow.jobs.build, "build job"); + build.steps = build.steps?.filter((step) => !/package-commit/.test(step.run ?? "")); + expect(runsSubcommand(workflow, "package-commit")).toBe(false); + expect(runsSubcommand(workflow, "npm-verdict next")).toBe(true); + expect(runsSubcommand(workflow, "npm-verdict stable")).toBe(false); + expect( + claimProblems( + page.replace( + "| `latest` | Every release cut |", + "| `latest` | Every green push to `main` |", + ), + ), + ).toEqual([ + 'the page says latest publishes on "Every green push to `main`", but post-green.yml runs no npm-verdict stable', ]); }); }); -/** The probe steps: each writes a `proceed=` verdict to GITHUB_OUTPUT for the steps after it to read. */ -const isProbe = (step: Step) => - /echo "proceed=(?:true|false)" >> "\$GITHUB_OUTPUT"/.test(step.run ?? ""); +/** Every `` a step writes to GITHUB_OUTPUT. */ +const outputsWritten = (step: Step): string[] => + [...(step.run ?? "").matchAll(/echo "([\w-]+)=[^"]*" >> "\$GITHUB_OUTPUT"/g)].map( + (m) => m[1] ?? "", + ); + +/** Every `steps..outputs.` a step reads, in its condition, env, with, or script. */ +const outputsRead = (step: Step): Array<[id: string, name: string]> => + [ + step.if ?? "", + step.run ?? "", + ...Object.values(step.env ?? {}), + ...Object.values(step.with ?? {}).map(String), + ].flatMap((text) => + [...text.matchAll(/\bsteps\.([\w-]+)\.outputs\.([\w-]+)/g)].map( + (m) => [m[1] ?? "", m[2] ?? ""] as [string, string], + ), + ); + +/** The probe steps: each writes a `proceed=` verdict for the steps after it to read. */ +const isProbe = (step: Step) => outputsWritten(step).includes("proceed"); /** - * Whether the step at `index` runs on a verdict: its condition is `steps..outputs. == 'true'` where `` names an - * earlier step that is the probe itself or is gated the same way (a chain back to the probe). + * Whether the step at `index` runs on a verdict: its condition is `steps..outputs. == 'true'` where an earlier step `` + * writes `` and is the probe itself or is gated the same way (a chain back to the probe). */ function gatedOnProbe(steps: Step[], index: number, probe: number): boolean { - const match = condition(steps[index]?.if).match(/^steps\.([\w-]+)\.outputs\.[\w-]+ == 'true'$/); + const match = condition(steps[index]?.if).match(/^steps\.([\w-]+)\.outputs\.([\w-]+) == 'true'$/); if (!match) return false; const source = steps.findIndex((step, at) => at < index && step.id === match[1]); - if (source < 0) return false; + if (source < 0 || !outputsWritten(steps[source] as Step).includes(match[2] ?? "")) return false; return source === probe || gatedOnProbe(steps, source, probe); } -/** The job ids whose steps after a probe are not gated on it, with the offending step names. */ -function ungatedAfterProbe(workflow: Workflow): string[] { +/** Every wiring fault in a job's outputs: a step after the probe not gated on it, a read of an output no earlier step writes, a written output nobody reads. */ +function wiringProblems(workflow: Workflow): string[] { return Object.entries(workflow.jobs).flatMap(([id, job]) => { const steps = job.steps ?? []; const probe = steps.findIndex(isProbe); - if (probe < 0) return []; - return steps.flatMap((step, index) => - index > probe && !gatedOnProbe(steps, index, probe) - ? [`${id}: "${step.name ?? step.uses ?? "unnamed step"}" runs whatever the probe found`] - : [], - ); + const label = (step: Step) => `"${step.name ?? step.uses ?? "unnamed step"}"`; + const problems: string[] = []; + steps.forEach((step, index) => { + if (probe >= 0 && index > probe && !gatedOnProbe(steps, index, probe)) { + problems.push(`${id}: ${label(step)} runs whatever the probe found`); + } + for (const [source, name] of outputsRead(step)) { + const writer = steps.findIndex((s, at) => at < index && s.id === source); + if (writer < 0 || !outputsWritten(steps[writer] as Step).includes(name)) { + problems.push( + `${id}: ${label(step)} reads steps.${source}.outputs.${name}, which no earlier step writes`, + ); + } + } + if (step.id !== undefined) { + for (const name of outputsWritten(step)) { + const read = steps + .slice(index + 1) + .some((later) => outputsRead(later).some(([s, n]) => s === step.id && n === name)); + if (!read) + problems.push(`${id}: ${label(step)} writes ${name}, which no later step reads`); + } + } + }); + return problems; }); } describe("post-green.yml", () => { const workflow = readWorkflow("post-green.yml"); const caller = must( - localCalls(CI).find((call) => call.file === "post-green.yml"), + HOOKS.find((call) => call.file === "post-green.yml"), "post-green caller", ).job; - test("every step after a probe runs on the probe's verdict", () => { + test("every step after a probe runs on its verdict, every read names a written output, every output is read", () => { const probes = Object.values(workflow.jobs).filter((job) => (job.steps ?? []).some(isProbe)); // Both jobs open with a probe today; a job without one is not judged here. expect(probes.length).toBe(Object.keys(workflow.jobs).length); - expect(ungatedAfterProbe(workflow)).toEqual([]); + expect(wiringProblems(workflow)).toEqual([]); }); - test.each<[string, (w: Workflow) => void]>([ + test.each<[string, (w: Workflow) => void, RegExp]>([ [ "the packaging step without its gate", (w) => delete must(must(w.jobs.build, "build").steps?.at(-1), "step").if, + /runs whatever the probe found/, ], [ "the confirmation gated on a step that is not gated itself", (w) => { const steps = must(must(w.jobs["publish-next"], "publish-next").steps, "steps"); must(steps.at(-1), "confirm").if = "steps.oidc-copy.outputs.proceed == 'true'"; - steps.splice(1, 0, { id: "oidc-copy", run: "echo" }); + steps.splice(1, 0, { id: "oidc-copy", run: 'echo "proceed=true" >> "$GITHUB_OUTPUT"' }); }, + /runs whatever the probe found/, ], [ "a gate that is not an equality on true", @@ -212,11 +410,33 @@ describe("post-green.yml", () => { must(must(w.jobs.build, "build").steps?.at(-1), "step").if = "always() || steps.token.outputs.proceed == 'true'"; }, + /runs whatever the probe found/, ], - ])("%s fails the gate relation (negative control)", (_case, mutate) => { + [ + "a gate on an output the probe never writes", + (w) => { + must(must(w.jobs.build, "build").steps?.at(-1), "step").if = + "steps.token.outputs.published == 'true'"; + }, + /reads steps\.token\.outputs\.published, which no earlier step writes/, + ], + [ + "the confirmation step gone, leaving the publish output unread", + (w) => must(w.jobs["publish-next"], "publish-next").steps?.pop(), + /writes published, which no later step reads/, + ], + [ + "the publish output no longer written", + (w) => { + const step = must(must(w.jobs["publish-next"], "publish-next").steps?.at(-2), "publish"); + step.run = step.run?.replace(/\n\s*echo "published=true" >> "\$GITHUB_OUTPUT"/, ""); + }, + /reads steps\.publish\.outputs\.published, which no earlier step writes/, + ], + ])("%s fails the wiring relation (negative control)", (_case, mutate, message) => { const drifted = structuredClone(workflow); mutate(drifted); - expect(ungatedAfterProbe(drifted)).not.toEqual([]); + expect(wiringProblems(drifted).join("\n")).toMatch(message); }); test("the judged sha the caller passes is the ref every checkout takes and the source every packaging step names", () => { @@ -239,6 +459,28 @@ describe("post-green.yml", () => { ).toBe(judged); } }); + + test("the probe tells a rejected PAT from a read ceiling by the same secret the checkout falls back from", () => { + const steps = must(workflow.jobs.build, "build job").steps ?? []; + const checkout = must( + steps.find((step) => (step.uses ?? "").startsWith("actions/checkout@")), + "checkout", + ); + const secret = must( + String(checkout.with?.token).match( + /^\$\{\{\s*secrets\.(\w+)\s*\|\|\s*github\.token\s*\}\}$/, + )?.[1], + "a `secrets.X || github.token` checkout token", + ); + const probe = must(steps.find(isProbe), "probe step"); + const flag = Object.entries(probe.env ?? {}).find(([, value]) => + value.includes(`secrets.${secret}`), + ); + expect(flag, `the probe's env does not read secrets.${secret}`).toBeDefined(); + expect(condition(flag?.[1])).toBe(`secrets.${secret} != ''`); + // The script branches on that variable, so the env name and the script agree. + expect(probe.run).toContain(`"$${flag?.[0]}" = "true"`); + }); }); /** The probe's stdout as the runner reads it: one entry per line. */ @@ -304,14 +546,13 @@ describe("the push probe under bash", () => { const run = must(must(steps.find(isProbe), "probe step").run, "probe run"); /** - * The fenced block: git's stderr, indented, between a stop-commands line keyed by a fresh 32-hex token and its own resume line, then one - * static error. The runner trims, so an indented "::error::forged" still reads as a command outside a fence; the indent only marks the - * lines as quoted text in the log. + * The fenced block: git's stderr, indented, between a stop-commands line keyed by a 32-hex token and its own resume line, then one + * static error. The runner trims, so an indented "::error::forged" still reads as a command outside a fence; the indent only marks + * the lines as quoted text in the log. Returns the token. */ - function expectFenced(lines: string[], inner: string[]): void { + function expectFenced(lines: string[], inner: string[]): string { const [open, header, ...rest] = lines; - const token = open?.match(FENCE_OPEN)?.[1]; - expect(token).toBeDefined(); + const token = must(open?.match(FENCE_OPEN)?.[1], "a stop-commands line with a 32-hex token"); expect(header).toBe("probe stderr:"); expect(rest.slice(0, inner.length)).toEqual(inner); expect(rest[inner.length]).toBe(`::${token}::`); @@ -321,14 +562,17 @@ describe("the push probe under bash", () => { for (const line of inner) { expect(line.startsWith(" ")).toBe(true); } + return token; } - test("a stderr that does not end in a newline still closes the fence on a line of its own", () => { + test("a stderr that does not end in a newline still closes the fence on a line of its own, under a token fresh per run", () => { const probe = runProbe(run, "refused", 1, true); expect(probe.status).toBe(1); - expectFenced(probe.lines, [" refused"]); + const token = expectFenced(probe.lines, [" refused"]); expect(probe.probeErrLeft).toBe(false); expect(probe.output).toBe(""); + // A fixed token is one remote text could name to resume command processing. + expect(expectFenced(runProbe(run, "refused", 1, true).lines, [" refused"])).not.toBe(token); }); test("a stderr carrying workflow-command text is confined to indented lines inside the fence", () => { diff --git a/test/docs/workflow-loader.ts b/test/docs/workflow-loader.ts index acd2a4be..111146aa 100644 --- a/test/docs/workflow-loader.ts +++ b/test/docs/workflow-loader.ts @@ -34,6 +34,7 @@ export interface Job { "timeout-minutes"?: number; permissions?: Record; concurrency?: Concurrency; + env?: Record; steps?: Step[]; uses?: string; with?: Record; From 70bfeef0d7f3f1659291b3b88a31a7611374c634 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:39:15 -0400 Subject: [PATCH 3/9] test(docs): a probe runs unconditionally, a channel's step runs on every call, npm is not configured through the env --- test/docs/npm-publish-workflows.test.ts | 10 ++++ test/docs/post-green-workflow.test.ts | 62 ++++++++++++++++++------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index 038169f4..e85c6675 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -126,6 +126,9 @@ function publisherProblems(next: RunJob, stableWorkflow: Workflow): string[] { for (const value of [...runnerInputs(next), ...runnerInputs(stable)]) { if (/secrets\.|NODE_AUTH_TOKEN|NPM_TOKEN/.test(value)) problems.push(`a token reaches npm: ${value}`); + // npm reads its config from the environment too, so a stray NPM_CONFIG_* can move the dist-tag or the registry. + if (/^npm_config_/i.test(value)) + problems.push(`npm is configured through the environment: ${value}`); } for (const [label, a, b] of sharedSteps(next, stable)) { if (JSON.stringify(body(a)) !== JSON.stringify(body(b))) @@ -187,6 +190,13 @@ describe("the npm publish jobs", () => { }, /a token reaches npm: NODE_AUTH_TOKEN/, ], + [ + "npm's dist-tag set through the job's environment", + (_n, w) => { + must(w.jobs[STABLE_JOB], "stable").env = { NPM_CONFIG_TAG: "next" }; + }, + /npm is configured through the environment: NPM_CONFIG_TAG/, + ], [ "a floor guard fixed in one job only", (n) => { diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 9ef6f9ac..312de792 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -2,7 +2,7 @@ * The hooks ci.yml calls after the all-green gate (post-green.yml, update-release.yml, update-release-pr.yml) push refs and publish * packages, so what they can do follows from where they are reachable and what each job is granted. The relations here: every hook is * reachable through workflow_call alone and its ci.yml caller sits downstream of all-green; a job's effective grant covers what its - * steps consume and nothing granted goes unconsumed; a hook job's own condition is the fork guard or nothing; post-green's judged sha + * steps consume; a hook job's own condition is the fork guard or nothing; post-green's judged sha * reaches every checkout and packaging step; every output a step writes is read by a later step, and every gate reads an output an * earlier step writes, back to the probe. The push probe also runs under bash against a stubbed git, since no pin shows what a branch * does. @@ -170,9 +170,12 @@ const effectiveGrant = (job: Job, workflow: Workflow, caller: Job): Record Workflow = readWorkflow, +): string[] { return hooks.flatMap(({ file, job: caller }) => { - const workflow = readWorkflow(file); + const workflow = read(file); return Object.entries(workflow.jobs).flatMap(([id, job]) => { const granted = effectiveGrant(job, workflow, caller); return (job.steps ?? []) @@ -224,17 +227,13 @@ describe("the hooks' grants", () => { }); test("an anchor job narrowed to read while its subcommand pushes fails (negative control)", () => { - const workflow = readWorkflow("update-release-pr.yml"); - const anchor = must(workflow.jobs.anchor, "anchor job"); - anchor.permissions = { contents: "read" }; - const caller = must( - HOOKS.find((hook) => hook.file === "update-release-pr.yml"), - "caller", - ).job; - const unmet = (anchor.steps ?? []) - .flatMap(consumedGrants) - .filter(([scope]) => effectiveGrant(anchor, workflow, caller)[scope] !== "write"); - expect(unmet.map(([scope]) => scope)).toEqual(["contents"]); + const narrowed = readWorkflow("update-release-pr.yml"); + must(narrowed.jobs.anchor, "anchor job").permissions = { contents: "read" }; + const read = (file: string) => + file === "update-release-pr.yml" ? narrowed : readWorkflow(file); + expect(grantProblems(HOOKS, read).join("\n")).toMatch( + /update-release-pr\.yml#anchor: .* needs contents: write/, + ); }); }); @@ -248,11 +247,19 @@ function hookFor(claim: string): { file: string; workflow: Workflow } { return { file: hook.file, workflow: readWorkflow(hook.file) }; } -/** Whether some step of the workflow runs the pipeline subcommand (with its argument, when one is named). */ +/** A step gate that reads a verdict (`steps..outputs. == 'true'`); every other condition can skip the step on the caller's own event. */ +const VERDICT_GATE = /^steps\.[\w-]+\.outputs\.[\w-]+ == 'true'$/; + +/** + * Whether some step of the workflow runs the pipeline subcommand (with its argument, when one is named) on every run its job takes: the + * step carries no condition, or a verdict gate the wiring relation ties to a probe. + */ const runsSubcommand = (workflow: Workflow, subcommand: string): boolean => Object.values(workflow.jobs).some((job) => - (job.steps ?? []).some((step) => - new RegExp(`release-pipeline\\.ts ${subcommand}(?![\\w-])`).test(step.run ?? ""), + (job.steps ?? []).some( + (step) => + new RegExp(`release-pipeline\\.ts ${subcommand}(?![\\w-])`).test(step.run ?? "") && + (step.if === undefined || VERDICT_GATE.test(condition(step.if))), ), ); @@ -295,6 +302,14 @@ describe("the library page's publishing claims", () => { expect(runsSubcommand(workflow, "package-commit")).toBe(false); expect(runsSubcommand(workflow, "npm-verdict next")).toBe(true); expect(runsSubcommand(workflow, "npm-verdict stable")).toBe(false); + // A publish step conditioned on the caller's event skips on the push that calls it, so the channel's claim fails. + const conditioned = readWorkflow("update-release.yml"); + for (const job of Object.values(conditioned.jobs)) { + for (const step of job.steps ?? []) { + if (/npm-verdict stable/.test(step.run ?? "")) step.if = "github.event_name == 'release'"; + } + } + expect(runsSubcommand(conditioned, "npm-verdict stable")).toBe(false); expect( claimProblems( page.replace( @@ -349,6 +364,12 @@ function wiringProblems(workflow: Workflow): string[] { const probe = steps.findIndex(isProbe); const label = (step: Step) => `"${step.name ?? step.uses ?? "unnamed step"}"`; const problems: string[] = []; + if (probe >= 0 && steps[probe]?.if !== undefined) { + // A probe that skips writes no verdict, and every gate on it reads empty: the whole job stands down quietly. + problems.push( + `${id}: the probe ${label(steps[probe] as Step)} runs under a condition of its own`, + ); + } steps.forEach((step, index) => { if (probe >= 0 && index > probe && !gatedOnProbe(steps, index, probe)) { problems.push(`${id}: ${label(step)} runs whatever the probe found`); @@ -425,6 +446,13 @@ describe("post-green.yml", () => { (w) => must(w.jobs["publish-next"], "publish-next").steps?.pop(), /writes published, which no later step reads/, ], + [ + "a probe under a condition of its own", + (w) => { + must(must(w.jobs.build, "build").steps?.[1], "probe").if = "github.event_name == 'release'"; + }, + /the probe "Check the token can push" runs under a condition of its own/, + ], [ "the publish output no longer written", (w) => { From 45bae39cd4793265c9893e00dd3367cd957a639a Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:40:42 -0400 Subject: [PATCH 4/9] test(docs): a workflow-level env reaches the publishers; sources are the steps that read SOURCE_SHA; the post-fence error carries none of git's words --- test/docs/npm-publish-workflows.test.ts | 60 +++++++++++++++++-------- test/docs/post-green-workflow.test.ts | 5 ++- test/docs/workflow-loader.ts | 1 + 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index e85c6675..f29bd7e2 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -63,10 +63,16 @@ const setupNode = (job: RunJob): Step => const STABLE_FILE = "update-release.yml"; const STABLE_JOB = "publish-npm"; -const publishers = () => ({ - next: runJob(readWorkflow("post-green.yml").jobs["publish-next"], "publish-next job"), - stableWorkflow: readWorkflow(STABLE_FILE), -}); +const NEXT_FILE = "post-green.yml"; +const NEXT_JOB = "publish-next"; +const publishers = () => { + const nextWorkflow = readWorkflow(NEXT_FILE); + return { + nextWorkflow, + next: runJob(nextWorkflow.jobs[NEXT_JOB], `${NEXT_JOB} job`), + stableWorkflow: readWorkflow(STABLE_FILE), + }; +}; /** The `owner/name` the manifest's repository URL names: the one repository whose CI is the trusted publisher. */ function manifestSlug(): string { @@ -101,7 +107,8 @@ function sharedSteps(next: RunJob, stable: RunJob): Array<[string, Step, Step]> const body = ({ if: _gate, id: _id, ...rest }: Step): Omit => rest; /** Every way the two publishers break their shared contract; the assertions and the negative controls read this one list. */ -function publisherProblems(next: RunJob, stableWorkflow: Workflow): string[] { +function publisherProblems(nextWorkflow: Workflow, stableWorkflow: Workflow): string[] { + const next = runJob(nextWorkflow.jobs[NEXT_JOB], `${NEXT_JOB} job`); const stable = runJob(stableWorkflow.jobs[STABLE_JOB], `${STABLE_JOB} job`); const problems: string[] = []; const guard = `github.repository == '${manifestSlug()}'`; @@ -123,7 +130,11 @@ function publisherProblems(next: RunJob, stableWorkflow: Workflow): string[] { if (JSON.stringify([stable.needs ?? []].flat().sort()) !== JSON.stringify(others)) { problems.push(`${STABLE_JOB} does not run after every other job (${others.join(", ")})`); } - for (const value of [...runnerInputs(next), ...runnerInputs(stable)]) { + // A workflow-level env reaches every step of every job, the publishers' included. + const workflowEnv = [nextWorkflow, stableWorkflow].flatMap((w) => + Object.entries(w.env ?? {}).flat(), + ); + for (const value of [...workflowEnv, ...runnerInputs(next), ...runnerInputs(stable)]) { if (/secrets\.|NODE_AUTH_TOKEN|NPM_TOKEN/.test(value)) problems.push(`a token reaches npm: ${value}`); // npm reads its config from the environment too, so a stray NPM_CONFIG_* can move the dist-tag or the registry. @@ -142,7 +153,7 @@ function publisherProblems(next: RunJob, stableWorkflow: Workflow): string[] { } describe("the npm publish jobs", () => { - const { next, stableWorkflow } = publishers(); + const { nextWorkflow, next, stableWorkflow } = publishers(); const stable = runJob(stableWorkflow.jobs[STABLE_JOB], `${STABLE_JOB} job`); test("both are guarded to the manifest's repository, take one lane, hand npm no token, share their steps' text, and name one registry; the stable one runs last", () => { @@ -153,10 +164,10 @@ describe("the npm publish jobs", () => { // workflow has jobs for the publish to wait on. expect(sharedSteps(next, stable).length).toBeGreaterThan(3); expect(Object.keys(stableWorkflow.jobs).length).toBeGreaterThan(2); - expect(publisherProblems(next, stableWorkflow)).toEqual([]); + expect(publisherProblems(nextWorkflow, stableWorkflow)).toEqual([]); }); - test.each<[string, (next: RunJob, stableWorkflow: Workflow) => void, RegExp]>([ + test.each<[string, (nextWorkflow: Workflow, stableWorkflow: Workflow) => void, RegExp]>([ [ "a stable publish open to forks", (_n, w) => delete must(w.jobs[STABLE_JOB], "stable").if, @@ -164,7 +175,8 @@ describe("the npm publish jobs", () => { ], [ "a pre-release publish on a lane of its own", - (n) => { + (w) => { + const n = must(w.jobs[NEXT_JOB], "next"); n.concurrency = { ...n.concurrency, group: "publish-next" }; }, /different lanes/, @@ -178,8 +190,10 @@ describe("the npm publish jobs", () => { ], [ "a token handed to setup-node", - (n) => { - setupNode(n).env = { NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}` }; + (w) => { + setupNode(runJob(w.jobs[NEXT_JOB], "next")).env = { + NODE_AUTH_TOKEN: `\${{ secrets.NPM_TOKEN }}`, + }; }, /a token reaches npm: NODE_AUTH_TOKEN/, ], @@ -197,10 +211,20 @@ describe("the npm publish jobs", () => { }, /npm is configured through the environment: NPM_CONFIG_TAG/, ], + [ + "a token in the workflow's own env, reaching every job", + (w) => { + w.env = { NODE_AUTH_TOKEN: `\${{ github.token }}` }; + }, + /a token reaches npm: NODE_AUTH_TOKEN/, + ], [ "a floor guard fixed in one job only", - (n) => { - const guard = stepNamed(n, "Require an npm that publishes through OIDC"); + (w) => { + const guard = stepNamed( + runJob(w.jobs[NEXT_JOB], "next"), + "Require an npm that publishes through OIDC", + ); guard.run = guard.run?.replace("11.5.1", "11.6.0"); }, /"Require an npm that publishes through OIDC" diverged/, @@ -213,10 +237,10 @@ describe("the npm publish jobs", () => { /different registries|diverged/, ], ])("%s fails its relation (negative control)", (_case, mutate, message) => { - const driftedNext = structuredClone(next); - const driftedWorkflow = structuredClone(stableWorkflow); - mutate(driftedNext, driftedWorkflow); - expect(publisherProblems(driftedNext, driftedWorkflow).join("\n")).toMatch(message); + const driftedNext = structuredClone(nextWorkflow); + const driftedStable = structuredClone(stableWorkflow); + mutate(driftedNext, driftedStable); + expect(publisherProblems(driftedNext, driftedStable).join("\n")).toMatch(message); }); }); diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 312de792..1787634f 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -474,7 +474,8 @@ describe("post-green.yml", () => { const judged = `\${{ inputs.${input} }}`; const steps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); const checkouts = steps.filter((step) => (step.uses ?? "").startsWith("actions/checkout@")); - const sources = steps.filter((step) => step.env?.SOURCE_SHA !== undefined); + // The steps that pass a source to the pipeline or to npm, found by the script that reads the variable, not by the env that sets it. + const sources = steps.filter((step) => /\$SOURCE_SHA\b/.test(step.run ?? "")); expect(checkouts.length).toBeGreaterThan(1); expect(sources.length).toBeGreaterThan(1); for (const step of checkouts) { @@ -589,6 +590,8 @@ describe("the push probe under bash", () => { expect(after[0]).toMatch(/^::error::/); for (const line of inner) { expect(line.startsWith(" ")).toBe(true); + // The error after the fence is static text: none of git's words reach a line the runner reads as a command. + expect(after[0]).not.toContain(line.trim()); } return token; } diff --git a/test/docs/workflow-loader.ts b/test/docs/workflow-loader.ts index 111146aa..41fb8833 100644 --- a/test/docs/workflow-loader.ts +++ b/test/docs/workflow-loader.ts @@ -49,6 +49,7 @@ export interface Workflow { on: Record; permissions?: Record; concurrency?: Concurrency; + env?: Record; jobs: Record; } interface CompositeAction { From fe6d1766d99479ff37f7a89c3ea04079b0f0d96c Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:46:02 -0400 Subject: [PATCH 5/9] test(docs): output wiring holds in every hook, no step ahead of a probe is conditioned, the library is built before it is published --- test/docs/npm-publish-workflows.test.ts | 28 ++++++++++++++++++++- test/docs/post-green-workflow.test.ts | 33 ++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index f29bd7e2..a6d4ca46 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -145,6 +145,22 @@ function publisherProblems(nextWorkflow: Workflow, stableWorkflow: Workflow): st if (JSON.stringify(body(a)) !== JSON.stringify(body(b))) problems.push(`"${label}" diverged between the publishers`); } + // The publish comes after every shared step: setup, setup-node, the floor guard, and the library build it publishes. + for (const [label, job] of [ + [NEXT_JOB, next], + [STABLE_JOB, stable], + ] as const) { + const publish = job.steps.findIndex((step) => /\bnpm\s+publish\b/.test(step.run ?? "")); + if (publish < 0) { + problems.push(`${label} has no npm publish step`); + continue; + } + for (const [name, a, b] of sharedSteps(next, stable)) { + const shared = job === next ? a : b; + if (job.steps.indexOf(shared) > publish) + problems.push(`${label}: "${name}" runs after the publish`); + } + } const registry = setupNode(next).with?.["registry-url"]; if (typeof registry !== "string" || setupNode(stable).with?.["registry-url"] !== registry) { problems.push("the publishers name different registries"); @@ -218,6 +234,16 @@ describe("the npm publish jobs", () => { }, /a token reaches npm: NODE_AUTH_TOKEN/, ], + [ + "a library build moved after the stable publish", + (_n, w) => { + const steps = must(w.jobs[STABLE_JOB], "stable").steps ?? []; + const build = steps.findIndex((step) => step.name === "Build the library"); + const [moved] = steps.splice(build, 1); + steps.push(moved as Step); + }, + /publish-npm: "Build the library" runs after the publish/, + ], [ "a floor guard fixed in one job only", (w) => { @@ -404,7 +430,7 @@ describe("the publish blocks under bash", () => { }, ], [ - "next: a skip verdict calls npm not at all, reports why, and leaves the confirmation ungated", + "next: a skip verdict calls npm not at all, reports why, and writes no publish output, so the confirmation skips", nextRun, "skip the registry's next is newer", { lines: ["::notice::the registry's next is newer"], status: 0, output: "" }, diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 1787634f..5ff98251 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -374,6 +374,10 @@ function wiringProblems(workflow: Workflow): string[] { if (probe >= 0 && index > probe && !gatedOnProbe(steps, index, probe)) { problems.push(`${id}: ${label(step)} runs whatever the probe found`); } + if (probe >= 0 && index < probe && step.if !== undefined) { + // A step the probe depends on (the checkout) that skips leaves the probe judging an empty workspace: it warns and stands down. + problems.push(`${id}: ${label(step)} runs under a condition of its own ahead of the probe`); + } for (const [source, name] of outputsRead(step)) { const writer = steps.findIndex((s, at) => at < index && s.id === source); if (writer < 0 || !outputsWritten(steps[writer] as Step).includes(name)) { @@ -403,11 +407,13 @@ describe("post-green.yml", () => { "post-green caller", ).job; - test("every step after a probe runs on its verdict, every read names a written output, every output is read", () => { + test("every step after a probe runs on its verdict, every read names a written output, every output is read, in every hook", () => { const probes = Object.values(workflow.jobs).filter((job) => (job.steps ?? []).some(isProbe)); - // Both jobs open with a probe today; a job without one is not judged here. + // Both post-green jobs open with a probe today; the release hooks have none, and their output reads are judged the same way. expect(probes.length).toBe(Object.keys(workflow.jobs).length); - expect(wiringProblems(workflow)).toEqual([]); + for (const { file } of HOOKS) { + expect(wiringProblems(readWorkflow(file)), file).toEqual([]); + } }); test.each<[string, (w: Workflow) => void, RegExp]>([ @@ -446,6 +452,14 @@ describe("post-green.yml", () => { (w) => must(w.jobs["publish-next"], "publish-next").steps?.pop(), /writes published, which no later step reads/, ], + [ + "the checkout ahead of the probe under a condition of its own", + (w) => { + must(must(w.jobs.build, "build").steps?.[0], "checkout").if = + "github.event_name == 'release'"; + }, + /runs under a condition of its own ahead of the probe/, + ], [ "a probe under a condition of its own", (w) => { @@ -467,6 +481,19 @@ describe("post-green.yml", () => { expect(wiringProblems(drifted).join("\n")).toMatch(message); }); + test("a verdict gate copied onto a hook without that probe fails the wiring there (negative control)", () => { + const stable = readWorkflow("update-release.yml"); + for (const job of Object.values(stable.jobs)) { + for (const step of job.steps ?? []) { + if (/npm-verdict stable/.test(step.run ?? "")) + step.if = "steps.oidc.outputs.proceed == 'true'"; + } + } + expect(wiringProblems(stable).join("\n")).toMatch( + /reads steps\.oidc\.outputs\.proceed, which no earlier step writes/, + ); + }); + test("the judged sha the caller passes is the ref every checkout takes and the source every packaging step names", () => { const [input, ...rest] = Object.keys(workflow.on.workflow_call?.inputs ?? {}); expect(rest, "post-green.yml takes more than the one judged sha").toEqual([]); From e65a4ffa5f93efefb81a5f087674e291833d7cf8 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:50:30 -0400 Subject: [PATCH 6/9] test(docs): a publisher step under a condition of its own fails; the probed job allows only a verdict gate --- test/docs/npm-publish-workflows.test.ts | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index a6d4ca46..5ae8c213 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -106,6 +106,22 @@ function sharedSteps(next: RunJob, stable: RunJob): Array<[string, Step, Step]> /** A step stripped of what legitimately differs between the two publishers: its gate and its id. */ const body = ({ if: _gate, id: _id, ...rest }: Step): Omit => rest; +/** A step gate that reads a verdict an earlier step wrote; every other condition can skip the step on the caller's own event. */ +const VERDICT_GATE = /^steps\.[\w-]+\.outputs\.[\w-]+ == 'true'$/; + +/** + * The steps of a publisher that run under a condition of their own: in the pre-release job (which opens with a probe) anything but a + * verdict gate, in the stable job (which has none) anything at all. A skipped build publishes a source-only tarball; a skipped guard + * publishes through an npm that cannot. + */ +function conditionedSteps(job: RunJob, probed: boolean): string[] { + return job.steps.flatMap((step) => + step.if !== undefined && !(probed && VERDICT_GATE.test(condition(step.if))) + ? [step.name ?? step.uses ?? "unnamed step"] + : [], + ); +} + /** Every way the two publishers break their shared contract; the assertions and the negative controls read this one list. */ function publisherProblems(nextWorkflow: Workflow, stableWorkflow: Workflow): string[] { const next = runJob(nextWorkflow.jobs[NEXT_JOB], `${NEXT_JOB} job`); @@ -117,6 +133,9 @@ function publisherProblems(nextWorkflow: Workflow, stableWorkflow: Workflow): st [STABLE_JOB, stable], ] as const) { if (condition(job.if) !== guard) problems.push(`${label} is not guarded to ${guard}`); + for (const name of conditionedSteps(job, job === next)) { + problems.push(`${label}: "${name}" runs under a condition of its own`); + } } if (JSON.stringify(next.concurrency) !== JSON.stringify(stable.concurrency)) { problems.push("the publishers take different lanes"); @@ -234,6 +253,22 @@ describe("the npm publish jobs", () => { }, /a token reaches npm: NODE_AUTH_TOKEN/, ], + [ + "a stable library build under a condition of its own", + (_n, w) => { + stepNamed(runJob(w.jobs[STABLE_JOB], "stable"), "Build the library").if = + "github.event_name == 'release'"; + }, + /publish-npm: "Build the library" runs under a condition of its own/, + ], + [ + "a pre-release step gated on the caller's event instead of the probe", + (w) => { + stepNamed(runJob(w.jobs[NEXT_JOB], "next"), "Build the library").if = + "github.event_name == 'push'"; + }, + /publish-next: "Build the library" runs under a condition of its own/, + ], [ "a library build moved after the stable publish", (_n, w) => { From 0e52285168bd634daf39b1f57ec394dfccf3d292 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 15:55:45 -0400 Subject: [PATCH 7/9] test(docs): a conditioned step in any hook job fails; the green-push hook is the one caller on a push to main behind the gate --- test/docs/post-green-workflow.test.ts | 59 +++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 5ff98251..1d562099 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -240,10 +240,22 @@ describe("the hooks' grants", () => { /** The hook a library-page claim about "every green push" or "every release cut" points at, by its ci.yml caller's condition. */ function hookFor(claim: string): { file: string; workflow: Workflow } { const release = /release cut/i.test(claim); - const hook = must( - HOOKS.find(({ job }) => /release_created == 'true'/.test(condition(job.if)) === release), - `a hook whose caller runs on ${release ? "a release cut" : "a green push"}`, - ); + // A green push: the caller's own condition names the push event and the main ref beside the gate's success; a release cut: it + // names release-please's release_created output. One caller each, or the claim points at nothing. + const runsOn = (job: Job) => { + const on = condition(job.if); + return release + ? /release_created == 'true'/.test(on) + : /needs\.all-green\.result == 'success'/.test(on) && + /github\.event_name == 'push'/.test(on) && + /github\.ref == 'refs\/heads\/main'/.test(on); + }; + const matching = HOOKS.filter(({ job }) => runsOn(job)); + expect( + matching.map((hook) => hook.file), + `exactly one hook caller runs on ${release ? "a release cut" : "a green push to main"}`, + ).toHaveLength(1); + const hook = matching[0] as LocalCall; return { file: hook.file, workflow: readWorkflow(hook.file) }; } @@ -295,6 +307,19 @@ describe("the library page's publishing claims", () => { expect(claimProblems(page)).toEqual([]); }); + test("a green-push caller widened to pull requests matches no claim (negative control)", () => { + const ci = structuredClone(CI); + must(ci.jobs["post-green"], "post-green caller").if = + "needs.all-green.result == 'success' && github.event_name == 'pull_request'"; + const widened = postGateHooks(ci).hooks.filter(({ job }) => { + const on = condition(job.if); + return ( + /github\.event_name == 'push'/.test(on) && /github\.ref == 'refs\/heads\/main'/.test(on) + ); + }); + expect(widened.map((hook) => hook.file)).not.toContain("post-green.yml"); + }); + test("a packaging step gone, or a channel's verdict gone, fails the claim (negative control)", () => { const { workflow } = hookFor("green push"); const build = must(workflow.jobs.build, "build job"); @@ -357,7 +382,10 @@ function gatedOnProbe(steps: Step[], index: number, probe: number): boolean { return source === probe || gatedOnProbe(steps, source, probe); } -/** Every wiring fault in a job's outputs: a step after the probe not gated on it, a read of an output no earlier step writes, a written output nobody reads. */ +/** + * Every wiring fault in a job's steps: a step conditioned on anything but a verdict (it would skip on the caller's own event), a step + * after the probe not gated on it, a read of an output no earlier step writes, a written output nobody reads. + */ function wiringProblems(workflow: Workflow): string[] { return Object.entries(workflow.jobs).flatMap(([id, job]) => { const steps = job.steps ?? []; @@ -374,9 +402,10 @@ function wiringProblems(workflow: Workflow): string[] { if (probe >= 0 && index > probe && !gatedOnProbe(steps, index, probe)) { problems.push(`${id}: ${label(step)} runs whatever the probe found`); } - if (probe >= 0 && index < probe && step.if !== undefined) { - // A step the probe depends on (the checkout) that skips leaves the probe judging an empty workspace: it warns and stands down. - problems.push(`${id}: ${label(step)} runs under a condition of its own ahead of the probe`); + if ((probe < 0 || index < probe) && step.if !== undefined) { + // A hook runs on the caller's event, so a step's own condition skips it quietly: retag-major on a release, or the checkout + // the probe judges an empty workspace without. + problems.push(`${id}: ${label(step)} runs under a condition of its own`); } for (const [source, name] of outputsRead(step)) { const writer = steps.findIndex((s, at) => at < index && s.id === source); @@ -458,7 +487,7 @@ describe("post-green.yml", () => { must(must(w.jobs.build, "build").steps?.[0], "checkout").if = "github.event_name == 'release'"; }, - /runs under a condition of its own ahead of the probe/, + /"actions\/checkout@[0-9a-f]+" runs under a condition of its own/, ], [ "a probe under a condition of its own", @@ -481,6 +510,18 @@ describe("post-green.yml", () => { expect(wiringProblems(drifted).join("\n")).toMatch(message); }); + test("a release-hook step under a condition of its own fails the wiring there (negative control)", () => { + const stable = readWorkflow("update-release.yml"); + for (const job of Object.values(stable.jobs)) { + for (const step of job.steps ?? []) { + if (/retag-major/.test(step.run ?? "")) step.if = "github.event_name == 'release'"; + } + } + expect(wiringProblems(stable).join("\n")).toMatch( + /"Move the major tag to the packaged commit" runs under a condition of its own/, + ); + }); + test("a verdict gate copied onto a hook without that probe fails the wiring there (negative control)", () => { const stable = readWorkflow("update-release.yml"); for (const job of Object.values(stable.jobs)) { From 9f5d1bb29b47b41b4358c2935554d79ca0d23a18 Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:03:11 -0400 Subject: [PATCH 8/9] test(docs): the confirmation block runs under bash over the union's outcomes; expressions normalized before comparison --- test/docs/npm-publish-workflows.test.ts | 62 +++++++++++++++++++++++++ test/docs/post-green-workflow.test.ts | 11 +++-- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index 5ae8c213..bd4e40d0 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -429,6 +429,68 @@ describe("the npm floor guard under bash", () => { }); }); +/** The outcome literals of the pipeline's ConfirmVerdict union, read from the type the script exports; the case block must name each. */ +function confirmOutcomes(): string[] { + const source = readFileSync(join(ROOT, ".github", "scripts", "release-pipeline.ts"), "utf8"); + const union = must( + source.match(/export type ConfirmVerdict =([\s\S]*?);\n/)?.[1], + "the ConfirmVerdict union", + ); + const outcomes = [...union.matchAll(/outcome: "([a-z]+)"/g)].map((m) => m[1] ?? ""); + expect(outcomes.length).toBeGreaterThan(2); + return outcomes; +} + +describe("the confirmation block under bash", () => { + const { next } = publishers(); + const confirm = must( + next.steps.find((step) => /release-pipeline\.ts npm-confirm\b/.test(step.run ?? "")), + "the npm-confirm step", + ); + const run = must(confirm.run, "confirm run"); + const outcomes = confirmOutcomes(); + + /** A bun that prints the given confirm line. */ + const stubBun = + (line: string) => + (bin: string): void => { + writeFileSync(join(bin, "bun"), `#!/bin/sh\nprintf '%s\\n' "${line}"\n`, { mode: 0o755 }); + }; + + test("the case block names every outcome the script can print, and the confirmation is gated on this job's own publish", () => { + for (const outcome of outcomes) { + expect(run, `the case block has no arm for "${outcome}"`).toContain(`${outcome}\\ *)`); + } + expect(condition(confirm.if)).toBe("steps.publish.outputs.published == 'true'"); + }); + + /** One run per outcome plus an unknown line: the run ends 1 for behind and the unknown, 0 otherwise, and the message class follows the outcome. */ + const expected: Record = { + settled: { status: 0, command: /^::notice::/ }, + unsettled: { status: 0, command: /^::warning::/ }, + behind: { status: 1, command: /^::error::/ }, + }; + test.each([...outcomes, "nonsense"])("a %s line", (outcome) => { + const step = runStep( + run, + { SOURCE_SHA: "b8df084c" }, + stubBun(`${outcome} next 2.0.1-main.446 is placed`), + ); + const want = expected[outcome] ?? { + status: 1, + command: /^::error::npm-confirm printed neither/, + }; + expect(step.status, `a ${outcome} line must exit ${want.status}`).toBe(want.status); + const commands = step.lines.filter((line) => line.startsWith("::")); + expect(commands).toHaveLength(1); + expect(commands[0]).toMatch(want.command); + }); + + test("every outcome of the union has an expected verdict here, so a new outcome fails until this table names it", () => { + expect(Object.keys(expected).sort()).toEqual([...outcomes].sort()); + }); +}); + describe("the publish blocks under bash", () => { const { next } = publishers(); const stable = runJob(readWorkflow(STABLE_FILE).jobs[STABLE_JOB], `${STABLE_JOB} job`); diff --git a/test/docs/post-green-workflow.test.ts b/test/docs/post-green-workflow.test.ts index 1d562099..ade79995 100644 --- a/test/docs/post-green-workflow.test.ts +++ b/test/docs/post-green-workflow.test.ts @@ -538,8 +538,9 @@ describe("post-green.yml", () => { test("the judged sha the caller passes is the ref every checkout takes and the source every packaging step names", () => { const [input, ...rest] = Object.keys(workflow.on.workflow_call?.inputs ?? {}); expect(rest, "post-green.yml takes more than the one judged sha").toEqual([]); - expect(caller.with?.[input ?? ""]).toBe(`\${{ github.sha }}`); - const judged = `\${{ inputs.${input} }}`; + // Compared as expressions, so a sync respelling the managed caller's braces or spacing is not a behavior change here. + expect(condition(caller.with?.[input ?? ""])).toBe("github.sha"); + const judged = `inputs.${input}`; const steps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); const checkouts = steps.filter((step) => (step.uses ?? "").startsWith("actions/checkout@")); // The steps that pass a source to the pipeline or to npm, found by the script that reads the variable, not by the env that sets it. @@ -547,11 +548,13 @@ describe("post-green.yml", () => { expect(checkouts.length).toBeGreaterThan(1); expect(sources.length).toBeGreaterThan(1); for (const step of checkouts) { - expect(step.with?.ref, "a checkout of something other than the judged sha").toBe(judged); + expect(condition(step.with?.ref), "a checkout of something other than the judged sha").toBe( + judged, + ); } for (const step of sources) { expect( - step.env?.SOURCE_SHA, + condition(step.env?.SOURCE_SHA), `"${step.name}" packages something other than the judged sha`, ).toBe(judged); } From 70fab5ff714fb76db1de224885c8cb7c23ff08cb Mon Sep 17 00:00:00 2001 From: Vivswan Shah <58091053+Vivswan@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:08:10 -0400 Subject: [PATCH 9/9] test(docs): drop two comments that restated the confirmation table and its stub --- test/docs/npm-publish-workflows.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/docs/npm-publish-workflows.test.ts b/test/docs/npm-publish-workflows.test.ts index bd4e40d0..9b44f393 100644 --- a/test/docs/npm-publish-workflows.test.ts +++ b/test/docs/npm-publish-workflows.test.ts @@ -450,7 +450,6 @@ describe("the confirmation block under bash", () => { const run = must(confirm.run, "confirm run"); const outcomes = confirmOutcomes(); - /** A bun that prints the given confirm line. */ const stubBun = (line: string) => (bin: string): void => { @@ -464,7 +463,6 @@ describe("the confirmation block under bash", () => { expect(condition(confirm.if)).toBe("steps.publish.outputs.published == 'true'"); }); - /** One run per outcome plus an unknown line: the run ends 1 for behind and the unknown, 0 otherwise, and the message class follows the outcome. */ const expected: Record = { settled: { status: 0, command: /^::notice::/ }, unsettled: { status: 0, command: /^::warning::/ },