diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5e71ab..d09fabf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,15 @@ on: - main pull_request: workflow_dispatch: + inputs: + pr-number: + description: Dependabot pull request to verify after a provenance refresh + required: false + type: string + expected-head: + description: Exact refreshed pull request head commit + required: false + type: string permissions: contents: read @@ -19,22 +28,90 @@ env: LFC_NPM_VERSION: "12.0.2" jobs: + source: + name: Resolve the verified source + runs-on: ubuntu-latest + timeout-minutes: 5 + + permissions: + contents: read + pull-requests: read + + outputs: + checkout-ref: ${{ steps.dispatch.outputs.checkout-ref || steps.default.outputs.checkout-ref }} + head: ${{ steps.dispatch.outputs.head || steps.default.outputs.head }} + base: ${{ steps.dispatch.outputs.base || steps.default.outputs.base }} + pr-number: ${{ steps.dispatch.outputs.pr-number || steps.default.outputs.pr-number }} + + steps: + - name: Set up exact Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.LFC_NODE_VERSION }} + package-manager-cache: false + + - name: Preserve the ordinary event source + id: default + if: ${{ github.event_name != 'workflow_dispatch' || (inputs.pr-number == '' && inputs.expected-head == '') }} + env: + LFC_CHECKOUT_REF: ${{ github.sha }} + LFC_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + LFC_BASE_SHA: ${{ github.event.pull_request.base.sha }} + LFC_PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + node --input-type=module --eval ' + import { appendFileSync } from "node:fs"; + const values = { + "checkout-ref": process.env.LFC_CHECKOUT_REF, + head: process.env.LFC_HEAD_SHA, + base: process.env.LFC_BASE_SHA ?? "", + "pr-number": process.env.LFC_PR_NUMBER ?? "" + }; + appendFileSync(process.env.GITHUB_OUTPUT, Object.entries(values).map(([key, value]) => `${key}=${value}\n`).join("")); + ' + + - name: Check out trusted dispatch validation without persisted credentials + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.pr-number != '' || inputs.expected-head != '') }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event.repository.default_branch }} + + - name: Validate the requested Dependabot head before installing dependencies + id: dispatch + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.pr-number != '' || inputs.expected-head != '') }} + env: + GITHUB_TOKEN: ${{ github.token }} + LFC_PR_NUMBER: ${{ inputs.pr-number }} + LFC_HEAD_SHA: ${{ inputs.expected-head }} + run: node scripts/screenshot-automation.mjs verify-dispatch + verify: name: Build, test, and verify package + needs: source + if: ${{ always() && !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 45 steps: + - name: Reject an invalid source selection + env: + LFC_SOURCE_RESULT: ${{ needs.source.result }} + run: test "${LFC_SOURCE_RESULT}" = success + - name: Check out source without persisted credentials uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + ref: ${{ github.sha }} - name: Reject high-severity dependency regressions - if: ${{ github.event_name == 'pull_request' }} + if: ${{ github.event_name == 'pull_request' || needs.source.outputs.pr-number != '' }} uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 with: fail-on-severity: high + base-ref: ${{ needs.source.outputs.base }} + head-ref: ${{ needs.source.outputs.head }} - name: Set up exact Node uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -53,8 +130,20 @@ jobs: run: npx playwright install --with-deps chromium webkit - name: Run the CI quality gate + id: quality-gate run: npm run check + - name: Verify dependency package after a failed quality gate + id: dependency-package + if: >- + ${{ + failure() && steps.quality-gate.outcome == 'failure' && + github.event_name == 'pull_request' && + github.event.pull_request.user.login == 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository + }} + run: npm run check:tarball + - name: Upload browser results and failure diagnostics if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -65,3 +154,75 @@ jobs: playwright-report/ test-results/ retention-days: 7 + + screenshot-candidate: + name: Capture Dependabot screenshot candidates + if: >- + ${{ + github.event_name == 'pull_request' && + github.event.pull_request.user.login == 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository + }} + runs-on: windows-2025 + timeout-minutes: 30 + + permissions: + contents: read + pull-requests: read + + env: + LFC_PR_NUMBER: ${{ github.event.pull_request.number }} + LFC_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + LFC_BASE_SHA: ${{ github.event.pull_request.base.sha }} + + steps: + - name: Check out the exact Dependabot head without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up exact Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.LFC_NODE_VERSION }} + package-manager-cache: false + + - name: Check whether the dependency diff needs screenshot evidence + id: eligibility + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/screenshot-automation.mjs candidate-eligibility + + - name: Set up exact npm + if: ${{ steps.eligibility.outputs.eligible == 'true' }} + working-directory: ${{ runner.temp }} + run: npm install --global "npm@$env:LFC_NPM_VERSION" --ignore-scripts + + - name: Install locked development tooling + if: ${{ steps.eligibility.outputs.eligible == 'true' }} + run: npm ci --ignore-scripts + + - name: Install pinned capture browser + if: ${{ steps.eligibility.outputs.eligible == 'true' }} + run: npx playwright install chromium + + - name: Capture candidates for pixel comparison and human review + if: ${{ steps.eligibility.outputs.eligible == 'true' }} + env: + LFC_BROWSER_TARGET_DATE: ${{ steps.eligibility.outputs.capture-date }} + run: npm run screenshots:candidate + + - name: Seal candidate provenance + if: ${{ steps.eligibility.outputs.eligible == 'true' }} + run: node scripts/screenshot-automation.mjs seal-candidate + + - name: Upload screenshot candidates and comparison gallery + if: ${{ always() && steps.eligibility.outputs.eligible == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: screenshot-candidate-${{ github.run_id }}-${{ github.run_attempt }} + path: .cache/screenshots/candidate/ + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/refresh-screenshot-evidence.yml b/.github/workflows/refresh-screenshot-evidence.yml new file mode 100644 index 0000000..01980d6 --- /dev/null +++ b/.github/workflows/refresh-screenshot-evidence.yml @@ -0,0 +1,75 @@ +name: Refresh unchanged screenshot evidence + +on: + workflow_run: + workflows: + - CI + types: + - completed + +permissions: + contents: read + +concurrency: + group: screenshot-evidence-${{ github.repository }}-${{ github.event.workflow_run.head_branch }} + queue: max + cancel-in-progress: false + +env: + LFC_NODE_VERSION: "24.19.0" + +jobs: + promote: + name: Promote identical Dependabot screenshot evidence + if: >- + ${{ + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion != 'cancelled' && + github.event.workflow_run.head_repository.full_name == github.repository + }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + permissions: + contents: write + pull-requests: read + actions: write + + env: + GITHUB_TOKEN: ${{ github.token }} + LFC_UPSTREAM_RUN_ID: ${{ github.event.workflow_run.id }} + LFC_UPSTREAM_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + LFC_UPSTREAM_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + LFC_CANDIDATE_DIRECTORY: ${{ runner.temp }}/screenshot-candidate + + steps: + - name: Check out trusted promotion code without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + + - name: Set up exact Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.LFC_NODE_VERSION }} + package-manager-cache: false + + - name: Validate the originating run and dependency pull request + id: context + run: node scripts/screenshot-automation.mjs promotion-context + + - name: Download the validated candidate artifact + if: ${{ steps.context.outputs.eligible == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + artifact-ids: ${{ steps.context.outputs.artifact-id }} + merge-multiple: true + path: ${{ runner.temp }}/screenshot-candidate + + - name: Refresh identical evidence and verify the new exact head + if: ${{ steps.context.outputs.eligible == 'true' }} + run: node scripts/screenshot-automation.mjs promote diff --git a/CHANGELOG.md b/CHANGELOG.md index d55c41d..b666070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Changed +- Narrowed screenshot freshness to build and capture dependency closures. Dependency-only Dependabot updates can refresh verified provenance automatically when every PNG is byte-identical; changed screenshots retain human review. - Expanded the visual specification for existing typography, count roles, agenda states, loading, and optional event layouts; corrected labels and documentation links. - **Migration:** Event titles are no longer trimmed. Any nonempty string of at most 1,024 UTF-16 code units is preserved exactly, including whitespace-only titles; applications are responsible for supplying meaningful visible and accessible text. - Reduced the default day padding to `clamp(0.25rem, 0.75cqi, 0.5rem)` for a more compact grid while retaining control target floors and intrinsic content growth. diff --git a/CONTRIBUTOR_COMMANDS.md b/CONTRIBUTOR_COMMANDS.md index 0326e35..b8c2daf 100644 --- a/CONTRIBUTOR_COMMANDS.md +++ b/CONTRIBUTOR_COMMANDS.md @@ -173,6 +173,10 @@ Prepare screenshot evidence after source and build inputs settle. The preparatio command skips current evidence, otherwise builds and validates a staged capture batch and produces a before/after gallery. Review every changed scene using the [screenshot contract](docs/screenshots/README.md). CI verification remains read-only. +Dependency-only Dependabot updates can refresh provenance automatically when every +captured PNG is byte-identical; changed images still require human review. See +[automatic dependency evidence](docs/screenshots/README.md#automatic-dependency-evidence) +for the eligibility checks and candidate gallery command. ```shell npm run screenshots:prepare diff --git a/docs/screenshots/README.md b/docs/screenshots/README.md index 2d05cf0..b8ea6b1 100644 --- a/docs/screenshots/README.md +++ b/docs/screenshots/README.md @@ -90,7 +90,23 @@ On a fresh Linux machine, add `--with-deps` to the Playwright install command wh All captures go into an ignored staging directory first. The updater validates the complete batch, checks that sources did not change during capture, and creates `.cache/screenshots/review/index.html` with before/after images plus `changes.json`. It replaces canonical images only after successful capture and validation, commits the manifest last, and restores originals if replacement fails. Failed capture batches remain under `.cache/screenshots/staging-*` for diagnosis; canonical evidence stays untouched. The replacement is rollback-capable, not a cross-file filesystem transaction, so interruption during replacement requires inspecting the retained backup before retrying. -Open the local review gallery and review changed scenes at native dimensions before staging files. CI runs read-only `check:screenshots`; preparation and capture reject attempts to accept stale evidence in CI. A capture lock prevents simultaneous updaters. After a forcibly terminated updater, verify no capture is still running before removing `.cache/screenshots/capture.lock`. +Open the local review gallery and review changed scenes at native dimensions before staging files. CI runs read-only `check:screenshots`; preparation and capture reject canonical acceptance in CI. CI can explicitly capture candidates under the automatic dependency workflow below. A capture lock prevents simultaneous updaters. After a forcibly terminated updater, verify no capture is still running before removing `.cache/screenshots/capture.lock`. + +### Automatic dependency evidence + +For same-repository Dependabot pull requests against `main`, CI checks whether the diff is limited to locked dependencies and existing exact-pinned development dependency versions. Changes to scripts, source, workflows, scenes, or screenshot images are ineligible for automatic acceptance. + +The capture job uses the pinned Playwright Chromium binary on `windows-2025`, with the workflow-selected Node and npm versions. It prepares candidates in the ignored `.cache/screenshots/candidate/` directory and uploads a seven-day artifact containing the before/after gallery, PNGs, candidate manifest, and exact run/base/head identities. It never replaces tracked evidence. Contributors can produce the same candidate layout locally: + +```shell +npm run screenshots:candidate +``` + +A separate workflow executes only trusted default-branch tooling, checks the originating CI attempt and current pull request, and recomputes the fingerprint without installing or executing changed dependencies. All required checks must pass; a failure solely due to the stale screenshot fingerprint is allowed only after independent tarball verification succeeds. Every candidate PNG must be byte-identical to its canonical image, and effective browser targets and scene definitions must remain unchanged. Only then does the workflow atomically commit refreshed provenance and dispatch CI for that exact new head. + +Changed screenshots, browser targets, missing evidence, unexpected files, or platform/font drift require human review. Download the candidate artifact from the CI run and open its `index.html` at native dimensions. Prepare and review canonical captures locally before committing any changed images. Automation does not approve or merge a pull request, and release verification remains read-only. + +The workflow becomes available after its trusted implementation is merged into `main`. The dependency branch must include the current base before capture. If main advances, update the branch with main and start fresh CI; rerunning an old attempt retains its old event base. Stale head or base results are discarded. A changed capture environment can require a reviewed baseline refresh rather than automatic provenance acceptance. Nine scenes capture settled states. The held-pull scene intentionally captures direct manipulation before release so the decorative adjacent-month/year lane can be reviewed; selection timing and post-release browser physics remain browser-test and supported-device concerns. Review every canonical scene after an intentional source-layout change. The four count scenes mount a fixture through the public built-package API, with empty, single-event, and multiple-event days. They retain the example's general palette but remove its independent overflow-theme class, preserving evidence for the contextual count defaults. The advanced scenes exercise the borderless overflow theme. @@ -112,7 +128,7 @@ The screenshot manifest is machine-owned and records, for every scene: - Example URL and deterministic setup steps. - Viewport and output dimensions. - SHA-256 hash. -- Source fingerprint covering package source, Vite configuration, the advanced route and shared stylesheet, build/server/capture helpers, locked compiler/browser dependencies, and effective JavaScript/CSS browser targets. The preparation scripts and their automatic `pre`/`post` hooks are included. Browser target evidence records its resolution date, but a date change alone does not require recapture. Other example routes, deployment metadata, Playwright's test configuration, root version metadata, and npm scripts outside that capture lifecycle are excluded. +- Source fingerprint covering package source, Vite configuration, the advanced route and shared stylesheet, build/server/capture helpers, and effective JavaScript/CSS browser targets. Locked development inputs include TypeScript, Vite, esbuild, and Playwright plus their resolved transitive, optional, and present peer dependencies. Unrelated lint/test dependency entries are excluded; a dependency shared with the build/capture graph remains included. The preparation and candidate scripts and their automatic `pre`/`post` hooks are included. Browser target evidence records its resolution date, but a date change alone does not require recapture. Other example routes, deployment metadata, Playwright's test configuration, root version metadata, and npm scripts outside that capture lifecycle are excluded. - Exact Node patch used for capture; validation accepts that provenance when it belongs to major version 24, while npm, Playwright, and Chromium remain exact-pinned. The Playwright value is derived from the exact `package.json` development dependency, and preparation rejects a stale local install before capture. - Documentation references and exact alt text. diff --git a/package-lock.json b/package-lock.json index a678766..5e4ce32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5009,9 +5009,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { diff --git a/package.json b/package.json index 84abee1..744e356 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "nightly:plan": "node scripts/nightly-release.mjs plan", "package:nightly": "node scripts/pack-nightly.mjs", "release:verify": "node scripts/nightly-release.mjs verify", + "screenshots:candidate": "node scripts/prepare-screenshots.mjs --candidate", "screenshots:prepare": "node scripts/prepare-screenshots.mjs", "screenshots:update": "node scripts/prepare-screenshots.mjs --force", "serve:repository": "node scripts/serve-repository.mjs", @@ -103,7 +104,7 @@ "test:browser:webkit": "npm run build && playwright test --project=webkit", "test:examples": "npm run build && npm run test:examples:built", "test:examples:built": "node scripts/run-advanced-smoke.mjs", - "test:tooling": "node --test scripts/tests/build-pages.test.mjs scripts/tests/calendar-workflow-measurement.test.mjs scripts/tests/check-sbom.test.mjs scripts/tests/dynamic-update-measurement.test.mjs scripts/tests/example-metadata.test.mjs scripts/tests/firefox-launch-options.test.mjs scripts/tests/markdown-diagrams.test.mjs scripts/tests/markdown-heading.test.mjs scripts/tests/node-version.test.mjs scripts/tests/ownership.test.mjs scripts/tests/package-entries.test.mjs scripts/tests/packed-extension-verification.test.mjs scripts/tests/publish-nightly-policy.test.mjs scripts/tests/release-artifacts.test.mjs scripts/tests/screenshot-contract.test.mjs scripts/tests/semver.test.mjs scripts/tests/serve-repository.test.mjs scripts/tests/styles.test.mjs scripts/tests/workflow-contracts.test.mjs scripts/tests/browser-targets.test.mjs scripts/tests/vite-build.test.mjs scripts/tests/package-bundle-policy.test.mjs scripts/tests/nightly-release.test.mjs scripts/tests/screenshot-preparation.test.mjs scripts/tests/remote-data-source.test.mjs scripts/tests/browser-target-documentation.test.mjs", + "test:tooling": "node --test scripts/tests/build-pages.test.mjs scripts/tests/calendar-workflow-measurement.test.mjs scripts/tests/check-sbom.test.mjs scripts/tests/dynamic-update-measurement.test.mjs scripts/tests/example-metadata.test.mjs scripts/tests/firefox-launch-options.test.mjs scripts/tests/markdown-diagrams.test.mjs scripts/tests/markdown-heading.test.mjs scripts/tests/node-version.test.mjs scripts/tests/ownership.test.mjs scripts/tests/package-entries.test.mjs scripts/tests/packed-extension-verification.test.mjs scripts/tests/publish-nightly-policy.test.mjs scripts/tests/release-artifacts.test.mjs scripts/tests/screenshot-contract.test.mjs scripts/tests/semver.test.mjs scripts/tests/serve-repository.test.mjs scripts/tests/styles.test.mjs scripts/tests/workflow-contracts.test.mjs scripts/tests/browser-targets.test.mjs scripts/tests/vite-build.test.mjs scripts/tests/package-bundle-policy.test.mjs scripts/tests/nightly-release.test.mjs scripts/tests/screenshot-preparation.test.mjs scripts/tests/remote-data-source.test.mjs scripts/tests/browser-target-documentation.test.mjs scripts/tests/screenshot-promotion.test.mjs scripts/tests/screenshot-automation.test.mjs", "test:unit": "node scripts/run-unit-tests.mjs", "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit", "typecheck:examples": "npm run build:package && npm run typecheck:examples:built", diff --git a/screenshots.manifest.json b/screenshots.manifest.json index 442b679..4685c06 100644 --- a/screenshots.manifest.json +++ b/screenshots.manifest.json @@ -7,7 +7,7 @@ "npm": "12.0.2", "playwright": "1.63.0" }, - "sourceFingerprint": "240a6c676cdcb272d0d176b469259418c2fd3c90cc28fd5fbf3bf579e8ed7d6f", + "sourceFingerprint": "01e2a155534cfca9923249ce50730cb901dcd3fd7720c250a23f89e0d1f13382", "scenes": [ { "id": "desktop-month-grid", @@ -217,7 +217,7 @@ "browserTargets": { "query": "baseline-widely-available", "effectiveQuery": "Vite 8.3.2: baseline-widely-available", - "resolvedAt": "2026-10-11", + "resolvedAt": "2026-10-12", "dataVersions": { "vite": "8.3.2", "esbuild": "0.28.2" diff --git a/scripts/check-package-policy.mjs b/scripts/check-package-policy.mjs index a3f38e7..242196f 100644 --- a/scripts/check-package-policy.mjs +++ b/scripts/check-package-policy.mjs @@ -1590,6 +1590,10 @@ await inspectWorkflowTree({ LFC_NPM_VERSION: String(referenceNpmVersion) }, + ".github/workflows/refresh-screenshot-evidence.yml": { + LFC_NODE_VERSION: + WORKFLOW_NODE_VERSION + }, ".github/workflows/rollback-examples.yml": { LFC_NODE_VERSION: WORKFLOW_NODE_VERSION, diff --git a/scripts/lib/screenshot-automation.mjs b/scripts/lib/screenshot-automation.mjs new file mode 100644 index 0000000..3b744de --- /dev/null +++ b/scripts/lib/screenshot-automation.mjs @@ -0,0 +1,375 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { appendFile, lstat, readFile, readdir, realpath, writeFile } from "node:fs/promises"; +import { basename, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { isScreenshotSourceInput, SCREENSHOT_SOURCE_INPUTS } from "./screenshot-fingerprint.mjs"; +import { dependencyScreenshotEligibility, planScreenshotPromotion } from "./screenshot-promotion.mjs"; + +const REPOSITORY_ROOT = fileURLToPath(new URL("../../", import.meta.url)); +const REPOSITORY = "tryagaindev/litefold-calendar"; +const MAX_FILE_BYTES = 4 * 1024 * 1024; +const MAX_LOG_BYTES = 16 * 1024 * 1024; +const execFileAsync = promisify(execFile); +const SHA = /^[a-f0-9]{40}$/u; + +function requireCondition(condition, message) { + if (!condition) { throw new Error(message); } +} + +function positiveInteger(value, label) { + requireCondition(typeof value === "string" && /^[1-9]\d*$/u.test(value) && + Number.isSafeInteger(Number(value)), label + " must be a positive integer."); + return Number(value); +} + +function commitSha(value, label) { + requireCondition(typeof value === "string" && SHA.test(value), label + " must be a full commit SHA."); + return value; +} + +function hash(contents) { + return createHash("sha256").update(contents).digest("hex"); +} + +/** Uses GitHub's authenticated API without forwarding credentials to redirected log hosts. */ +export function githubRequest(environment, fetchImplementation = fetch) { + requireCondition(environment.GITHUB_REPOSITORY === REPOSITORY, "Unexpected repository."); + requireCondition(typeof environment.GITHUB_TOKEN === "string" && environment.GITHUB_TOKEN.length > 0, + "GitHub API authentication is required."); + return async (path, { method = "GET", body, text = false } = {}) => { + requireCondition(path === "/graphql" || path.startsWith("/repos/" + REPOSITORY + "/"), + "Unexpected API destination."); + let response = await fetchImplementation("https://api.github.com" + path, { + method, redirect: "manual", + headers: { + Authorization: "Bearer " + environment.GITHUB_TOKEN, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + ...(body === undefined ? {} : { "Content-Type": "application/json" }) + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }) + }); + if (text && response.status === 302) { + const destination = new URL(response.headers.get("location")); + requireCondition(destination.protocol === "https:" && !destination.username && !destination.password, + "Invalid log download destination."); + response = await fetchImplementation(destination, { redirect: "error" }); + } + requireCondition(response.ok, "GitHub API request failed (" + response.status + ")."); + if (response.status === 204) { return undefined; } + const value = await response.text(); + requireCondition(Buffer.byteLength(value) <= (text ? MAX_LOG_BYTES : MAX_FILE_BYTES * 2), + "GitHub API response exceeds the evidence limit."); + return text ? value : JSON.parse(value); + }; +} + +async function output(environment, values) { + if (environment.GITHUB_OUTPUT) { + await appendFile(environment.GITHUB_OUTPUT, Object.entries(values) + .map(([key, value]) => key + "=" + String(value) + "\n").join("")); + } +} + +async function summary(environment, message) { + console.log(message); + if (environment.GITHUB_STEP_SUMMARY) { await appendFile(environment.GITHUB_STEP_SUMMARY, message + "\n"); } +} + +async function fileAt(request, path, sha) { + const file = await request("/repos/" + REPOSITORY + "/contents/" + path + "?ref=" + sha); + requireCondition(file.type === "file" && file.encoding === "base64" && SHA.test(file.sha), + "Repository evidence must be a regular Git blob."); + const contents = Buffer.from(file.content, "base64"); + requireCondition(contents.length <= MAX_FILE_BYTES, "Repository evidence exceeds the file limit."); + return { contents, sha: file.sha, value: JSON.parse(contents.toString("utf8")) }; +} + +async function pullRequestContext(request, number, run) { + const pullRequest = await request("/repos/" + REPOSITORY + "/pulls/" + number); + const changedFiles = []; + for (let page = 1; page <= 2; page += 1) { + const files = await request("/repos/" + REPOSITORY + "/pulls/" + number + "/files?per_page=100&page=" + page); + requireCondition(Array.isArray(files), "Invalid pull request file inventory."); + changedFiles.push(...files); + if (files.length < 100) { break; } + } + //Classify paths before downloading any branch-controlled document. + if (changedFiles.length > 3 || changedFiles.some((file) => + !["package.json", "package-lock.json", "screenshots.manifest.json"].includes(file.filename))) { + return { eligible: false, reason: "The pull request includes changes beyond dependency provenance." }; + } + const [basePackage, headPackage, baseLock, headLock, baseline, current] = await Promise.all([ + fileAt(request, "package.json", pullRequest.base.sha), + fileAt(request, "package.json", pullRequest.head.sha), + fileAt(request, "package-lock.json", pullRequest.base.sha), + fileAt(request, "package-lock.json", pullRequest.head.sha), + fileAt(request, "screenshots.manifest.json", pullRequest.base.sha), + fileAt(request, "screenshots.manifest.json", pullRequest.head.sha) + ]); + const input = { + repository: REPOSITORY, pullRequest, run, changedFiles, + basePackage: basePackage.value, headPackage: headPackage.value, + baseLock: baseLock.value, headLock: headLock.value, + baselineManifest: baseline.value, currentManifest: current.value + }; + const eligibility = dependencyScreenshotEligibility(input); + if (eligibility.eligible) { + //The PR inventory is a merge-base diff; copied base inputs require that base to be an ancestor. + const comparison = await request("/repos/" + REPOSITORY + "/compare/" + + pullRequest.base.sha + "..." + pullRequest.head.sha); + if (comparison.merge_base_commit?.sha !== pullRequest.base.sha) { + return { eligible: false, reason: "Update the dependency branch with current main before capturing evidence." }; + } + } + return { ...eligibility, input, headPackage, headLock, current }; +} + +/** Accepts only a complete quality gate, or its sole stale-fingerprint failure with a green package check. */ +export function classifyScreenshotQuality(jobs, log = "") { + const candidate = jobs.find((job) => job.name === "Capture Dependabot screenshot candidates"); + const verify = jobs.find((job) => job.name === "Build, test, and verify package"); + const source = jobs.find((job) => job.name === "Resolve the verified source"); + if (candidate?.conclusion !== "success" || source?.conclusion !== "success") { return false; } + if (verify?.conclusion === "success") { return true; } + if (verify?.conclusion !== "failure") { return false; } + const quality = verify.steps?.find((step) => step.name === "Run the CI quality gate"); + const packageCheck = verify.steps?.find((step) => step.name === "Verify dependency package after a failed quality gate"); + if (quality?.conclusion !== "failure" || packageCheck?.conclusion !== "success" || + verify.steps.some((step) => step.conclusion === "failure" && step !== quality)) { return false; } + const lines = log.split(/\r?\n/u).map((line) => line.replace(/^\d{4}-\d\d-\d\dT\S+\s/u, "")); + const failures = lines.filter((line) => line.startsWith("##[error]")); + const start = lines.lastIndexOf("Screenshot validation failed:"); + if (start === -1 || failures.length !== 1 || + !failures[0].includes("Process completed with exit code 1.")) { return false; } + const validation = lines.slice(start + 1).findIndex((line) => line.startsWith("##[error]")); + if (validation < 0) { return false; } + const diagnostics = lines.slice(start + 1, start + 1 + validation).filter((line) => line.trim() !== ""); + return diagnostics.length === 1 && + diagnostics[0] === "- Screenshot source fingerprint is stale; run npm run screenshots:prepare."; +} + +async function upstreamContext(request, environment) { + const runId = positiveInteger(environment.LFC_UPSTREAM_RUN_ID, "Upstream run"); + const attempt = positiveInteger(environment.LFC_UPSTREAM_RUN_ATTEMPT, "Upstream attempt"); + const head = commitSha(environment.LFC_UPSTREAM_HEAD_SHA, "Upstream head"); + const run = await request("/repos/" + REPOSITORY + "/actions/runs/" + runId); + if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== head || run.event !== "pull_request" || + run.path !== ".github/workflows/ci.yml" || run.status !== "completed" || + !["success", "failure"].includes(run.conclusion) || + typeof run.created_at !== "string" || Number.isNaN(Date.parse(run.created_at)) || + run.repository?.full_name !== REPOSITORY || run.head_repository?.full_name !== REPOSITORY) { + return { eligible: false, reason: "The originating CI run is stale or ineligible." }; + } + const linked = run.pull_requests ?? []; + if (linked.length !== 1) { return { eligible: false, reason: "CI must identify exactly one pull request." }; } + const context = await pullRequestContext(request, linked[0].number, run); + if (!context.eligible) { return context; } + if (context.input.pullRequest.head.ref !== run.head_branch) { + return { eligible: false, reason: "The originating branch changed." }; + } + const jobs = (await request("/repos/" + REPOSITORY + "/actions/runs/" + runId + + "/attempts/" + attempt + "/jobs?per_page=100")).jobs; + requireCondition(Array.isArray(jobs) && jobs.length < 100, "Invalid CI job inventory."); + const verify = jobs.find((job) => job.name === "Build, test, and verify package"); + const log = verify?.conclusion === "failure" + ? await request("/repos/" + REPOSITORY + "/actions/jobs/" + verify.id + "/logs", { text: true }) : ""; + if (!classifyScreenshotQuality(jobs, log)) { + return { eligible: false, reason: "Required checks failed beyond screenshot freshness." }; + } + const artifacts = (await request("/repos/" + REPOSITORY + "/actions/runs/" + runId + + "/artifacts?per_page=100")).artifacts; + requireCondition(Array.isArray(artifacts) && artifacts.length < 100, "Invalid artifact inventory."); + const matches = artifacts.filter((artifact) => + artifact.name === "screenshot-candidate-" + runId + "-" + attempt && !artifact.expired); + if (matches.length !== 1 || !Number.isSafeInteger(matches[0].id) || matches[0].id <= 0 || + !Number.isSafeInteger(matches[0].size_in_bytes) || matches[0].size_in_bytes <= 0 || + matches[0].size_in_bytes > 20 * 1024 * 1024 || + matches[0].workflow_run?.id !== runId || matches[0].workflow_run?.head_sha !== head) { + return { eligible: false, reason: "A unique exact-run candidate artifact is required." }; + } + return { ...context, artifact: matches[0] }; +} + +async function readBoundedFile(path) { + const metadata = await lstat(path); + requireCondition(metadata.isFile() && !metadata.isSymbolicLink() && metadata.size <= MAX_FILE_BYTES, + "Evidence must be a bounded regular file."); + return readFile(path); +} + +/** Rejects unexpected paths and links before consuming any downloaded evidence. */ +export async function inspectScreenshotArtifact(directory, manifest) { + const metadata = await lstat(directory); + requireCondition(metadata.isDirectory() && !metadata.isSymbolicLink(), "Candidate root must be a regular directory."); + const root = await realpath(directory); + const expected = new Set(["index.html", "changes.json", "screenshots.manifest.json", "report.json"]); + for (const scene of manifest.scenes) { + for (const side of ["before", "after"]) { expected.add(side + "/" + basename(scene.output)); } + } + const found = new Set(); + async function inspect(path, prefix = "") { + for (const entry of await readdir(path, { withFileTypes: true })) { + const name = prefix + entry.name; + const target = join(path, entry.name); + const metadata = await lstat(target); + requireCondition(!metadata.isSymbolicLink(), "Candidate links are forbidden."); + if (metadata.isDirectory()) { + requireCondition(prefix === "" && ["before", "after"].includes(entry.name), + "Unexpected candidate directory."); + await inspect(target, name + "/"); + } else { + requireCondition(expected.has(name), "Unexpected candidate file."); + await readBoundedFile(target); + found.add(name); + } + } + } + await inspect(root); + requireCondition(found.size === expected.size, "The candidate evidence is incomplete."); + return root; +} + +async function repositorySources(root) { + const files = []; + async function collect(path) { + const metadata = await lstat(join(root, path)); + requireCondition(!metadata.isSymbolicLink(), "Screenshot source links are forbidden."); + if (metadata.isDirectory()) { + for (const entry of await readdir(join(root, path))) { await collect(path + "/" + entry); } + } else if (isScreenshotSourceInput(path)) { + files.push({ path, contents: await readBoundedFile(join(root, path)) }); + } + } + for (const input of SCREENSHOT_SOURCE_INPUTS) { await collect(input); } + return files; +} + +async function readSourceHead(root) { + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root }); + return stdout.trim(); +} + +async function promote(context, request, environment, root, readHead) { + const { input } = context; + if (await readHead(root) !== input.pullRequest.base.sha) { + await summary(environment, "Main advanced since capture; update the dependency branch with main, then rerun CI."); + return; + } + const candidateDirectory = resolve(environment.LFC_CANDIDATE_DIRECTORY ?? ""); + const temporaryRoot = resolve(environment.RUNNER_TEMP ?? ""); + const child = relative(temporaryRoot, candidateDirectory); + requireCondition(environment.RUNNER_TEMP && environment.LFC_CANDIDATE_DIRECTORY && child !== "" && + child !== ".." && !child.startsWith(".." + sep) && !isAbsolute(child), + "Candidate evidence must stay in the runner temporary directory."); + const artifactRoot = await inspectScreenshotArtifact(candidateDirectory, input.baselineManifest); + const [candidateManifest, report] = await Promise.all(["screenshots.manifest.json", "report.json"] + .map(async (name) => JSON.parse((await readBoundedFile(join(artifactRoot, name))).toString("utf8")))); + const canonicalHashes = {}; + const baseCaptureHashes = {}; + const candidateHashes = {}; + for (const scene of input.baselineManifest.scenes) { + canonicalHashes[scene.output] = hash(await readBoundedFile(join(root, scene.output))); + baseCaptureHashes[scene.output] = hash(await readBoundedFile(join(artifactRoot, "before", basename(scene.output)))); + candidateHashes[scene.output] = hash(await readBoundedFile(join(artifactRoot, "after", basename(scene.output)))); + } + const baseSourceFiles = await repositorySources(root); + const sourceFiles = baseSourceFiles.map((file) => ({ + ...file, contents: file.path === "package.json" ? context.headPackage.contents : + file.path === "package-lock.json" ? context.headLock.contents : file.contents + })); + const plan = planScreenshotPromotion({ ...input, candidateManifest, report, canonicalHashes, + baseCaptureHashes, candidateHashes, baseSourceFiles, sourceFiles, + expectedCapture: { node: environment.LFC_NODE_VERSION, resolvedAt: input.run.created_at.slice(0, 10) } }); + if (plan.status !== "promote") { + await summary(environment, plan.status === "needs-review" + ? "Screenshots require human review. Download the candidate gallery from the originating CI run." + : (plan.reason ?? "Screenshot evidence is already current.")); + return; + } + const fresh = await request("/repos/" + REPOSITORY + "/pulls/" + input.pullRequest.number); + if (fresh.state !== "open" || fresh.head.sha !== input.pullRequest.head.sha || + fresh.base.sha !== input.pullRequest.base.sha || fresh.head.repo?.full_name !== REPOSITORY) { + await summary(environment, "Pull request changed during verification; no evidence was committed."); + return; + } + //GraphQL expectedHeadOid atomically rejects a concurrent push, rebase, or rewind. + const mutation = "mutation($input:CreateCommitOnBranchInput!){createCommitOnBranch(input:$input){commit{oid}}}"; + const commit = await request("/graphql", { + method: "POST", body: { + query: mutation, variables: { input: { + branch: { repositoryNameWithOwner: REPOSITORY, branchName: fresh.head.ref }, + expectedHeadOid: input.pullRequest.head.sha, + message: { headline: "chore(deps): refresh unchanged screenshot provenance" }, + fileChanges: { additions: [{ + path: "screenshots.manifest.json", + contents: Buffer.from(JSON.stringify(plan.manifest, null, 2) + "\n").toString("base64") + }] } + } } + } + }); + const newHead = commit.data?.createCommitOnBranch?.commit?.oid; + requireCondition(!commit.errors && SHA.test(newHead), "The pull request changed before the atomic evidence commit."); + await request("/repos/" + REPOSITORY + "/actions/workflows/ci.yml/dispatches", { + method: "POST", body: { + ref: fresh.head.ref, + inputs: { "pr-number": String(fresh.number), "expected-head": newHead } + } + }); + await summary(environment, "Refreshed unchanged screenshot provenance and dispatched CI for " + newHead + "."); +} + +/** Runs one narrowly scoped capture, promotion, or exact-head dispatch operation. */ +export async function runScreenshotAutomation({ command, environment = process.env, + request, root = REPOSITORY_ROOT, readHead = readSourceHead }) { + requireCondition(environment.GITHUB_REPOSITORY === REPOSITORY, "Unexpected repository."); + if (command !== "seal-candidate") { request ??= githubRequest(environment); } + if (command === "promotion-context" || command === "promote") { + const context = await upstreamContext(request, environment); + await output(environment, { eligible: context.eligible, "artifact-id": context.artifact?.id ?? "" }); + if (!context.eligible) { await summary(environment, context.reason); return; } + if (command === "promote") { await promote(context, request, environment, root, readHead); } + return; + } + const number = positiveInteger(environment.LFC_PR_NUMBER, "Pull request"); + const head = commitSha(environment.LFC_HEAD_SHA, "Pull request head"); + if (command === "verify-dispatch") { + requireCondition(environment.GITHUB_EVENT_NAME === "workflow_dispatch" && environment.GITHUB_SHA === head, + "Dispatch must identify the exact current pull request head."); + const context = await pullRequestContext(request, number, { + event: "pull_request", repository: { full_name: REPOSITORY }, + id: positiveInteger(environment.GITHUB_RUN_ID, "CI run"), run_attempt: 1, head_sha: head + }); + requireCondition(context.eligible, context.reason ?? "Dispatch head changed."); + await output(environment, { "checkout-ref": head, head, base: context.input.pullRequest.base.sha, "pr-number": number }); + return; + } + const base = commitSha(environment.LFC_BASE_SHA, "Pull request base"); + if (command === "seal-candidate") { + const directory = join(root, ".cache", "screenshots", "candidate"); + try { await lstat(join(directory, "screenshots.manifest.json")); } + catch (error) { if (error.code === "ENOENT") { return; } throw error; } + const report = { version: 1, repository: REPOSITORY, prNumber: number, headSha: head, baseSha: base, + runId: positiveInteger(environment.GITHUB_RUN_ID, "CI run"), + runAttempt: positiveInteger(environment.GITHUB_RUN_ATTEMPT, "CI attempt") }; + await writeFile(join(directory, "report.json"), JSON.stringify(report, null, 2) + "\n"); + return; + } + requireCondition(command === "candidate-eligibility", "Unknown screenshot automation operation."); + const run = { event: "pull_request", repository: { full_name: REPOSITORY }, head_sha: head, + id: positiveInteger(environment.GITHUB_RUN_ID, "CI run"), + run_attempt: positiveInteger(environment.GITHUB_RUN_ATTEMPT, "CI attempt") }; + const context = await pullRequestContext(request, number, run); + const eligible = context.eligible && context.input.pullRequest.base.sha === base; + const upstream = eligible ? await request("/repos/" + REPOSITORY + "/actions/runs/" + run.id) : undefined; + if (eligible) { + requireCondition(upstream.id === run.id && upstream.head_sha === head && + upstream.run_attempt === run.run_attempt && typeof upstream.created_at === "string" && + !Number.isNaN(Date.parse(upstream.created_at)), "Capture run identity changed."); + } + await output(environment, { eligible, "capture-date": upstream?.created_at?.slice(0, 10) ?? "" }); + if (!eligible) { await summary(environment, context.reason ?? "The pull request base changed."); } +} diff --git a/scripts/lib/screenshot-fingerprint.mjs b/scripts/lib/screenshot-fingerprint.mjs new file mode 100644 index 0000000..d15e0d6 --- /dev/null +++ b/scripts/lib/screenshot-fingerprint.mjs @@ -0,0 +1,193 @@ +import { createHash } from "node:crypto"; +import { extname } from "node:path"; + +const SOURCE_EXTENSIONS = new Set([".css", ".html", ".js", ".json", ".mjs", ".ts"]); +const GENERATED_SOURCE_FILES = new Set(["examples/advanced/main.js", "examples/metadata.json"]); +const CAPTURE_DEPENDENCIES = Object.freeze(["@playwright/test", "esbuild", "typescript", "vite"]); +const PACKAGE_NAME = /^(?:@[a-z\d._-]+\/)?[a-z\d._-]+$/iu; +const PACKAGE_PARENT = /(?:^|\/)node_modules\/(?:@[^/]+\/)?[^/]+$/u; +const SCREENSHOT_PACKAGE_SCRIPTS = Object.freeze([ + "prebuild:package", "build:package", "postbuild:package", + "prebuild:examples:advanced", "build:examples:advanced", "postbuild:examples:advanced", + "prescreenshots:update", "screenshots:update", "postscreenshots:update", + "prescreenshots:prepare", "screenshots:prepare", "postscreenshots:prepare", + "prescreenshots:candidate", "screenshots:candidate", "postscreenshots:candidate" +]); + +/** Repository-owned files that can change the canonical advanced example captures. */ +export const SCREENSHOT_SOURCE_INPUTS = Object.freeze([ + "package.json", "package-lock.json", "tsconfig.base.json", "tsconfig.build.json", + "vite.config.mjs", "src", "examples/example.css", "examples/advanced/index.html", + "examples/advanced/main.ts", "examples/advanced/theme.css", "examples/advanced/tsconfig.json", + "scripts/build-advanced-example.mjs", "scripts/build-package.mjs", + "scripts/lib/advanced-example-build.mjs", "scripts/lib/browser-targets.mjs", + "scripts/lib/node-version.mjs", "scripts/lib/package-entries.mjs", "scripts/lib/process.mjs", + "scripts/lib/styles.mjs", "scripts/lib/screenshot-fingerprint.mjs", + "scripts/lib/screenshot-preparation.mjs", "scripts/prepare-screenshots.mjs", + "scripts/screenshot-contract.mjs", "scripts/screenshot-scenes.mjs", + "scripts/serve-repository.mjs", "scripts/update-screenshots.mjs" +]); + +function assertObject(value, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must contain a JSON object.`); + } + return value; +} + +function dependencyMap(value, label) { + const entries = Object.entries(value === undefined ? {} : assertObject(value, label)); + for (const [name, requirement] of entries) { + if (!PACKAGE_NAME.test(name) || name === "." || name === ".." || typeof requirement !== "string") { + throw new Error(`${label} contains an invalid dependency: ${name}.`); + } + } + return Object.fromEntries(entries); +} + +function captureRootManifest(manifest) { + const result = { ...manifest }; + if (Object.hasOwn(result, "version")) { result.version = ""; } + if (Object.hasOwn(result, "devDependencies")) { + const dependencies = dependencyMap(result.devDependencies, "devDependencies"); + result.devDependencies = Object.fromEntries(CAPTURE_DEPENDENCIES + .filter((name) => Object.hasOwn(dependencies, name)).map((name) => [name, dependencies[name]])); + } + if (Object.hasOwn(result, "scripts")) { + assertObject(result.scripts, "scripts"); + result.scripts = Object.fromEntries(SCREENSHOT_PACKAGE_SCRIPTS + .filter((name) => Object.hasOwn(result.scripts, name)).map((name) => [name, result.scripts[name]])); + } + return result; +} + +function lockedPackages(lock) { + assertObject(lock, "package-lock.json"); + if (lock.lockfileVersion !== 2 && lock.lockfileVersion !== 3) { + throw new Error("Screenshot fingerprints require npm lockfile version 2 or 3."); + } + const packages = assertObject(lock.packages, "package-lock.json packages"); + assertObject(packages[""], "package-lock.json root package"); + return packages; +} + +function resolveDependency(packages, from, name) { + let directory = from; + for (;;) { + const candidate = `${directory === "" ? "" : `${directory}/`}node_modules/${name}`; + if (Object.hasOwn(packages, candidate)) { return candidate; } + if (directory === "") { return undefined; } + const parent = directory.replace(PACKAGE_PARENT, ""); + if (parent === directory) { throw new Error(`Unsupported npm package path: ${directory}.`); } + directory = parent; + } +} + +function captureLockfile(lock) { + const packages = lockedPackages(lock); + const root = captureRootManifest(packages[""]); + const selected = new Map([["", root]]); + const pending = []; + function include(from, name, required) { + const path = resolveDependency(packages, from, name); + if (path === undefined) { + if (required) { throw new Error(`Missing locked screenshot dependency ${name} from ${from || "root"}.`); } + return; + } + if (selected.has(path)) { return; } + const entry = assertObject(packages[path], `Locked package ${path}`); + if (entry.link) { throw new Error(`Linked screenshot dependencies are unsupported: ${path}.`); } + selected.set(path, entry); + pending.push(path); + } + function includeDependencies(from, entry, development = false) { + const optional = dependencyMap(entry.optionalDependencies, `${from || "root"} optionalDependencies`); + const required = dependencyMap(entry.dependencies, `${from || "root"} dependencies`); + for (const name of Object.keys(required)) { include(from, name, !Object.hasOwn(optional, name)); } + for (const name of Object.keys(optional)) { include(from, name, false); } + for (const name of Object.keys(dependencyMap(entry.peerDependencies, `${from || "root"} peerDependencies`))) { + include(from, name, false); + } + if (development) { + for (const name of Object.keys(root.devDependencies ?? {})) { include(from, name, true); } + } + } + includeDependencies("", root, true); + for (let index = 0; index < pending.length; index++) { + const path = pending[index]; + includeDependencies(path, selected.get(path)); + } + const result = { ...lock, packages: Object.fromEntries(selected) }; + if (Object.hasOwn(result, "version")) { result.version = ""; } + //The legacy v2 dependency tree duplicates packages and includes unrelated development tooling. + delete result.dependencies; + return result; +} + +function stableJson(value) { + return JSON.stringify(value, (_key, entry) => entry !== null && typeof entry === "object" && !Array.isArray(entry) + ? Object.fromEntries(Object.keys(entry).sort().map((key) => [key, entry[key]])) : entry); +} + +/** Selects supported source files while excluding generated example identity and output. */ +export function isScreenshotSourceFile(repositoryRelativePath) { + const path = repositoryRelativePath.replaceAll("\\", "/"); + return !GENERATED_SOURCE_FILES.has(path) && SOURCE_EXTENSIONS.has(extname(path).toLowerCase()); +} + +/** Selects the capture lifecycle and package source without including unrelated examples or tests. */ +export function isScreenshotSourceInput(repositoryRelativePath) { + const path = repositoryRelativePath.replaceAll("\\", "/"); + return isScreenshotSourceFile(path) && (SCREENSHOT_SOURCE_INPUTS.includes(path) || path.startsWith("src/")); +} + +/** Keeps execution metadata and runtime dependencies, selecting only capture/build development dependencies. */ +export function canonicalizeScreenshotSource(repositoryRelativePath, contents) { + const path = repositoryRelativePath.replaceAll("\\", "/"); + const bytes = Buffer.isBuffer(contents) ? contents : Buffer.from(contents); + if (path !== "package.json" && path !== "package-lock.json") { return bytes; } + const manifest = assertObject(JSON.parse(bytes.toString("utf8")), path); + return Buffer.from(stableJson(path === "package.json" ? captureRootManifest(manifest) : captureLockfile(manifest))); +} + +/** Returns the exact locked versions of declared capture/build roots without loading their code. */ +export function captureDependencyVersions(lock) { + const packages = lockedPackages(lock); + const roots = dependencyMap(packages[""].devDependencies, "root devDependencies"); + return Object.fromEntries(CAPTURE_DEPENDENCIES.filter((name) => Object.hasOwn(roots, name)).map((name) => { + const path = resolveDependency(packages, "", name); + const entry = path === undefined ? undefined : assertObject(packages[path], `Locked package ${path}`); + if (entry?.link || typeof entry?.version !== "string" || + !/^\d+\.\d+\.\d+(?:-[\da-z.-]+)?(?:\+[\da-z.-]+)?$/iu.test(entry.version)) { + throw new Error(`Screenshot dependency ${name} must resolve to an exact locked version.`); + } + return [name, entry.version]; + })); +} + +/** Hashes {path, contents} files and effective browser targets; requires only built-in Node modules. */ +export function fingerprintScreenshotFiles(files, browserTargets) { + assertObject(browserTargets, "Browser targets"); + const paths = new Set(); + const normalized = files.map((file) => { + const path = file.path.replaceAll("\\", "/"); + if (path.split("/").some((part) => part === "." || part === ".." || part === "") || + !isScreenshotSourceInput(path) || paths.has(path)) { + throw new Error(`Invalid or duplicate screenshot source: ${path}.`); + } + paths.add(path); + return { ...file, path }; + }).sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0); + const targets = { query: browserTargets.query, dataVersions: browserTargets.dataVersions, + browsers: browserTargets.browsers, javascript: browserTargets.javascript, css: browserTargets.css }; + const hash = createHash("sha256"); + hash.update(createHash("sha256").update(stableJson(targets)).digest("hex")); + hash.update("\0"); + for (const file of normalized) { + hash.update(file.path); + hash.update("\0"); + hash.update(canonicalizeScreenshotSource(file.path, file.contents)); + hash.update("\0"); + } + return hash.digest("hex"); +} diff --git a/scripts/lib/screenshot-preparation.mjs b/scripts/lib/screenshot-preparation.mjs index 5e60a2e..c4a4afe 100644 --- a/scripts/lib/screenshot-preparation.mjs +++ b/scripts/lib/screenshot-preparation.mjs @@ -1,21 +1,47 @@ import { copyFile, mkdir, rename, rm, writeFile } from "node:fs/promises"; import { basename, join } from "node:path"; -/** Builds once only when evidence is stale. CI can verify evidence but never accept it. */ +/** Builds stale evidence once; CI may capture candidates but can never accept canonical evidence. */ export async function prepareScreenshots({ checkCurrent, build, capture, verify, force = false, - ci = Boolean(process.env["CI"]) }) { + candidate = false, ci = Boolean(process.env["CI"]) }) { if (!force && await checkCurrent()) { return "current"; } - if (ci) { + if (ci && !candidate) { throw new Error("Screenshot evidence is stale. Prepare and review captures locally; CI cannot accept them."); } await build(); - await capture(); + await capture({ candidate }); + if (candidate) { + return "candidate"; + } await verify(); return "updated"; } +/** Publishes a validated gallery, keeping candidate captures separate from canonical evidence. */ +export async function finalizeScreenshotCapture({ cacheDirectory, stagingDirectory, + screenshotDirectory, manifestPath, previousManifest, manifest, readDimensions, hashFile, + computeSourceFingerprint, candidate = false, ci = Boolean(process.env["CI"]), + replaceBatch = replaceScreenshotBatch }) { + if (ci && !candidate) { + throw new Error("CI must verify screenshot evidence; canonical captures can only be prepared locally."); + } + await validateScreenshotBatch({ directory: stagingDirectory, manifest, readDimensions, hashFile }); + if (await computeSourceFingerprint() !== manifest.sourceFingerprint) { + throw new Error("Screenshot sources changed during capture; canonical evidence was preserved. Retry after edits settle."); + } + const directory = join(cacheDirectory, candidate ? "candidate" : "review"); + const review = await writeScreenshotReview({ directory, beforeDirectory: screenshotDirectory, + afterDirectory: stagingDirectory, previousManifest, manifest }); + if (candidate) { + await writeFile(join(directory, "screenshots.manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + } else { + await replaceBatch({ stagingDirectory, screenshotDirectory, manifestPath, manifest }); + } + return { directory, review }; +} + function escapeHtml(value) { return String(value).replaceAll("&", "&").replaceAll("<", "<") .replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'"); diff --git a/scripts/lib/screenshot-promotion.mjs b/scripts/lib/screenshot-promotion.mjs new file mode 100644 index 0000000..7f43e44 --- /dev/null +++ b/scripts/lib/screenshot-promotion.mjs @@ -0,0 +1,252 @@ +import { isDeepStrictEqual } from "node:util"; + +import { + captureDependencyVersions, fingerprintScreenshotFiles, isScreenshotSourceInput, + SCREENSHOT_SOURCE_INPUTS +} from "./screenshot-fingerprint.mjs"; + +const SHA256 = /^[a-f0-9]{64}$/u; +const COMMIT_SHA = /^[a-f0-9]{40}$/u; +const EXACT_VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/u; +const MANIFEST_KEYS = ["version", "state", "toolchain", "sourceFingerprint", "scenes", "browserTargets"]; +const TARGET_KEYS = ["query", "effectiveQuery", "resolvedAt", "dataVersions", "browsers", "javascript", "css"]; +const SCENE_KEYS = ["id", "route", "output", "viewport", "hasTouch", "setup", "alt", "references", "sha256"]; +const REPORT_KEYS = ["version", "repository", "prNumber", "headSha", "baseSha", "runId", "runAttempt"]; +const ALLOWED_PATHS = new Set(["package.json", "package-lock.json", "screenshots.manifest.json"]); + +function requireCondition(condition, message) { + if (!condition) { throw new Error(message); } +} + +function hasExactKeys(value, keys) { + return value !== null && typeof value === "object" && !Array.isArray(value) && + isDeepStrictEqual(Object.keys(value).sort(), [...keys].sort()); +} + +function without(value, keys) { + return Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key))); +} + +function validateIdentity({ repository, pullRequest, run }) { + requireCondition(typeof repository === "string" && /^[^/\s]+\/[^/\s]+$/u.test(repository), + "Repository identity is invalid."); + requireCondition(pullRequest?.state === "open" && pullRequest.user?.login === "dependabot[bot]", + "Only an open Dependabot pull request is eligible."); + requireCondition(pullRequest.head?.repo?.full_name === repository && + pullRequest.base?.repo?.full_name === repository && pullRequest.base.ref === "main", + "Pull request must originate in this repository and target main."); + requireCondition(Number.isSafeInteger(pullRequest.number) && pullRequest.number > 0 && + COMMIT_SHA.test(pullRequest.head.sha) && COMMIT_SHA.test(pullRequest.base.sha), + "Pull request commit identities are invalid."); + requireCondition(run?.event === "pull_request" && run.repository?.full_name === repository && + run.head_sha === pullRequest.head.sha && Number.isSafeInteger(run.id) && run.id > 0 && + Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0, + "Workflow run does not identify the current pull request head."); +} + +function validateDependencyChanges({ changedFiles, basePackage, headPackage, baseLock, headLock }) { + requireCondition(Array.isArray(changedFiles) && changedFiles.length > 0 && changedFiles.length <= 3, + "Changed file inventory is invalid."); + const paths = changedFiles.map((file) => typeof file === "string" ? file : file.filename); + requireCondition(new Set(paths).size === paths.length && paths.every((path) => ALLOWED_PATHS.has(path)) && + changedFiles.every((file) => typeof file === "string" || file.status === "modified"), + "Only modified dependency manifests and existing screenshot provenance are eligible."); + requireCondition(basePackage?.devDependencies !== null && typeof basePackage?.devDependencies === "object" && + headPackage?.devDependencies !== null && typeof headPackage?.devDependencies === "object" && + !Array.isArray(basePackage.devDependencies) && !Array.isArray(headPackage.devDependencies), + "Package development dependencies are invalid."); + requireCondition(isDeepStrictEqual(without(basePackage, ["devDependencies"]), + without(headPackage, ["devDependencies"])), "Package changes exceed development dependency versions."); + const names = Object.keys(basePackage.devDependencies).sort(); + requireCondition(isDeepStrictEqual(names, Object.keys(headPackage.devDependencies).sort()) && + names.every((name) => typeof headPackage.devDependencies[name] === "string" && + EXACT_VERSION.test(headPackage.devDependencies[name])), + "Only existing, exactly pinned development dependency versions may change."); + requireCondition(baseLock?.packages?.[""] !== undefined && headLock?.packages?.[""] !== undefined && + isDeepStrictEqual(without(baseLock, ["packages"]), without(headLock, ["packages"])) && + isDeepStrictEqual(without(baseLock.packages[""], ["devDependencies"]), + without(headLock.packages[""], ["devDependencies"])) && + isDeepStrictEqual(baseLock.packages[""].devDependencies, basePackage.devDependencies) && + isDeepStrictEqual(headLock.packages[""].devDependencies, headPackage.devDependencies), + "Lockfile root and metadata must match the dependency-only package update."); + requireCondition(!isDeepStrictEqual(baseLock, headLock) && paths.includes("package-lock.json"), + "There is no locked dependency update to promote."); + requireCondition(isDeepStrictEqual(basePackage, headPackage) || paths.includes("package.json"), + "Package update is absent from the changed file inventory."); +} + +/** Checks API-provided PR identities and dependency-only changes without executing repository code. */ +export function dependencyScreenshotEligibility(input) { + try { + validateIdentity(input); + const changes = validateDependencyOnlyChanges(input); + if (!changes.eligible) { return changes; } + return { eligible: true }; + } catch (error) { return { eligible: false, reason: error.message }; } +} + +/** Classifies dependency changes for an unprivileged capture job and a trusted writer alike. */ +export function validateDependencyOnlyChanges(input) { + try { + validateDependencyChanges(input); + if (input.baselineManifest !== undefined || input.currentManifest !== undefined) { + validateManifest(input.baselineManifest, "Baseline"); + validateManifest(input.currentManifest, "Current"); + requireCondition(isDeepStrictEqual( + without(input.baselineManifest, ["sourceFingerprint", "toolchain", "browserTargets"]), + without(input.currentManifest, ["sourceFingerprint", "toolchain", "browserTargets"])), + "Current screenshot evidence changes more than provenance."); + for (const key of ["query", "browsers", "javascript", "css"]) { + requireCondition(isDeepStrictEqual(input.baselineManifest.browserTargets[key], + input.currentManifest.browserTargets[key]), "Current screenshot browser targets changed."); + } + } + return { eligible: true }; + } catch (error) { return { eligible: false, reason: error.message }; } +} + +function validateManifest(manifest, label) { + requireCondition(hasExactKeys(manifest, MANIFEST_KEYS) && manifest.version === 1 && + manifest.state === "final" && SHA256.test(manifest.sourceFingerprint), `${label} manifest schema is invalid.`); + requireCondition(hasExactKeys(manifest.toolchain, ["browser", "node", "npm", "playwright"]) && + manifest.toolchain.browser === "chromium" && /^24\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/u.test(manifest.toolchain.node) && + EXACT_VERSION.test(manifest.toolchain.npm) && EXACT_VERSION.test(manifest.toolchain.playwright), + `${label} capture toolchain is invalid.`); + const targets = manifest.browserTargets; + requireCondition(hasExactKeys(targets, TARGET_KEYS) && hasExactKeys(targets.dataVersions, ["vite", "esbuild"]) && + EXACT_VERSION.test(targets.dataVersions.vite) && EXACT_VERSION.test(targets.dataVersions.esbuild) && + targets.query === "baseline-widely-available" && + targets.effectiveQuery === `Vite ${targets.dataVersions.vite}: ${targets.query}` && + typeof targets.resolvedAt === "string" && /^\d{4}-\d{2}-\d{2}$/u.test(targets.resolvedAt) && + Number.isFinite(Date.parse(`${targets.resolvedAt}T00:00:00.000Z`)) && + new Date(`${targets.resolvedAt}T00:00:00.000Z`).toISOString().slice(0, 10) === targets.resolvedAt && + ["browsers", "javascript", "css"].every((key) => Array.isArray(targets[key]) && targets[key].length > 0 && + targets[key].every((target) => typeof target === "string" && /^[a-z]+\d+(?:\.\d+)*$/u.test(target))), + `${label} browser target evidence is invalid.`); + requireCondition(Array.isArray(manifest.scenes) && manifest.scenes.length > 0 && manifest.scenes.length <= 50, + `${label} scene inventory is invalid.`); + const ids = new Set(); + const outputs = new Set(); + for (const scene of manifest.scenes) { + requireCondition(hasExactKeys(scene, SCENE_KEYS) && typeof scene.id === "string" && + /^[a-z0-9-]+$/u.test(scene.id) && !ids.has(scene.id) && !outputs.has(scene.output) && + typeof scene.output === "string" && /^docs\/screenshots\/[a-z0-9-]+\.png$/u.test(scene.output) && + scene.route === "/examples/advanced/" && hasExactKeys(scene.viewport, ["width", "height"]) && + [scene.viewport.width, scene.viewport.height].every((dimension) => Number.isSafeInteger(dimension) && dimension > 0) && + typeof scene.hasTouch === "boolean" && Array.isArray(scene.setup) && scene.setup.length > 0 && + scene.setup.every((step) => typeof step === "string" && step.length > 0) && + typeof scene.alt === "string" && scene.alt.length > 0 && Array.isArray(scene.references) && + scene.references.length > 0 && scene.references.every((reference) => typeof reference === "string" && + reference.endsWith(".md")) && SHA256.test(scene.sha256), `${label} scene contract is invalid.`); + ids.add(scene.id); + outputs.add(scene.output); + } +} + +function validateSourceFiles(files, packageManifest, lock, label) { + requireCondition(Array.isArray(files), `${label} source inventory is invalid.`); + const byPath = new Map(); + for (const file of files) { + requireCondition(typeof file?.path === "string" && isScreenshotSourceInput(file.path) && + !byPath.has(file.path) && (typeof file.contents === "string" || Buffer.isBuffer(file.contents)), + `${label} source inventory has an unexpected or duplicate input.`); + byPath.set(file.path, file.contents); + } + requireCondition(SCREENSHOT_SOURCE_INPUTS.every((path) => byPath.has(path) || + [...byPath.keys()].some((name) => name.startsWith(`${path}/`))), `${label} source inventory is incomplete.`); + for (const [path, value] of [["package.json", packageManifest], ["package-lock.json", lock]]) { + requireCondition(isDeepStrictEqual(JSON.parse(String(byPath.get(path))), value), + `${label} source inventory does not match ${path}.`); + } +} + +function validateToolchain(manifest, packageManifest, lock, expectedNpm, label) { + const versions = captureDependencyVersions(lock); + requireCondition(manifest.toolchain.npm === expectedNpm && packageManifest.packageManager === `npm@${expectedNpm}` && + manifest.toolchain.playwright === packageManifest.devDependencies["@playwright/test"] && + manifest.toolchain.playwright === versions["@playwright/test"] && + manifest.browserTargets.dataVersions.vite === versions.vite && + manifest.browserTargets.dataVersions.esbuild === versions.esbuild, + `${label} capture provenance does not match its locked toolchain.`); +} + +function validateHashMap(hashes, manifest, label) { + requireCondition(hasExactKeys(hashes, manifest.scenes.map((scene) => scene.output)) && + Object.values(hashes).every((value) => typeof value === "string" && SHA256.test(value)), + `${label} image hash inventory is invalid.`); +} + +/** + * Plans a provenance-only update using API identities and externally hashed PNG files. + * The caller must load both source inventories from exact Git commits, derive image hashes + * from actual canonical/candidate bytes, verify workflow identity and quality checks, and + * provide expectedCapture from trusted workflow configuration and the API run creation date. + * Untrusted artifact manifests are validated as data; their fields never execute as code. + */ +export function planScreenshotPromotion(input) { + const eligibility = dependencyScreenshotEligibility(input); + if (!eligibility.eligible) { return { status: "ineligible", reason: eligibility.reason }; } + try { + const { repository, pullRequest, run, report, basePackage, headPackage, baseLock, headLock, + baselineManifest, currentManifest, candidateManifest, canonicalHashes, baseCaptureHashes, + candidateHashes, sourceFiles, baseSourceFiles, expectedCapture } = input; + requireCondition(hasExactKeys(report, REPORT_KEYS) && report.version === 1 && + report.repository === repository && report.prNumber === pullRequest.number && + report.headSha === pullRequest.head.sha && report.baseSha === pullRequest.base.sha && + report.runId === run.id && report.runAttempt === run.run_attempt, + "Candidate report does not identify this exact base, head, and workflow attempt."); + validateManifest(baselineManifest, "Baseline"); + validateManifest(currentManifest, "Current"); + validateManifest(candidateManifest, "Candidate"); + requireCondition(hasExactKeys(expectedCapture, ["node", "resolvedAt"]) && + typeof run.created_at === "string" && expectedCapture.resolvedAt === run.created_at.slice(0, 10) && + candidateManifest.toolchain.node === expectedCapture.node && + candidateManifest.browserTargets.resolvedAt === expectedCapture.resolvedAt, + "Candidate runtime and capture date do not match the trusted workflow run."); + const expectedNpm = baselineManifest.toolchain.npm; + validateToolchain(baselineManifest, basePackage, baseLock, expectedNpm, "Baseline"); + validateToolchain(candidateManifest, headPackage, headLock, expectedNpm, "Candidate"); + requireCondition(isDeepStrictEqual(without(baselineManifest, ["sourceFingerprint", "toolchain", "browserTargets"]), + without(currentManifest, ["sourceFingerprint", "toolchain", "browserTargets"])), + "Current screenshot evidence changes more than provenance."); + validateSourceFiles(baseSourceFiles, basePackage, baseLock, "Base"); + validateSourceFiles(sourceFiles, headPackage, headLock, "Head"); + requireCondition(fingerprintScreenshotFiles(baseSourceFiles, baselineManifest.browserTargets) === + baselineManifest.sourceFingerprint, "Baseline screenshot fingerprint is stale."); + const fingerprint = fingerprintScreenshotFiles(sourceFiles, candidateManifest.browserTargets); + requireCondition(fingerprint === candidateManifest.sourceFingerprint, "Candidate screenshot fingerprint is incorrect."); + for (const key of ["query", "browsers", "javascript", "css"]) { + if (!isDeepStrictEqual(baselineManifest.browserTargets[key], candidateManifest.browserTargets[key])) { + return { status: "needs-review", reason: "Effective browser targets changed." }; + } + } + if (!isDeepStrictEqual(baselineManifest.scenes.map((scene) => without(scene, ["sha256"])), + candidateManifest.scenes.map((scene) => without(scene, ["sha256"])))) { + return { status: "needs-review", reason: "Screenshot scene definitions changed." }; + } + validateHashMap(canonicalHashes, baselineManifest, "Canonical"); + validateHashMap(baseCaptureHashes, baselineManifest, "Base capture"); + validateHashMap(candidateHashes, baselineManifest, "Candidate capture"); + for (const scene of baselineManifest.scenes) { + if (canonicalHashes[scene.output] !== scene.sha256 || baseCaptureHashes[scene.output] !== scene.sha256) { + return { status: "needs-review", reason: "Before evidence does not match canonical screenshot bytes." }; + } + const candidate = candidateManifest.scenes.find((value) => value.id === scene.id); + requireCondition(candidate.sha256 === candidateHashes[scene.output], + "Candidate screenshot hash does not match its PNG bytes."); + if (candidateHashes[scene.output] !== scene.sha256) { + return { status: "needs-review", reason: "Candidate screenshots contain visual changes." }; + } + } + const manifest = { ...structuredClone(baselineManifest), sourceFingerprint: fingerprint, + toolchain: structuredClone(candidateManifest.toolchain), + browserTargets: structuredClone(candidateManifest.browserTargets) }; + if (currentManifest.sourceFingerprint === fingerprint && + currentManifest.toolchain.npm === expectedNpm && + currentManifest.toolchain.playwright === candidateManifest.toolchain.playwright && + isDeepStrictEqual(currentManifest.browserTargets.dataVersions, candidateManifest.browserTargets.dataVersions)) { + return { status: "current" }; + } + return { status: "promote", manifest }; + } catch (error) { return { status: "ineligible", reason: error.message }; } +} diff --git a/scripts/prepare-screenshots.mjs b/scripts/prepare-screenshots.mjs index 0ba4c69..c827aae 100644 --- a/scripts/prepare-screenshots.mjs +++ b/scripts/prepare-screenshots.mjs @@ -8,13 +8,15 @@ assertSupportedNode(); assertPinnedNpm(); await assertPinnedPlaywright(); const arguments_ = process.argv.slice(2); -if (arguments_.some((argument) => argument !== "--force")) { - throw new Error("Usage: npm run screenshots:prepare [-- --force]"); +if (arguments_.some((argument) => argument !== "--force" && argument !== "--candidate") || + new Set(arguments_).size !== arguments_.length) { + throw new Error("Usage: npm run screenshots:prepare [-- --force] [--candidate]"); } const verify = () => run(process.execPath, ["scripts/check-screenshots.mjs"]); const result = await prepareScreenshots({ force: arguments_.includes("--force"), + candidate: arguments_.includes("--candidate"), checkCurrent: async () => { try { await run(process.execPath, ["scripts/check-screenshots.mjs"], { capture: true }); @@ -29,9 +31,12 @@ const result = await prepareScreenshots({ throw new Error("Screenshot sources changed during build. Retry after edits settle."); } }, - capture: () => run(process.execPath, ["scripts/update-screenshots.mjs"]), + capture: ({ candidate }) => run(process.execPath, + ["scripts/update-screenshots.mjs", ...(candidate ? ["--candidate"] : [])]), verify }); console.log(result === "current" ? "Screenshot evidence is current; no build or capture was needed." - : "Screenshot evidence prepared. Review .cache/screenshots/review/index.html before committing."); + : result === "candidate" + ? "Screenshot candidate prepared. Review .cache/screenshots/candidate/index.html; canonical evidence was preserved." + : "Screenshot evidence prepared. Review .cache/screenshots/review/index.html before committing."); diff --git a/scripts/screenshot-automation.mjs b/scripts/screenshot-automation.mjs new file mode 100644 index 0000000..c774b9d --- /dev/null +++ b/scripts/screenshot-automation.mjs @@ -0,0 +1,3 @@ +import { runScreenshotAutomation } from "./lib/screenshot-automation.mjs"; + +await runScreenshotAutomation({ command: process.argv[2] }); diff --git a/scripts/screenshot-contract.mjs b/scripts/screenshot-contract.mjs index 148f0bb..5f62a61 100644 --- a/scripts/screenshot-contract.mjs +++ b/scripts/screenshot-contract.mjs @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { readdir, readFile, stat } from "node:fs/promises"; -import { extname, isAbsolute, relative, resolve, sep } from "node:path"; +import { isAbsolute, relative, resolve, sep } from "node:path"; import { assertSupportedNodeVersion, @@ -8,8 +8,16 @@ import { SUPPORTED_NODE_RANGE } from "./lib/node-version.mjs"; import { REPOSITORY_ROOT } from "./lib/process.mjs"; -import { browserTargetsFingerprint } from "./lib/browser-targets.mjs"; +import { resolveBrowserTargets } from "./lib/browser-targets.mjs"; import { parseSemVer } from "./lib/semver.mjs"; +import { + fingerprintScreenshotFiles, isScreenshotSourceInput, SCREENSHOT_SOURCE_INPUTS +} from "./lib/screenshot-fingerprint.mjs"; + +export { + canonicalizeScreenshotSource, captureDependencyVersions, fingerprintScreenshotFiles, + isScreenshotSourceFile, isScreenshotSourceInput, SCREENSHOT_SOURCE_INPUTS +} from "./lib/screenshot-fingerprint.mjs"; export const REQUIRED_NPM_VERSION = "12.0.2"; const PACKAGE_MANIFEST = JSON.parse(await readFile(resolve(REPOSITORY_ROOT, "package.json"), "utf8")); @@ -29,53 +37,6 @@ export const EXPECTED_SCENES = Object.freeze([ "count-keyboard-focus" ]); -const SOURCE_EXTENSIONS = new Set([".css", ".html", ".js", ".json", ".mjs", ".ts"]); -const GENERATED_SOURCE_FILES = new Set([ - "examples/advanced/main.js", - "examples/metadata.json" -]); -const SCREENSHOT_SOURCE_DIRECTORIES = new Set(["src"]); -const SCREENSHOT_PACKAGE_SCRIPTS = Object.freeze([ - "prebuild:package", - "build:package", - "postbuild:package", - "prebuild:examples:advanced", - "build:examples:advanced", - "postbuild:examples:advanced", - "prescreenshots:update", - "screenshots:update", - "postscreenshots:update", - "prescreenshots:prepare", - "screenshots:prepare", - "postscreenshots:prepare" -]); -export const SCREENSHOT_SOURCE_INPUTS = Object.freeze([ - "package.json", - "package-lock.json", - "tsconfig.base.json", - "tsconfig.build.json", - "vite.config.mjs", - "src", - "examples/example.css", - "examples/advanced/index.html", - "examples/advanced/main.ts", - "examples/advanced/theme.css", - "examples/advanced/tsconfig.json", - "scripts/build-advanced-example.mjs", - "scripts/build-package.mjs", - "scripts/lib/advanced-example-build.mjs", - "scripts/lib/browser-targets.mjs", - "scripts/lib/node-version.mjs", - "scripts/lib/package-entries.mjs", - "scripts/lib/process.mjs", - "scripts/lib/styles.mjs", - "scripts/lib/screenshot-preparation.mjs", - "scripts/prepare-screenshots.mjs", - "scripts/screenshot-contract.mjs", - "scripts/screenshot-scenes.mjs", - "scripts/serve-repository.mjs", - "scripts/update-screenshots.mjs" -]); const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); export function assertSupportedNode() { @@ -128,52 +89,6 @@ function isRepositoryPath(path) { return child !== "" && child !== ".." && !child.startsWith(`..${sep}`) && !isAbsolute(child); } -export function isScreenshotSourceFile(repositoryRelativePath) { - const normalizedPath = repositoryRelativePath.replaceAll("\\", "/"); - return !GENERATED_SOURCE_FILES.has(normalizedPath) && - SOURCE_EXTENSIONS.has(extname(normalizedPath).toLowerCase()); -} - -export function isScreenshotSourceInput(repositoryRelativePath) { - const normalizedPath = repositoryRelativePath.replaceAll("\\", "/"); - return isScreenshotSourceFile(normalizedPath) && ( - SCREENSHOT_SOURCE_INPUTS.includes(normalizedPath) || - [...SCREENSHOT_SOURCE_DIRECTORIES].some((directory) => - normalizedPath.startsWith(`${directory}/`) - ) - ); -} - -export function canonicalizeScreenshotSource(repositoryRelativePath, contents) { - const normalizedPath = repositoryRelativePath.replaceAll("\\", "/"); - if (normalizedPath !== "package.json" && normalizedPath !== "package-lock.json") { - return Buffer.isBuffer(contents) ? contents : Buffer.from(contents); - } - - const manifest = JSON.parse(Buffer.isBuffer(contents) ? contents.toString("utf8") : contents); - if (manifest === null || typeof manifest !== "object" || Array.isArray(manifest)) { - throw new Error(`${normalizedPath} must contain a JSON object.`); - } - if (Object.hasOwn(manifest, "version")) { - manifest.version = ""; - } - if (normalizedPath === "package.json" && - manifest.scripts !== null && typeof manifest.scripts === "object" && - !Array.isArray(manifest.scripts)) { - manifest.scripts = Object.fromEntries(SCREENSHOT_PACKAGE_SCRIPTS - .filter((name) => Object.hasOwn(manifest.scripts, name)) - .map((name) => [name, manifest.scripts[name]])); - } - if (normalizedPath === "package-lock.json" && - manifest.packages?.[""] !== null && - typeof manifest.packages?.[""] === "object" && - !Array.isArray(manifest.packages[""]) && - Object.hasOwn(manifest.packages[""], "version")) { - manifest.packages[""].version = ""; - } - return Buffer.from(JSON.stringify(manifest)); -} - async function collectFiles(path) { const metadata = await stat(path); if (metadata.isFile()) { @@ -192,24 +107,15 @@ async function collectFiles(path) { } export async function computeSourceFingerprint() { - const files = (await Promise.all(SCREENSHOT_SOURCE_INPUTS.map((input) => - collectFiles(resolve(REPOSITORY_ROOT, input))))).flat().sort(); - const hash = createHash("sha256"); - hash.update(browserTargetsFingerprint()); - hash.update("\0"); - for (const file of files) { + const paths = (await Promise.all(SCREENSHOT_SOURCE_INPUTS.map((input) => + collectFiles(resolve(REPOSITORY_ROOT, input))))).flat(); + const files = await Promise.all(paths.map(async (file) => { if (!isRepositoryPath(file)) { throw new Error(`Screenshot source escaped the repository: ${file}`); } - hash.update(relative(REPOSITORY_ROOT, file).replaceAll("\\", "/")); - hash.update("\0"); - hash.update(canonicalizeScreenshotSource( - relative(REPOSITORY_ROOT, file).replaceAll("\\", "/"), - await readFile(file) - )); - hash.update("\0"); - } - return hash.digest("hex"); + return { path: relative(REPOSITORY_ROOT, file).replaceAll("\\", "/"), contents: await readFile(file) }; + })); + return fingerprintScreenshotFiles(files, resolveBrowserTargets()); } export async function readScreenshotManifest() { diff --git a/scripts/tests/screenshot-automation.test.mjs b/scripts/tests/screenshot-automation.test.mjs new file mode 100644 index 0000000..a0cecc6 --- /dev/null +++ b/scripts/tests/screenshot-automation.test.mjs @@ -0,0 +1,437 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, rmdir, symlink, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import test from "node:test"; + +import { + classifyScreenshotQuality, githubRequest, inspectScreenshotArtifact, runScreenshotAutomation +} from "../lib/screenshot-automation.mjs"; +import { fingerprintScreenshotFiles, SCREENSHOT_SOURCE_INPUTS } from "../lib/screenshot-fingerprint.mjs"; + +const REPOSITORY = "tryagaindev/litefold-calendar"; +const API = `/repos/${REPOSITORY}`; +const BASE_SHA = "a".repeat(40); +const HEAD_SHA = "b".repeat(40); +const NEW_HEAD_SHA = "c".repeat(40); +const SCENE_OUTPUT = "docs/screenshots/desktop-month-grid-1440x900.png"; +const IMAGE = Buffer.from("canonical image fixture"); +const STALE_LOG = [ + "2026-10-12T00:00:00.000Z Screenshot validation failed:", + "2026-10-12T00:00:00.000Z - Screenshot source fingerprint is stale; run npm run screenshots:prepare.", + "2026-10-12T00:00:00.000Z ##[error]Process completed with exit code 1." +].join("\n"); + +function hash(value) { return createHash("sha256").update(value).digest("hex"); } + +function response(value, status = 200, location) { + return { ok: status >= 200 && status < 300, status, + headers: { get: (name) => name === "location" ? location : null }, + text: async () => typeof value === "string" ? value : JSON.stringify(value) }; +} + +function qualityJobs() { + return [ + { name: "Resolve the verified source", conclusion: "success" }, + { name: "Capture Dependabot screenshot candidates", conclusion: "success" }, + { id: 99, name: "Build, test, and verify package", conclusion: "failure", steps: [ + { name: "Run the CI quality gate", conclusion: "failure" }, + { name: "Verify dependency package after a failed quality gate", conclusion: "success" } + ] } + ]; +} + +function fixture() { + const basePackage = { name: "fixture", version: "1.0.0", packageManager: "npm@12.0.2", + scripts: { "build:package": "node scripts/build-package.mjs" }, devDependencies: { + "@playwright/test": "1.63.0", typescript: "6.0.3", vite: "8.3.2", esbuild: "0.28.2" + } }; + const baseLock = { name: "fixture", version: "1.0.0", lockfileVersion: 3, requires: true, + packages: { "": { name: "fixture", version: "1.0.0", devDependencies: { ...basePackage.devDependencies } }, + ...Object.fromEntries(Object.entries(basePackage.devDependencies).map(([name, version]) => + [`node_modules/${name}`, { version, dev: true }])) } }; + const headPackage = structuredClone(basePackage); + headPackage.devDependencies.vite = "8.3.3"; + const headLock = structuredClone(baseLock); + headLock.packages[""].devDependencies.vite = "8.3.3"; + headLock.packages["node_modules/vite"].version = "8.3.3"; + const browserTargets = { query: "baseline-widely-available", + effectiveQuery: "Vite 8.3.2: baseline-widely-available", resolvedAt: "2026-10-11", + dataVersions: { vite: "8.3.2", esbuild: "0.28.2" }, browsers: ["chrome111", "safari16.4"], + javascript: ["chrome111", "safari16.4"], css: ["chrome111", "safari16.4"] }; + const sourceFiles = SCREENSHOT_SOURCE_INPUTS.map((path) => ({ + path: path === "src" ? "src/index.ts" : path, + contents: path === "package.json" ? JSON.stringify(basePackage) : + path === "package-lock.json" ? JSON.stringify(baseLock) : "fixture source\n" + })); + const baselineManifest = { version: 1, state: "final", + toolchain: { browser: "chromium", node: "24.19.0", npm: "12.0.2", playwright: "1.63.0" }, + sourceFingerprint: fingerprintScreenshotFiles(sourceFiles, browserTargets), browserTargets, + scenes: [{ id: "desktop-month-grid", route: "/examples/advanced/", output: SCENE_OUTPUT, + viewport: { width: 1440, height: 900 }, hasTouch: false, setup: ["wait-ready"], + alt: "Desktop advanced calendar example", references: ["README.md"], sha256: hash(IMAGE) }] }; + const candidateManifest = structuredClone(baselineManifest); + candidateManifest.browserTargets.dataVersions.vite = "8.3.3"; + candidateManifest.browserTargets.effectiveQuery = "Vite 8.3.3: baseline-widely-available"; + candidateManifest.browserTargets.resolvedAt = "2026-10-12"; + candidateManifest.sourceFingerprint = fingerprintScreenshotFiles(sourceFiles.map((file) => ({ ...file, + contents: file.path === "package.json" ? JSON.stringify(headPackage) : + file.path === "package-lock.json" ? JSON.stringify(headLock) : file.contents + })), candidateManifest.browserTargets); + const branch = "dependabot/npm_and_yarn/vite-8.3.3"; + return { basePackage, headPackage, baseLock, headLock, sourceFiles, baselineManifest, candidateManifest, + currentManifest: structuredClone(baselineManifest), + pullRequest: { number: 38, state: "open", user: { login: "dependabot[bot]" }, + head: { sha: HEAD_SHA, ref: branch, repo: { full_name: REPOSITORY } }, + base: { sha: BASE_SHA, ref: "main", repo: { full_name: REPOSITORY } } }, + run: { id: 42, run_attempt: 1, event: "pull_request", path: ".github/workflows/ci.yml", + status: "completed", conclusion: "failure", head_sha: HEAD_SHA, head_branch: branch, + created_at: "2026-10-12T00:00:00Z", repository: { full_name: REPOSITORY }, + head_repository: { full_name: REPOSITORY }, pull_requests: [{ number: 38 }] }, + report: { version: 1, repository: REPOSITORY, prNumber: 38, headSha: HEAD_SHA, + baseSha: BASE_SHA, runId: 42, runAttempt: 1 }, + changedFiles: [{ filename: "package.json", status: "modified" }, + { filename: "package-lock.json", status: "modified" }], + jobs: qualityJobs(), log: STALE_LOG, + artifacts: [{ id: 700, name: "screenshot-candidate-42-1", expired: false, size_in_bytes: 2000, + workflow_run: { id: 42, head_sha: HEAD_SHA } }] }; +} + +function mockRequest(data, { freshPullRequest, commitResponse } = {}) { + const calls = []; + let pullReads = 0; + async function request(path, options = {}) { + calls.push({ path, ...structuredClone(options) }); + if (path === `${API}/pulls/38`) { + pullReads += 1; + return structuredClone(pullReads > 1 && freshPullRequest ? freshPullRequest : data.pullRequest); + } + if (path === `${API}/pulls/38/files?per_page=100&page=1`) { return structuredClone(data.changedFiles); } + if (path === `${API}/compare/${data.pullRequest.base.sha}...${data.pullRequest.head.sha}`) { + return { merge_base_commit: { sha: data.mergeBaseSha ?? data.pullRequest.base.sha } }; + } + if (path === `${API}/actions/runs/42`) { return structuredClone(data.run); } + if (path === `${API}/actions/runs/42/attempts/1/jobs?per_page=100`) { return { jobs: structuredClone(data.jobs) }; } + if (path === `${API}/actions/jobs/99/logs`) { assert.equal(options.text, true); return data.log; } + if (path === `${API}/actions/runs/42/artifacts?per_page=100`) { return { artifacts: structuredClone(data.artifacts) }; } + if (path === "/graphql") { + return commitResponse ?? { data: { createCommitOnBranch: { commit: { oid: NEW_HEAD_SHA } } } }; + } + if (path === `${API}/actions/workflows/ci.yml/dispatches`) { return undefined; } + const content = path.match(/\/contents\/([^?]+)\?ref=([a-f0-9]{40})$/u); + if (content) { + const [, name, sha] = content; + const values = sha === BASE_SHA ? { + "package.json": data.basePackage, "package-lock.json": data.baseLock, + "screenshots.manifest.json": data.baselineManifest + } : { + "package.json": data.headPackage, "package-lock.json": data.headLock, + "screenshots.manifest.json": data.currentManifest + }; + assert.ok(Object.hasOwn(values, name), `Unexpected evidence path: ${path}`); + return { type: "file", encoding: "base64", sha: "d".repeat(40), + content: Buffer.from(JSON.stringify(values[name])).toString("base64") }; + } + throw new Error(`Unexpected API request: ${path}`); + } + return { request, calls }; +} + +function noWrites(calls) { assert.deepEqual(calls.filter((call) => call.method && call.method !== "GET"), []); } + +async function temporaryDirectory(t) { + const directory = await mkdtemp(join(tmpdir(), "lfc-screenshot-automation-")); + t.after(async () => { await rm(directory, { recursive: true, force: true }); }); + return directory; +} + +async function write(path, contents) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, contents); +} + +async function candidateDirectory(t, data) { + const temporary = await temporaryDirectory(t); + const directory = join(temporary, "candidate"); + await write(join(directory, "index.html"), "Candidate gallery"); + await write(join(directory, "changes.json"), "[]\n"); + await write(join(directory, "screenshots.manifest.json"), JSON.stringify(data.candidateManifest)); + await write(join(directory, "report.json"), JSON.stringify(data.report)); + for (const side of ["before", "after"]) { + await write(join(directory, side, basename(SCENE_OUTPUT)), IMAGE); + } + return { temporary, directory }; +} + +async function promotionFixture(t) { + const data = fixture(); + const root = await temporaryDirectory(t); + for (const file of data.sourceFiles) { await write(join(root, file.path), file.contents); } + await write(join(root, SCENE_OUTPUT), IMAGE); + const { temporary, directory } = await candidateDirectory(t, data); + const environment = { GITHUB_REPOSITORY: REPOSITORY, LFC_NODE_VERSION: "24.19.0", + LFC_UPSTREAM_RUN_ID: "42", LFC_UPSTREAM_RUN_ATTEMPT: "1", LFC_UPSTREAM_HEAD_SHA: HEAD_SHA, + RUNNER_TEMP: temporary, LFC_CANDIDATE_DIRECTORY: directory }; + return { data, root, directory, environment }; +} + +void test("GitHub log redirects never receive the API bearer token", async () => { + const calls = []; + const request = githubRequest({ GITHUB_REPOSITORY: REPOSITORY, GITHUB_TOKEN: "test-api-secret" }, + async (url, options) => { + calls.push({ url: String(url), options }); + return calls.length === 1 ? response("", 302, "https://logs.example.test/download") : response("job log"); + }); + assert.equal(await request(`${API}/actions/jobs/99/logs`, { text: true }), "job log"); + assert.equal(calls[0].options.redirect, "manual"); + assert.equal(calls[0].options.headers.Authorization, "Bearer test-api-secret"); + assert.deepEqual(calls[1], { url: "https://logs.example.test/download", options: { redirect: "error" } }); +}); + +void test("GitHub transport rejects unrelated destinations, unsafe log redirects, and oversized evidence", async () => { + const environment = { GITHUB_REPOSITORY: REPOSITORY, GITHUB_TOKEN: "test-api-secret" }; + for (const location of ["http://logs.example.test/download", "https://user:secret@logs.example.test/download"]) { + let calls = 0; + const request = githubRequest(environment, async () => { calls += 1; return response("", 302, location); }); + await assert.rejects(request(`${API}/actions/jobs/99/logs`, { text: true }), /Invalid log download destination/u); + assert.equal(calls, 1); + } + for (const text of [false, true]) { + const limit = text ? 16 * 1024 * 1024 : 8 * 1024 * 1024; + const request = githubRequest(environment, async () => response("x".repeat(limit + 1))); + await assert.rejects(request(`${API}/actions/jobs/99/logs`, { text }), /evidence limit/u); + } + const request = githubRequest(environment, async () => { throw new Error("Unexpected fetch"); }); + await assert.rejects(request("/repos/another/repository/pulls/1"), /Unexpected API destination/u); + assert.throws(() => githubRequest({ ...environment, GITHUB_REPOSITORY: "another/repository" }), /Unexpected repository/u); + assert.throws(() => githubRequest({ GITHUB_REPOSITORY: REPOSITORY }), /authentication/u); +}); + +void test("only the sole stale screenshot failure plus successful package validation qualifies", () => { + assert.equal(classifyScreenshotQuality(qualityJobs(), STALE_LOG), true); + const green = qualityJobs(); + green[2].conclusion = "success"; + assert.equal(classifyScreenshotQuality(green), true); + for (const log of [ + "", STALE_LOG.replace("fingerprint is stale", "image hash is incorrect"), + "##[error]Earlier test failure\n" + STALE_LOG, + STALE_LOG + "\n##[error]A second check failed", + STALE_LOG.replace("##[error]", "- A second screenshot diagnostic\n##[error]"), + STALE_LOG.replace("exit code 1.", "exit code 2.") + ]) { assert.equal(classifyScreenshotQuality(qualityJobs(), log), false, log); } + for (const mutate of [ + (jobs) => { jobs[0].conclusion = "failure"; }, + (jobs) => { jobs[1].conclusion = "skipped"; }, + (jobs) => { jobs[2].conclusion = "cancelled"; }, + (jobs) => { jobs[2].steps[1].conclusion = "failure"; }, + (jobs) => { jobs[2].steps[1].conclusion = "skipped"; }, + (jobs) => { jobs[2].steps.push({ name: "Another check", conclusion: "failure" }); } + ]) { + const jobs = qualityJobs(); mutate(jobs); + assert.equal(classifyScreenshotQuality(jobs, STALE_LOG), false); + } +}); + +void test("candidate artifacts require the exact complete file inventory", async (t) => { + const data = fixture(); + const { directory } = await candidateDirectory(t, data); + assert.equal(await inspectScreenshotArtifact(directory, data.baselineManifest), directory); + await write(join(directory, "unexpected.txt"), "unexpected"); + await assert.rejects(inspectScreenshotArtifact(directory, data.baselineManifest), /Unexpected candidate file/u); + await rm(join(directory, "unexpected.txt")); + await rm(join(directory, "before", basename(SCENE_OUTPUT))); + await assert.rejects(inspectScreenshotArtifact(directory, data.baselineManifest), /incomplete/u); +}); + +void test("candidate evidence rejects links and nested directories", async (t) => { + const data = fixture(); + const { temporary, directory } = await candidateDirectory(t, data); + const nested = join(directory, "after", "nested"); + await mkdir(nested); + await assert.rejects(inspectScreenshotArtifact(directory, data.baselineManifest), /Unexpected candidate directory/u); + await rmdir(nested); + const linked = join(directory, "linked"); + await symlink(join(directory, "after"), linked, "junction"); + await assert.rejects(inspectScreenshotArtifact(directory, data.baselineManifest), /links are forbidden/u); + if (process.platform === "win32") { await rmdir(linked); } + else { await unlink(linked); } + const rootLink = join(temporary, "linked-root"); + await symlink(directory, rootLink, "junction"); + await assert.rejects(inspectScreenshotArtifact(rootLink, data.baselineManifest), /regular directory/u); +}); + +void test("candidate eligibility binds live author, base, head, and API changed paths", async (t) => { + for (const [label, mutate] of [ + ["valid dependency update", () => {}], + ["changed head", (data) => { data.pullRequest.head.sha = NEW_HEAD_SHA; }], + ["changed base", (data) => { data.pullRequest.base.sha = NEW_HEAD_SHA; }], + ["non-Dependabot author", (data) => { data.pullRequest.user.login = "contributor"; }], + ["source change", (data) => { data.changedFiles.push({ filename: "src/index.ts", status: "modified" }); }] + ]) { + const data = fixture(); mutate(data); + const output = join(await temporaryDirectory(t), "output"); + const { request, calls } = mockRequest(data); + await runScreenshotAutomation({ command: "candidate-eligibility", request, environment: { + GITHUB_REPOSITORY: REPOSITORY, GITHUB_RUN_ID: "42", GITHUB_RUN_ATTEMPT: "1", GITHUB_OUTPUT: output, + LFC_PR_NUMBER: "38", LFC_HEAD_SHA: HEAD_SHA, LFC_BASE_SHA: BASE_SHA + } }); + const eligible = label === "valid dependency update"; + assert.equal(await readFile(output, "utf8"), `eligible=${eligible}\ncapture-date=${eligible ? "2026-10-12" : ""}\n`, label); + noWrites(calls); + if (label === "source change") { assert.equal(calls.some((call) => call.path.includes("/contents/")), false); } + } +}); + +void test("dispatch checkout accepts only an eligible PR at the exact triggering SHA", async (t) => { + const data = fixture(); + const environment = { GITHUB_REPOSITORY: REPOSITORY, GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_SHA: HEAD_SHA, GITHUB_RUN_ID: "42", LFC_PR_NUMBER: "38", LFC_HEAD_SHA: HEAD_SHA }; + const output = join(await temporaryDirectory(t), "output"); + const success = mockRequest(data); + await runScreenshotAutomation({ command: "verify-dispatch", request: success.request, + environment: { ...environment, GITHUB_OUTPUT: output } }); + assert.equal(await readFile(output, "utf8"), `checkout-ref=${HEAD_SHA}\nhead=${HEAD_SHA}\nbase=${BASE_SHA}\npr-number=38\n`); + noWrites(success.calls); + for (const [label, mutate] of [ + ["wrong triggering SHA", (_data, env) => { env.GITHUB_SHA = BASE_SHA; }], + ["wrong trigger", (_data, env) => { env.GITHUB_EVENT_NAME = "pull_request"; }], + ["changed live head", (data) => { data.pullRequest.head.sha = NEW_HEAD_SHA; }], + ["other author", (data) => { data.pullRequest.user.login = "contributor"; }], + ["unexpected PR path", (data) => { data.changedFiles.push({ filename: "scripts/update-screenshots.mjs", status: "modified" }); }] + ]) { + const rejected = fixture(); const env = { ...environment }; mutate(rejected, env); + const mock = mockRequest(rejected); + await assert.rejects(runScreenshotAutomation({ command: "verify-dispatch", request: mock.request, environment: env }), undefined, label); + noWrites(mock.calls); + } +}); + +void test("capture, promotion, and dispatch reject a PR branch missing the current main base", async (t) => { + for (const command of ["candidate-eligibility", "promote", "verify-dispatch"]) { + const data = fixture(); + data.mergeBaseSha = "e".repeat(40); + const output = join(await temporaryDirectory(t), "output"); + const { request, calls } = mockRequest(data); + const environment = { + GITHUB_REPOSITORY: REPOSITORY, GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_SHA: HEAD_SHA, + GITHUB_RUN_ID: "42", GITHUB_RUN_ATTEMPT: "1", GITHUB_OUTPUT: output, + LFC_PR_NUMBER: "38", LFC_HEAD_SHA: HEAD_SHA, LFC_BASE_SHA: BASE_SHA, + LFC_UPSTREAM_RUN_ID: "42", LFC_UPSTREAM_RUN_ATTEMPT: "1", LFC_UPSTREAM_HEAD_SHA: HEAD_SHA + }; + const operation = runScreenshotAutomation({ command, request, environment }); + if (command === "verify-dispatch") { + await assert.rejects(operation, /current main/u); + } else { + await operation; + assert.equal(await readFile(output, "utf8"), command === "promote" + ? "eligible=false\nartifact-id=\n" : "eligible=false\ncapture-date=\n", command); + } + assert.equal(calls.some((call) => call.path === `${API}/compare/${BASE_SHA}...${HEAD_SHA}`), true, command); + noWrites(calls); + } +}); + +void test("forged runs and nonunique or stale artifacts never authorize provider writes", async (t) => { + for (const [label, mutate] of [ + ["wrong run", (data) => { data.run.id = 43; }], + ["wrong attempt", (data) => { data.run.run_attempt = 2; }], + ["old head", (data) => { data.run.head_sha = BASE_SHA; }], + ["wrong workflow", (data) => { data.run.path = ".github/workflows/other.yml"; }], + ["wrong event", (data) => { data.run.event = "push"; }], + ["running", (data) => { data.run.status = "in_progress"; }], + ["foreign source", (data) => { data.run.head_repository.full_name = "fork/repo"; }], + ["ambiguous PR", (data) => { data.run.pull_requests.push({ number: 39 }); }], + ["changed branch", (data) => { data.run.head_branch = "different"; }], + ["additional failed check", (data) => { data.log = "##[error]Unit tests failed\n" + data.log; }], + ["expired artifact", (data) => { data.artifacts[0].expired = true; }], + ["wrong artifact run", (data) => { data.artifacts[0].workflow_run.id = 43; }], + ["wrong artifact head", (data) => { data.artifacts[0].workflow_run.head_sha = BASE_SHA; }], + ["oversized artifact", (data) => { data.artifacts[0].size_in_bytes = 21 * 1024 * 1024; }], + ["duplicate artifact", (data) => { data.artifacts.push(structuredClone(data.artifacts[0])); }] + ]) { + const data = fixture(); mutate(data); + const { request, calls } = mockRequest(data); + const output = join(await temporaryDirectory(t), "output"); + await runScreenshotAutomation({ command: "promote", request, environment: { + GITHUB_REPOSITORY: REPOSITORY, LFC_UPSTREAM_RUN_ID: "42", LFC_UPSTREAM_RUN_ATTEMPT: "1", + LFC_UPSTREAM_HEAD_SHA: HEAD_SHA, GITHUB_OUTPUT: output + } }); + assert.equal(await readFile(output, "utf8"), "eligible=false\nartifact-id=\n", label); + noWrites(calls); + } +}); + +void test("unchanged images promote only the manifest atomically and dispatch CI for the new head", async (t) => { + const { data, environment, root } = await promotionFixture(t); + const { request, calls } = mockRequest(data); + await runScreenshotAutomation({ command: "promote", environment, request, root, readHead: async () => BASE_SHA }); + const writes = calls.filter((call) => call.method === "POST"); + assert.equal(writes.length, 2); + assert.equal(writes[0].path, "/graphql"); + assert.match(writes[0].body.query, /createCommitOnBranch/u); + const mutation = writes[0].body.variables.input; + assert.deepEqual(mutation.branch, { repositoryNameWithOwner: REPOSITORY, branchName: data.pullRequest.head.ref }); + assert.equal(mutation.expectedHeadOid, HEAD_SHA); + assert.deepEqual(Object.keys(mutation.fileChanges), ["additions"]); + assert.equal(mutation.fileChanges.additions.length, 1); + assert.equal(mutation.fileChanges.additions[0].path, "screenshots.manifest.json"); + const manifest = JSON.parse(Buffer.from(mutation.fileChanges.additions[0].contents, "base64").toString("utf8")); + assert.deepEqual(manifest.scenes, data.baselineManifest.scenes); + assert.equal(manifest.sourceFingerprint, data.candidateManifest.sourceFingerprint); + assert.deepEqual(writes[1], { path: `${API}/actions/workflows/ci.yml/dispatches`, method: "POST", body: { + ref: data.pullRequest.head.ref, inputs: { "pr-number": "38", "expected-head": NEW_HEAD_SHA } + } }); + assert.deepEqual(await readFile(join(root, SCENE_OUTPUT)), IMAGE); +}); + +void test("changed candidate images require human review and make no provider writes", async (t) => { + const { data, environment, root, directory } = await promotionFixture(t); + const changedImage = Buffer.from("changed image fixture"); + data.candidateManifest.scenes[0].sha256 = hash(changedImage); + await write(join(directory, "screenshots.manifest.json"), JSON.stringify(data.candidateManifest)); + await write(join(directory, "after", basename(SCENE_OUTPUT)), changedImage); + const { request, calls } = mockRequest(data); + await runScreenshotAutomation({ command: "promote", environment, request, root, readHead: async () => BASE_SHA }); + noWrites(calls); +}); + +void test("main advancing or a concurrent PR push prevents the evidence commit", async (t) => { + for (const kind of ["main", "head", "base"]) { + const { data, environment, root } = await promotionFixture(t); + const fresh = structuredClone(data.pullRequest); + if (kind === "head") { fresh.head.sha = NEW_HEAD_SHA; } + if (kind === "base") { fresh.base.sha = NEW_HEAD_SHA; } + const { request, calls } = mockRequest(data, { freshPullRequest: fresh }); + await runScreenshotAutomation({ command: "promote", environment, request, root, + readHead: async () => kind === "main" ? NEW_HEAD_SHA : BASE_SHA }); + noWrites(calls); + } +}); + +void test("atomic commit rejection never dispatches CI on an unverified head", async (t) => { + const { data, environment, root } = await promotionFixture(t); + const { request, calls } = mockRequest(data, { commitResponse: { errors: [{ message: "Branch moved" }] } }); + await assert.rejects(runScreenshotAutomation({ command: "promote", environment, request, root, + readHead: async () => BASE_SHA }), /atomic evidence commit/u); + assert.deepEqual(calls.filter((call) => call.method === "POST").map((call) => call.path), ["/graphql"]); +}); + +void test("forged candidate reports, manifest fields, and image hashes never write", async (t) => { + for (const [label, mutate] of [ + ["old report head", (data) => { data.report.headSha = BASE_SHA; }], + ["different report base", (data) => { data.report.baseSha = HEAD_SHA; }], + ["different report attempt", (data) => { data.report.runAttempt = 2; }], + ["forged fingerprint", (data) => { data.candidateManifest.sourceFingerprint = "e".repeat(64); }], + ["unknown manifest command", (data) => { data.candidateManifest.command = "hostile"; }], + ["image hash mismatch", (data) => { data.candidateManifest.scenes[0].sha256 = "e".repeat(64); }] + ]) { + const { data, environment, root, directory } = await promotionFixture(t); mutate(data); + await write(join(directory, "report.json"), JSON.stringify(data.report)); + await write(join(directory, "screenshots.manifest.json"), JSON.stringify(data.candidateManifest)); + const { request, calls } = mockRequest(data); + await runScreenshotAutomation({ command: "promote", environment, request, root, readHead: async () => BASE_SHA }); + noWrites(calls); + assert.equal(calls.some((call) => call.path === "/graphql"), false, label); + } +}); diff --git a/scripts/tests/screenshot-contract.test.mjs b/scripts/tests/screenshot-contract.test.mjs index 1c82664..901a618 100644 --- a/scripts/tests/screenshot-contract.test.mjs +++ b/scripts/tests/screenshot-contract.test.mjs @@ -6,6 +6,8 @@ import test from "node:test"; import { REPOSITORY_ROOT } from "../lib/process.mjs"; import { canonicalizeScreenshotSource, + captureDependencyVersions, + fingerprintScreenshotFiles, isScreenshotSourceFile, isScreenshotSourceInput, pinnedPlaywrightVersion, @@ -59,6 +61,7 @@ void test("screenshot fingerprints select the exact advanced route and build con "scripts/lib/package-entries.mjs", "scripts/lib/process.mjs", "scripts/lib/styles.mjs", + "scripts/lib/screenshot-fingerprint.mjs", "scripts/lib/screenshot-preparation.mjs", "scripts/prepare-screenshots.mjs", "scripts/screenshot-contract.mjs", @@ -159,7 +162,13 @@ void test("screenshot fingerprints ignore root versions and nonvisual package sc "postbuild:examples:advanced", "prescreenshots:update", "screenshots:update", - "postscreenshots:update" + "postscreenshots:update", + "prescreenshots:prepare", + "screenshots:prepare", + "postscreenshots:prepare", + "prescreenshots:candidate", + "screenshots:candidate", + "postscreenshots:candidate" ]) { assert.notDeepEqual( canonicalizeScreenshotSource("package.json", JSON.stringify(packageManifest)), @@ -197,3 +206,142 @@ void test("screenshot fingerprints ignore root versions and nonvisual package sc canonicalizeScreenshotSource("package-lock.json", JSON.stringify(changedDependency)) ); }); + +function dependencyFixture() { + return { + lockfileVersion: 3, name: "fixture", version: "1.0.0", + packages: { + "": { name: "fixture", version: "1.0.0", devDependencies: { + "@playwright/test": "1.63.0", esbuild: "0.28.2", typescript: "6.0.3", vite: "8.3.2", + eslint: "10.11.0", jsdom: "30.1.1", stylelint: "17.16.0" + } }, + "node_modules/@playwright/test": { version: "1.63.0", dependencies: { playwright: "1.63.0" } }, + "node_modules/playwright": { version: "1.63.0", dependencies: { "playwright-core": "1.63.0" }, + optionalDependencies: { fsevents: "2.3.2" } }, + "node_modules/playwright-core": { version: "1.63.0" }, + "node_modules/esbuild": { version: "0.28.2", optionalDependencies: { "@esbuild/linux-x64": "0.28.2" } }, + "node_modules/@esbuild/linux-x64": { version: "0.28.2", os: ["linux"], cpu: ["x64"] }, + "node_modules/typescript": { version: "6.0.3" }, + "node_modules/vite": { version: "8.3.2", dependencies: { "source-map-js": "^1.2.1" }, + peerDependencies: { "@types/node": ">=24", sass: "*" } }, + "node_modules/@types/node": { version: "24.19.1" }, + "node_modules/source-map-js": { version: "1.2.1", integrity: "sha512-source" }, + "node_modules/eslint": { version: "10.11.0" }, + "node_modules/jsdom": { version: "30.1.1", dependencies: { "source-map-js": "^1.2.1" } }, + "node_modules/stylelint": { version: "17.16.0" } + } + }; +} + +function canonicalLock(lock) { + return canonicalizeScreenshotSource("package-lock.json", JSON.stringify(lock)); +} + +void test("unrelated lint and test dependencies do not stale captures, but shared rendering dependencies do", () => { + const lock = dependencyFixture(); + const changed = structuredClone(lock); + for (const name of ["eslint", "jsdom", "stylelint"]) { + changed.packages[""].devDependencies[name] = "99.0.0"; + changed.packages[`node_modules/${name}`] = { version: "99.0.0", integrity: "new", license: "new" }; + } + assert.deepEqual(canonicalLock(changed), canonicalLock(lock)); + assert.deepEqual(canonicalizeScreenshotSource("package.json", JSON.stringify(changed.packages[""])), + canonicalizeScreenshotSource("package.json", JSON.stringify(lock.packages[""]))); + changed.packages["node_modules/source-map-js"].integrity = "sha512-changed"; + assert.notDeepEqual(canonicalLock(changed), canonicalLock(lock)); + for (const name of ["@playwright/test", "typescript", "vite", "esbuild"]) { + const compilerChanged = structuredClone(lock.packages[""]); + compilerChanged.devDependencies[name] = "99.0.0"; + assert.notDeepEqual(canonicalizeScreenshotSource("package.json", JSON.stringify(compilerChanged)), + canonicalizeScreenshotSource("package.json", JSON.stringify(lock.packages[""])), name); + } +}); + +void test("lock traversal follows nested, hoisted, scoped, optional, peer, and cyclic dependencies", () => { + const lock = dependencyFixture(); + lock.packages["node_modules/vite"].dependencies = { plugin: "1", "@scope/renderer": "1" }; + lock.packages["node_modules/vite/node_modules/plugin"] = { + version: "1.0.0", dependencies: { "source-map-js": "2", vite: "8" }, + optionalDependencies: { "optional-renderer": "1" } + }; + lock.packages["node_modules/vite/node_modules/source-map-js"] = { version: "2.0.0" }; + lock.packages["node_modules/vite/node_modules/@scope/renderer"] = { + version: "1.0.0", dependencies: { "hoisted-renderer": "1" } + }; + lock.packages["node_modules/hoisted-renderer"] = { version: "1.0.0" }; + lock.packages["node_modules/optional-renderer"] = { version: "1.0.0" }; + const selected = JSON.parse(canonicalLock(lock)).packages; + for (const path of ["node_modules/vite/node_modules/plugin", "node_modules/vite/node_modules/source-map-js", + "node_modules/vite/node_modules/@scope/renderer", "node_modules/hoisted-renderer", + "node_modules/optional-renderer", "node_modules/@types/node", "node_modules/@esbuild/linux-x64"]) { + assert.ok(Object.hasOwn(selected, path), path); + } + assert.equal(Object.hasOwn(selected, "node_modules/source-map-js"), false); + for (const path of ["node_modules/@types/node", "node_modules/@esbuild/linux-x64", "node_modules/optional-renderer"]) { + const changed = structuredClone(lock); + changed.packages[path].version = "2.0.0"; + assert.notDeepEqual(canonicalLock(changed), canonicalLock(lock), path); + } +}); + +void test("missing required and linked capture dependencies fail; absent optional and peer packages are allowed", () => { + const lock = dependencyFixture(); + assert.doesNotThrow(() => canonicalLock(lock)); + for (const path of ["node_modules/@playwright/test", "node_modules/source-map-js"]) { + const missing = structuredClone(lock); + delete missing.packages[path]; + assert.throws(() => canonicalLock(missing), /Missing locked screenshot dependency/u, path); + } + const linked = structuredClone(lock); + linked.packages["node_modules/vite"] = { link: true, resolved: "packages/vite" }; + assert.throws(() => canonicalLock(linked), /Linked screenshot dependencies/u); + const optionalOverride = structuredClone(lock); + optionalOverride.packages["node_modules/vite"].dependencies.missing = "1"; + optionalOverride.packages["node_modules/vite"].optionalDependencies = { missing: "1" }; + assert.doesNotThrow(() => canonicalLock(optionalOverride)); + assert.throws(() => canonicalLock({ ...lock, lockfileVersion: 1 }), /lockfile version/u); +}); + +void test("runtime dependency fields and reachable runtime lock entries remain fingerprint inputs", () => { + const lock = dependencyFixture(); + lock.packages[""].dependencies = { "calendar-runtime": "1.0.0" }; + lock.packages["node_modules/calendar-runtime"] = { version: "1.0.0" }; + const changed = structuredClone(lock); + changed.packages["node_modules/calendar-runtime"].version = "2.0.0"; + assert.notDeepEqual(canonicalLock(changed), canonicalLock(lock)); + for (const field of ["dependencies", "optionalDependencies", "peerDependencies", "overrides", "exports", "engines"]) { + const manifest = { name: "fixture", [field]: { "calendar-runtime": "1.0.0" } }; + assert.notDeepEqual(canonicalizeScreenshotSource("package.json", JSON.stringify(manifest)), + canonicalizeScreenshotSource("package.json", JSON.stringify({ ...manifest, [field]: {} })), field); + } +}); + +void test("capture root versions come from exact locked records without loading dependency code", () => { + const lock = dependencyFixture(); + assert.deepEqual(captureDependencyVersions(lock), { + "@playwright/test": "1.63.0", esbuild: "0.28.2", typescript: "6.0.3", vite: "8.3.2" + }); + for (const entry of [{ version: "^1.63.0" }, { link: true, version: "1.63.0" }, {}]) { + const changed = structuredClone(lock); + changed.packages["node_modules/@playwright/test"] = entry; + assert.throws(() => captureDependencyVersions(changed), /exact locked version/u); + } +}); + +void test("pure fingerprints are stable across file and JSON key ordering, but retain content and browser inputs", async () => { + const source = await readFile(new URL("../lib/screenshot-fingerprint.mjs", import.meta.url)); + const pure = await import(`data:text/javascript;base64,${source.toString("base64")}`); + const targets = { query: "baseline-widely-available", dataVersions: { vite: "8.3.2", esbuild: "0.28.2" }, + browsers: ["chrome111"], javascript: ["chrome111"], css: ["chrome111"], resolvedAt: "2026-10-12" }; + const files = [{ path: "src/index.ts", contents: "export {};" }, + { path: "package.json", contents: JSON.stringify({ name: "fixture", engines: { node: "24" } }) }]; + const expected = fingerprintScreenshotFiles(files, targets); + assert.equal(pure.fingerprintScreenshotFiles(files, targets), expected); + assert.equal(fingerprintScreenshotFiles([...files].reverse(), { ...targets, resolvedAt: "2026-10-13" }), expected); + assert.equal(fingerprintScreenshotFiles([files[0], { ...files[1], + contents: JSON.stringify({ engines: { node: "24" }, name: "fixture" }) }], targets), expected); + assert.notEqual(fingerprintScreenshotFiles([{ ...files[0], contents: "export const value = 1;" }, files[1]], targets), expected); + assert.notEqual(fingerprintScreenshotFiles(files, { ...targets, css: ["chrome112"] }), expected); + assert.throws(() => fingerprintScreenshotFiles([...files, files[0]], targets), /duplicate/u); + assert.throws(() => fingerprintScreenshotFiles([{ path: "src/../README.md", contents: "" }], targets), /Invalid/u); +}); diff --git a/scripts/tests/screenshot-preparation.test.mjs b/scripts/tests/screenshot-preparation.test.mjs index 733c258..2b1d7ec 100644 --- a/scripts/tests/screenshot-preparation.test.mjs +++ b/scripts/tests/screenshot-preparation.test.mjs @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; -import { mkdtemp, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { access, copyFile, mkdtemp, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { prepareScreenshots, replaceScreenshotBatch, validateScreenshotBatch, +import { finalizeScreenshotCapture, prepareScreenshots, replaceScreenshotBatch, validateScreenshotBatch, writeScreenshotReview } from "../lib/screenshot-preparation.mjs"; void test("current evidence skips builds and capture; stale evidence builds once before capture", async () => { @@ -29,6 +30,22 @@ void test("CI cannot accept stale or forced evidence, and failed builds never ca assert.equal(captures, 0); }); +void test("CI candidate preparation builds and captures without verifying canonical freshness", async () => { + const calls = []; + const options = { candidate: true, ci: true, checkCurrent: async () => false, + build: async () => { calls.push("build"); }, + capture: async (mode) => { calls.push(mode); }, + verify: async () => { throw new Error("Canonical evidence must stay stale."); } }; + assert.equal(await prepareScreenshots(options), "candidate"); + assert.deepEqual(calls, ["build", { candidate: true }]); + calls.length = 0; + assert.equal(await prepareScreenshots({ ...options, checkCurrent: async () => true }), "current"); + assert.deepEqual(calls, []); + assert.equal(await prepareScreenshots({ ...options, force: true, + checkCurrent: async () => { throw new Error("Forced capture must not check freshness."); } }), "candidate"); + assert.deepEqual(calls, ["build", { candidate: true }]); +}); + async function fixture(t) { const root = await mkdtemp(join(tmpdir(), "calendar-screenshots-")); t.after(() => rm(root, { recursive: true, force: true })); @@ -94,3 +111,73 @@ void test("review gallery records changed scenes and preserves before and after assert.equal(await readFile(join(review, "after", "one.png"), "utf8"), "new one.png"); assert.match(await readFile(join(review, "index.html"), "utf8"), /Before: Calendar scene one/u); }); + +async function hashFile(path) { + return createHash("sha256").update(await readFile(path)).digest("hex"); +} + +async function captureFixture(t, { changed = false } = {}) { + const f = await fixture(t); + const previousManifest = structuredClone(f.manifest); + for (const scene of previousManifest.scenes) { + scene.sha256 = await hashFile(join(f.canonical, `${scene.id}.png`)); + if (!changed || scene.id !== "one") { + await copyFile(join(f.canonical, `${scene.id}.png`), join(f.staging, `${scene.id}.png`)); + } + } + for (const scene of f.manifest.scenes) { + scene.sha256 = await hashFile(join(f.staging, `${scene.id}.png`)); + } + f.manifest.sourceFingerprint = "captured-source"; + const options = { cacheDirectory: join(f.root, "cache"), stagingDirectory: f.staging, + screenshotDirectory: f.canonical, manifestPath: f.manifestPath, previousManifest, + manifest: f.manifest, readDimensions: async () => ({ width: 390, height: 844 }), hashFile, + computeSourceFingerprint: async () => f.manifest.sourceFingerprint }; + return { ...f, options }; +} + +void test("CI candidates retain complete changed and unchanged galleries without replacing canonical files", async (t) => { + for (const changed of [false, true]) { + const f = await captureFixture(t, { changed }); + let replacements = 0; + const { directory, review } = await finalizeScreenshotCapture({ ...f.options, + candidate: true, ci: true, replaceBatch: async () => { replacements++; } }); + assert.equal(replacements, 0); + assert.equal(directory, join(f.options.cacheDirectory, "candidate")); + assert.deepEqual(review.changed, changed ? ["one"] : []); + assert.equal(await readFile(f.manifestPath, "utf8"), "old manifest"); + assert.deepEqual(JSON.parse(await readFile(join(directory, "screenshots.manifest.json"), "utf8")), f.manifest); + assert.deepEqual(JSON.parse(await readFile(join(directory, "changes.json"), "utf8")), review); + for (const scene of f.manifest.scenes) { + const filename = `${scene.id}.png`; + assert.equal(await readFile(join(f.canonical, filename), "utf8"), `old ${filename}`); + assert.equal(await readFile(join(directory, "before", filename), "utf8"), `old ${filename}`); + assert.equal(await hashFile(join(directory, "after", filename)), scene.sha256); + assert.equal(review.scenes.find((capture) => capture.id === scene.id).after, `after/${filename}`); + } + } +}); + +void test("CI rejects canonical finalization before writing a gallery or replacing evidence", async (t) => { + const f = await captureFixture(t); + let replacements = 0; + await assert.rejects(finalizeScreenshotCapture({ ...f.options, ci: true, + replaceBatch: async () => { replacements++; } }), /canonical captures can only be prepared locally/u); + assert.equal(replacements, 0); + assert.equal(await readFile(f.manifestPath, "utf8"), "old manifest"); + await assert.rejects(access(join(f.options.cacheDirectory, "review")), { code: "ENOENT" }); +}); + +void test("source drift and invalid batches do not publish candidate evidence or replace canonical files", async (t) => { + for (const invalidBatch of [false, true]) { + const f = await captureFixture(t, { changed: true }); + await assert.rejects(finalizeScreenshotCapture({ ...f.options, candidate: true, ci: true, + ...(invalidBatch ? { hashFile: async () => "incorrect-hash" } : + { computeSourceFingerprint: async () => "modified-source" }) }), + invalidBatch ? /SHA-256/u : /sources changed during capture/u); + assert.equal(await readFile(f.manifestPath, "utf8"), "old manifest"); + assert.equal(await readFile(join(f.canonical, "one.png"), "utf8"), "old one.png"); + assert.equal(await readFile(join(f.staging, "one.png"), "utf8"), "new one.png"); + await assert.rejects(access(join(f.options.cacheDirectory, "candidate")), { code: "ENOENT" }); + } +}); diff --git a/scripts/tests/screenshot-promotion.test.mjs b/scripts/tests/screenshot-promotion.test.mjs new file mode 100644 index 0000000..b62b5f4 --- /dev/null +++ b/scripts/tests/screenshot-promotion.test.mjs @@ -0,0 +1,247 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { fingerprintScreenshotFiles, SCREENSHOT_SOURCE_INPUTS } from "../lib/screenshot-fingerprint.mjs"; +import { + dependencyScreenshotEligibility, planScreenshotPromotion, validateDependencyOnlyChanges +} from "../lib/screenshot-promotion.mjs"; + +const BASE_SHA = "a".repeat(40); +const HEAD_SHA = "b".repeat(40); +const IMAGE_HASH = "c".repeat(64); +const CHANGED_IMAGE_HASH = "d".repeat(64); +const SCENE_OUTPUT = "docs/screenshots/desktop-month-grid-1440x900.png"; + +function sourceFiles(packageManifest, lock) { + return SCREENSHOT_SOURCE_INPUTS.map((path) => ({ path: path === "src" ? "src/index.ts" : path, + contents: path === "package.json" ? JSON.stringify(packageManifest) : + path === "package-lock.json" ? JSON.stringify(lock) : "fixture source\n" })); +} + +function fixture() { + const basePackage = { name: "fixture", version: "1.0.0", packageManager: "npm@12.0.2", + scripts: { "build:package": "node scripts/build-package.mjs" }, + devDependencies: { "@playwright/test": "1.63.0", typescript: "6.0.3", vite: "8.3.2", + esbuild: "0.28.2", stylelint: "17.16.0" } }; + const baseLock = { name: "fixture", version: "1.0.0", lockfileVersion: 3, requires: true, + packages: { "": { name: "fixture", version: "1.0.0", devDependencies: { ...basePackage.devDependencies } }, + ...Object.fromEntries(Object.entries(basePackage.devDependencies).map(([name, version]) => + [`node_modules/${name}`, { version, dev: true }])) } }; + const headPackage = structuredClone(basePackage); + headPackage.devDependencies.vite = "8.3.3"; + const headLock = structuredClone(baseLock); + headLock.packages[""].devDependencies.vite = "8.3.3"; + headLock.packages["node_modules/vite"].version = "8.3.3"; + const browserTargets = { query: "baseline-widely-available", + effectiveQuery: "Vite 8.3.2: baseline-widely-available", resolvedAt: "2026-10-11", + dataVersions: { vite: "8.3.2", esbuild: "0.28.2" }, browsers: ["chrome111", "safari16.4"], + javascript: ["chrome111", "safari16.4"], css: ["chrome111", "safari16.4"] }; + const baseSourceFiles = sourceFiles(basePackage, baseLock); + const baselineManifest = { version: 1, state: "final", + toolchain: { browser: "chromium", node: "24.19.0", npm: "12.0.2", playwright: "1.63.0" }, + sourceFingerprint: fingerprintScreenshotFiles(baseSourceFiles, browserTargets), + scenes: [{ id: "desktop-month-grid", output: SCENE_OUTPUT, route: "/examples/advanced/", + viewport: { width: 1440, height: 900 }, hasTouch: false, setup: ["wait-ready"], + alt: "Desktop advanced calendar example", references: ["README.md"], sha256: IMAGE_HASH }], + browserTargets }; + const candidateManifest = structuredClone(baselineManifest); + candidateManifest.toolchain.node = "24.21.0"; + candidateManifest.browserTargets.dataVersions.vite = "8.3.3"; + candidateManifest.browserTargets.effectiveQuery = "Vite 8.3.3: baseline-widely-available"; + candidateManifest.browserTargets.resolvedAt = "2026-10-12"; + const headSourceFiles = sourceFiles(headPackage, headLock); + candidateManifest.sourceFingerprint = fingerprintScreenshotFiles(headSourceFiles, candidateManifest.browserTargets); + return { repository: "owner/repo", pullRequest: { number: 38, state: "open", + user: { login: "dependabot[bot]" }, head: { sha: HEAD_SHA, repo: { full_name: "owner/repo" } }, + base: { sha: BASE_SHA, ref: "main", repo: { full_name: "owner/repo" } } }, + run: { id: 42, run_attempt: 1, event: "pull_request", repository: { full_name: "owner/repo" }, + head_sha: HEAD_SHA, created_at: "2026-10-12T01:00:00Z" }, + expectedCapture: { node: "24.21.0", resolvedAt: "2026-10-12" }, + report: { version: 1, repository: "owner/repo", prNumber: 38, headSha: HEAD_SHA, + baseSha: BASE_SHA, runId: 42, runAttempt: 1 }, + changedFiles: [{ filename: "package.json", status: "modified" }, + { filename: "package-lock.json", status: "modified" }], + basePackage, headPackage, baseLock, headLock, baselineManifest, + currentManifest: structuredClone(baselineManifest), candidateManifest, + canonicalHashes: { [SCENE_OUTPUT]: IMAGE_HASH }, baseCaptureHashes: { [SCENE_OUTPUT]: IMAGE_HASH }, + candidateHashes: { [SCENE_OUTPUT]: IMAGE_HASH }, sourceFiles: headSourceFiles, baseSourceFiles }; +} + +void test("promotion rebuilds provenance while retaining canonical scenes and leaving input data untouched", () => { + const input = fixture(); + const before = structuredClone(input); + const result = planScreenshotPromotion(input); + assert.equal(result.status, "promote", result.reason); + assert.deepEqual(result.manifest.scenes, input.baselineManifest.scenes); + assert.deepEqual(result.manifest.toolchain, input.candidateManifest.toolchain); + assert.deepEqual(result.manifest.browserTargets, input.candidateManifest.browserTargets); + assert.equal(result.manifest.sourceFingerprint, input.candidateManifest.sourceFingerprint); + assert.deepEqual(input, before); + result.manifest.scenes[0].alt = "changed result"; + assert.equal(input.baselineManifest.scenes[0].alt, "Desktop advanced calendar example"); +}); + +void test("already promoted current evidence does not create another commit", () => { + const input = fixture(); + input.currentManifest = planScreenshotPromotion(input).manifest; + input.changedFiles.push({ filename: "screenshots.manifest.json", status: "modified" }); + assert.deepEqual(planScreenshotPromotion(input), { status: "current" }); +}); + +void test("an unrelated locked linter update keeps the screenshot fingerprint current", () => { + const input = fixture(); + input.headPackage = structuredClone(input.basePackage); + input.headPackage.devDependencies.stylelint = "17.16.1"; + input.headLock = structuredClone(input.baseLock); + input.headLock.packages[""].devDependencies.stylelint = "17.16.1"; + input.headLock.packages["node_modules/stylelint"].version = "17.16.1"; + input.sourceFiles = sourceFiles(input.headPackage, input.headLock); + input.candidateManifest = structuredClone(input.baselineManifest); + input.candidateManifest.toolchain.node = "24.21.0"; + input.candidateManifest.browserTargets.resolvedAt = "2026-10-12"; + input.candidateManifest.sourceFingerprint = fingerprintScreenshotFiles(input.sourceFiles, + input.candidateManifest.browserTargets); + assert.equal(input.candidateManifest.sourceFingerprint, input.baselineManifest.sourceFingerprint); + assert.deepEqual(planScreenshotPromotion(input), { status: "current" }); +}); + +void test("a transitive lock-only update is eligible", () => { + const input = fixture(); + input.headPackage = structuredClone(input.basePackage); + input.headLock = structuredClone(input.baseLock); + input.headLock.packages["node_modules/source-map-js"] = { version: "1.2.2", dev: true }; + input.changedFiles = ["package-lock.json"]; + assert.equal(validateDependencyOnlyChanges(input).eligible, true); +}); + +void test("capture eligibility rejects changed scripts, added dependencies, and nondependency files", () => { + for (const mutate of [ + (input) => { input.headPackage.scripts["build:package"] = "node hostile.mjs"; }, + (input) => { input.headPackage.devDependencies["new-dependency"] = "1.0.0"; }, + (input) => { input.headPackage.devDependencies.vite = "^8.3.3"; }, + (input) => { input.changedFiles.push("scripts/hostile.mjs"); }, + (input) => { input.changedFiles[0].status = "renamed"; }, + (input) => { input.headLock.packages[""].devDependencies.vite = "8.3.4"; }, + (input) => { input.headLock.version = "2.0.0"; }, + (input) => { input.changedFiles = ["package-lock.json", "package-lock.json"]; } + ]) { + const input = fixture(); + mutate(input); + assert.equal(validateDependencyOnlyChanges(input).eligible, false); + assert.equal(planScreenshotPromotion(input).status, "ineligible"); + } +}); + +void test("author, repository, target, and current run head all bind capture eligibility", () => { + for (const mutate of [ + (input) => { input.pullRequest.user.login = "contributor"; }, + (input) => { input.pullRequest.state = "closed"; }, + (input) => { input.pullRequest.head.repo.full_name = "fork/repo"; }, + (input) => { input.pullRequest.base.ref = "release"; }, + (input) => { input.run.event = "workflow_dispatch"; }, + (input) => { input.run.repository.full_name = "fork/repo"; }, + (input) => { input.run.head_sha = BASE_SHA; } + ]) { + const input = fixture(); + mutate(input); + assert.equal(dependencyScreenshotEligibility(input).eligible, false); + } +}); + +void test("stale or forged artifact identities cannot promote current evidence", () => { + for (const mutate of [ + (input) => { input.report.headSha = BASE_SHA; }, + (input) => { input.report.baseSha = HEAD_SHA; }, + (input) => { input.report.repository = "fork/repo"; }, + (input) => { input.report.prNumber = 39; }, + (input) => { input.report.runId = 41; }, + (input) => { input.report.runAttempt = 2; }, + (input) => { input.report.command = "arbitrary code"; } + ]) { + const input = fixture(); + mutate(input); + assert.equal(planScreenshotPromotion(input).status, "ineligible"); + } +}); + +void test("unknown artifact fields and invalid capture provenance are rejected", () => { + for (const mutate of [ + (input) => { input.candidateManifest.command = "arbitrary code"; }, + (input) => { input.candidateManifest.scenes[0].command = "arbitrary code"; }, + (input) => { input.candidateManifest.toolchain.command = "arbitrary code"; }, + (input) => { input.candidateManifest.browserTargets.command = "arbitrary code"; }, + (input) => { input.candidateManifest.toolchain.node = "25.0.0"; }, + (input) => { input.candidateManifest.toolchain.npm = "12.0.3"; }, + (input) => { input.candidateManifest.toolchain.playwright = "1.62.0"; }, + (input) => { input.candidateManifest.browserTargets.dataVersions.vite = "8.3.4"; }, + (input) => { input.candidateManifest.browserTargets.resolvedAt = "2026-02-30"; }, + (input) => { input.candidateManifest.sourceFingerprint = CHANGED_IMAGE_HASH; }, + (input) => { input.baselineManifest.sourceFingerprint = CHANGED_IMAGE_HASH; } + ]) { + const input = fixture(); + mutate(input); + assert.equal(planScreenshotPromotion(input).status, "ineligible"); + } +}); + +void test("source inventories must be complete, unique, and tied to the exact package data", () => { + for (const mutate of [ + (input) => { input.sourceFiles = input.sourceFiles.filter((file) => file.path !== "package-lock.json"); }, + (input) => { input.sourceFiles.push(input.sourceFiles[0]); }, + (input) => { input.sourceFiles.push({ path: "../../hostile.mjs", contents: "code" }); }, + (input) => { input.sourceFiles.find((file) => file.path === "package.json").contents = JSON.stringify(input.basePackage); }, + (input) => { input.baseSourceFiles[0].contents = "{}"; } + ]) { + const input = fixture(); + mutate(input); + assert.equal(planScreenshotPromotion(input).status, "ineligible"); + } +}); + +void test("valid-looking runtime and date claims must match trusted workflow capture inputs", () => { + for (const mutate of [ + (input) => { delete input.expectedCapture; }, + (input) => { input.candidateManifest.toolchain.node = "24.999.999"; }, + (input) => { input.candidateManifest.browserTargets.resolvedAt = "1900-01-01"; }, + (input) => { input.expectedCapture.resolvedAt = "2026-10-11"; }, + (input) => { input.run.created_at = "2026-10-11T01:00:00Z"; } + ]) { + const input = fixture(); + mutate(input); + assert.equal(planScreenshotPromotion(input).status, "ineligible"); + } +}); + +void test("a valid target change requires review even if images are unchanged", () => { + const input = fixture(); + input.candidateManifest.browserTargets.css = ["chrome112", "safari16.4"]; + input.candidateManifest.sourceFingerprint = fingerprintScreenshotFiles(input.sourceFiles, + input.candidateManifest.browserTargets); + assert.deepEqual(planScreenshotPromotion(input), { status: "needs-review", reason: "Effective browser targets changed." }); +}); + +void test("changed scene setup requires review, and canonical scene edits in the PR are ineligible", () => { + const input = fixture(); + input.candidateManifest.scenes[0].setup.push("new-state"); + assert.equal(planScreenshotPromotion(input).status, "needs-review"); + input.currentManifest.scenes[0].sha256 = CHANGED_IMAGE_HASH; + assert.equal(planScreenshotPromotion(input).status, "ineligible"); +}); + +void test("changed or mismatched PNG bytes never authorize a provenance refresh", () => { + const changed = fixture(); + changed.candidateManifest.scenes[0].sha256 = CHANGED_IMAGE_HASH; + changed.candidateHashes[SCENE_OUTPUT] = CHANGED_IMAGE_HASH; + assert.deepEqual(planScreenshotPromotion(changed), { status: "needs-review", reason: "Candidate screenshots contain visual changes." }); + const forged = fixture(); + forged.candidateHashes[SCENE_OUTPUT] = CHANGED_IMAGE_HASH; + assert.equal(planScreenshotPromotion(forged).status, "ineligible"); + for (const key of ["canonicalHashes", "baseCaptureHashes"]) { + const drifted = fixture(); + drifted[key][SCENE_OUTPUT] = CHANGED_IMAGE_HASH; + assert.equal(planScreenshotPromotion(drifted).status, "needs-review"); + } + const unknown = fixture(); + unknown.candidateHashes["../../manifest.json"] = IMAGE_HASH; + assert.equal(planScreenshotPromotion(unknown).status, "ineligible"); +}); diff --git a/scripts/tests/workflow-contracts.test.mjs b/scripts/tests/workflow-contracts.test.mjs index e66de16..0db9a64 100644 --- a/scripts/tests/workflow-contracts.test.mjs +++ b/scripts/tests/workflow-contracts.test.mjs @@ -483,6 +483,7 @@ void test("all third-party workflow actions are pinned to full commits", async ( "ci.yml", "deploy-examples.yml", "publish-nightly.yml", + "refresh-screenshot-evidence.yml", "rollback-examples.yml" ]) { const source = await workflow(name); @@ -499,11 +500,65 @@ void test("CI rejects high-severity dependency regressions on pull requests", as assert.match(source, /^permissions:\s*\n\s*contents: read/mu); assert.match( verify, - /name: Reject high-severity dependency regressions[\s\S]*?if: \$\{\{ github\.event_name == 'pull_request' \}\}[\s\S]*?uses: actions\/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5\.0\.0[\s\S]*?fail-on-severity: high/u + /name: Reject high-severity dependency regressions[\s\S]*?if: \$\{\{ github\.event_name == 'pull_request' \|\| needs\.source\.outputs\.pr-number != '' \}\}[\s\S]*?uses: actions\/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5\.0\.0[\s\S]*?fail-on-severity: high/u ); + assert.match(verify, /base-ref: \$\{\{ needs\.source\.outputs\.base \}\}/u); + assert.match(verify, /head-ref: \$\{\{ needs\.source\.outputs\.head \}\}/u); assert.doesNotMatch(source, /pull-requests: write|issues: write/u); }); +void test("CI dispatch validation resolves an exact Dependabot head before installing dependencies", async () => { + const source = await workflow("ci.yml"); + const selection = job(source, "source"); + const verify = job(source, "verify"); + assert.match(trigger(source), /workflow_dispatch:[\s\S]*?pr-number:[\s\S]*?expected-head:/u); + assert.match(selection, /LFC_CHECKOUT_REF: \$\{\{ github\.sha \}\}/u); + assert.match(selection, /LFC_HEAD_SHA: \$\{\{ github\.event\.pull_request\.head\.sha \|\| github\.sha \}\}/u); + assert.match(selection, /LFC_BASE_SHA: \$\{\{ github\.event\.pull_request\.base\.sha \}\}/u); + assert.match(selection, /Check out trusted dispatch validation[\s\S]*?persist-credentials: false[\s\S]*?ref: \$\{\{ github\.event\.repository\.default_branch \}\}/u); + assert.match(selection, /github\.event_name == 'workflow_dispatch' && \(inputs\.pr-number != '' \|\| inputs\.expected-head != ''\)/u); + assert.match(selection, /LFC_PR_NUMBER: \$\{\{ inputs\.pr-number \}\}[\s\S]*?LFC_HEAD_SHA: \$\{\{ inputs\.expected-head \}\}[\s\S]*?node scripts\/screenshot-automation\.mjs verify-dispatch/u); + assert.doesNotMatch(selection, /npm ci|npm install|npm run|contents: write|id-token: write/u); + assert.match(verify, /name: Build, test, and verify package[\s\S]*?needs: source[\s\S]*?if: \$\{\{ always\(\) && !cancelled\(\) \}\}/u); + assert.match(verify, /Reject an invalid source selection[\s\S]*?needs\.source\.result[\s\S]*?test "\$\{LFC_SOURCE_RESULT\}" = success[\s\S]*?actions\/checkout@/u); + assert.match(verify, /ref: \$\{\{ github\.sha \}\}[\s\S]*?npm ci --ignore-scripts/u); + assert.doesNotMatch(verify, /ref: \$\{\{ needs\.source\.outputs\.checkout-ref \}\}/u); + assert.match(verify, /name: Run the CI quality gate\s*\n\s*id: quality-gate\s*\n\s*run: npm run check/u); +}); + +void test("Dependabot screenshot capture is read-only and retains changed candidates for human review", async () => { + const source = await workflow("ci.yml"); + const candidate = job(source, "screenshot-candidate"); + const verify = job(source, "verify"); + assert.match(candidate, /github\.event_name == 'pull_request'[\s\S]*?github\.event\.pull_request\.user\.login == 'dependabot\[bot\]'[\s\S]*?github\.event\.pull_request\.head\.repo\.full_name == github\.repository/u); + assert.match(candidate, /runs-on: windows-2025/u); + assert.match(candidate, /permissions:\s*\n\s*contents: read\s*\n\s*pull-requests: read/u); + assert.doesNotMatch(candidate, /(?:contents|actions|pull-requests|issues|id-token): write|git (?:push|commit)|screenshots:prepare|pull_request_target/u); + assert.match(candidate, /persist-credentials: false[\s\S]*?ref: \$\{\{ github\.event\.pull_request\.head\.sha \}\}/u); + assert.match(candidate, /candidate-eligibility[\s\S]*?npm ci --ignore-scripts[\s\S]*?npx playwright install chromium[\s\S]*?npm run screenshots:candidate[\s\S]*?screenshot-automation\.mjs seal-candidate/u); + assert.match(candidate, /Capture candidates for pixel comparison and human review[\s\S]*?LFC_BROWSER_TARGET_DATE: \$\{\{ steps\.eligibility\.outputs\.capture-date \}\}[\s\S]*?npm run screenshots:candidate/u); + assert.match(candidate, /Upload screenshot candidates and comparison gallery[\s\S]*?always\(\)[\s\S]*?screenshot-candidate-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}[\s\S]*?path: \.cache\/screenshots\/candidate\/[\s\S]*?include-hidden-files: true/u); + assert.match(verify, /id: dependency-package[\s\S]*?failure\(\) && steps\.quality-gate\.outcome == 'failure'[\s\S]*?dependabot\[bot\][\s\S]*?head\.repo\.full_name == github\.repository[\s\S]*?run: npm run check:tarball/u); +}); + +void test("screenshot promotion consumes a validated artifact without executing pull request code", async () => { + const source = await workflow("refresh-screenshot-evidence.yml"); + const promote = job(source, "promote"); + assert.match(trigger(source), /workflow_run:[\s\S]*?workflows:\s*\n\s*- CI[\s\S]*?types:\s*\n\s*- completed/u); + assert.doesNotMatch(trigger(source), /pull_request(?:_target)?:|workflow_dispatch:|push:/u); + assert.match(promote, /workflow_run\.event == 'pull_request'[\s\S]*?workflow_run\.head_repository\.full_name == github\.repository/u); + assert.match(promote, /permissions:\s*\n\s*contents: write\s*\n\s*pull-requests: read\s*\n\s*actions: write/u); + assert.doesNotMatch(source, /id-token: write|pull-requests: write|issues: write|secrets\.|npm (?:ci|install|run|publish)|npx |git push/u); + assert.equal(occurrences(promote, /actions\/checkout@/gu), 1); + assert.match(promote, /Check out trusted promotion code[\s\S]*?persist-credentials: false[\s\S]*?ref: \$\{\{ github\.sha \}\}/u); + assert.doesNotMatch(promote, /ref: \$\{\{ github\.event\.workflow_run\.head_sha \}\}/u); + assert.match(promote, /LFC_UPSTREAM_RUN_ID: \$\{\{ github\.event\.workflow_run\.id \}\}/u); + assert.match(promote, /LFC_UPSTREAM_RUN_ATTEMPT: \$\{\{ github\.event\.workflow_run\.run_attempt \}\}/u); + assert.match(promote, /LFC_UPSTREAM_HEAD_SHA: \$\{\{ github\.event\.workflow_run\.head_sha \}\}/u); + assert.match(promote, /screenshot-automation\.mjs promotion-context[\s\S]*?steps\.context\.outputs\.eligible == 'true'[\s\S]*?actions\/download-artifact@[0-9a-f]{40}[\s\S]*?run-id: \$\{\{ github\.event\.workflow_run\.id \}\}[\s\S]*?artifact-ids: \$\{\{ steps\.context\.outputs\.artifact-id \}\}[\s\S]*?screenshot-automation\.mjs promote/u); + assert.match(promote, /LFC_CANDIDATE_DIRECTORY: \$\{\{ runner\.temp \}\}\/screenshot-candidate/u); +}); + void test("hosted gates run Chromium and WebKit while local Playwright retains Firefox", async () => { const [ciSource, publishSource, packageSource] = await Promise.all([ workflow("ci.yml"), @@ -596,9 +651,10 @@ void test("Windows outer-sandbox mode changes only Firefox launch options", { void test("workflow artifacts use bounded purpose-specific retention", async () => { const expectations = new Map([ - ["ci.yml", [7]], + ["ci.yml", [7, 7]], ["deploy-examples.yml", [1, 1]], ["publish-nightly.yml", [7, 30, 30]], + ["refresh-screenshot-evidence.yml", []], ["rollback-examples.yml", [1]] ]); for (const [name, expected] of expectations) { @@ -612,6 +668,7 @@ void test("artifact downloads use the official Node 24 action", async () => { const workflows = [ "deploy-examples.yml", "publish-nightly.yml", + "refresh-screenshot-evidence.yml", "rollback-examples.yml" ]; const expected = @@ -624,7 +681,7 @@ void test("artifact downloads use the official Node 24 action", async () => { assert.equal(match[1], expected, name); } } - assert.equal(downloads, 5); + assert.equal(downloads, 6); }); void test("workflow dependency caches stay disabled", async () => { @@ -632,6 +689,7 @@ void test("workflow dependency caches stay disabled", async () => { "ci.yml", "deploy-examples.yml", "publish-nightly.yml", + "refresh-screenshot-evidence.yml", "rollback-examples.yml" ]) { const source = await workflow(name); diff --git a/scripts/update-screenshots.mjs b/scripts/update-screenshots.mjs index e785eb8..59b1663 100644 --- a/scripts/update-screenshots.mjs +++ b/scripts/update-screenshots.mjs @@ -5,9 +5,7 @@ import { chromium } from "@playwright/test"; import { resolveBrowserTargets } from "./lib/browser-targets.mjs"; import { REPOSITORY_ROOT } from "./lib/process.mjs"; -import { - replaceScreenshotBatch, validateScreenshotBatch, writeScreenshotReview -} from "./lib/screenshot-preparation.mjs"; +import { finalizeScreenshotCapture } from "./lib/screenshot-preparation.mjs"; import { assertSupportedNode, assertPinnedNpm, assertPinnedPlaywright, computeSourceFingerprint, readPngDimensions, readScreenshotManifest, REQUIRED_NPM_VERSION, @@ -20,7 +18,12 @@ import { startRepositoryServer } from "./serve-repository.mjs"; assertSupportedNode(); assertPinnedNpm(); await assertPinnedPlaywright(); -if (process.env["CI"]) { +const arguments_ = process.argv.slice(2); +if (arguments_.length > 1 || arguments_.some((argument) => argument !== "--candidate")) { + throw new Error("Usage: node scripts/update-screenshots.mjs [--candidate]"); +} +const candidate = arguments_.includes("--candidate"); +if (process.env["CI"] && !candidate) { throw new Error("CI must verify screenshot evidence; canonical captures can only be prepared locally."); } @@ -49,6 +52,9 @@ let repositoryServer; let browser; try { + if (candidate) { + await rm(join(cacheDirectory, "candidate"), { recursive: true, force: true }); + } const stagingDirectory = await mkdtemp(join(cacheDirectory, "staging-")); const sourceFingerprint = await computeSourceFingerprint(); repositoryServer = await startRepositoryServer({ port: 0 }); @@ -95,19 +101,13 @@ try { }; const finalErrors = validateScreenshotManifest(updatedManifest, { final: true }); if (finalErrors.length > 0) { throw new Error(finalErrors.join("\n")); } - await validateScreenshotBatch({ directory: stagingDirectory, manifest: updatedManifest, - readDimensions: readPngDimensions, hashFile: sha256File }); - if (await computeSourceFingerprint() !== sourceFingerprint) { - throw new Error("Screenshot sources changed during capture; canonical evidence was preserved. Retry after edits settle."); - } - const review = await writeScreenshotReview({ directory: join(cacheDirectory, "review"), - beforeDirectory: SCREENSHOT_DIRECTORY, afterDirectory: stagingDirectory, - previousManifest, manifest: updatedManifest }); - await replaceScreenshotBatch({ stagingDirectory, screenshotDirectory: SCREENSHOT_DIRECTORY, - manifestPath: SCREENSHOT_MANIFEST_PATH, manifest: updatedManifest }); + const { directory, review } = await finalizeScreenshotCapture({ cacheDirectory, stagingDirectory, + screenshotDirectory: SCREENSHOT_DIRECTORY, manifestPath: SCREENSHOT_MANIFEST_PATH, + previousManifest, manifest: updatedManifest, readDimensions: readPngDimensions, + hashFile: sha256File, computeSourceFingerprint, candidate }); await rm(stagingDirectory, { recursive: true, force: true }); - console.log(`Finalized ${updatedScenes.length} screenshots; ${review.changed.length} changed. ` + - `Review ${relative(REPOSITORY_ROOT, join(cacheDirectory, "review", "index.html"))}.`); + console.log(`${candidate ? "Prepared candidate for" : "Finalized"} ${updatedScenes.length} screenshots; ` + + `${review.changed.length} changed. Review ${relative(REPOSITORY_ROOT, join(directory, "index.html"))}.`); } finally { try { await browser?.close(); } finally {