diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7851752c7d9..94d7151bd64 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /src/scripts/webmcp.ts @cloudflare/ai-search @cloudflare/content-engineering /src/util/api.ts @cloudflare/content-engineering /src/util/search.ts @cloudflare/ai-search @cloudflare/content-engineering +/openapi.lock.json @cloudflare/content-engineering package.json @cloudflare/content-engineering # 1.1.1.1 diff --git a/.github/workflows/bump-openapi-schema.yml b/.github/workflows/bump-openapi-schema.yml new file mode 100644 index 00000000000..919f5257edb --- /dev/null +++ b/.github/workflows/bump-openapi-schema.yml @@ -0,0 +1,160 @@ +name: Bump OpenAPI schema + +# Weekly PR updating openapi.lock.json to the newest versioned schema snapshot +# in middlecache, so upstream schema changes reach the docs through review +# instead of breaking every open PR at once. The PR needs a content-engineering +# approval (see .github/CODEOWNERS); CI on the PR is the validation that every +# still resolves against the new schema. + +on: + schedule: + # Mondays 14:00 UTC — after the daily middlecache graduation (~23:13 UTC), + # so the run sees a complete weekend of upstream schema changes. + - cron: "0 14 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: bump-openapi-schema + cancel-in-progress: false + +jobs: + bump: + name: Check for schema update + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + # Pushes use the App token (below), never the workflow token. + persist-credentials: false + + - name: Set up pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5 + with: + version: 11 + + - name: Set up node + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: 24.x + + - name: Compare lock file against middlecache + id: bump + env: + SUMMARY_PATH: ${{ runner.temp }}/bump-summary.json + PR_BODY_PATH: ${{ runner.temp }}/pr-body.md + run: | + set -euo pipefail + # Dependency-free script (node builtins only), so it runs via dlx + # without installing the full dependency tree. + pnpm dlx tsx@4.23.15 bin/bump-openapi-lock.ts + cat "$SUMMARY_PATH" + + status=$(jq -r .status "$SUMMARY_PATH") + echo "status=$status" >> "$GITHUB_OUTPUT" + echo "new_sha=$(jq -r '.new_sha // empty' "$SUMMARY_PATH")" >> "$GITHUB_OUTPUT" + + if [ "$status" = "skipped" ]; then + echo "::warning::Schema bump skipped: $(jq -r .reason "$SUMMARY_PATH") (pin $(jq -r .sha "$SUMMARY_PATH"), middlecache latest $(jq -r .latest_sha "$SUMMARY_PATH"))" + fi + + - name: Generate App token + if: steps.bump.outputs.status == 'bumped' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.CLOUDFLARE_DOCS_BOT_APP_ID }} + private-key: ${{ secrets.CLOUDFLARE_DOCS_BOT_APP_PRIVATE_KEY }} + + - name: Open or update the bump PR + if: steps.bump.outputs.status == 'bumped' + env: + # PRs and pushes made with the App token (unlike GITHUB_TOKEN) + # trigger CI on the bump PR. + GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_REPO: ${{ github.repository }} + NEW_SHA: ${{ steps.bump.outputs.new_sha }} + PR_BODY_PATH: ${{ runner.temp }}/pr-body.md + run: | + set -euo pipefail + + BRANCH=bot/openapi-schema-bump + BASE=production + BOT_NAME="cloudflare-docs-bot[bot]" + BOT_EMAIL="cloudflare-docs-bot[bot]@users.noreply.github.com" + TITLE="chore: bump pinned OpenAPI schema to ${NEW_SHA:0:8}" + + # Supply the App token to a single git invocation via a header, the + # same way actions/checkout does, so it is never written to disk. + basic_auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 -w0) + echo "::add-mask::$basic_auth" + git_auth() { git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $basic_auth" "$@"; } + + open_pr=$(gh pr list --head "$BRANCH" --base "$BASE" --state open --json number --jq '.[0].number // empty') + + # The bump branch is reused across weeks. Only rewrite it when every + # non-merge commit on it is bot-authored; merges of production (the + # "Update branch" button) are safe to discard. + remote_sha=$(git ls-remote --heads origin "refs/heads/$BRANCH" | cut -f1) + if [ -n "$remote_sha" ]; then + human_commits=$(gh api "repos/$GH_REPO/compare/$BASE...$BRANCH" \ + | jq --arg bot "$BOT_EMAIL" '[.commits[] | select((.parents | length) == 1) | select(.commit.author.email != $bot)] | length') + if [ "$human_commits" -gt 0 ]; then + if [ -n "$open_pr" ]; then + echo "::warning::$BRANCH has manual commits; not force-pushing. Commented on #$open_pr." + gh pr comment "$open_pr" --body "A newer schema snapshot (\`$NEW_SHA\`) is available, but this branch has manual commits, so the bot did not force-push. Merge this PR (or drop the manual commits), then re-run the **Bump OpenAPI schema** workflow." + exit 0 + fi + echo "::error::$BRANCH has manual commits but no open PR. Open a PR for it or delete the branch, then re-run this workflow." + exit 1 + fi + fi + + # Rebuild the branch from the current production tip so the PR only + # ever contains the lock change. --force discards the working-tree + # lock (saved above), which git would otherwise refuse to overwrite + # if the lock differs between the checked-out commit and the tip. + cp openapi.lock.json "$RUNNER_TEMP/openapi.lock.json" + git fetch --no-tags --depth=1 origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" + git checkout --force -B "$BRANCH" "origin/$BASE" + cp "$RUNNER_TEMP/openapi.lock.json" openapi.lock.json + + if git diff --quiet -- openapi.lock.json; then + echo "$BASE already has this pin; nothing to do." + exit 0 + fi + + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + git add openapi.lock.json + git commit --no-verify -m "$TITLE" + + # Skip the push (and a redundant CI run) when the branch already has + # exactly this change on top of the current production tip. + same_tree=false + if [ -n "$remote_sha" ]; then + git fetch --no-tags --depth=2 origin "+refs/heads/$BRANCH:refs/remotes/origin/$BRANCH" + if [ "$(git rev-parse "origin/$BRANCH^{tree}")" = "$(git rev-parse "HEAD^{tree}")" ]; then + same_tree=true + fi + fi + + if [ "$same_tree" = true ]; then + echo "$BRANCH is already up to date; not pushing." + else + # The lease fails the push if someone pushed to the branch after + # the manual-commit check above. + git_auth push --force-with-lease="refs/heads/$BRANCH:$remote_sha" origin "HEAD:refs/heads/$BRANCH" + fi + + if [ -n "$open_pr" ]; then + gh pr edit "$open_pr" --title "$TITLE" --body-file "$PR_BODY_PATH" + echo "Updated #$open_pr" + else + gh pr create --base "$BASE" --head "$BRANCH" --title "$TITLE" --body-file "$PR_BODY_PATH" + fi diff --git a/AGENTS.md b/AGENTS.md index 5d4edf5fa06..4d487d55568 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,10 +45,19 @@ cloudflare-docs/ │ # skills/ is in .gitignore and is NOT committed to the repository. ├── .flue/ # Flue cloudflare-docs-bot — see .flue/AGENTS.md ├── astro.config.ts # Astro + Nimbus configuration +├── openapi.lock.json # Pinned Cloudflare API schema version (see "OpenAPI schema pinning") ├── package.json └── tsconfig.json ``` +## OpenAPI schema pinning + +`` renders against the Cloudflare API OpenAPI schema pinned by the repo-root `openapi.lock.json` (an upstream [`cloudflare/api-schemas`](https://github.com/cloudflare/api-schemas) commit SHA plus the snapshot's sha256). `prebuild`/`predev` download and verify the pinned snapshot from middlecache; a weekly workflow (`.github/workflows/bump-openapi-schema.yml` + `bin/bump-openapi-lock.ts`) opens a PR when a newer snapshot exists. Source: `src/util/openapi-schema.ts`. + +- Builds fail if an `` path/method does not exist in the pinned schema. Fix the page to match the current API, or merge the pending bump PR. +- Set `OPENAPI_SCHEMA=latest` to render against the newest published snapshot (escape hatch; CI always uses the pin). +- Snapshots expire after 365 days. A build that 404s on the pinned snapshot is on a stale branch — rebase onto `production`. + ## Content — writing and editing docs ### File locations diff --git a/bin/bump-openapi-lock.ts b/bin/bump-openapi-lock.ts new file mode 100644 index 00000000000..ec41d707af6 --- /dev/null +++ b/bin/bump-openapi-lock.ts @@ -0,0 +1,458 @@ +#!/usr/bin/env tsx + +/** + * Compare the pinned schema in openapi.lock.json against the newest versioned + * snapshot published by middlecache, verify the new snapshot's archive sha256, + * and rewrite the lock file. + * + * Outputs, for .github/workflows/bump-openapi-schema.yml: + * - $SUMMARY_PATH: JSON summary; `status` is "bumped", "up-to-date", or + * "skipped" (with a `reason`). + * - $PR_BODY_PATH: markdown PR body (only when status is "bumped"), with + * the operations added/removed upstream and any `` in + * src/content that no longer resolves. + * Either falls back to stdout when its env var is unset (local runs). + * + * Deliberately dependency-free (node builtins only, no repo imports) so the + * workflow can run it via `pnpm dlx tsx` without installing the dependency + * tree. + */ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/"; +const SCHEMA_BASE = "v1/cloudflare-api-schemas"; +const UPSTREAM_REPO = "https://github.com/cloudflare/api-schemas"; +const REPO_ROOT = fileURLToPath(new URL("../", import.meta.url)); +const LOCK_PATH = join(REPO_ROOT, "openapi.lock.json"); +const CONTENT_DIR = join(REPO_ROOT, "src", "content"); + +const SHA_PATTERN = /^[0-9a-f]{40}$/; +const SHA256_PATTERN = /^[0-9a-f]{64}$/; +const HTTP_METHODS = ["get", "put", "post", "delete", "patch", "head"]; +/** Keeps the PR body well under GitHub's 65,536-character limit. */ +const MAX_LISTED_OPERATIONS = 100; + +interface Lock { + sha: string; + sha256?: string; + committed_at: string; +} + +interface LatestManifest { + source_sha: string; + source_committed_at: string; +} + +interface VersionManifest extends LatestManifest { + path_count: number; + api_version: string; + files: { "openapi.tar.gz": { size_bytes: number; sha256: string } }; +} + +interface Snapshot { + sha: string; + manifest: VersionManifest; + operations: Set; +} + +interface DocsReference { + operation: string; + file: string; +} + +const snapshotUrl = (path: string) => + `${MIDDLECACHE_BASE_URL}${SCHEMA_BASE}/${path}`; + +const shortSha = (sha: string) => sha.slice(0, 8); + +const fetchOk = async (url: string): Promise => { + // Request the identity encoding so middlecache serves the bytes as-is + // rather than on-the-fly brotli. + const response = await fetch(url, { + headers: { "Accept-Encoding": "identity" }, + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + throw new Error( + `Failed to fetch ${url}: HTTP ${response.status} ${response.statusText}`, + ); + } + return response; +}; + +const fetchLatestManifest = async (): Promise => { + const manifest = await (await fetchOk(snapshotUrl("manifest.json"))).json(); + if (!SHA_PATTERN.test(String(manifest?.source_sha))) { + throw new Error( + `middlecache manifest.json reports no usable source_sha (${JSON.stringify(manifest?.source_sha)}) — is the versioned-snapshots pipeline deployed?`, + ); + } + if (Number.isNaN(Date.parse(String(manifest.source_committed_at)))) { + throw new Error( + `middlecache manifest.json reports no usable source_committed_at (${JSON.stringify(manifest.source_committed_at)}).`, + ); + } + return manifest; +}; + +const fetchVersionManifest = async (sha: string): Promise => { + const manifest: VersionManifest = await ( + await fetchOk(snapshotUrl(`versions/${sha}/manifest.json`)) + ).json(); + if (manifest.source_sha !== sha) { + throw new Error( + `middlecache manifest for ${sha} reports source_sha ${manifest.source_sha}; the snapshot is mislabeled.`, + ); + } + if ( + !SHA256_PATTERN.test(String(manifest.files?.["openapi.tar.gz"]?.sha256)) + ) { + throw new Error(`Version manifest for ${sha} has no usable sha256.`); + } + return manifest; +}; + +/** `METHOD /path` for every operation in an OpenAPI document. */ +const listOperations = (openapi: { + paths?: Record>; +}): Set => { + const operations = new Set(); + for (const [path, item] of Object.entries(openapi.paths ?? {})) { + for (const method of HTTP_METHODS) { + if (item?.[method]) { + operations.add(`${method.toUpperCase()} ${path}`); + } + } + } + return operations; +}; + +/** + * Download the snapshot for `sha`, verify its archive against + * `expectedSha256` (or the version manifest's), and list its operations. + */ +const loadSnapshot = async ( + sha: string, + scratchDir: string, + expectedSha256?: string, +): Promise => { + const manifest = await fetchVersionManifest(sha); + const manifestSha256 = manifest.files["openapi.tar.gz"].sha256; + if (expectedSha256 && expectedSha256 !== manifestSha256) { + throw new Error( + `Expected sha256 ${expectedSha256} for ${sha}, but its version manifest reports ${manifestSha256}. The snapshot may have been replaced — investigate before bumping.`, + ); + } + + const dir = join(scratchDir, sha); + fs.mkdirSync(dir, { recursive: true }); + const archivePath = join(dir, "openapi.tar.gz"); + const response = await fetchOk(snapshotUrl(`versions/${sha}/openapi.tar.gz`)); + await writeFile(archivePath, Buffer.from(await response.arrayBuffer())); + + const actualSha256 = createHash("sha256") + .update(await readFile(archivePath)) + .digest("hex"); + if (actualSha256 !== manifestSha256) { + throw new Error( + `Downloaded archive sha256 for ${sha} (${actualSha256}) does not match its version manifest (${manifestSha256}).`, + ); + } + + // The archive is verified above, and only the one named member is + // extracted into a private scratch directory. + const tar = spawnSync( + "tar", + ["-xzf", archivePath, "-C", dir, "openapi.json"], + { + encoding: "utf8", + }, + ); + if (tar.status !== 0 || tar.error) { + throw new Error( + `Could not extract openapi.json from the ${sha} snapshot: ${tar.stderr?.trim() || tar.error?.message}`, + ); + } + const openapi = JSON.parse(await readFile(join(dir, "openapi.json"), "utf8")); + return { sha, manifest, operations: listOperations(openapi) }; +}; + +const walkMdx = (dir: string): string[] => + fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name); + if (entry.isDirectory()) return walkMdx(path); + return entry.name.endsWith(".mdx") ? [path] : []; + }); + +/** + * Best-effort scan of `` usages. Every + * current usage passes both as string literals; anything else is skipped and + * still caught by the CI build. + */ +const scanDocsReferences = (): DocsReference[] => { + const references: DocsReference[] = []; + for (const file of walkMdx(CONTENT_DIR)) { + const blocks = fs.readFileSync(file, "utf8").split(" { + let raw: string; + try { + raw = fs.readFileSync(LOCK_PATH, "utf8"); + } catch { + return null; + } + const parsed = JSON.parse(raw) as Partial; + if (!parsed.sha || !SHA_PATTERN.test(parsed.sha)) { + throw new Error( + `openapi.lock.json has no usable sha (${JSON.stringify(parsed.sha)}) — fix or delete it before bumping.`, + ); + } + return parsed as Lock; +}; + +const commitLink = (sha: string) => + `[\`${shortSha(sha)}\`](${UPSTREAM_REPO}/commit/${sha})`; + +const operationList = (title: string, operations: string[]): string[] => { + if (operations.length === 0) return []; + const listed = operations.slice(0, MAX_LISTED_OPERATIONS); + const more = operations.length - listed.length; + return [ + "
", + `${title} (${operations.length})`, + "", + ...listed.map((op) => `- \`${op}\``), + ...(more > 0 ? [`- …and ${more} more`] : []), + "", + "
", + "", + ]; +}; + +const renderBody = ({ + lock, + next, + previous, + brokenReferences, + referenceCount, +}: { + lock: Lock | null; + next: Snapshot; + previous: Snapshot | null; + brokenReferences: DocsReference[]; + referenceCount: number; +}): string => { + const lines: string[] = ["## Summary", ""]; + + if (lock?.sha === next.sha) { + lines.push( + `- Adds the verified archive sha256 to the existing pin ${commitLink(next.sha)}.`, + "", + ); + } else { + lines.push( + `- Pins \`openapi.lock.json\` to ${commitLink(next.sha)} (committed ${next.manifest.source_committed_at}).`, + ); + if (lock) { + lines.push( + `- Previous pin: ${commitLink(lock.sha)} (committed ${lock.committed_at}).`, + `- Upstream diff: ${UPSTREAM_REPO}/compare/${lock.sha}...${next.sha}`, + ); + } + lines.push(""); + + const before = (value: string | number | undefined) => + previous ? String(value) : "n/a"; + lines.push( + "## Schema changes", + "", + "| | Before | After |", + "| --- | --- | --- |", + `| API version | ${before(previous?.manifest.api_version)} | ${next.manifest.api_version} |`, + `| Paths | ${before(previous?.manifest.path_count)} | ${next.manifest.path_count} |`, + `| Operations | ${before(previous?.operations.size)} | ${next.operations.size} |`, + "", + ); + + if (previous) { + const added = [...next.operations] + .filter((op) => !previous.operations.has(op)) + .sort(); + const removed = [...previous.operations] + .filter((op) => !next.operations.has(op)) + .sort(); + lines.push( + `- Operations added: ${added.length}`, + `- Operations removed: ${removed.length}`, + "", + ...operationList("Removed operations", removed), + ...operationList("Added operations", added), + ); + } else if (lock) { + lines.push( + "The previous snapshot is no longer on middlecache, so operations added/removed could not be computed.", + "", + ); + } + } + + lines.push("## Docs impact", ""); + if (brokenReferences.length === 0) { + lines.push( + `All ${referenceCount} \`\` usages in \`src/content\` resolve against the new schema (best-effort scan; the CI build is authoritative).`, + "", + ); + } else { + lines.push( + `${brokenReferences.length} \`\` usages do not resolve against the new schema. The build on this PR fails until they are updated:`, + "", + ...brokenReferences + .slice(0, MAX_LISTED_OPERATIONS) + .map((ref) => `- \`${ref.operation}\` in \`${ref.file}\``), + "", + ); + } + + lines.push( + "## Review", + "", + "- CI on this PR renders every `` against the new schema.", + "- If the build fails, fix the affected pages to match the upstream change. Do not revert the pin: snapshots expire after 365 days.", + "- Needs a `@cloudflare/content-engineering` approval (`.github/CODEOWNERS`).", + "", + ); + return lines.join("\n"); +}; + +const writeOutput = async (envVar: string, content: string) => { + const path = process.env[envVar]; + if (path) { + await writeFile(path, content, "utf8"); + } else { + console.log(content); + } +}; + +const finish = async (summary: Record, body?: string) => { + await writeOutput("SUMMARY_PATH", `${JSON.stringify(summary, null, 2)}\n`); + if (body !== undefined) { + await writeOutput("PR_BODY_PATH", body); + } +}; + +const main = async (): Promise => { + const lock = readCurrentLock(); + const latest = await fetchLatestManifest(); + const newSha = latest.source_sha; + + if (lock?.sha === newSha && lock.sha256) { + // Re-check the pin against middlecache weekly so a replaced snapshot + // is noticed before a build trips over it. + const manifest = await fetchVersionManifest(newSha); + if (manifest.files["openapi.tar.gz"].sha256 !== lock.sha256) { + throw new Error( + `openapi.lock.json sha256 (${lock.sha256}) does not match the middlecache manifest for ${newSha} (${manifest.files["openapi.tar.gz"].sha256}). The pinned snapshot may have been replaced — investigate before updating the lock.`, + ); + } + console.log(`Lock is up to date (${newSha}).`); + await finish({ status: "up-to-date", sha: newSha }); + return; + } + + if ( + lock && + lock.sha !== newSha && + Date.parse(latest.source_committed_at) < Date.parse(lock.committed_at) + ) { + console.warn( + `middlecache's latest snapshot ${newSha} (committed ${latest.source_committed_at}) is older than the pin ${lock.sha} (committed ${lock.committed_at}); not downgrading.`, + ); + await finish({ + status: "skipped", + reason: "downgrade", + sha: lock.sha, + latest_sha: newSha, + }); + return; + } + + const scratchDir = await mkdtemp(join(tmpdir(), "openapi-bump-")); + try { + // Verify the new archive before pinning it, so a bad publish can never + // enter the lock file. + const next = await loadSnapshot(newSha, scratchDir); + + let previous: Snapshot | null = null; + if (lock && lock.sha !== newSha) { + try { + previous = await loadSnapshot(lock.sha, scratchDir, lock.sha256); + } catch (err) { + console.warn( + `Could not load the previous snapshot ${lock.sha} for the diff: ${(err as Error).message}`, + ); + } + } + + const references = scanDocsReferences(); + const brokenReferences = references.filter( + (ref) => !next.operations.has(ref.operation), + ); + + const newLock: Lock = { + sha: newSha, + sha256: next.manifest.files["openapi.tar.gz"].sha256, + committed_at: next.manifest.source_committed_at, + }; + fs.writeFileSync( + LOCK_PATH, + `${JSON.stringify(newLock, null, "\t")}\n`, + "utf8", + ); + + console.log( + `Bumped lock: ${lock?.sha ?? "none"} -> ${newSha} (${brokenReferences.length} broken usages)`, + ); + await finish( + { + status: "bumped", + old_sha: lock?.sha ?? null, + new_sha: newSha, + committed_at: next.manifest.source_committed_at, + broken_references: brokenReferences.length, + }, + renderBody({ + lock, + next, + previous, + brokenReferences, + referenceCount: references.length, + }), + ); + } finally { + await rm(scratchDir, { recursive: true, force: true }); + } +}; + +main().catch((err: unknown) => { + console.error(`Error: ${err instanceof Error ? err.message : err}`); + process.exit(1); +}); diff --git a/bin/fetch-openapi.ts b/bin/fetch-openapi.ts index 485d7072a4d..ea355c6543e 100644 --- a/bin/fetch-openapi.ts +++ b/bin/fetch-openapi.ts @@ -1,17 +1,15 @@ #!/usr/bin/env tsx -import fs from "fs"; - import { fetchOpenApiSchema, - getOpenApiJsonPath, + getEffectiveSchemaId, + getSchemaMode, + readLock, } from "../src/util/openapi-schema"; // --soft: warn and continue on failure instead of exiting non-zero. // Used by the predev hook so a network failure doesn't block local development. -// --force: re-fetch even if the schema already exists. const soft = process.argv.includes("--soft"); -const force = process.argv.includes("--force"); const fail = (message: string): never => { if (soft) { @@ -24,21 +22,19 @@ const fail = (message: string): never => { process.exit(1); }; -const openapiFile = getOpenApiJsonPath(); +// The archive is cached in .tmp and verified on every run, so re-invoking this +// script never re-downloads unless the cache is missing or corrupted. +try { + const mode = + getSchemaMode() === "latest" + ? "OPENAPI_SCHEMA=latest" + : `pinned ${(await readLock()).sha}`; -if (fs.existsSync(openapiFile) && !force) { console.log( - "OpenAPI schema already exists, skipping fetch. (run `pnpm tsx bin/fetch-openapi.ts --force` to re-fetch)", + `Fetching Cloudflare API OpenAPI schema from middlecache (${mode})`, ); - process.exit(0); -} - -console.log("Fetching Cloudflare API OpenAPI schema from middlecache"); - -try { await fetchOpenApiSchema(); + console.log(`OpenAPI schema ready (${await getEffectiveSchemaId()})`); } catch (err) { fail(`fetch failed: ${err}`); } - -console.log("OpenAPI schema ready"); diff --git a/openapi.lock.json b/openapi.lock.json new file mode 100644 index 00000000000..d600fd935ed --- /dev/null +++ b/openapi.lock.json @@ -0,0 +1,5 @@ +{ + "sha": "c548640dcc3a2df3dca706dfaffbd41d616a44fb", + "sha256": "03638f649511e81b6f1bf673bce9c72964a7039f5d75eb9d39c2639a990d324d", + "committed_at": "2026-09-25T19:12:28Z" +} diff --git a/src/pages/[...slug].astro b/src/pages/[...slug].astro index ec5079fff22..c22ce4a0f9c 100644 --- a/src/pages/[...slug].astro +++ b/src/pages/[...slug].astro @@ -14,6 +14,7 @@ import { config } from "virtual:nimbus/config"; import { docsSidebarTransform, getCfBreadcrumbs } from "../util/sidebar"; import { components } from "../mdx-components"; import { getOgImage } from "~/util/og"; +import { getEffectiveSchemaId } from "~/util/openapi-schema"; import { getDirectoryEntryBySection } from "~/util/directory"; import { pageHasRuntimeHeadings, @@ -27,14 +28,27 @@ const NOINDEX_PRODUCTS = ["email-security"]; export const prerender = true; export const getStaticPaths = (async (...args: Parameters) => { const paths = await getDocsStaticPaths(...args); + // APIRequest output on docs pages is derived from the pinned OpenAPI schema + // rather than the page's MDX, so the schema id must ride along in the + // incremental-build cacheKey or a schema bump would serve stale curl examples. + const schemaId = await getEffectiveSchemaId(); // Docs pages whose URLs are owned by a custom route (not the catch-all). // These remain in the docs collection for sidebar metadata but should not // be rendered by [...slug].astro to avoid route conflicts. const blocklist = new Set(["waf/change-log/changelog"]); - return paths.filter((p) => { - const slug = p.params.slug; - return !(typeof slug === "string" && blocklist.has(slug)); - }); + return ( + paths + .filter((p) => { + const slug = p.params.slug; + return !(typeof slug === "string" && blocklist.has(slug)); + }) + // A path without a cacheKey opts out of reuse; keep it that way. + .map((p) => + p.cacheKey === undefined + ? p + : { ...p, cacheKey: `${p.cacheKey}:openapi-${schemaId}` }, + ) + ); }) satisfies GetStaticPaths; const { entry, Content, headings } = await getDocsPageProps(Astro); diff --git a/src/util/api.ts b/src/util/api.ts index 3752547a5b7..d6ffea7b8d2 100644 --- a/src/util/api.ts +++ b/src/util/api.ts @@ -1,12 +1,13 @@ /** * OpenAPI schema loader for the APIRequest component. * - * The schema is fetched by `bin/fetch-openapi.ts` from the `prebuild` and - * `prebuild:incremental` hooks (see package.json). `getSchema` reads the local - * copy and fails loudly if it is missing, so a build invoked without the - * pre-step is caught early instead of silently downloading mid-render. The - * dereferenced result is memoized so the deref runs once per build, not per - * component instance. + * The build renders against the schema pinned by `openapi.lock.json`; the + * snapshot is downloaded and verified by `bin/fetch-openapi.ts` from the + * `prebuild` and `prebuild:incremental` hooks (see package.json). `getSchema` + * reads the local copy and fails loudly if it is missing, so a build invoked + * without the pre-step is caught early instead of silently downloading + * mid-render. The dereferenced result is memoized so the deref runs once per + * build, not per component instance. */ import SwaggerParser from "@apidevtools/swagger-parser"; import type { OpenAPI } from "openapi-types"; @@ -16,14 +17,14 @@ import { getOpenApiJsonPath } from "./openapi-schema"; let schemaPromise: Promise | undefined; const loadSchema = async (): Promise => { - const openapiFile = getOpenApiJsonPath(); + const openapiFile = await getOpenApiJsonPath(); let raw: string; try { raw = await readFile(openapiFile, "utf8"); } catch (cause) { throw new Error( - `OpenAPI schema not found at ${openapiFile}. Run \`pnpm run build\` (or \`pnpm run build:incremental\`) so the prebuild hook fetches it first.`, + `OpenAPI schema not found at ${openapiFile}. Run \`pnpm run fetch:assets\` so the snapshot pinned by openapi.lock.json is downloaded first (the prebuild hook does this automatically).`, { cause }, ); } diff --git a/src/util/custom-loaders.ts b/src/util/custom-loaders.ts index 8562c89eb3d..4d40d51113d 100644 --- a/src/util/custom-loaders.ts +++ b/src/util/custom-loaders.ts @@ -21,8 +21,19 @@ const MAX_DOWNLOAD_ATTEMPTS = 3; // cleanup can delete a sibling call's freshly-written output. const inFlightDownloads = new Map>(); +/** Non-2xx response from a download; `status` lets callers special-case 404s. */ +export class HttpError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = "HttpError"; + } +} + /** - * Resolve the repo-root `.tmp/` directory used for downloaded artifacts. + * Resolve the repo-root directory (the one holding `package.json`). * * The tsx prebuild scripts and the bundled prerender resolve `import.meta.url` * to different locations (source files vs `dist/.prerender/chunks/`), so the @@ -30,19 +41,22 @@ const inFlightDownloads = new Map>(); * `package.json` is found. Falls back to the current working directory for * runtimes where `import.meta.url` is not a `file://` URL (e.g. Vitest). */ -export const getDotTmpPath = () => { +export const getRepoRoot = (): string => { try { const moduleDir = dirname(fileURLToPath(import.meta.url)); const root = findRepoRoot(moduleDir); if (root) { - return join(root, ".tmp"); + return root; } } catch { // not a file:// URL (e.g. under Vitest) } - return join(process.cwd(), ".tmp"); + return process.cwd(); }; +/** Repo-root `.tmp/` directory used for downloaded artifacts. */ +export const getDotTmpPath = () => join(getRepoRoot(), ".tmp"); + const findRepoRoot = (startDir: string): string | undefined => { let dir = startDir; for (;;) { @@ -130,8 +144,9 @@ const downloadWithRetry = async ( }); if (!response.ok) { - throw new Error( + throw new HttpError( `Failed to download ${url}: HTTP ${response.status} ${response.statusText}`, + response.status, ); } @@ -161,7 +176,9 @@ const downloadWithRetry = async ( } catch (err) { fs.rmSync(destination, { force: true }); fs.rmSync(`${destination}.tmp`, { force: true }); - if (attempt === MAX_DOWNLOAD_ATTEMPTS) { + // A 404 is not transient; retrying only delays the error. + const notFound = err instanceof HttpError && err.status === 404; + if (notFound || attempt === MAX_DOWNLOAD_ATTEMPTS) { throw err; } console.warn( diff --git a/src/util/openapi-schema.node.test.ts b/src/util/openapi-schema.node.test.ts new file mode 100644 index 00000000000..a264853e86e --- /dev/null +++ b/src/util/openapi-schema.node.test.ts @@ -0,0 +1,486 @@ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + test, + vi, +} from "vitest"; +import fs from "fs"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { dirname, join } from "node:path"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; + +const TEST_ROOT = await mkdtemp(join(tmpdir(), "openapi-schema-root-")); +const TEST_TMP = await mkdtemp(join(tmpdir(), "openapi-schema-tmp-")); +const TEST_SCHEMA_ROOT = join( + TEST_TMP, + "middlecache", + "v1", + "cloudflare-api-schemas", +); +const TEST_VERSIONS_DIR = join(TEST_SCHEMA_ROOT, "versions"); +const TEST_ARCHIVES_DIR = join(TEST_SCHEMA_ROOT, "archives"); +const TEST_LATEST_SOURCE = join(TEST_SCHEMA_ROOT, "latest-source.json"); + +// Hermetic overrides: the real getRepoRoot/getDotTmpPath resolve to the repo +// (and .tmp would collide with real dev caches), and the real download helper +// joins its destination with the real .tmp. Point both at temp dirs and stand +// in a minimal download helper with the same "skip if present, else fetch and +// validate" contract. +vi.mock("./custom-loaders", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getRepoRoot: () => TEST_ROOT, + getDotTmpPath: () => TEST_TMP, + downloadToDotTempIfNotPresent: async ( + url: string, + destinationPath: string, + options: { + validate?: (filePath: string) => Promise; + } = {}, + ) => { + const destination = join(TEST_TMP, destinationPath); + if (fs.existsSync(destination)) { + await options.validate?.(destination); + return; + } + fs.mkdirSync(dirname(destination), { recursive: true }); + const response = await fetch(url); + if (!response.ok) { + throw new actual.HttpError( + `Failed to download ${url}: HTTP ${response.status} ${response.statusText}`, + response.status, + ); + } + fs.writeFileSync(destination, Buffer.from(await response.arrayBuffer())); + await options.validate?.(destination); + }, + }; +}); + +const { + fetchOpenApiSchema, + getEffectiveSchemaId, + getLockPath, + getOpenApiJsonPath, + getSchemaMode, + readLock, +} = await import("./openapi-schema"); + +const SHA = "a".repeat(40); +const OTHER_SHA = "b".repeat(40); + +const lockFixture = { + sha: SHA, + committed_at: "2026-09-25T19:12:28Z", +}; + +const writeLock = (lock: Record) => { + fs.writeFileSync(join(TEST_ROOT, "openapi.lock.json"), JSON.stringify(lock)); +}; + +const buildSchemaArchive = (dir: string): { path: string; sha256: string } => { + fs.rmSync(dir, { recursive: true, force: true }); + fs.mkdirSync(join(dir, "src"), { recursive: true }); + fs.writeFileSync( + join(dir, "src", "openapi.json"), + JSON.stringify({ + openapi: "3.0.3", + info: { version: "4.0.0" }, + paths: {}, + }), + ); + fs.writeFileSync( + join(dir, "src", "openapi.yaml"), + "openapi: 3.0.3\ninfo:\n version: 4.0.0\n", + ); + const result = spawnSync( + "tar", + [ + "-czf", + join(dir, "openapi.tar.gz"), + "-C", + join(dir, "src"), + "openapi.json", + "openapi.yaml", + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + if (result.status !== 0 || result.error) { + throw new Error( + `Failed to build fixture archive: ${result.stderr?.toString()}`, + ); + } + return { + path: join(dir, "openapi.tar.gz"), + sha256: createHash("sha256") + .update(fs.readFileSync(join(dir, "openapi.tar.gz"))) + .digest("hex"), + }; +}; + +const COMMITTED_AT = "2026-09-25T19:12:28Z"; + +const versionManifest = (sha: string, archiveSha256: string) => ({ + openapi_version: "3.0.3", + api_version: "4.0.0", + path_count: 1, + source_sha: sha, + source_committed_at: COMMITTED_AT, + files: { + "openapi.tar.gz": { size_bytes: 1, sha256: archiveSha256 }, + }, +}); + +/** + * Stubbed middlecache. Serves the top-level manifest (pointing at `latestSha`), + * `versions/{sha}/manifest.json` for each entry in `versions`, and the fixture + * archive for any known version. Everything else 404s. + */ +const serveMiddlecache = ({ + versions, + latestSha, +}: { + versions: Record; + latestSha?: string; +}) => + vi.fn(async (input: RequestInfo | URL) => { + const href = String(input instanceof Request ? input.url : input); + const path = new URL(href).pathname.replace( + "/v1/cloudflare-api-schemas/", + "", + ); + if (path === "manifest.json" && latestSha) { + return Response.json({ + source_sha: latestSha, + source_committed_at: COMMITTED_AT, + }); + } + const match = path.match(/^versions\/([0-9a-f]{40})\/(.+)$/); + const version = match && versions[match[1]]; + if (match && version) { + if (match[2] === "manifest.json") { + return Response.json( + versionManifest(version.manifestSha ?? match[1], version.sha256), + ); + } + if (match[2] === "openapi.tar.gz") { + return new Response(fs.readFileSync(fixtureArchive.path)); + } + } + return new Response("not found", { status: 404, statusText: "Not Found" }); + }); + +const fetchedPaths = (fetchMock: ReturnType) => + fetchMock.mock.calls.map((call) => + new URL(String(call[0])).pathname.replace( + "/v1/cloudflare-api-schemas/", + "", + ), + ); + +const fixtureArchive = { path: "", sha256: "" }; + +beforeAll(() => { + const built = buildSchemaArchive(join(TEST_TMP, "__archive-fixture__")); + fixtureArchive.path = built.path; + fixtureArchive.sha256 = built.sha256; +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + fs.rmSync(TEST_SCHEMA_ROOT, { recursive: true, force: true }); +}); + +afterAll(() => { + fs.rmSync(TEST_ROOT, { recursive: true, force: true }); + fs.rmSync(TEST_TMP, { recursive: true, force: true }); +}); + +describe("getSchemaMode", () => { + test("defaults to pinned", () => { + expect(getSchemaMode()).toBe("pinned"); + }); + + test("follows OPENAPI_SCHEMA=latest", () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + expect(getSchemaMode()).toBe("latest"); + }); + + test("any other value still pins", () => { + vi.stubEnv("OPENAPI_SCHEMA", "pinned"); + expect(getSchemaMode()).toBe("pinned"); + }); +}); + +describe("readLock", () => { + test("reads a valid lock file", async () => { + writeLock({ ...lockFixture, sha256: "c".repeat(64) }); + const lock = await readLock(); + expect(lock.sha).toBe(SHA); + expect(lock.sha256).toBe("c".repeat(64)); + }); + + test("exposes the lock path at the repo root", () => { + expect(getLockPath()).toBe(join(TEST_ROOT, "openapi.lock.json")); + }); + + test("throws when the lock file is missing", async () => { + fs.rmSync(join(TEST_ROOT, "openapi.lock.json"), { force: true }); + await expect(readLock()).rejects.toThrow(/lock file not found/); + }); + + test("throws on a malformed SHA", async () => { + writeLock({ ...lockFixture, sha: "not-a-sha" }); + await expect(readLock()).rejects.toThrow(/invalid/); + }); + + test("throws on a malformed sha256", async () => { + writeLock({ ...lockFixture, sha256: "xyz" }); + await expect(readLock()).rejects.toThrow(/invalid/); + }); + + test("throws on a malformed committed_at", async () => { + writeLock({ ...lockFixture, committed_at: "last tuesday" }); + await expect(readLock()).rejects.toThrow(/invalid/); + }); +}); + +describe("getOpenApiJsonPath", () => { + test("points at the pinned version's extraction directory", async () => { + writeLock(lockFixture); + expect(await getOpenApiJsonPath()).toBe( + join(TEST_VERSIONS_DIR, SHA, "openapi.json"), + ); + }); + + test("points at the resolved latest version in latest mode", async () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + fs.mkdirSync(TEST_SCHEMA_ROOT, { recursive: true }); + fs.writeFileSync( + TEST_LATEST_SOURCE, + JSON.stringify({ + source_sha: OTHER_SHA, + source_committed_at: COMMITTED_AT, + }), + ); + expect(await getOpenApiJsonPath()).toBe( + join(TEST_VERSIONS_DIR, OTHER_SHA, "openapi.json"), + ); + }); + + test("throws in latest mode when nothing has been fetched", async () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + await expect(getOpenApiJsonPath()).rejects.toThrow( + /no latest schema has been fetched/, + ); + }); +}); + +describe("getEffectiveSchemaId", () => { + test("returns the pinned SHA", async () => { + writeLock(lockFixture); + expect(await getEffectiveSchemaId()).toBe(SHA); + }); + + test("returns the resolved latest SHA in latest mode", async () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + fs.mkdirSync(TEST_SCHEMA_ROOT, { recursive: true }); + fs.writeFileSync( + TEST_LATEST_SOURCE, + JSON.stringify({ + source_sha: OTHER_SHA, + source_committed_at: COMMITTED_AT, + }), + ); + expect(await getEffectiveSchemaId()).toBe(OTHER_SHA); + }); + + test("falls back to a stable id when the latest source is unknown", async () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + expect(await getEffectiveSchemaId()).toBe("unknown"); + }); +}); + +describe("fetchOpenApiSchema (pinned)", () => { + test("downloads, verifies, and extracts without a manifest lookup when the lock has sha256", async () => { + writeLock({ ...lockFixture, sha256: fixtureArchive.sha256 }); + const fetchMock = serveMiddlecache({ + versions: { [SHA]: { sha256: fixtureArchive.sha256 } }, + }); + vi.stubGlobal("fetch", fetchMock); + + const jsonPath = await fetchOpenApiSchema(); + + expect(jsonPath).toBe(join(TEST_VERSIONS_DIR, SHA, "openapi.json")); + expect(JSON.parse(fs.readFileSync(jsonPath, "utf8"))).toMatchObject({ + openapi: "3.0.3", + }); + expect(fetchedPaths(fetchMock)).toEqual([`versions/${SHA}/openapi.tar.gz`]); + }); + + test("re-verifies a cached archive with no network access", async () => { + writeLock({ ...lockFixture, sha256: fixtureArchive.sha256 }); + const fetchMock = serveMiddlecache({ + versions: { [SHA]: { sha256: fixtureArchive.sha256 } }, + }); + vi.stubGlobal("fetch", fetchMock); + + await fetchOpenApiSchema(); + fetchMock.mockClear(); + await fetchOpenApiSchema(); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + test("fails loudly when the archive does not match the lock's sha256", async () => { + writeLock({ ...lockFixture, sha256: "d".repeat(64) }); + vi.stubGlobal( + "fetch", + serveMiddlecache({ versions: { [SHA]: { sha256: "d".repeat(64) } } }), + ); + + await expect(fetchOpenApiSchema()).rejects.toThrow(/sha256 mismatch/); + }); + + test("fails loudly when a cached archive was tampered with", async () => { + writeLock({ ...lockFixture, sha256: fixtureArchive.sha256 }); + vi.stubGlobal( + "fetch", + serveMiddlecache({ + versions: { [SHA]: { sha256: fixtureArchive.sha256 } }, + }), + ); + await fetchOpenApiSchema(); + + fs.appendFileSync(join(TEST_ARCHIVES_DIR, `${SHA}.openapi.tar.gz`), "x"); + // The test download helper does not retry, so the corrupt cache must + // surface as a sha256 failure rather than be silently re-used. + await expect(fetchOpenApiSchema()).rejects.toThrow(/sha256 mismatch/); + }); + + test("verifies against the version manifest when the lock has no sha256", async () => { + writeLock(lockFixture); + const fetchMock = serveMiddlecache({ + versions: { [SHA]: { sha256: fixtureArchive.sha256 } }, + }); + vi.stubGlobal("fetch", fetchMock); + + const jsonPath = await fetchOpenApiSchema(); + + expect(JSON.parse(fs.readFileSync(jsonPath, "utf8"))).toMatchObject({ + info: { version: "4.0.0" }, + }); + expect(fetchedPaths(fetchMock)).toEqual([ + `versions/${SHA}/manifest.json`, + `versions/${SHA}/openapi.tar.gz`, + ]); + }); + + test("fails when the version manifest is mislabeled", async () => { + writeLock(lockFixture); + vi.stubGlobal( + "fetch", + serveMiddlecache({ + versions: { + [SHA]: { manifestSha: OTHER_SHA, sha256: fixtureArchive.sha256 }, + }, + }), + ); + + await expect(fetchOpenApiSchema()).rejects.toThrow(/mislabeled/); + }); + + test("explains retention when the pinned snapshot is gone", async () => { + writeLock({ ...lockFixture, sha256: fixtureArchive.sha256 }); + vi.stubGlobal("fetch", serveMiddlecache({ versions: {} })); + + const error = await fetchOpenApiSchema().catch((err: Error) => err); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/expire after 365 days/); + expect((error as Error).message).toMatch(/rebase onto production/); + expect(String((error as Error).cause)).toMatch(/HTTP 404/); + }); + + test("explains retention when the lock has no sha256 and the manifest is gone", async () => { + writeLock(lockFixture); + vi.stubGlobal("fetch", serveMiddlecache({ versions: {} })); + + await expect(fetchOpenApiSchema()).rejects.toThrow(/expire after 365 days/); + }); +}); + +describe("fetchOpenApiSchema (latest)", () => { + test("resolves the newest SHA, fetches that versioned snapshot, and records it", async () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + const fetchMock = serveMiddlecache({ + latestSha: OTHER_SHA, + versions: { [OTHER_SHA]: { sha256: fixtureArchive.sha256 } }, + }); + vi.stubGlobal("fetch", fetchMock); + + const jsonPath = await fetchOpenApiSchema(); + + expect(jsonPath).toBe(join(TEST_VERSIONS_DIR, OTHER_SHA, "openapi.json")); + expect(fs.existsSync(jsonPath)).toBe(true); + expect(fetchedPaths(fetchMock)).toEqual([ + "manifest.json", + `versions/${OTHER_SHA}/manifest.json`, + `versions/${OTHER_SHA}/openapi.tar.gz`, + ]); + expect(JSON.parse(fs.readFileSync(TEST_LATEST_SOURCE, "utf8"))).toEqual({ + source_sha: OTHER_SHA, + source_committed_at: COMMITTED_AT, + }); + expect(await getOpenApiJsonPath()).toBe(jsonPath); + expect(await getEffectiveSchemaId()).toBe(OTHER_SHA); + }); + + test("leaves pinned snapshots and cached archives intact", async () => { + writeLock({ ...lockFixture, sha256: fixtureArchive.sha256 }); + vi.stubGlobal( + "fetch", + serveMiddlecache({ + latestSha: OTHER_SHA, + versions: { + [SHA]: { sha256: fixtureArchive.sha256 }, + [OTHER_SHA]: { sha256: fixtureArchive.sha256 }, + }, + }), + ); + await fetchOpenApiSchema(); + + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + await fetchOpenApiSchema(); + + expect(fs.existsSync(join(TEST_VERSIONS_DIR, SHA, "openapi.json"))).toBe( + true, + ); + expect( + fs.existsSync(join(TEST_ARCHIVES_DIR, `${SHA}.openapi.tar.gz`)), + ).toBe(true); + expect( + fs.existsSync(join(TEST_VERSIONS_DIR, OTHER_SHA, "openapi.json")), + ).toBe(true); + }); + + test("does not record a latest SHA when the snapshot fetch fails", async () => { + vi.stubEnv("OPENAPI_SCHEMA", "latest"); + vi.stubGlobal( + "fetch", + serveMiddlecache({ + latestSha: OTHER_SHA, + versions: { [OTHER_SHA]: { sha256: "d".repeat(64) } }, + }), + ); + + await expect(fetchOpenApiSchema()).rejects.toThrow(/sha256 mismatch/); + expect(fs.existsSync(TEST_LATEST_SOURCE)).toBe(false); + }); +}); diff --git a/src/util/openapi-schema.ts b/src/util/openapi-schema.ts index 0ffd5ab5fa9..2e4cf6168dd 100644 --- a/src/util/openapi-schema.ts +++ b/src/util/openapi-schema.ts @@ -1,86 +1,319 @@ /** * Download and extract the Cloudflare API OpenAPI schema from middlecache. * - * Used by `bin/fetch-openapi.ts`, which runs from the `prebuild`, - * `prebuild:incremental`, and `predev` hooks (see package.json). + * Builds render `` against a schema pinned by the repo-root + * `openapi.lock.json` (an upstream `cloudflare/api-schemas` commit SHA plus + * the snapshot's sha256), so schema changes land through a reviewed bump PR + * instead of breaking every build at once. `bin/fetch-openapi.ts` — run from + * the prebuild, `prebuild:incremental`, and predev hooks — downloads the + * pinned snapshot and verifies its sha256. + * + * Set `OPENAPI_SCHEMA=latest` to render against the newest published snapshot + * instead (escape hatch, not used in CI). Latest mode resolves the newest + * snapshot's SHA from middlecache's top-level manifest and then fetches that + * versioned snapshot exactly like a pin, so both modes share one verified + * on-disk layout: + * + * .tmp/middlecache/v1/cloudflare-api-schemas/ + * archives/{sha}.openapi.tar.gz downloaded archive (cache) + * versions/{sha}/openapi.json extracted schema + * latest-source.json SHA resolved by the last latest-mode fetch + * + * The effective schema id (pinned SHA, or the resolved latest SHA) feeds + * Astro's incremental-build cacheKey in `src/pages/[...slug].astro` so a + * schema bump invalidates cached pages whose curl examples are derived from it. */ -import { mkdir, readFile, rename, rm } from "node:fs/promises"; -import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import * as z from "zod"; import { + HttpError, downloadToDotTempIfNotPresent, extractTarGz, getDotTmpPath, + getRepoRoot, } from "./custom-loaders"; const MIDDLECACHE_BASE_URL = "https://middlecache.ced.cloudflare.com/"; -const OPENAPI_ARCHIVE_PATH = "v1/cloudflare-api-schemas/openapi.tar.gz"; -const OPENAPI_JSON_PATH = "v1/cloudflare-api-schemas/openapi.json"; +const SCHEMA_BASE = "v1/cloudflare-api-schemas"; + +/** Snapshots older than this are expired by an R2 lifecycle rule. */ +const SNAPSHOT_RETENTION_DAYS = 365; + +const SHA_PATTERN = /^[0-9a-f]{40}$/; +const gitSha = z.string().regex(SHA_PATTERN); +const sha256Hex = z.string().regex(/^[0-9a-f]{64}$/); +const timestamp = z.iso.datetime({ offset: true }); + +const lockSchema = z.object({ + /** Upstream cloudflare/api-schemas commit SHA the build pins to. */ + sha: gitSha, + /** + * sha256 of the pinned snapshot's openapi.tar.gz, set by + * `bin/bump-openapi-lock.ts` after verifying the archive. When present it + * is authoritative and the build needs no manifest lookup; when absent + * (a hand-edited pin) the version manifest supplies it. + */ + sha256: sha256Hex.optional(), + /** Upstream commit time of the pinned SHA (RFC 3339). */ + committed_at: timestamp, +}); + +export type OpenApiLock = z.infer; + +const versionManifestSchema = z.object({ + source_sha: gitSha, + source_committed_at: timestamp, + files: z.object({ + "openapi.tar.gz": z.object({ + size_bytes: z.number(), + sha256: sha256Hex, + }), + }), +}); + +const latestSourceSchema = z.object({ + source_sha: gitSha, + source_committed_at: timestamp, +}); -export const getOpenApiExtractDir = () => - join(getDotTmpPath(), "middlecache", "v1", "cloudflare-api-schemas"); +type LatestSource = z.infer; -export const getOpenApiJsonPath = () => - join(getOpenApiExtractDir(), "openapi.json"); +export type SchemaMode = "pinned" | "latest"; + +export const getSchemaMode = (): SchemaMode => + process.env.OPENAPI_SCHEMA === "latest" ? "latest" : "pinned"; + +export const getLockPath = (): string => + join(getRepoRoot(), "openapi.lock.json"); /** - * Download the schema into `.tmp` and return the path to `openapi.json`. - * - * Prefers the gzip-compressed archive (integrity-checkable via gzip CRC32), - * falling back to the raw openapi.json. Downloads are validated (extract + - * parse) and retried by `downloadToDotTempIfNotPresent`. + * Read and validate the schema pin. Fails loudly in pinned mode (the default) + * so a build never silently renders against an unintended schema. */ -export const fetchOpenApiSchema = async (): Promise => { - const extractDir = getOpenApiExtractDir(); +export const readLock = async (): Promise => { + const lockPath = getLockPath(); + let raw: string; try { - await downloadToDotTempIfNotPresent( - `${MIDDLECACHE_BASE_URL}${OPENAPI_ARCHIVE_PATH}`, - `middlecache/${OPENAPI_ARCHIVE_PATH}`, - { - validate: async (archivePath) => { - // Extract into a staging directory, then promote it on success - // so a failed extract or parse never leaves stale or partial - // files in the destination that could mask a fresh failure. - const stagingDir = join( - getDotTmpPath(), - "middlecache", - "v1", - ".openapi-staging", - ); - await rm(stagingDir, { recursive: true, force: true }); - try { - await mkdir(stagingDir, { recursive: true }); - await extractTarGz(archivePath, stagingDir); - const raw = await readFile( - join(stagingDir, "openapi.json"), - "utf8", - ); - void JSON.parse(raw); - await rm(extractDir, { recursive: true, force: true }); - await rename(stagingDir, extractDir); - } catch (err) { - await rm(stagingDir, { recursive: true, force: true }); - throw err; - } - }, - }, + raw = await readFile(lockPath, "utf8"); + } catch (cause) { + throw new Error(`OpenAPI schema lock file not found at ${lockPath}.`, { + cause, + }); + } + + let json: unknown; + try { + json = JSON.parse(raw); + } catch (cause) { + throw new Error(`openapi.lock.json is not valid JSON (${lockPath}).`, { + cause, + }); + } + + const parsed = lockSchema.safeParse(json); + if (!parsed.success) { + throw new Error(`openapi.lock.json is invalid: ${parsed.error.message}`); + } + return parsed.data; +}; + +const schemaRoot = () => join(getDotTmpPath(), "middlecache", SCHEMA_BASE); +const versionDir = (sha: string) => join(schemaRoot(), "versions", sha); +// Outside versions/ so a leftover staging dir never looks like a snapshot. +const stagingDir = (sha: string) => join(schemaRoot(), ".staging", sha); +const latestSourcePath = () => join(schemaRoot(), "latest-source.json"); + +/** Relative to `.tmp/`, as `downloadToDotTempIfNotPresent` expects. */ +const archiveDestination = (sha: string) => + `middlecache/${SCHEMA_BASE}/archives/${sha}.openapi.tar.gz`; + +const readLatestSource = async (): Promise => { + try { + return latestSourceSchema.parse( + JSON.parse(await readFile(latestSourcePath(), "utf8")), ); - } catch (err) { - console.warn( - `Failed to fetch OpenAPI archive, falling back to raw openapi.json: ${(err as Error).message}`, + } catch { + return undefined; + } +}; + +/** Path of the extracted openapi.json for the active schema mode. */ +export const getOpenApiJsonPath = async (): Promise => { + if (getSchemaMode() === "pinned") { + return join(versionDir((await readLock()).sha), "openapi.json"); + } + const latest = await readLatestSource(); + if (!latest) { + throw new Error( + `OPENAPI_SCHEMA=latest is set but no latest schema has been fetched (${latestSourcePath()} is missing). Run \`OPENAPI_SCHEMA=latest pnpm run fetch:assets\` first.`, + ); + } + return join(versionDir(latest.source_sha), "openapi.json"); +}; + +/** + * Stable identifier for the schema the build renders against: the pinned SHA, + * or the SHA resolved by the last latest-mode fetch. "unknown" (only possible + * in latest mode before `bin/fetch-openapi.ts` has ever succeeded) keeps the + * cache key stable until the schema is fetched. + */ +export const getEffectiveSchemaId = async (): Promise => { + if (getSchemaMode() === "pinned") { + return (await readLock()).sha; + } + return (await readLatestSource())?.source_sha ?? "unknown"; +}; + +const sha256File = async (filePath: string): Promise => + createHash("sha256") + .update(await readFile(filePath)) + .digest("hex"); + +/** Fetch a middlecache JSON document and validate it against `schema`. */ +const fetchManifest = async ( + path: string, + schema: z.ZodType, +): Promise => { + const url = `${MIDDLECACHE_BASE_URL}${SCHEMA_BASE}/${path}`; + // Request the identity encoding so middlecache serves the bytes as-is + // rather than on-the-fly brotli (see downloadToDotTempIfNotPresent). + const response = await fetch(url, { + headers: { "Accept-Encoding": "identity" }, + }); + if (!response.ok) { + throw new HttpError( + `Failed to fetch ${url}: HTTP ${response.status} ${response.statusText}`, + response.status, + ); + } + + let json: unknown; + try { + json = await response.json(); + } catch (cause) { + throw new Error(`${url} is not valid JSON.`, { cause }); + } + + const parsed = schema.safeParse(json); + if (!parsed.success) { + throw new Error(`${url} is invalid: ${parsed.error.message}`); + } + return parsed.data; +}; + +/** Resolve the archive sha256 for `sha` from its write-once version manifest. */ +const fetchVersionSha256 = async (sha: string): Promise => { + const manifest = await fetchManifest( + `versions/${sha}/manifest.json`, + versionManifestSchema, + ); + if (manifest.source_sha !== sha) { + throw new Error( + `middlecache manifest for ${sha} reports source_sha ${manifest.source_sha}; the snapshot is mislabeled.`, ); - await downloadToDotTempIfNotPresent( - `${MIDDLECACHE_BASE_URL}${OPENAPI_JSON_PATH}`, - `middlecache/${OPENAPI_JSON_PATH}`, - { - validate: async (filePath) => { - const raw = await readFile(filePath, "utf8"); - void JSON.parse(raw); - }, + } + return manifest.files["openapi.tar.gz"].sha256; +}; + +/** + * Extract an already-verified archive into a per-SHA staging directory and + * promote it on success, so a failed extract or parse never leaves partial + * files in the destination that could mask a fresh failure. + */ +const extractVerified = async (archivePath: string, sha: string) => { + const staging = stagingDir(sha); + const destination = versionDir(sha); + + await rm(staging, { recursive: true, force: true }); + try { + await mkdir(staging, { recursive: true }); + await extractTarGz(archivePath, staging); + void JSON.parse(await readFile(join(staging, "openapi.json"), "utf8")); + await rm(destination, { recursive: true, force: true }); + await mkdir(dirname(destination), { recursive: true }); + await rename(staging, destination); + } catch (err) { + await rm(staging, { recursive: true, force: true }); + throw err; + } +}; + +/** + * Download (or re-verify the cached copy of) the snapshot for `sha`, check its + * sha256, and extract it to `versions/{sha}/`. + */ +const fetchVersion = async (sha: string, expectedSha256: string) => { + await downloadToDotTempIfNotPresent( + `${MIDDLECACHE_BASE_URL}${SCHEMA_BASE}/versions/${sha}/openapi.tar.gz`, + archiveDestination(sha), + { + validate: async (archivePath) => { + const actual = await sha256File(archivePath); + if (actual !== expectedSha256) { + throw new Error( + `sha256 mismatch for schema snapshot ${sha}: expected ${expectedSha256}, got ${actual}`, + ); + } + await extractVerified(archivePath, sha); }, + }, + ); +}; + +const fetchPinnedSchema = async (): Promise => { + const lock = await readLock(); + try { + // The lock's sha256 was verified against the archive by the bump job, + // so a cached archive can be re-verified without any network access. + await fetchVersion( + lock.sha, + lock.sha256 ?? (await fetchVersionSha256(lock.sha)), ); + } catch (err) { + if (err instanceof HttpError && err.status === 404) { + throw new Error( + `Pinned OpenAPI snapshot ${lock.sha} (committed ${lock.committed_at}) is not on middlecache. Snapshots expire after ${SNAPSHOT_RETENTION_DAYS} days; if this branch is old, rebase onto production to pick up the current pin.`, + { cause: err }, + ); + } + throw err; } + return join(versionDir(lock.sha), "openapi.json"); +}; + +const fetchLatestSchema = async (): Promise => { + const latest = await fetchManifest("manifest.json", latestSourceSchema); + await fetchVersion( + latest.source_sha, + await fetchVersionSha256(latest.source_sha), + ); + + // Written last, and atomically, so getOpenApiJsonPath only ever points at + // a fully extracted snapshot. + const record: LatestSource = { + source_sha: latest.source_sha, + source_committed_at: latest.source_committed_at, + }; + const tmpPath = `${latestSourcePath()}.tmp`; + await writeFile(tmpPath, `${JSON.stringify(record, null, "\t")}\n`, "utf8"); + await rename(tmpPath, latestSourcePath()); + + return join(versionDir(latest.source_sha), "openapi.json"); +}; - return getOpenApiJsonPath(); +/** + * Fetch (or re-verify) the schema for the active mode and return the path of + * the extracted openapi.json. Downloads are cached in `.tmp` and verified on + * every call, so a corrupted cache fails the build instead of rendering. + */ +export const fetchOpenApiSchema = async (): Promise => { + if (getSchemaMode() === "pinned") { + return await fetchPinnedSchema(); + } + return await fetchLatestSchema(); };