diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 0d09f8f..445837a 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -20,12 +20,16 @@ jobs: uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.14 + - name: Install inert verifier dependencies + run: bun install --frozen-lockfile --ignore-scripts - name: Download the exact low-privilege build uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: marketplace-release-${{ github.sha }} path: dist - name: Reverify and stage strict catalogs + env: + CONVAX_MARKETPLACE_CHANGED: dist/release-plan.json run: | bun tooling/verify-marketplace-output.mjs dist/catalog bun tooling/verify-product-lock-input.mjs dist/product-lock-input.json diff --git a/.github/workflows/release-on-main.yml b/.github/workflows/release-on-main.yml index b88f1ef..e4677d4 100644 --- a/.github/workflows/release-on-main.yml +++ b/.github/workflows/release-on-main.yml @@ -34,12 +34,17 @@ jobs: env: BASE_SHA: ${{ github.event.before }} run: | - bun tooling/marketplace-release.mjs \ - --base "$BASE_SHA" \ - --head "$GITHUB_SHA" \ - --output dist/release-plan.json - echo "count=$(jq length dist/release-plan.json)" >> "$GITHUB_OUTPUT" + set -euo pipefail + mkdir -p dist/production + export PATH="$PWD/node_modules/.bin:$PATH" + convax-marketplace changed . --base "$BASE_SHA" > dist/release-plan.json + count="$(jq length dist/release-plan.json)" + echo "count=$count" >> "$GITHUB_OUTPUT" echo "CONVAX_MARKETPLACE_CHANGED=dist/release-plan.json" >> "$GITHUB_ENV" + if [ "$count" != "0" ]; then + git show "$BASE_SHA:marketplace.json" > dist/production/trusted-marketplace.json + echo "CONVAX_MARKETPLACE_TRUSTED_DESCRIPTOR=dist/production/trusted-marketplace.json" >> "$GITHUB_ENV" + fi - name: Fetch the current production sequence input if: steps.plan.outputs.count != '0' run: bun tooling/fetch-marketplace-previous.mjs diff --git a/bun.lock b/bun.lock index 2b58f1c..e3088da 100644 --- a/bun.lock +++ b/bun.lock @@ -8,7 +8,8 @@ "acorn": "8.17.0", }, "devDependencies": { - "@convax/marketplace-kit": "0.1.0", + "@convax/marketplace": "0.1.1", + "@convax/marketplace-kit": "0.1.1", }, }, "packages/plugins/chatcut": { @@ -56,7 +57,7 @@ }, "packages/plugins/nexus-service": { "name": "@microvoid/convax-plugin-nexus-service", - "version": "0.3.11", + "version": "0.3.12", "dependencies": { "@microvoid/convax-nexus-mcp": "workspace:*", }, @@ -220,9 +221,9 @@ }, }, "packages": { - "@convax/marketplace": ["@convax/marketplace@0.1.0", "", { "dependencies": { "ajv": "8.20.0" } }, "sha512-Etzca0NT5o2is6HMbupFP0xyFzP1DrTzCXWVNSsFmTXgID2UTaR4D4g5tDb8qfyy7TjNIOdQqnhPUbmzObVSdA=="], + "@convax/marketplace": ["@convax/marketplace@0.1.1", "", { "dependencies": { "ajv": "8.20.0" } }, "sha512-FHrcTrHlmywDbWfm8N5LndUQLuYpHohw2eqGaQBrnevOHNmoOQ/sgSIAUm8QvSWdn2cZ/HKl1IdT1VtJMT4wAg=="], - "@convax/marketplace-kit": ["@convax/marketplace-kit@0.1.0", "", { "dependencies": { "@convax/marketplace": "^0.1.0" }, "bin": { "convax-marketplace": "dist/cli.js" } }, "sha512-tHlWRQURnRuLyFiajCpeBA/KYukLQlh++nFJC+/q+yqXIwBgkCuGvGTz9BhJ1eklbM8hDnS/dwN7+SHr8v1vag=="], + "@convax/marketplace-kit": ["@convax/marketplace-kit@0.1.1", "", { "dependencies": { "@convax/marketplace": "^0.1.1" }, "bin": { "convax-marketplace": "dist/cli.js" } }, "sha512-+XZynFr4B2lRLA8ZFVUiA9MFbAplhXdIzgd3+mmz8/HWfvHVW0jWt5zeVLVIjUVMvIJbrlUmQp2pjSkJyWaD+g=="], "@floating-ui/core": ["@floating-ui/core@1.8.0", "", { "dependencies": { "@floating-ui/utils": "^0.2.12" } }, "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ=="], diff --git a/docs/packaging.md b/docs/packaging.md index d61120b..64236fe 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -173,9 +173,16 @@ disable a compromised version for new installs, publish a reviewed higher packag version with `yanked: true`. Existing immutable assets remain available for inventory, recovery, and audit. -The serialized workflow fetches and strictly validates the current production v2 -Registry as an explicit sequence input. The one-time bootstrap accepts strict v1 -only after an exact v2 HTTP 404 and inherits only its sequence high-water mark. +The serialized workflow takes one bounded snapshot of the production descriptor, +Registry v2, Showcase v2, Registry v1, and Showcase v1 endpoints. A steady-state +release requires all five endpoints to return strict matching documents. The one-time +legacy bootstrap requires the exact `404/404/404/200/200` state and binds the current +descriptor to the previous protected `main` revision. It losslessly lifts the v1 +Plugin and Skill entries and proves the v1 projection round-trips before replacing +only explicitly selected identities. Unselected production entries remain +byte-for-byte canonical metadata; source-only or source-ahead packages wait for +their own versioned publication. Every other status matrix fails closed, including +an all-404 response from an already deployed Official Marketplace. Every other network or validation failure stops publication. The Kit then writes one grouped directory per immutable package Release plus one content-addressed Registry metadata Release. A changed Storyboard source also publishes the matching diff --git a/package.json b/package.json index 3d59bc2..dba911d 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "acorn": "8.17.0" }, "devDependencies": { - "@convax/marketplace-kit": "0.1.0" + "@convax/marketplace": "0.1.1", + "@convax/marketplace-kit": "0.1.1" }, "packageManager": "bun@1.3.14", "workspaces": [ diff --git a/packages/plugins/nexus-service/convax-package.json b/packages/plugins/nexus-service/convax-package.json index 4501426..00f838c 100644 --- a/packages/plugins/nexus-service/convax-package.json +++ b/packages/plugins/nexus-service/convax-package.json @@ -4,7 +4,7 @@ "id": "nexus-service", "name": "Nexus · OpenRouter", "description": "Connects Convax to the Convax Workspace in Nexus for OpenRouter chat and live image-model generation through short-lived Data Tokens.", - "version": "0.3.11", + "version": "0.3.12", "license": "MIT", "compatibility": { "pluginSchema": "convax.plugin/7", diff --git a/packages/plugins/nexus-service/package.json b/packages/plugins/nexus-service/package.json index 8d4df31..d895744 100644 --- a/packages/plugins/nexus-service/package.json +++ b/packages/plugins/nexus-service/package.json @@ -1,6 +1,6 @@ { "name": "@microvoid/convax-plugin-nexus-service", - "version": "0.3.11", + "version": "0.3.12", "private": true, "type": "module", "dependencies": { diff --git a/packages/plugins/nexus-service/package/manifest.json b/packages/plugins/nexus-service/package/manifest.json index 709bec3..cc263de 100644 --- a/packages/plugins/nexus-service/package/manifest.json +++ b/packages/plugins/nexus-service/package/manifest.json @@ -3,7 +3,7 @@ "id": "nexus-service", "name": "Nexus · OpenRouter", "description": "Connects Convax to the Convax Workspace in Nexus for OpenRouter chat and live image-model generation through short-lived Data Tokens.", - "version": "0.3.11", + "version": "0.3.12", "contributes": { "generation": { "models": [ diff --git a/tooling/fetch-marketplace-previous.mjs b/tooling/fetch-marketplace-previous.mjs index 3afb1a4..d641529 100644 --- a/tooling/fetch-marketplace-previous.mjs +++ b/tooling/fetch-marketplace-previous.mjs @@ -1,5 +1,11 @@ import { promises as fs } from "node:fs" import path from "node:path" +import { + canonicalJson, + parseMarketplaceDescriptor, + parseShowcaseV1, + parseShowcaseV2, +} from "@convax/marketplace" const maximumBytes = 8 * 1024 * 1024 @@ -34,17 +40,28 @@ async function responseBytes(response, label) { return result } -function parseSequenceInput(bytes, mode, parser) { - let registry +function parseJsonBytes(bytes, label) { try { - registry = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) } catch (cause) { - throw new Error(`${mode} is not valid UTF-8 JSON`, { cause }) + throw new Error(`${label} is not valid UTF-8 JSON`, { cause }) } +} + +function parseDescriptorInput(bytes, label) { try { - registry = parser(registry) + return parseMarketplaceDescriptor(parseJsonBytes(bytes, label)) } catch (cause) { - throw new Error(`${mode} strict validation failed`, { cause }) + throw new Error(`${label} strict validation failed`, { cause }) + } +} + +function parseSequenceInput(bytes, mode, parser) { + let registry + try { + registry = parser(parseJsonBytes(bytes, `production Registry ${mode}`)) + } catch (cause) { + throw new Error(`production Registry ${mode} strict validation failed`, { cause }) } const revisionPattern = mode === "v2" ? /^[a-f0-9]{64}$/ : /^[a-f0-9]{40}$/ if ( @@ -58,11 +75,22 @@ function parseSequenceInput(bytes, mode, parser) { !Array.isArray(registry.packages) || (mode === "v2" && registry.marketplaceId !== "convax-official") ) { - throw new Error(`${mode} is not a strict sequence input`) + throw new Error(`production Registry ${mode} is not a strict sequence input`) } return registry } +function parseShowcaseInput(bytes, mode, registry, descriptor) { + try { + const value = parseJsonBytes(bytes, `production Showcase ${mode}`) + return mode === "v2" + ? parseShowcaseV2(value, registry, descriptor) + : parseShowcaseV1(value, registry, descriptor) + } catch (cause) { + throw new Error(`production Showcase ${mode} strict validation failed`, { cause }) + } +} + async function fetchExact(fetchImpl, url, label) { return fetchImpl(url, { headers: { accept: "application/json" }, @@ -73,67 +101,171 @@ async function fetchExact(fetchImpl, url, label) { }) } -async function writeSnapshot(outputDirectory, name, bytes) { +async function writeSnapshots(outputDirectory, snapshots) { await fs.mkdir(outputDirectory, { recursive: true }) - const output = path.join(outputDirectory, name) - const temporary = `${output}.tmp-${process.pid}` - await fs.writeFile(temporary, bytes, { mode: 0o600 }) - await fs.rename(temporary, output) - return output + const nonce = `${process.pid}-${Date.now()}` + const pending = [] + try { + for (const [name, bytes] of Object.entries(snapshots)) { + const output = path.join(outputDirectory, name) + const temporary = `${output}.tmp-${nonce}` + await fs.writeFile(temporary, bytes, { mode: 0o600 }) + pending.push({ output, temporary }) + } + for (const { output, temporary } of pending) await fs.rename(temporary, output) + } catch (cause) { + await Promise.all(pending.map(({ temporary }) => fs.rm(temporary, { force: true }))) + throw cause + } + return Object.fromEntries( + Object.keys(snapshots).map((name) => [name, path.join(outputDirectory, name)]), + ) } -export async function fetchPreviousRegistry({ +export async function fetchPreviousMarketplace({ fetchImpl = fetch, outputDirectory, parseV1, parseV2, - v1Url, - v2Url, + trustedDescriptorBytes, + urls, }) { if (typeof parseV1 !== "function" || typeof parseV2 !== "function") { throw new Error("strict Registry v1 and v2 parsers are required") } - const v2Response = await fetchExact(fetchImpl, v2Url, "production Registry v2") - if (v2Response.status === 200) { - const bytes = await responseBytes(v2Response, "production Registry v2") - const registry = parseSequenceInput(bytes, "v2", parseV2) - const snapshot = await writeSnapshot(outputDirectory, "registry-v2.json", bytes) - return { mode: "v2", registry, snapshot } + const trustedDescriptor = parseDescriptorInput( + trustedDescriptorBytes, + "trusted base Marketplace descriptor", + ) + const endpoints = [ + ["descriptor", urls.descriptor], + ["Registry v2", urls.registryV2], + ["Showcase v2", urls.showcaseV2], + ["Registry v1", urls.registryV1], + ["Showcase v1", urls.showcaseV1], + ] + if (endpoints.some(([, url]) => typeof url !== "string" || url.length === 0)) { + throw new Error("all five production Marketplace endpoint URLs are required") + } + const responses = await Promise.all( + endpoints.map(([label, url]) => fetchExact(fetchImpl, url, `production ${label}`)), + ) + const status = responses.map((response) => response.status).join("/") + const steady = "200/200/200/200/200" + const bootstrap = "404/404/404/200/200" + if (status !== steady && status !== bootstrap) { + throw new Error( + `production Marketplace baseline is inconsistent: expected ${steady} or ${bootstrap}, got ${status}`, + ) } - if (v2Response.status !== 404) { - throw new Error(`production Registry v2 returned HTTP ${v2Response.status}`) + + if (status === steady) { + const [descriptorBytes, registryV2Bytes, showcaseV2Bytes, registryV1Bytes, showcaseV1Bytes] = + await Promise.all(responses.map((response, index) => responseBytes(response, `production ${endpoints[index][0]}`))) + const descriptor = parseDescriptorInput(descriptorBytes, "production Marketplace descriptor") + if (canonicalJson(descriptor) !== canonicalJson(trustedDescriptor)) { + throw new Error("production Marketplace descriptor differs from the trusted base descriptor") + } + const registryV2 = parseSequenceInput(registryV2Bytes, "v2", parseV2) + const showcaseV2 = parseShowcaseInput(showcaseV2Bytes, "v2", registryV2, descriptor) + const registryV1 = parseSequenceInput(registryV1Bytes, "v1", parseV1) + const showcaseV1 = parseShowcaseInput(showcaseV1Bytes, "v1", registryV1, descriptor) + const snapshots = await writeSnapshots(outputDirectory, { + "marketplace.json": descriptorBytes, + "registry-v2.json": registryV2Bytes, + "showcase-v2.json": showcaseV2Bytes, + "registry-v1.json": registryV1Bytes, + "showcase-v1.json": showcaseV1Bytes, + }) + return { + mode: "v2", + descriptor, + registryV1, + registryV2, + showcaseV1, + showcaseV2, + snapshots, + } } - const v1Response = await fetchExact(fetchImpl, v1Url, "production Registry v1 bootstrap") - if (v1Response.status !== 200) { - throw new Error(`production Registry v1 bootstrap returned HTTP ${v1Response.status}`) + const registryV1Bytes = await responseBytes(responses[3], "production Registry v1") + const showcaseV1Bytes = await responseBytes(responses[4], "production Showcase v1") + const registryV1 = parseSequenceInput(registryV1Bytes, "v1", parseV1) + const showcaseV1 = parseShowcaseInput( + showcaseV1Bytes, + "v1", + registryV1, + trustedDescriptor, + ) + const snapshots = await writeSnapshots(outputDirectory, { + "marketplace.json": trustedDescriptorBytes, + "registry-v1.json": registryV1Bytes, + "showcase-v1.json": showcaseV1Bytes, + }) + return { + mode: "bootstrap-v1", + descriptor: trustedDescriptor, + registryV1, + showcaseV1, + snapshots, } - const bytes = await responseBytes(v1Response, "production Registry v1 bootstrap") - const registry = parseSequenceInput(bytes, "v1", parseV1) - const snapshot = await writeSnapshot(outputDirectory, "registry-v1.json", bytes) - return { mode: "bootstrap-v1", registry, snapshot } +} + +export function cacheBusted(url, runId, runAttempt) { + const value = new URL(url) + value.searchParams.set("run", `${runId ?? "local"}-${runAttempt ?? "1"}`) + return value.href } async function main() { const kit = await import("@convax/marketplace-kit") - const run = encodeURIComponent(process.env.GITHUB_RUN_ID ?? "local") - const result = await fetchPreviousRegistry({ + const trustedDescriptorPath = process.env.CONVAX_MARKETPLACE_TRUSTED_DESCRIPTOR + if (!trustedDescriptorPath) { + throw new Error("CONVAX_MARKETPLACE_TRUSTED_DESCRIPTOR is required") + } + const trustedDescriptorBytes = new Uint8Array(await fs.readFile(path.resolve(trustedDescriptorPath))) + const descriptor = parseDescriptorInput( + trustedDescriptorBytes, + "trusted base Marketplace descriptor", + ) + const runId = process.env.GITHUB_RUN_ID + const runAttempt = process.env.GITHUB_RUN_ATTEMPT + const pagesBase = `https://${descriptor.repository.owner.toLowerCase()}.github.io/${descriptor.repository.name}/` + const showcaseV1Url = new URL("../v1/index.json", descriptor.showcase.v2.url).href + const result = await fetchPreviousMarketplace({ outputDirectory: path.resolve("dist/production"), parseV1: kit.parseRegistryV1, parseV2: kit.parseRegistryV2, - v1Url: `https://microvoid.github.io/convax-plugins/registry/v1/index.json?run=${run}`, - v2Url: `https://microvoid.github.io/convax-plugins/registry/v2/index.json?run=${run}`, + trustedDescriptorBytes, + urls: { + descriptor: cacheBusted(new URL("marketplace.json", pagesBase).href, runId, runAttempt), + registryV1: cacheBusted(descriptor.registry.v1.url, runId, runAttempt), + registryV2: cacheBusted(descriptor.registry.v2.url, runId, runAttempt), + showcaseV1: cacheBusted(showcaseV1Url, runId, runAttempt), + showcaseV2: cacheBusted(descriptor.showcase.v2.url, runId, runAttempt), + }, }) const environment = process.env.GITHUB_ENV if (!environment) throw new Error("GITHUB_ENV is required for the publication workflow") - const variable = result.mode === "v2" - ? "CONVAX_MARKETPLACE_PREVIOUS" - : "CONVAX_MARKETPLACE_BOOTSTRAP_PREVIOUS_V1" - await fs.appendFile(environment, `${variable}=${path.relative(process.cwd(), result.snapshot)}\n`) + const relative = (name) => path.relative(process.cwd(), result.snapshots[name]) + const environmentLines = result.mode === "v2" + ? [ + `CONVAX_MARKETPLACE_PREVIOUS_DESCRIPTOR=${relative("marketplace.json")}`, + `CONVAX_MARKETPLACE_PREVIOUS=${relative("registry-v2.json")}`, + `CONVAX_MARKETPLACE_PREVIOUS_SHOWCASE=${relative("showcase-v2.json")}`, + `CONVAX_MARKETPLACE_PREVIOUS_V1=${relative("registry-v1.json")}`, + `CONVAX_MARKETPLACE_PREVIOUS_SHOWCASE_V1=${relative("showcase-v1.json")}`, + ] + : [ + `CONVAX_MARKETPLACE_PREVIOUS_DESCRIPTOR=${relative("marketplace.json")}`, + `CONVAX_MARKETPLACE_BOOTSTRAP_PREVIOUS_V1=${relative("registry-v1.json")}`, + `CONVAX_MARKETPLACE_PREVIOUS_SHOWCASE_V1=${relative("showcase-v1.json")}`, + ] + await fs.appendFile(environment, `${environmentLines.join("\n")}\n`) console.log( result.mode === "v2" - ? `Using strict production Registry v2 sequence ${result.registry.sequence}.` - : `Bootstrapping Registry v2 from strict Registry v1 sequence ${result.registry.sequence}.`, + ? `Using complete production Marketplace sequence ${result.registryV2.sequence}.` + : `Bootstrapping Registry v2 from complete legacy sequence ${result.registryV1.sequence}.`, ) } diff --git a/tooling/marketplace-output.test.js b/tooling/marketplace-output.test.js index 20889e2..b4de2b7 100644 --- a/tooling/marketplace-output.test.js +++ b/tooling/marketplace-output.test.js @@ -3,6 +3,8 @@ import { createHash } from "node:crypto" import { promises as fs } from "node:fs" import os from "node:os" import path from "node:path" +import { canonicalJson, projectRegistryV1 } from "@convax/marketplace" +import { releaseTagForPackage } from "@convax/marketplace-kit" import { verifyMarketplaceOutput } from "./verify-marketplace-output.mjs" const temporaryDirectories = [] @@ -18,8 +20,7 @@ function sha256(bytes) { } function mcpTag(id, version) { - const key = sha256(Buffer.from(`mcp-server\0${id}`, "utf8")) - return `mcp-server-${key.slice(0, 16)}-v${version}` + return releaseTagForPackage({ kind: "mcp-server", id, version }) } async function writeFixture() { @@ -28,7 +29,7 @@ async function writeFixture() { const definitions = [ { kind: "plugin", id: "fixture-plugin", version: "1.0.0" }, { kind: "skill", id: "fixture-skill", version: "2.0.0" }, - { kind: "mcp-server", id: "io.example/fixture", version: "2026.07" }, + { kind: "mcp-server", id: "io.example/fixture", version: "2026.7.0" }, ] const releasePlan = [] const packages = [] @@ -50,25 +51,29 @@ async function writeFixture() { url, } releasePlan.push({ tag, assets: [artifact] }) + const serverJson = { + name: definition.id, + description: "Fixture MCP server", + version: definition.version, + remotes: [{ type: "streamable-http", url: "https://example.com/mcp" }], + } packages.push({ ...definition, compatibility: { convax: ">=0.1.0" }, - presentation: { name: definition.id }, + presentation: { + name: definition.id, + description: `${definition.kind} fixture`, + }, yanked: false, delivery: definition.kind === "mcp-server" ? { - kind: "mcp-managed-stdio", - serverJson: {}, - serverJsonSha256: "0".repeat(64), - extension: {}, - extensionSha256: "1".repeat(64), - companions: [{ - target: "darwin-arm64", - command: name, - url, - size: bytes.length, - sha256: sha256(bytes), - }], + kind: "mcp-http", + serverJson, + serverJsonSha256: sha256(Buffer.from(`${canonicalJson(serverJson)}\n`)), + runtime: { + endpoint: "https://example.com/mcp", + transport: "streamable-http", + }, } : { kind: "artifact", @@ -76,28 +81,26 @@ async function writeFixture() { size: bytes.length, sha256: sha256(bytes), }, + ...(definition.kind === "plugin" + ? { + manifest: { + schema: "convax.plugin/6", + id: definition.id, + version: definition.version, + }, + } + : {}), }) } - const registryV2Bytes = Buffer.from( - `${JSON.stringify({ - schema: "convax.registry/2", - marketplaceId: "convax-official", - sequence: 1, - revision: "a".repeat(64), - packages, - })}\n`, - ) - const registryV1Bytes = Buffer.from( - `${JSON.stringify({ - schema: "convax.registry/1", - sequence: 1, - revision: "a".repeat(40), - packages: packages - .filter((entry) => entry.kind !== "mcp-server") - .map(({ kind, id, version }) => ({ kind, id, version })), - })}\n`, - ) - const marketplaceBytes = Buffer.from(`${JSON.stringify({ + const registryV2 = { + schema: "convax.registry/2", + marketplaceId: "convax-official", + sequence: 2, + revision: sha256(Buffer.from(canonicalJson(packages))), + packages, + } + const registryV1 = projectRegistryV1(registryV2, "a".repeat(40)) + const descriptor = { schema: "convax.marketplace/1", id: "convax-official", name: "Convax Official", @@ -112,23 +115,35 @@ async function writeFixture() { }, compatibility: { convax: ">=0.1.0" }, delivery: { kind: "github-pages-releases" }, - })}\n`) - const showcaseBytes = Buffer.from(`${JSON.stringify({ + } + const showcaseV2 = { schema: "convax.showcase/2", marketplaceId: "convax-official", - revision: "a".repeat(64), + revision: registryV2.revision, packages: [], - })}\n`) + } + const showcaseV1 = { + schema: "convax.showcase/1", + sequence: registryV1.sequence, + revision: registryV1.revision, + packages: [], + } + const registryV2Bytes = Buffer.from(`${JSON.stringify(registryV2)}\n`) + const registryV1Bytes = Buffer.from(`${JSON.stringify(registryV1)}\n`) + const marketplaceBytes = Buffer.from(`${JSON.stringify(descriptor)}\n`) + const showcaseV2Bytes = Buffer.from(`${JSON.stringify(showcaseV2)}\n`) + const showcaseV1Bytes = Buffer.from(`${JSON.stringify(showcaseV1)}\n`) await fs.writeFile(path.join(catalogDirectory, "marketplace.json"), marketplaceBytes) await fs.writeFile(path.join(catalogDirectory, "registry-v2.json"), registryV2Bytes) await fs.writeFile(path.join(catalogDirectory, "registry-v1.json"), registryV1Bytes) - await fs.writeFile(path.join(catalogDirectory, "showcase-v2.json"), showcaseBytes) - const metadataTag = `registry-v2-${"a".repeat(64)}` + await fs.writeFile(path.join(catalogDirectory, "showcase-v2.json"), showcaseV2Bytes) + await fs.writeFile(path.join(catalogDirectory, "showcase-v1.json"), showcaseV1Bytes) + const metadataTag = `registry-v2-${registryV2.revision}` const metadataAssets = [] for (const [name, bytes] of [ ["marketplace.json", marketplaceBytes], ["registry-v2.json", registryV2Bytes], - ["showcase-v2.json", showcaseBytes], + ["showcase-v2.json", showcaseV2Bytes], ]) { const relativePath = `releases/${metadataTag}/${name}` const url = `${repository}/${metadataTag}/${name}` @@ -154,13 +169,23 @@ async function writeFixture() { ["marketplace.json", marketplaceBytes], ["registry/v1/index.json", registryV1Bytes], ["registry/v2/index.json", registryV2Bytes], - ["showcase/v2/index.json", showcaseBytes], + ["showcase/v1/index.json", showcaseV1Bytes], + ["showcase/v2/index.json", showcaseV2Bytes], ]) { const output = path.join(catalogDirectory, "site", ...relativePath.split("/")) await fs.mkdir(path.dirname(output), { recursive: true }) await fs.writeFile(output, bytes) } - return { catalogDirectory, releasePlan } + return { + catalogDirectory, + descriptor, + metadataTag, + registryV1, + registryV2, + releasePlan, + showcaseV1, + showcaseV2, + } } async function rewriteMetadataFixture(catalogDirectory, name, value, sitePath) { @@ -183,6 +208,92 @@ async function rewriteMetadataFixture(catalogDirectory, name, value, sitePath) { await fs.writeFile(planPath, `${JSON.stringify(plan)}\n`) } +async function writeSelectiveFixture({ + contextProductionPreviousVersion = "0.9.0", + contextReleaseTag = "plugin-fixture-plugin-v1.0.0", + contextSourcePreviousVersion = "0.9.0", +} = {}) { + const fixture = await writeFixture() + const selectedPackage = fixture.registryV2.packages.find((entry) => + entry.kind === "plugin" && entry.id === "fixture-plugin") + const baselinePackages = structuredClone(fixture.registryV2.packages) + const baselinePlugin = baselinePackages.find((entry) => + entry.kind === "plugin" && entry.id === "fixture-plugin") + baselinePlugin.version = "0.9.0" + baselinePlugin.manifest.version = "0.9.0" + baselinePlugin.delivery.url = baselinePlugin.delivery.url.replace( + "plugin-fixture-plugin-v1.0.0", + "plugin-fixture-plugin-v0.9.0", + ) + const baselineRegistry = { + schema: "convax.registry/2", + marketplaceId: "convax-official", + sequence: 1, + revision: sha256(Buffer.from(canonicalJson(baselinePackages))), + packages: baselinePackages, + } + const baselineShowcase = { + schema: "convax.showcase/2", + marketplaceId: "convax-official", + revision: baselineRegistry.revision, + packages: [], + } + const legacyRegistry = projectRegistryV1(baselineRegistry, "b".repeat(40)) + const legacyShowcase = { + schema: "convax.showcase/1", + sequence: legacyRegistry.sequence, + revision: legacyRegistry.revision, + packages: [], + } + const selectedVersions = [{ + id: selectedPackage.id, + kind: selectedPackage.kind, + previousVersion: "0.9.0", + releaseTag: releaseTagForPackage(selectedPackage), + version: selectedPackage.version, + }] + const selectionContext = { + schema: "convax.marketplace-selection-context/1", + descriptor: fixture.descriptor, + selectedPackages: [{ + kind: selectedPackage.kind, + id: selectedPackage.id, + version: selectedPackage.version, + sourcePreviousVersion: contextSourcePreviousVersion, + productionPreviousVersion: contextProductionPreviousVersion, + releaseTag: contextReleaseTag, + }], + baseline: { + mode: "v2", + registry: baselineRegistry, + showcase: baselineShowcase, + }, + legacy: { registry: legacyRegistry, showcase: legacyShowcase }, + } + await fs.writeFile( + path.join(fixture.catalogDirectory, "selection-context.json"), + `${JSON.stringify(selectionContext)}\n`, + ) + await fs.writeFile( + path.join(fixture.catalogDirectory, "release-plan.json"), + `${JSON.stringify({ + schema: "convax.release-plan/1", + releases: fixture.releasePlan.filter((release) => + release.tag === selectedVersions[0].releaseTag || release.tag === fixture.metadataTag), + })}\n`, + ) + for (const release of fixture.releasePlan) { + if (release.tag === selectedVersions[0].releaseTag || release.tag === fixture.metadataTag) { + continue + } + await fs.rm(path.join(fixture.catalogDirectory, "releases", release.tag), { + recursive: true, + force: true, + }) + } + return { ...fixture, selectedVersions, selectionContext } +} + afterAll(async () => { await Promise.all(temporaryDirectories.map((directory) => fs.rm(directory, { recursive: true, force: true }))) @@ -259,7 +370,7 @@ describe("published Marketplace output closure", () => { .rejects.toThrow("release-plan tags differ from Registry v2") }) - test("verifies only selected package Releases while retaining the complete Registry", async () => { + test("rejects a selected package subset without its production baseline context", async () => { const fixture = await writeFixture() const selected = fixture.releasePlan[0] const metadata = fixture.releasePlan.at(-1) @@ -272,7 +383,6 @@ describe("published Marketplace output closure", () => { ) const selectedVersions = [{ id: "fixture-plugin", - itemKey: sha256(Buffer.from("plugin\0fixture-plugin", "utf8")), kind: "plugin", previousVersion: "0.9.0", releaseTag: "plugin-fixture-plugin-v1.0.0", @@ -282,18 +392,54 @@ describe("published Marketplace output closure", () => { await expect(verifyMarketplaceOutput( fixture.catalogDirectory, { selectedVersions }, + )).rejects.toThrow("requires a production selection context") + }) + + test("accepts only selected package bytes while preserving unselected production references", async () => { + const fixture = await writeSelectiveFixture() + await expect(verifyMarketplaceOutput( + fixture.catalogDirectory, + { selectedVersions: fixture.selectedVersions }, )).resolves.toEqual({ packages: 3, releaseAssets: 4, releaseTags: 2, v1Packages: 2, }) + }) + + test("rejects selection context fields that diverge from the source release plan", async () => { + const sourceVersionFixture = await writeSelectiveFixture({ + contextSourcePreviousVersion: "0.8.0", + }) + await expect(verifyMarketplaceOutput( + sourceVersionFixture.catalogDirectory, + { selectedVersions: sourceVersionFixture.selectedVersions }, + )).rejects.toThrow( + "selection context plugin/fixture-plugin differs from selected version changes", + ) - selectedVersions[0].releaseTag = "plugin-fixture-plugin-v9.9.9" + const releaseTagFixture = await writeSelectiveFixture({ + contextReleaseTag: "plugin-fixture-plugin-v1.0.1", + }) + await expect(verifyMarketplaceOutput( + releaseTagFixture.catalogDirectory, + { selectedVersions: releaseTagFixture.selectedVersions }, + )).rejects.toThrow( + "selection context plugin/fixture-plugin differs from selected version changes", + ) + }) + + test("lets the Kit reject a forged production previous version", async () => { + const fixture = await writeSelectiveFixture({ + contextProductionPreviousVersion: "0.8.0", + }) await expect(verifyMarketplaceOutput( fixture.catalogDirectory, - { selectedVersions }, - )).rejects.toThrow("selected version change plugin/fixture-plugin differs from Registry v2") + { selectedVersions: fixture.selectedVersions }, + )).rejects.toThrow( + "selected package plugin/fixture-plugin does not match production baseline", + ) }) test("rejects a descriptor or Showcase that changes the Official publication identity", async () => { @@ -339,37 +485,20 @@ describe("published Marketplace output closure", () => { path.join(fixture.catalogDirectory, asset.path), Buffer.from("plugin/fixture-plugin@1.0.0"), ) - const registryPath = path.join(fixture.catalogDirectory, "registry-v2.json") - const registry = JSON.parse(await fs.readFile(registryPath, "utf8")) - registry.packages[0].delivery.url = registry.packages[0].delivery.url.replace( - "plugin-fixture-plugin-v1.0.0", - "plugin-wrong-v1.0.0", - ) - const changedRegistry = `${JSON.stringify(registry)}\n` - await fs.writeFile(registryPath, changedRegistry) - await fs.writeFile( - path.join(fixture.catalogDirectory, "site/registry/v2/index.json"), - changedRegistry, - ) + const assetPath = path.join(fixture.catalogDirectory, asset.path) + await fs.rename(assetPath, `${assetPath}.misplaced`) + await expect(verifyMarketplaceOutput(fixture.catalogDirectory)).rejects.toThrow() + }) + + test("rejects Release directory bytes that are absent from the exact plan", async () => { + const fixture = await writeFixture() + const release = fixture.releasePlan[0] await fs.writeFile( - path.join( - fixture.catalogDirectory, - "releases", - `registry-v2-${"a".repeat(64)}`, - "registry-v2.json", - ), - changedRegistry, + path.join(fixture.catalogDirectory, "releases", release.tag, "unplanned.bin"), + "unverified", ) - const planPath = path.join(fixture.catalogDirectory, "release-plan.json") - const plan = JSON.parse(await fs.readFile(planPath, "utf8")) - const registryAsset = plan.releases - .find((entry) => entry.tag === `registry-v2-${"a".repeat(64)}`) - .assets.find((entry) => entry.name === "registry-v2.json") - registryAsset.size = Buffer.byteLength(changedRegistry) - registryAsset.sha256 = sha256(Buffer.from(changedRegistry)) - await fs.writeFile(planPath, `${JSON.stringify(plan)}\n`) await expect(verifyMarketplaceOutput(fixture.catalogDirectory)) - .rejects.toThrow("has no exact local Release asset") + .rejects.toThrow("directory contents differ from its exact assets") }) test("rejects a Pages tree that differs from the descriptor-addressed flat catalogs", async () => { @@ -380,5 +509,13 @@ describe("published Marketplace output closure", () => { ) await expect(verifyMarketplaceOutput(fixture.catalogDirectory)) .rejects.toThrow("Pages registry v2 differs from its verified flat catalog") + + const showcaseFixture = await writeFixture() + await fs.writeFile( + path.join(showcaseFixture.catalogDirectory, "site/showcase/v1/index.json"), + "changed", + ) + await expect(verifyMarketplaceOutput(showcaseFixture.catalogDirectory)) + .rejects.toThrow("Pages showcase v1 differs from its verified flat catalog") }) }) diff --git a/tooling/marketplace-release.mjs b/tooling/marketplace-release.mjs deleted file mode 100644 index 7378ec1..0000000 --- a/tooling/marketplace-release.mjs +++ /dev/null @@ -1,382 +0,0 @@ -import { createHash } from "node:crypto" -import { execFileSync } from "node:child_process" -import { promises as fs } from "node:fs" -import path from "node:path" -import { fileURLToPath } from "node:url" - -const collections = [ - { directory: "mcp-servers", kind: "mcp-server", metadata: "server.json" }, - { directory: "plugins", kind: "plugin", metadata: "convax-package.json" }, - { directory: "skills", kind: "skill", metadata: "convax-package.json" }, -] - -function sha256(input) { - return createHash("sha256").update(input).digest("hex") -} - -function itemKey(kind, id) { - return sha256(Buffer.from(`${kind}\0${id}`, "utf8")) -} - -async function collectPackageFiles(directory, relative = "") { - const entries = await fs.readdir(directory, { withFileTypes: true }) - const files = [] - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name, "en"))) { - if (entry.name === "dist" || entry.name === "node_modules" || entry.name === ".DS_Store") continue - const absolute = path.join(directory, entry.name) - const nextRelative = relative ? `${relative}/${entry.name}` : entry.name - if (entry.isDirectory()) files.push(...await collectPackageFiles(absolute, nextRelative)) - else if (entry.isFile()) files.push({ path: nextRelative, bytes: await fs.readFile(absolute) }) - else throw new Error(`${absolute}: package source must contain only regular files and directories`) - } - return files -} - -async function collectOptionalFiles(directory, label) { - const state = await fs.lstat(directory).catch((cause) => { - if (cause?.code === "ENOENT") return undefined - throw cause - }) - if (!state) return [] - if (!state.isDirectory() || state.isSymbolicLink()) { - throw new Error(`${label} must be a real directory`) - } - return collectPackageFiles(directory) -} - -function digestFiles(files) { - const hash = createHash("sha256") - for (const file of files) { - const pathBytes = Buffer.from(file.path, "utf8") - const size = Buffer.alloc(8) - size.writeBigUInt64BE(BigInt(file.bytes.length)) - hash.update(pathBytes) - hash.update(Buffer.from([0])) - hash.update(size) - hash.update(file.bytes) - } - return hash.digest("hex") -} - -function parseIdentity(kind, metadata, label) { - const id = kind === "mcp-server" ? metadata.name : metadata.id - if (typeof id !== "string" || id.length === 0) throw new Error(`${label}: missing package identity`) - if (typeof metadata.version !== "string" || metadata.version.length === 0) { - throw new Error(`${label}: missing package version`) - } - if (kind !== "mcp-server" && metadata.kind !== kind) { - throw new Error(`${label}: metadata kind does not match its collection`) - } - return { id, version: metadata.version } -} - -function pluginCompanionSourceRoots(metadata, label) { - if (metadata.companions === undefined) return [] - if (!Array.isArray(metadata.companions)) throw new Error(`${label}: companions must be an array`) - return metadata.companions.map((companion, index) => { - const source = companion?.source - if ( - typeof source !== "string" || - !/^packages\/tools\/[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(source) - ) { - throw new Error(`${label}: companion ${index} source must name one packages/tools directory`) - } - return source - }) -} - -function pluginOwnedSkillNames(manifest, label) { - const skills = manifest?.contributes?.skills - if (skills === undefined) return [] - if (!Array.isArray(skills)) throw new Error(`${label}: contributes.skills must be an array`) - return skills.map((skill, index) => { - if ( - typeof skill?.name !== "string" || - !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(skill.name) - ) { - throw new Error(`${label}: owned Skill ${index} has an invalid name`) - } - return skill.name - }) -} - -async function readOptionalJson(file) { - return JSON.parse(await fs.readFile(file, "utf8").catch((cause) => { - if (cause?.code === "ENOENT") return "null" - throw cause - })) -} - -async function pluginLinkedSourceRoots(workspaceRoot, packageRoot, metadata, label) { - const roots = pluginCompanionSourceRoots(metadata, label) - const manifest = await readOptionalJson(path.join(packageRoot, "package", "manifest.json")) - for (const skillName of pluginOwnedSkillNames(manifest, label)) { - const skillRoot = `packages/skills/${skillName}` - const skillMetadata = await readOptionalJson(path.join( - workspaceRoot, - skillRoot, - "convax-package.json", - )) - if ( - skillMetadata?.kind !== "skill" || - skillMetadata.id !== skillName || - skillMetadata.ownerPluginId !== metadata.id - ) { - throw new Error(`${label}: owned Skill ${skillName} does not bind back to ${metadata.id}`) - } - roots.push(skillRoot) - } - return [...new Set(roots)].sort((left, right) => left.localeCompare(right, "en")) -} - -export async function packageVersionSnapshot(workspaceRoot) { - const result = new Map() - for (const collection of collections) { - const collectionRoot = path.join(workspaceRoot, "packages", collection.directory) - const directories = await fs.readdir(collectionRoot, { withFileTypes: true }).catch((cause) => { - if (cause?.code === "ENOENT") return [] - throw cause - }) - for (const directory of directories.sort((left, right) => left.name.localeCompare(right.name, "en"))) { - if (!directory.isDirectory() || directory.name.startsWith(".")) continue - const packageRoot = path.join(collectionRoot, directory.name) - const metadataPath = path.join(packageRoot, collection.metadata) - const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")) - const label = `${collection.directory}/${directory.name}` - const identity = parseIdentity(collection.kind, metadata, label) - const key = `${collection.kind}\0${identity.id}` - if (result.has(key)) throw new Error(`${label}: duplicate package identity`) - const sourceFiles = (await collectPackageFiles(packageRoot)).map((file) => ({ - ...file, - path: `${label}/${file.path}`, - })) - if (collection.kind === "plugin") { - for (const linkedRoot of await pluginLinkedSourceRoots( - workspaceRoot, - packageRoot, - metadata, - label, - )) { - sourceFiles.push(...(await collectPackageFiles(path.join(workspaceRoot, linkedRoot))) - .map((file) => ({ ...file, path: `${linkedRoot}/${file.path}` }))) - } - } - const keyDigest = itemKey(collection.kind, identity.id) - if (collection.kind === "mcp-server") { - const companionRoot = `.marketplace/companion-inputs/${keyDigest}` - sourceFiles.push(...(await collectOptionalFiles( - path.join(workspaceRoot, companionRoot), - "managed MCP companion input", - )) - .map((file) => ({ ...file, path: `${companionRoot}/${file.path}` }))) - } - result.set(key, { - digest: digestFiles(sourceFiles.sort((left, right) => - left.path.localeCompare(right.path, "en"))), - directory: label, - id: identity.id, - itemKey: keyDigest, - kind: collection.kind, - version: identity.version, - }) - } - } - return result -} - -function releaseTagFor(item) { - return item.kind === "mcp-server" - ? `mcp-server-${item.itemKey.slice(0, 16)}-v${item.version}` - : `${item.kind}-${item.id}-v${item.version}` -} - -export function changedPackageVersions(previous, current) { - for (const [key, item] of previous) { - if (!current.has(key)) { - throw new Error( - `${item.kind}/${item.id}@${item.version} was removed; publish a reviewed yanked version instead`, - ) - } - } - const changes = [] - for (const [key, item] of current) { - const old = previous.get(key) - if (old?.version === item.version) { - if (old.digest !== item.digest) { - throw new Error(`${item.kind}/${item.id}@${item.version} changed without a version change`) - } - continue - } - changes.push({ - id: item.id, - itemKey: item.itemKey, - kind: item.kind, - previousVersion: old?.version, - releaseTag: releaseTagFor(item), - version: item.version, - }) - } - return changes.sort((left, right) => - `${left.kind}\0${left.id}`.localeCompare(`${right.kind}\0${right.id}`, "en")) -} - -function git(repositoryRoot, args) { - return execFileSync("git", args, { - cwd: repositoryRoot, - encoding: args.includes("-z") ? "buffer" : "utf8", - maxBuffer: 64 * 1024 * 1024, - }) -} - -export function gitTreePackageSnapshot(repositoryRoot, revision) { - const tree = git(repositoryRoot, [ - "ls-tree", - "-r", - "-z", - revision, - "--", - "packages/plugins", - "packages/skills", - "packages/mcp-servers", - "packages/tools", - ".marketplace/companion-inputs", - ]) - const filesByPackage = new Map() - const companionFilesByItemKey = new Map() - for (const record of tree.toString("utf8").split("\0").filter(Boolean)) { - const match = /^[0-7]{6} blob ([a-f0-9]{40})\t(.+)$/.exec(record) - if (!match) continue - const [, blob, file] = match - const parts = file.split("/") - if ( - parts[0] === ".marketplace" && - parts[1] === "companion-inputs" && - parts.length >= 5 - ) { - const files = companionFilesByItemKey.get(parts[2]) ?? [] - files.push({ blob, path: parts.slice(3).join("/") }) - companionFilesByItemKey.set(parts[2], files) - continue - } - if (parts.length < 4 || parts[2].startsWith(".")) continue - const packageRoot = parts.slice(0, 3).join("/") - const files = filesByPackage.get(packageRoot) ?? [] - files.push({ blob, path: parts.slice(3).join("/") }) - filesByPackage.set(packageRoot, files) - } - const result = new Map() - for (const [packageRoot, files] of filesByPackage) { - const [, collection, directory] = packageRoot.split("/") - const definition = collections.find((item) => item.directory === collection) - if (!definition) continue - const metadataRecord = files.find((item) => item.path === definition.metadata) - if (!metadataRecord) { - throw new Error(`${packageRoot}: missing ${definition.metadata}`) - } - const metadata = JSON.parse(git(repositoryRoot, [ - "show", - `${revision}:${packageRoot}/${definition.metadata}`, - ])) - const identity = parseIdentity(definition.kind, metadata, packageRoot) - const key = `${definition.kind}\0${identity.id}` - if (result.has(key)) throw new Error(`${packageRoot}: duplicate package identity`) - const sourceFiles = files.map((file) => ({ - ...file, - path: `${packageRoot}/${file.path}`, - })) - if (definition.kind === "plugin") { - for (const linkedRoot of pluginCompanionSourceRoots(metadata, packageRoot)) { - const linked = filesByPackage.get(linkedRoot) - if (!linked) throw new Error(`${packageRoot}: missing linked companion source ${linkedRoot}`) - sourceFiles.push(...linked.map((file) => ({ - ...file, - path: `${linkedRoot}/${file.path}`, - }))) - } - const manifestRecord = files.find((file) => file.path === "package/manifest.json") - const manifest = manifestRecord - ? JSON.parse(git(repositoryRoot, [ - "show", - `${revision}:${packageRoot}/package/manifest.json`, - ])) - : undefined - for (const skillName of pluginOwnedSkillNames(manifest, packageRoot)) { - const skillRoot = `packages/skills/${skillName}` - const linked = filesByPackage.get(skillRoot) - if (!linked) throw new Error(`${packageRoot}: missing owned Skill ${skillName}`) - const skillMetadataRecord = linked.find((file) => file.path === "convax-package.json") - if (!skillMetadataRecord) throw new Error(`${skillRoot}: missing convax-package.json`) - const skillMetadata = JSON.parse(git(repositoryRoot, [ - "show", - `${revision}:${skillRoot}/convax-package.json`, - ])) - if ( - skillMetadata.kind !== "skill" || - skillMetadata.id !== skillName || - skillMetadata.ownerPluginId !== metadata.id - ) { - throw new Error(`${packageRoot}: owned Skill ${skillName} does not bind back to ${metadata.id}`) - } - sourceFiles.push(...linked.map((file) => ({ - ...file, - path: `${skillRoot}/${file.path}`, - }))) - } - } - const keyDigest = itemKey(definition.kind, identity.id) - if (definition.kind === "mcp-server") { - const companionRoot = `.marketplace/companion-inputs/${keyDigest}` - sourceFiles.push(...(companionFilesByItemKey.get(keyDigest) ?? []).map((file) => ({ - ...file, - path: `${companionRoot}/${file.path}`, - }))) - } - const digest = sha256(sourceFiles - .sort((left, right) => left.path.localeCompare(right.path, "en")) - .map((file) => `${file.path}\0${file.blob}\n`) - .join("")) - result.set(key, { - digest, - directory: `${collection}/${directory}`, - id: identity.id, - itemKey: keyDigest, - kind: definition.kind, - version: identity.version, - }) - } - return result -} - -function parseCliArgs(argv) { - const result = {} - for (let index = 0; index < argv.length; index += 1) { - const argument = argv[index] - if (!argument.startsWith("--")) throw new Error(`Unexpected argument ${argument}`) - const key = argument.slice(2) - if (!["base", "head", "output"].includes(key) || result[key] !== undefined) { - throw new Error(`Unsupported or duplicate argument ${argument}`) - } - const value = argv[index + 1] - if (!value || value.startsWith("--")) throw new Error(`Missing value for ${argument}`) - result[key] = value - index += 1 - } - if (!result.base || !result.output) throw new Error("Usage: marketplace-release --base --output [--head ]") - return result -} - -async function main(argv) { - const args = parseCliArgs(argv) - const repositoryRoot = path.resolve(fileURLToPath(new URL("..", import.meta.url))) - const head = args.head ?? "HEAD" - const changes = changedPackageVersions( - gitTreePackageSnapshot(repositoryRoot, args.base), - gitTreePackageSnapshot(repositoryRoot, head), - ) - const output = path.resolve(repositoryRoot, args.output) - await fs.mkdir(path.dirname(output), { recursive: true }) - await fs.writeFile(output, `${JSON.stringify(changes, null, 2)}\n`) - console.log(`Selected ${changes.length} version-change release${changes.length === 1 ? "" : "s"}.`) -} - -if (import.meta.main) await main(process.argv.slice(2)) diff --git a/tooling/marketplace-release.test.js b/tooling/marketplace-release.test.js deleted file mode 100644 index d03ec79..0000000 --- a/tooling/marketplace-release.test.js +++ /dev/null @@ -1,362 +0,0 @@ -import { afterAll, describe, expect, test } from "bun:test" -import { createHash } from "node:crypto" -import { execFileSync } from "node:child_process" -import { promises as fs } from "node:fs" -import os from "node:os" -import path from "node:path" -import { - changedPackageVersions, - gitTreePackageSnapshot, - packageVersionSnapshot, -} from "./marketplace-release.mjs" -import { composePublicationPlan } from "./publication-plan.mjs" - -const temporaryDirectories = [] - -async function temporaryDirectory() { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), "convax-marketplace-release-")) - temporaryDirectories.push(directory) - return directory -} - -async function writePackage(root, kind, id, version, body = {}) { - const directory = path.join(root, "packages", kind, id) - await fs.mkdir(directory, { recursive: true }) - const marker = kind === "mcp-servers" - ? { name: `io.github.microvoid/${id}`, description: `${id} server`, version, ...body } - : { - schema: "convax.package/2", - kind: kind === "plugins" ? "plugin" : "skill", - id, - name: id, - description: `${id} package`, - version, - ...body, - } - await fs.writeFile( - path.join(directory, kind === "mcp-servers" ? "server.json" : "convax-package.json"), - `${JSON.stringify(marker, null, 2)}\n`, - ) -} - -async function writePluginClosure(root, version = "1.0.0") { - await writePackage(root, "plugins", "closed-plugin", version, { - companions: [{ - command: "closed-tool", - version: "1.0.0", - source: "packages/tools/closed-tool", - targets: [{ platform: "darwin", arch: "arm64", path: "dist/closed-tool" }], - }], - }) - await fs.mkdir(path.join(root, "packages/plugins/closed-plugin/package"), { recursive: true }) - await fs.writeFile( - path.join(root, "packages/plugins/closed-plugin/package/manifest.json"), - `${JSON.stringify({ - schema: "convax.plugin/4", - id: "closed-plugin", - version, - name: "Closed Plugin", - contributes: { skills: [{ name: "closed-skill", path: "skills/closed-skill" }] }, - }, null, 2)}\n`, - ) - await writePackage(root, "skills", "closed-skill", version, { - ownerPluginId: "closed-plugin", - }) - await fs.mkdir(path.join(root, "packages/skills/closed-skill/package"), { recursive: true }) - await fs.writeFile( - path.join(root, "packages/skills/closed-skill/package/SKILL.md"), - "---\nname: closed-skill\n---\n\n# Closed Skill\n", - ) - await fs.mkdir(path.join(root, "packages/tools/closed-tool/src"), { recursive: true }) - await fs.writeFile(path.join(root, "packages/tools/closed-tool/src/main.ts"), "export const version = 1\n") -} - -async function writeManagedMcpCompanion(root, id, bytes) { - const itemKey = createHash("sha256") - .update(Buffer.from(`mcp-server\0${id}`, "utf8")) - .digest("hex") - const companion = path.join( - root, - ".marketplace", - "companion-inputs", - itemKey, - "darwin-arm64", - "fixture-mcp", - ) - await fs.mkdir(path.dirname(companion), { recursive: true }) - await fs.writeFile(companion, bytes) -} - -afterAll(async () => { - await Promise.all(temporaryDirectories.map((directory) => - fs.rm(directory, { recursive: true, force: true }))) -}) - -describe("default-branch version-change release selection", () => { - test("returns an exact empty plan when every package version and byte is unchanged", async () => { - const unchanged = await temporaryDirectory() - await writePackage(unchanged, "plugins", "example-plugin", "1.0.0") - await writePackage(unchanged, "skills", "example-skill", "2.0.0") - await writePackage(unchanged, "mcp-servers", "example-http", "2026.07") - const snapshot = await packageVersionSnapshot(unchanged) - - expect(changedPackageVersions(snapshot, snapshot)).toEqual([]) - }) - - test("selects Plugin, Skill, and MCP Server version changes with stable release identities", async () => { - const previous = await temporaryDirectory() - const current = await temporaryDirectory() - for (const root of [previous, current]) { - await writePackage(root, "plugins", "example-plugin", root === previous ? "1.0.0" : "1.1.0") - await writePackage(root, "skills", "example-skill", "2.0.0") - await writePackage(root, "mcp-servers", "example-http", root === previous ? "2026.07" : "2026.08") - } - - const changes = changedPackageVersions( - await packageVersionSnapshot(previous), - await packageVersionSnapshot(current), - ) - - expect(changes).toEqual([ - { - id: "io.github.microvoid/example-http", - itemKey: expect.stringMatching(/^[a-f0-9]{64}$/), - kind: "mcp-server", - previousVersion: "2026.07", - releaseTag: expect.stringMatching(/^mcp-server-[a-f0-9]{16}-v2026\.08$/), - version: "2026.08", - }, - { - id: "example-plugin", - itemKey: expect.stringMatching(/^[a-f0-9]{64}$/), - kind: "plugin", - previousVersion: "1.0.0", - releaseTag: "plugin-example-plugin-v1.1.0", - version: "1.1.0", - }, - ]) - }) - - test("rejects changed immutable package bytes without a version change", async () => { - const previous = await temporaryDirectory() - const current = await temporaryDirectory() - await writePackage(previous, "skills", "example-skill", "1.0.0", { description: "old bytes" }) - await writePackage(current, "skills", "example-skill", "1.0.0", { description: "new bytes" }) - - const previousSnapshot = await packageVersionSnapshot(previous) - const currentSnapshot = await packageVersionSnapshot(current) - expect(() => changedPackageVersions(previousSnapshot, currentSnapshot)) - .toThrow("changed without a version change") - }) - - test("requires a reviewed yanked version instead of silently removing a package", async () => { - const previous = await temporaryDirectory() - const current = await temporaryDirectory() - await writePackage(previous, "skills", "removed-skill", "1.0.0") - - const previousSnapshot = await packageVersionSnapshot(previous) - const currentSnapshot = await packageVersionSnapshot(current) - expect(() => changedPackageVersions(previousSnapshot, currentSnapshot)) - .toThrow("skill/removed-skill@1.0.0 was removed") - }) - - test("binds linked companion and owned Skill sources to the Plugin version", async () => { - const previous = await temporaryDirectory() - const current = await temporaryDirectory() - await writePluginClosure(previous) - await writePluginClosure(current) - - await fs.writeFile( - path.join(current, "packages/tools/closed-tool/src/main.ts"), - "export const version = 2\n", - ) - const previousSnapshot = await packageVersionSnapshot(previous) - let currentSnapshot = await packageVersionSnapshot(current) - expect(() => changedPackageVersions(previousSnapshot, currentSnapshot)) - .toThrow("plugin/closed-plugin@1.0.0 changed without a version change") - - await fs.writeFile( - path.join(current, "packages/tools/closed-tool/src/main.ts"), - "export const version = 1\n", - ) - await fs.writeFile( - path.join(current, "packages/skills/closed-skill/package/SKILL.md"), - "---\nname: closed-skill\n---\n\n# Changed Skill\n", - ) - currentSnapshot = await packageVersionSnapshot(current) - expect(() => changedPackageVersions(previousSnapshot, currentSnapshot)) - .toThrow("plugin/closed-plugin@1.0.0 changed without a version change") - }) - - test("binds scaffold-owned managed MCP companion inputs to the MCP Server version", async () => { - const previous = await temporaryDirectory() - const current = await temporaryDirectory() - const id = "io.github.microvoid/managed-example" - for (const root of [previous, current]) { - await writePackage(root, "mcp-servers", "managed-example", "1.0.0", { - name: id, - }) - } - await writeManagedMcpCompanion(previous, id, "previous companion bytes") - await writeManagedMcpCompanion(current, id, "changed companion bytes") - - const previousSnapshot = await packageVersionSnapshot(previous) - const currentSnapshot = await packageVersionSnapshot(current) - expect(() => changedPackageVersions( - previousSnapshot, - currentSnapshot, - )).toThrow("mcp-server/io.github.microvoid/managed-example@1.0.0 changed without a version change") - }) - - test("uses the same managed MCP companion closure for the production Git-tree selector", async () => { - const repository = await temporaryDirectory() - const id = "io.github.microvoid/git-managed-example" - await writePackage(repository, "mcp-servers", "git-managed-example", "1.0.0", { - name: id, - }) - await writeManagedMcpCompanion(repository, id, "previous companion bytes") - const git = (args) => execFileSync("git", args, { - cwd: repository, - encoding: "utf8", - }).trim() - git(["init"]) - git(["config", "user.email", "fixture@example.test"]) - git(["config", "user.name", "Fixture"]) - git(["add", "."]) - git(["commit", "-m", "initial"]) - const previousRevision = git(["rev-parse", "HEAD"]) - - await writeManagedMcpCompanion(repository, id, "changed companion bytes") - git(["add", "."]) - git(["commit", "-m", "change companion"]) - const currentRevision = git(["rev-parse", "HEAD"]) - - expect(() => changedPackageVersions( - gitTreePackageSnapshot(repository, previousRevision), - gitTreePackageSnapshot(repository, currentRevision), - )).toThrow("mcp-server/io.github.microvoid/git-managed-example@1.0.0 changed without a version change") - }) - - test("does not follow a scaffold-owned managed MCP companion root outside the repository", async () => { - const repository = await temporaryDirectory() - const outside = await temporaryDirectory() - const id = "io.github.microvoid/symlinked-managed-example" - await writePackage(repository, "mcp-servers", "symlinked-managed-example", "1.0.0", { - name: id, - }) - await fs.writeFile(path.join(outside, "outside-companion"), "outside bytes") - const itemKey = createHash("sha256") - .update(Buffer.from(`mcp-server\0${id}`, "utf8")) - .digest("hex") - const inputs = path.join(repository, ".marketplace", "companion-inputs") - await fs.mkdir(inputs, { recursive: true }) - await fs.symlink(outside, path.join(inputs, itemKey)) - - await expect(packageVersionSnapshot(repository)) - .rejects.toThrow("managed MCP companion input must be a real directory") - }) - - test("publishes only from a protected default-branch version change", async () => { - const workflow = await fs.readFile(path.join( - import.meta.dir, - "..", - ".github/workflows/release-on-main.yml", - ), "utf8") - const pages = await fs.readFile(path.join( - import.meta.dir, - "..", - ".github/workflows/pages.yml", - ), "utf8") - expect(workflow).toContain("branches: [main]") - expect(workflow).not.toContain("tags:") - expect(workflow).toContain("bun tooling/marketplace-release.mjs") - expect(workflow).toContain("--base \"$BASE_SHA\"") - expect(workflow).toContain("permissions:\n contents: read") - expect(workflow).toContain("attestations: write") - expect(workflow).toContain("contents: write") - expect(workflow).toContain("id-token: write") - expect(workflow).toContain("dist/catalog/releases/$tag") - expect(workflow).toContain("uses: ./.github/workflows/pages.yml") - expect(workflow).toContain("fetch-marketplace-previous.mjs") - expect(workflow).toContain("publication-plan.mjs") - expect(workflow).toContain("gh release download") - expect(workflow).toContain("cmp \"$asset\"") - expect(workflow).not.toContain("already exists; immutable versions are never overwritten") - expect(workflow.match(/if: steps\.plan\.outputs\.count != '0'/g)?.length).toBeGreaterThanOrEqual(5) - expect(workflow).toContain("if: needs.verify.outputs.count != '0'") - expect(workflow).toContain("needs.publish.result == 'success'") - expect(workflow).not.toContain("needs.publish.result == 'skipped'") - expect(workflow).not.toContain("pull_request_target") - expect(pages).toContain("workflow_call:") - expect(pages).not.toContain("workflow_run:") - expect(pages).not.toContain("concurrency:") - }) - - test("publishes changed packages with one metadata Release and the changed Builtin bundle", () => { - const selected = [ - { - kind: "plugin", - id: "ffmpeg-tools", - releaseTag: "plugin-ffmpeg-tools-v0.3.1", - }, - { - kind: "skill", - id: "canvas-storyboard", - releaseTag: "skill-canvas-storyboard-v0.1.0", - }, - ] - const catalog = { - schema: "convax.release-plan/1", - releases: [ - { - tag: "plugin-ffmpeg-tools-v0.3.1", - assets: [{ path: "releases/plugin-ffmpeg-tools-v0.3.1/plugin.zip" }], - }, - { - tag: "skill-canvas-storyboard-v0.1.0", - assets: [{ path: "releases/skill-canvas-storyboard-v0.1.0/skill.zip" }], - }, - { - tag: `registry-v2-${"a".repeat(64)}`, - assets: [{ path: `releases/registry-v2-${"a".repeat(64)}/registry-v2.json` }], - }, - ], - } - const builtin = { - schema: "convax.release-plan/1", - releases: [{ - tag: "builtin-release", - assets: [{ path: "releases/builtin-release/convax-builtin-bundle.zip" }], - }], - } - expect(composePublicationPlan({ builtin, catalog, selected })).toEqual({ - schema: "convax.publication-plan/1", - releases: [ - { - directory: "builtin/releases/builtin-release", - tag: "builtin-release", - }, - { - directory: "catalog/releases/plugin-ffmpeg-tools-v0.3.1", - tag: "plugin-ffmpeg-tools-v0.3.1", - }, - { - directory: `catalog/releases/registry-v2-${"a".repeat(64)}`, - tag: `registry-v2-${"a".repeat(64)}`, - }, - { - directory: "catalog/releases/skill-canvas-storyboard-v0.1.0", - tag: "skill-canvas-storyboard-v0.1.0", - }, - ], - }) - expect(() => composePublicationPlan({ - builtin, - catalog: { - ...catalog, - releases: catalog.releases.filter((entry) => !entry.tag.startsWith("registry-v2-")), - }, - selected, - })).toThrow("exactly one Registry metadata Release") - }) -}) diff --git a/tooling/official-marketplace-build.mjs b/tooling/official-marketplace-build.mjs index 014fbc4..58913ee 100644 --- a/tooling/official-marketplace-build.mjs +++ b/tooling/official-marketplace-build.mjs @@ -2,10 +2,16 @@ import { spawnSync } from "node:child_process" import path from "node:path" import { fileURLToPath } from "node:url" -export function officialBuildArgs({ bootstrapPreviousV1, changed, previous, v1Revision }) { - if (bootstrapPreviousV1 && previous) { - throw new Error("Official build accepts exactly one previous Registry mode") - } +export function officialBuildArgs({ + bootstrapPreviousV1, + changed, + previous, + previousDescriptor, + previousShowcase, + previousShowcaseV1, + previousV1, + v1Revision, +}) { if (typeof v1Revision !== "string" || !/^[a-f0-9]{40}$/.test(v1Revision)) { throw new Error("Official v1 revision must be an exact Git SHA") } @@ -16,18 +22,66 @@ export function officialBuildArgs({ bootstrapPreviousV1, changed, previous, v1Re "dist/catalog", "--official", ] - if (changed) args.push("--changed", changed) - if (previous) return [...args, "--previous", previous, "--v1-revision", v1Revision] - if (bootstrapPreviousV1) { + if (!changed) { + if ( + bootstrapPreviousV1 || + previous || + previousDescriptor || + previousShowcase || + previousShowcaseV1 || + previousV1 + ) { + throw new Error("Official production baselines require an explicit changed selection") + } + return [...args, "--initial", "--v1-revision", v1Revision] + } + if (!previousDescriptor) { + throw new Error("Selective Official build requires the trusted previous descriptor") + } + if ((bootstrapPreviousV1 === undefined) === (previous === undefined)) { + throw new Error("Selective Official build requires exactly one previous Registry mode") + } + if (!previousShowcaseV1) { + throw new Error("Selective Official build requires the previous Showcase v1") + } + if (previous) { + if (!previousShowcase || !previousV1) { + throw new Error("Selective Official v2 build requires complete v2 and v1 baselines") + } return [ ...args, - "--bootstrap-previous-v1", - bootstrapPreviousV1, + "--changed", + changed, + "--previous-descriptor", + previousDescriptor, + "--previous", + previous, + "--previous-showcase", + previousShowcase, + "--previous-v1", + previousV1, + "--previous-showcase-v1", + previousShowcaseV1, "--v1-revision", v1Revision, ] } - return [...args, "--initial", "--v1-revision", v1Revision] + if (previousShowcase || previousV1) { + throw new Error("Selective Official v1 bootstrap cannot consume a v2 baseline") + } + return [ + ...args, + "--changed", + changed, + "--previous-descriptor", + previousDescriptor, + "--bootstrap-previous-v1", + bootstrapPreviousV1, + "--previous-showcase-v1", + previousShowcaseV1, + "--v1-revision", + v1Revision, + ] } export function officialBuildInvocation(args) { @@ -48,6 +102,10 @@ function main() { bootstrapPreviousV1: process.env.CONVAX_MARKETPLACE_BOOTSTRAP_PREVIOUS_V1, changed: process.env.CONVAX_MARKETPLACE_CHANGED, previous: process.env.CONVAX_MARKETPLACE_PREVIOUS, + previousDescriptor: process.env.CONVAX_MARKETPLACE_PREVIOUS_DESCRIPTOR, + previousShowcase: process.env.CONVAX_MARKETPLACE_PREVIOUS_SHOWCASE, + previousShowcaseV1: process.env.CONVAX_MARKETPLACE_PREVIOUS_SHOWCASE_V1, + previousV1: process.env.CONVAX_MARKETPLACE_PREVIOUS_V1, v1Revision, }) const invocation = officialBuildInvocation(args) diff --git a/tooling/official-marketplace.test.js b/tooling/official-marketplace.test.js index c4f3d82..24c6d80 100644 --- a/tooling/official-marketplace.test.js +++ b/tooling/official-marketplace.test.js @@ -11,7 +11,10 @@ import { officialBuildArgs, officialBuildInvocation, } from "./official-marketplace-build.mjs" -import { fetchPreviousRegistry } from "./fetch-marketplace-previous.mjs" +import { + cacheBusted, + fetchPreviousMarketplace, +} from "./fetch-marketplace-previous.mjs" import { root, sha256 } from "./lib.mjs" describe("Official and Builtin marketplace source", () => { @@ -108,6 +111,10 @@ describe("Official and Builtin marketplace source", () => { expect(officialBuildArgs({ changed: "dist/release-plan.json", previous: "dist/production/registry-v2.json", + previousDescriptor: "dist/production/marketplace.json", + previousShowcase: "dist/production/showcase-v2.json", + previousShowcaseV1: "dist/production/showcase-v1.json", + previousV1: "dist/production/registry-v1.json", v1Revision: "a".repeat(40), })).toEqual([ "build-index", @@ -117,13 +124,24 @@ describe("Official and Builtin marketplace source", () => { "--official", "--changed", "dist/release-plan.json", + "--previous-descriptor", + "dist/production/marketplace.json", "--previous", "dist/production/registry-v2.json", + "--previous-showcase", + "dist/production/showcase-v2.json", + "--previous-v1", + "dist/production/registry-v1.json", + "--previous-showcase-v1", + "dist/production/showcase-v1.json", "--v1-revision", "a".repeat(40), ]) expect(officialBuildArgs({ bootstrapPreviousV1: "dist/production/registry-v1.json", + changed: "dist/release-plan.json", + previousDescriptor: "dist/production/marketplace.json", + previousShowcaseV1: "dist/production/showcase-v1.json", v1Revision: "a".repeat(40), })).toEqual([ "build-index", @@ -131,8 +149,14 @@ describe("Official and Builtin marketplace source", () => { "--out", "dist/catalog", "--official", + "--changed", + "dist/release-plan.json", + "--previous-descriptor", + "dist/production/marketplace.json", "--bootstrap-previous-v1", "dist/production/registry-v1.json", + "--previous-showcase-v1", + "dist/production/showcase-v1.json", "--v1-revision", "a".repeat(40), ]) @@ -148,9 +172,20 @@ describe("Official and Builtin marketplace source", () => { ]) expect(() => officialBuildArgs({ bootstrapPreviousV1: "dist/production/registry-v1.json", + changed: "dist/release-plan.json", previous: "dist/production/registry-v2.json", + previousDescriptor: "dist/production/marketplace.json", + previousShowcaseV1: "dist/production/showcase-v1.json", v1Revision: "a".repeat(40), })).toThrow("exactly one previous Registry mode") + expect(() => officialBuildArgs({ + previous: "dist/production/registry-v2.json", + v1Revision: "a".repeat(40), + })).toThrow("require an explicit changed selection") + expect(() => officialBuildArgs({ + changed: "dist/release-plan.json", + v1Revision: "a".repeat(40), + })).toThrow("trusted previous descriptor") expect(() => officialBuildArgs({ v1Revision: "bad" })) .toThrow("v1 revision must be an exact Git SHA") }) @@ -173,101 +208,137 @@ describe("Official and Builtin marketplace source", () => { }) }) - test("prefers production v2 and bootstraps from strict v1 only after an exact v2 404", async () => { + test("accepts only complete steady or complete legacy Marketplace snapshots", async () => { const output = await fs.mkdtemp(path.join(os.tmpdir(), "convax-marketplace-previous-")) try { - const v2Bytes = JSON.stringify({ + const source = await loadOfficialMarketplaceSource(root) + const trustedDescriptorBytes = new TextEncoder().encode(JSON.stringify(source.descriptor)) + const registryV2 = { schema: "convax.registry/2", marketplaceId: "convax-official", sequence: 45, revision: "a".repeat(64), packages: [], - }) + } + const showcaseV2 = { + schema: "convax.showcase/2", + marketplaceId: "convax-official", + revision: registryV2.revision, + packages: [], + } + const registryV1 = { + schema: "convax.registry/1", + sequence: 45, + revision: "b".repeat(40), + packages: [], + } + const showcaseV1 = { + schema: "convax.showcase/1", + sequence: registryV1.sequence, + revision: registryV1.revision, + packages: [], + } + const urls = { + descriptor: "https://example.test/marketplace.json", + registryV2: "https://example.test/registry/v2/index.json", + showcaseV2: "https://example.test/showcase/v2/index.json", + registryV1: "https://example.test/registry/v1/index.json", + showcaseV1: "https://example.test/showcase/v1/index.json", + } + const bodies = new Map([ + [urls.descriptor, source.descriptor], + [urls.registryV2, registryV2], + [urls.showcaseV2, showcaseV2], + [urls.registryV1, registryV1], + [urls.showcaseV1, showcaseV1], + ]) let requests = [] - const v2 = await fetchPreviousRegistry({ + const steady = await fetchPreviousMarketplace({ fetchImpl: async (url) => { requests.push(url) - return new Response(v2Bytes, { status: 200 }) + return new Response(JSON.stringify(bodies.get(url)), { status: 200 }) }, - outputDirectory: output, + outputDirectory: path.join(output, "steady"), parseV1: strictRegistryParser(1), parseV2: strictRegistryParser(2), - v1Url: "https://example.test/v1", - v2Url: "https://example.test/v2", + trustedDescriptorBytes, + urls, }) - expect(v2.mode).toBe("v2") - expect(requests).toEqual(["https://example.test/v2"]) + expect(steady.mode).toBe("v2") + expect(requests).toEqual(Object.values(urls)) + expect(Object.keys(steady.snapshots).sort()).toEqual([ + "marketplace.json", + "registry-v1.json", + "registry-v2.json", + "showcase-v1.json", + "showcase-v2.json", + ]) requests = [] - const bootstrap = await fetchPreviousRegistry({ + const bootstrap = await fetchPreviousMarketplace({ fetchImpl: async (url) => { requests.push(url) - return url.endsWith("/v2") - ? new Response("", { status: 404 }) - : new Response(JSON.stringify({ - schema: "convax.registry/1", - sequence: 44, - revision: "b".repeat(40), - packages: [], - }), { status: 200 }) + if ([urls.descriptor, urls.registryV2, urls.showcaseV2].includes(url)) { + return new Response("", { status: 404 }) + } + return new Response(JSON.stringify(bodies.get(url)), { status: 200 }) }, - outputDirectory: output, + outputDirectory: path.join(output, "bootstrap"), parseV1: strictRegistryParser(1), parseV2: strictRegistryParser(2), - v1Url: "https://example.test/v1", - v2Url: "https://example.test/v2", + trustedDescriptorBytes, + urls, }) expect(bootstrap.mode).toBe("bootstrap-v1") - expect(requests).toEqual(["https://example.test/v2", "https://example.test/v1"]) - - requests = [] - await expect(fetchPreviousRegistry({ - fetchImpl: async (url) => { - requests.push(url) - return new Response("", { status: 503 }) - }, - outputDirectory: output, - parseV1: strictRegistryParser(1), - parseV2: strictRegistryParser(2), - v1Url: "https://example.test/v1", - v2Url: "https://example.test/v2", - })).rejects.toThrow("v2 returned HTTP 503") - expect(requests).toEqual(["https://example.test/v2"]) + expect(requests).toEqual(Object.values(urls)) + expect(Object.keys(bootstrap.snapshots).sort()).toEqual([ + "marketplace.json", + "registry-v1.json", + "showcase-v1.json", + ]) + expect(JSON.parse(await fs.readFile( + bootstrap.snapshots["marketplace.json"], + "utf8", + ))).toEqual(source.descriptor) + } finally { + await fs.rm(output, { recursive: true, force: true }) + } + }) - await expect(fetchPreviousRegistry({ - fetchImpl: async (url) => url.endsWith("/v2") - ? new Response("", { status: 404 }) - : new Response(JSON.stringify({ - schema: "convax.registry/1", - sequence: 0, - revision: "bad", - packages: [], - }), { status: 200 }), - outputDirectory: output, - parseV1: strictRegistryParser(1), - parseV2: strictRegistryParser(2), - v1Url: "https://example.test/v1", - v2Url: "https://example.test/v2", - })).rejects.toThrow("v1 is not a strict sequence input") + test("uses a distinct production snapshot cache key for every workflow attempt", () => { + expect(cacheBusted("https://example.test/registry/v2/index.json", "123", "1")) + .toBe("https://example.test/registry/v2/index.json?run=123-1") + expect(cacheBusted("https://example.test/registry/v2/index.json?source=pages", "123", "2")) + .toBe("https://example.test/registry/v2/index.json?source=pages&run=123-2") + expect(cacheBusted("https://example.test/registry/v2/index.json", undefined, undefined)) + .toBe("https://example.test/registry/v2/index.json?run=local-1") + }) - await expect(fetchPreviousRegistry({ - fetchImpl: async () => new Response(JSON.stringify({ - schema: "convax.registry/2", - marketplaceId: "convax-official", - sequence: 45, - revision: "a".repeat(64), - packages: [ - { kind: "skill", id: "duplicate" }, - { kind: "skill", id: "duplicate" }, - ], - unexpected: true, - }), { status: 200 }), - outputDirectory: output, + test("fails closed without writing snapshots for every other five-endpoint state", async () => { + const output = await fs.mkdtemp(path.join(os.tmpdir(), "convax-marketplace-incomplete-")) + try { + const source = await loadOfficialMarketplaceSource(root) + const urls = { + descriptor: "https://example.test/marketplace.json", + registryV2: "https://example.test/registry/v2/index.json", + showcaseV2: "https://example.test/showcase/v2/index.json", + registryV1: "https://example.test/registry/v1/index.json", + showcaseV1: "https://example.test/showcase/v1/index.json", + } + const rejectedOutput = path.join(output, "rejected") + await expect(fetchPreviousMarketplace({ + fetchImpl: async (url) => new Response("", { + status: url === urls.showcaseV2 ? 503 : 200, + }), + outputDirectory: rejectedOutput, parseV1: strictRegistryParser(1), parseV2: strictRegistryParser(2), - v1Url: "https://example.test/v1", - v2Url: "https://example.test/v2", - })).rejects.toThrow("strict validation failed") + trustedDescriptorBytes: new TextEncoder().encode(JSON.stringify(source.descriptor)), + urls, + })).rejects.toThrow( + "expected 200/200/200/200/200 or 404/404/404/200/200, got 200/200/503/200/200", + ) + await expect(fs.stat(rejectedOutput)).rejects.toMatchObject({ code: "ENOENT" }) } finally { await fs.rm(output, { recursive: true, force: true }) } @@ -282,8 +353,19 @@ describe("Official and Builtin marketplace source", () => { path.join(root, ".github/workflows/pages.yml"), "utf8", ) + const fetchPrevious = await fs.readFile( + path.join(root, "tooling/fetch-marketplace-previous.mjs"), + "utf8", + ) expect(releaseWorkflow).toContain("branches: [main]") expect(releaseWorkflow).not.toContain("pull_request_target") + expect(releaseWorkflow).toContain('convax-marketplace changed . --base "$BASE_SHA"') + expect(releaseWorkflow).not.toContain("tooling/marketplace-release.mjs") + expect(releaseWorkflow).toContain( + 'git show "$BASE_SHA:marketplace.json" > dist/production/trusted-marketplace.json', + ) + expect(fetchPrevious).toContain('const steady = "200/200/200/200/200"') + expect(fetchPrevious).toContain('const bootstrap = "404/404/404/200/200"') expect(releaseWorkflow).toContain("if: steps.plan.outputs.count != '0'") expect(releaseWorkflow).toContain("if: needs.verify.outputs.count != '0'") expect(releaseWorkflow).toContain("needs: [verify, publish]") @@ -294,6 +376,9 @@ describe("Official and Builtin marketplace source", () => { expect(pagesWorkflow).toContain( "bun tooling/verify-product-lock-input.mjs dist/product-lock-input.json", ) + expect(pagesWorkflow).toContain( + "CONVAX_MARKETPLACE_CHANGED: dist/release-plan.json", + ) expect(releaseWorkflow).toContain( "cp schemas/*.json dist/catalog/site/schemas/", ) diff --git a/tooling/product-lock-output.test.js b/tooling/product-lock-output.test.js index cc152c8..e134607 100644 --- a/tooling/product-lock-output.test.js +++ b/tooling/product-lock-output.test.js @@ -3,6 +3,7 @@ import { createHash } from "node:crypto" import { promises as fs } from "node:fs" import os from "node:os" import path from "node:path" +import { canonicalJson, projectRegistryV1 } from "@convax/marketplace" import { verifyProductLockInput } from "./verify-product-lock-input.mjs" const temporaryDirectories = [] @@ -12,32 +13,75 @@ function sha256(bytes) { return createHash("sha256").update(bytes).digest("hex") } -async function writeRelease(root, area, tag, name, bytes) { - const relativePath = `${area}/releases/${tag}/${name}` - await fs.mkdir(path.join(root, area, "releases", tag), { recursive: true }) +async function writeRelease(root, area, tag, name, bytes, { inherited = false } = {}) { + const digest = sha256(bytes) + const relativePath = inherited + ? `${area}/inherited/${digest}/${name}` + : `${area}/releases/${tag}/${name}` + await fs.mkdir(path.dirname(path.join(root, relativePath)), { recursive: true }) await fs.writeFile(path.join(root, relativePath), bytes) return { path: relativePath, + url: `${repository}/${tag}/${name}`, + size: bytes.length, + sha256: digest, + } +} + +function registry(packages, sequence) { + return { + schema: "convax.registry/2", + marketplaceId: "convax-official", + sequence, + revision: sha256(Buffer.from(canonicalJson(packages))), + packages, + } +} + +function releaseReference(tag, name, bytes) { + return { url: `${repository}/${tag}/${name}`, size: bytes.length, sha256: sha256(bytes), } } -async function writeFixture() { +async function writeFixture({ inheritedFfmpeg = false } = {}) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "convax-product-lock-output-")) temporaryDirectories.push(root) - const revision = "a".repeat(64) - const metadataTag = `registry-v2-${revision}` const pluginTag = "plugin-ffmpeg-tools-v0.3.1" const skillTag = "skill-ffmpeg-canvas-v0.3.1" const builtinTag = `builtin-${"b".repeat(64)}` - const plugin = await writeRelease(root, "catalog", pluginTag, "convax-plugin-ffmpeg-tools-0.3.1.zip", Buffer.from("plugin")) - const skill = await writeRelease(root, "catalog", skillTag, "convax-skill-ffmpeg-canvas-0.3.1.zip", Buffer.from("skill")) - const companion = await writeRelease(root, "catalog", pluginTag, "convax-companion-convax-ffmpeg-mcp-0.2.0-darwin-arm64", Buffer.from("companion")) + const plugin = await writeRelease( + root, + "catalog", + pluginTag, + "convax-plugin-ffmpeg-tools-0.3.1.zip", + Buffer.from("plugin"), + { inherited: inheritedFfmpeg }, + ) + const skill = await writeRelease( + root, + "catalog", + skillTag, + "convax-skill-ffmpeg-canvas-0.3.1.zip", + Buffer.from("skill"), + { inherited: inheritedFfmpeg }, + ) + const companion = await writeRelease( + root, + "catalog", + pluginTag, + "convax-companion-convax-ffmpeg-mcp-0.2.0-darwin-arm64", + Buffer.from("companion"), + { inherited: inheritedFfmpeg }, + ) const descriptorValue = { schema: "convax.marketplace/1", id: "convax-official", + name: "Convax Official", + publisher: { name: "Microvoid" }, + repository: { owner: "microvoid", name: "convax-plugins" }, registry: { v2: { url: "https://microvoid.github.io/convax-plugins/registry/v2/index.json" }, v1: { url: "https://microvoid.github.io/convax-plugins/registry/v1/index.json" }, @@ -45,38 +89,159 @@ async function writeFixture() { showcase: { v2: { url: "https://microvoid.github.io/convax-plugins/showcase/v2/index.json" }, }, + compatibility: { convax: ">=0.1.0" }, + delivery: { kind: "github-pages-releases" }, } - const registryValue = { - schema: "convax.registry/2", - marketplaceId: "convax-official", - sequence: 45, - revision, - packages: [ - { - kind: "plugin", + const ffmpegPackages = [ + { + kind: "plugin", + id: "ffmpeg-tools", + version: "0.3.1", + compatibility: { convax: ">=0.1.0" }, + presentation: { name: "FFmpeg Tools", description: "FFmpeg fixture Plugin." }, + delivery: { kind: "artifact", url: plugin.url, size: plugin.size, sha256: plugin.sha256 }, + yanked: false, + companions: [{ + command: "convax-ffmpeg-mcp", + version: "0.2.0", + targets: [{ + platform: "darwin", + arch: "arm64", + artifact: { url: companion.url, size: companion.size, sha256: companion.sha256 }, + }], + }], + manifest: { + schema: "convax.plugin/4", id: "ffmpeg-tools", + name: "FFmpeg Tools", + description: "FFmpeg fixture Plugin.", version: "0.3.1", - delivery: { kind: "artifact", url: plugin.url, size: plugin.size, sha256: plugin.sha256 }, - companions: [{ - command: "convax-ffmpeg-mcp", - version: "0.2.0", - targets: [{ - platform: "darwin", - arch: "arm64", - artifact: { url: companion.url, size: companion.size, sha256: companion.sha256 }, - }], - }], - manifest: { contributes: { skills: [{ name: "ffmpeg-canvas", path: "skills/ffmpeg-canvas" }] } }, + contributes: { skills: [{ name: "ffmpeg-canvas", path: "skills/ffmpeg-canvas" }] }, + runtime: { type: "mcp-stdio", command: "convax-ffmpeg-mcp" }, }, - { - kind: "skill", - id: "ffmpeg-canvas", - version: "0.3.1", - delivery: { kind: "artifact", url: skill.url, size: skill.size, sha256: skill.sha256 }, + }, + { + kind: "skill", + id: "ffmpeg-canvas", + version: "0.3.1", + compatibility: { convax: ">=0.1.0" }, + presentation: { name: "FFmpeg Canvas", description: "FFmpeg fixture Skill." }, + delivery: { kind: "artifact", url: skill.url, size: skill.size, sha256: skill.sha256 }, + yanked: false, + ownerPluginId: "ffmpeg-tools", + }, + ] + let selectedVersions + let selectionContext + let selectedRelease + let registryValue + if (inheritedFfmpeg) { + const currentVersion = "0.3.12" + const previousVersion = "0.3.8" + const sourcePreviousVersion = "0.3.11" + const releaseTag = `plugin-nexus-service-v${currentVersion}` + const currentBytes = Buffer.from("current nexus") + selectedRelease = await writeRelease( + root, + "catalog", + releaseTag, + `convax-plugin-nexus-service-${currentVersion}.zip`, + currentBytes, + ) + const nexusPackage = (version, delivery) => ({ + kind: "plugin", + id: "nexus-service", + version, + compatibility: { convax: ">=0.1.0" }, + presentation: { name: "Nexus Service", description: "Nexus fixture Plugin." }, + delivery: { kind: "artifact", ...delivery }, + yanked: false, + manifest: { + schema: "convax.plugin/6", + id: "nexus-service", + name: "Nexus Service", + description: "Nexus fixture Plugin.", + version, + }, + }) + const baselineRegistry = registry([ + ...ffmpegPackages, + nexusPackage( + previousVersion, + releaseReference( + `plugin-nexus-service-v${previousVersion}`, + `convax-plugin-nexus-service-${previousVersion}.zip`, + Buffer.from("production nexus"), + ), + ), + ], 45) + const baselineShowcase = { + schema: "convax.showcase/2", + marketplaceId: "convax-official", + revision: baselineRegistry.revision, + packages: [], + } + const baselineRegistryV1 = projectRegistryV1(baselineRegistry, "c".repeat(40)) + const baselineShowcaseV1 = { + schema: "convax.showcase/1", + sequence: baselineRegistryV1.sequence, + revision: baselineRegistryV1.revision, + packages: [], + } + registryValue = registry([ + ...ffmpegPackages, + nexusPackage(currentVersion, { + url: selectedRelease.url, + size: selectedRelease.size, + sha256: selectedRelease.sha256, + }), + ], 46) + selectedVersions = [{ + kind: "plugin", + id: "nexus-service", + version: currentVersion, + previousVersion: sourcePreviousVersion, + releaseTag, + }] + selectionContext = { + schema: "convax.marketplace-selection-context/1", + descriptor: descriptorValue, + selectedPackages: [{ + kind: "plugin", + id: "nexus-service", + version: currentVersion, + sourcePreviousVersion, + productionPreviousVersion: previousVersion, + releaseTag, + }], + baseline: { + mode: "v2", + registry: baselineRegistry, + showcase: baselineShowcase, + }, + legacy: { + registry: baselineRegistryV1, + showcase: baselineShowcaseV1, }, - ], + } + } else { + registryValue = registry(ffmpegPackages, 45) + } + const revision = registryValue.revision + const metadataTag = `registry-v2-${revision}` + const showcaseValue = { + schema: "convax.showcase/2", + marketplaceId: "convax-official", + revision, + packages: [], + } + const registryV1Value = projectRegistryV1(registryValue, "a".repeat(40)) + const showcaseV1Value = { + schema: "convax.showcase/1", + sequence: registryV1Value.sequence, + revision: registryV1Value.revision, + packages: [], } - const showcaseValue = { schema: "convax.showcase/2", revision, packages: [] } const metadata = {} for (const [name, value] of [ ["marketplace.json", descriptorValue], @@ -88,6 +253,20 @@ async function writeFixture() { await fs.writeFile(path.join(root, "catalog", name), bytes) metadata[name] = await writeRelease(root, "catalog", metadataTag, name, bytes) } + await fs.writeFile( + path.join(root, "catalog", "registry-v1.json"), + `${JSON.stringify(registryV1Value)}\n`, + ) + await fs.writeFile( + path.join(root, "catalog", "showcase-v1.json"), + `${JSON.stringify(showcaseV1Value)}\n`, + ) + if (selectionContext) { + await fs.writeFile( + path.join(root, "catalog", "selection-context.json"), + `${JSON.stringify(selectionContext)}\n`, + ) + } const builtin = await writeRelease( root, "builtin", @@ -133,9 +312,7 @@ async function writeFixture() { ) } await writePlan("catalog", [ - plugin, - skill, - companion, + ...(inheritedFfmpeg ? [selectedRelease] : [plugin, skill, companion]), metadata["marketplace.json"], metadata["registry-v2.json"], metadata["showcase-v2.json"], @@ -171,7 +348,7 @@ async function writeFixture() { }], })}\n`, ) - return { root } + return { revision, root, selectedVersions } } afterAll(async () => { @@ -190,7 +367,35 @@ describe("Convax product lock input closure", () => { }) }) - test("verifies an existing preinstall outside a selected package release-plan", async () => { + test("closes inherited FFmpeg bytes for an unrelated selected Plugin release", async () => { + const fixture = await writeFixture({ inheritedFfmpeg: true }) + const lockPath = path.join(fixture.root, "product-lock-input.json") + await expect(verifyProductLockInput(lockPath, { + selectedVersions: fixture.selectedVersions, + })).resolves.toEqual({ + builtinReservations: 1, + preinstalledPackages: 1, + verifiedArtifacts: 7, + }) + + const changed = structuredClone(fixture.selectedVersions) + changed[0].previousVersion = "0.3.10" + await expect(verifyProductLockInput(lockPath, { selectedVersions: changed })) + .rejects.toThrow("selection context differs from selected version changes") + }) + + test("rejects tampered inherited FFmpeg bytes", async () => { + const fixture = await writeFixture({ inheritedFfmpeg: true }) + const lockPath = path.join(fixture.root, "product-lock-input.json") + const lock = JSON.parse(await fs.readFile(lockPath, "utf8")) + await fs.writeFile(path.join(fixture.root, lock.packages[0].artifact.path), "PLUGIN") + + await expect(verifyProductLockInput(lockPath, { + selectedVersions: fixture.selectedVersions, + })).rejects.toThrow("preinstalled Plugin artifact bytes differ from Registry v2") + }) + + test("rejects an existing preinstall outside the plan without a production baseline", async () => { const fixture = await writeFixture() const planPath = path.join(fixture.root, "catalog", "release-plan.json") const plan = JSON.parse(await fs.readFile(planPath, "utf8")) @@ -199,22 +404,8 @@ describe("Convax product lock input closure", () => { await fs.writeFile(planPath, `${JSON.stringify(plan)}\n`) const lockPath = path.join(fixture.root, "product-lock-input.json") - await expect(verifyProductLockInput(lockPath)).resolves.toEqual({ - builtinReservations: 1, - preinstalledPackages: 1, - verifiedArtifacts: 7, - }) - - const pluginPath = path.join( - fixture.root, - "catalog", - "releases", - "plugin-ffmpeg-tools-v0.3.1", - "convax-plugin-ffmpeg-tools-0.3.1.zip", - ) - await fs.writeFile(pluginPath, "PLUGIN") await expect(verifyProductLockInput(lockPath)) - .rejects.toThrow("preinstalled Plugin artifact bytes differ from Registry v2") + .rejects.toThrow("absent from the catalog release-plan") }) test("rejects a source swap or widened preinstall policy", async () => { @@ -254,7 +445,7 @@ describe("Convax product lock input closure", () => { hardlinked.root, "catalog", "releases", - `registry-v2-${"a".repeat(64)}`, + `registry-v2-${hardlinked.revision}`, "marketplace.json", ) await fs.link(descriptor, path.join(hardlinked.root, "descriptor-hardlink.json")) @@ -276,6 +467,17 @@ describe("Convax product lock input closure", () => { await expect(verifyProductLockInput(path.join(symlinked.root, "product-lock-input.json"))) .rejects.toThrow("single-link regular no-follow file") + const contextAlias = await writeFixture() + const contextTarget = path.join(contextAlias.root, "selection-context-target.json") + await fs.writeFile(contextTarget, "{}\n") + await fs.symlink( + contextTarget, + path.join(contextAlias.root, "catalog", "selection-context.json"), + ) + await expect(verifyProductLockInput( + path.join(contextAlias.root, "product-lock-input.json"), + )).rejects.toThrow("selection context must be an existing single-link regular no-follow file") + const oversized = await writeFixture() const builtinPath = path.join( oversized.root, diff --git a/tooling/publication-plan.test.js b/tooling/publication-plan.test.js new file mode 100644 index 0000000..8fbba6d --- /dev/null +++ b/tooling/publication-plan.test.js @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test" +import { composePublicationPlan } from "./publication-plan.mjs" + +describe("Marketplace publication plan", () => { + test("publishes selected packages, one metadata Release, and the changed Builtin bundle", () => { + const selected = [ + { + kind: "plugin", + id: "ffmpeg-tools", + releaseTag: "plugin-ffmpeg-tools-v0.3.1", + }, + { + kind: "skill", + id: "canvas-storyboard", + releaseTag: "skill-canvas-storyboard-v0.1.0", + }, + ] + const metadataTag = `registry-v2-${"a".repeat(64)}` + const catalog = { + schema: "convax.release-plan/1", + releases: [ + { + tag: "plugin-ffmpeg-tools-v0.3.1", + assets: [{ path: "releases/plugin-ffmpeg-tools-v0.3.1/plugin.zip" }], + }, + { + tag: "skill-canvas-storyboard-v0.1.0", + assets: [{ path: "releases/skill-canvas-storyboard-v0.1.0/skill.zip" }], + }, + { + tag: metadataTag, + assets: [{ path: `releases/${metadataTag}/registry-v2.json` }], + }, + ], + } + const builtin = { + schema: "convax.release-plan/1", + releases: [{ + tag: "builtin-release", + assets: [{ path: "releases/builtin-release/convax-builtin-bundle.zip" }], + }], + } + + expect(composePublicationPlan({ builtin, catalog, selected })).toEqual({ + schema: "convax.publication-plan/1", + releases: [ + { directory: "builtin/releases/builtin-release", tag: "builtin-release" }, + { + directory: "catalog/releases/plugin-ffmpeg-tools-v0.3.1", + tag: "plugin-ffmpeg-tools-v0.3.1", + }, + { directory: `catalog/releases/${metadataTag}`, tag: metadataTag }, + { + directory: "catalog/releases/skill-canvas-storyboard-v0.1.0", + tag: "skill-canvas-storyboard-v0.1.0", + }, + ], + }) + expect(() => composePublicationPlan({ + builtin, + catalog: { + ...catalog, + releases: catalog.releases.filter((entry) => entry.tag !== metadataTag), + }, + selected, + })).toThrow("exactly one Registry metadata Release") + }) +}) diff --git a/tooling/verify-marketplace-output.mjs b/tooling/verify-marketplace-output.mjs index 6ba53a5..e5e7d82 100644 --- a/tooling/verify-marketplace-output.mjs +++ b/tooling/verify-marketplace-output.mjs @@ -2,6 +2,21 @@ import { createHash } from "node:crypto" import { promises as fs } from "node:fs" import path from "node:path" import { fileURLToPath } from "node:url" +import { + canonicalJson, + parseMarketplaceDescriptor, + parseRegistryV1, + parseRegistryV2, + parseShowcaseV1, + parseShowcaseV2, + projectRegistryV1, +} from "@convax/marketplace" +import { + assertSelectiveMarketplaceClosure, + packageIdentity, + parseMarketplaceSelectionContext, + releaseTagForPackage, +} from "@convax/marketplace-kit" const releaseBase = "https://github.com/microvoid/convax-plugins/releases/download/" const digestPattern = /^[a-f0-9]{64}$/ @@ -18,6 +33,29 @@ async function readJson(file, label) { } } +async function readOptionalJson(file, label) { + try { + return await readJson(file, label) + } catch (error) { + if (error.cause?.code === "ENOENT") return undefined + throw error + } +} + +function exactKeys(value, keys, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + const actual = Object.keys(value).sort() + const expected = [...keys].sort() + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + throw new Error(`${label} has unsupported or missing fields`) + } +} + function projectedPackages(registry, version) { const label = `Registry v${version}` const topLevelKeys = version === 1 @@ -73,48 +111,46 @@ function assertEqualMaps(left, right, identityMessage, versionMessage) { } } -function expectedReleaseTag(entry) { - if (entry.kind === "mcp-server") { - const key = sha256(Buffer.from(`mcp-server\0${entry.id}`, "utf8")) - return `mcp-server-${key.slice(0, 16)}-v${entry.version}` - } - return `${entry.kind}-${entry.id}-v${entry.version}` -} - function selectedPackageTags(registryPackages, selectedVersions) { - const packageTags = new Set(registryPackages.map(expectedReleaseTag)) + const packageTags = new Set(registryPackages.map(releaseTagForPackage)) if (selectedVersions === undefined) return packageTags - if (!Array.isArray(selectedVersions)) { - throw new Error("selected version changes must be an array") + if ( + !Array.isArray(selectedVersions) || + selectedVersions.length === 0 || + selectedVersions.length > 16_384 + ) { + throw new Error("selected version changes must be a bounded non-empty array") } const registryByIdentity = new Map( - registryPackages.map((entry) => [`${entry.kind}\0${entry.id}`, entry]), + registryPackages.map((entry) => [packageIdentity(entry), entry]), ) const selectedIdentities = new Set() const selectedTags = new Set() for (const entry of selectedVersions) { + const expectedFields = [ + "id", + "kind", + ...(entry?.previousVersion === undefined ? [] : ["previousVersion"]), + "releaseTag", + "version", + ] + exactKeys(entry, expectedFields, "selected version change") if ( - !entry || - typeof entry !== "object" || - Array.isArray(entry) || typeof entry.kind !== "string" || typeof entry.id !== "string" || typeof entry.version !== "string" || - typeof entry.itemKey !== "string" || typeof entry.releaseTag !== "string" || (entry.previousVersion !== undefined && typeof entry.previousVersion !== "string") ) { throw new Error("selected version change is incomplete") } - const identity = `${entry.kind}\0${entry.id}` + const identity = packageIdentity(entry) const registryEntry = registryByIdentity.get(identity) - const expectedItemKey = sha256(Buffer.from(identity, "utf8")) if ( !registryEntry || registryEntry.version !== entry.version || - entry.itemKey !== expectedItemKey || - entry.releaseTag !== expectedReleaseTag(registryEntry) + entry.releaseTag !== releaseTagForPackage(registryEntry) ) { throw new Error(`selected version change ${entry.kind}/${entry.id} differs from Registry v2`) } @@ -127,6 +163,33 @@ function selectedPackageTags(registryPackages, selectedVersions) { return selectedTags } +function assertSelectionPlanMatchesContext(selectedVersions, context) { + if (selectedVersions === undefined) { + throw new Error("selection context requires selected version changes") + } + const plannedByIdentity = new Map( + selectedVersions.map((entry) => [packageIdentity(entry), entry]), + ) + if (plannedByIdentity.size !== context.selectedPackages.length) { + throw new Error("selection context differs from selected version changes") + } + for (const selected of context.selectedPackages) { + const planned = plannedByIdentity.get(packageIdentity(selected)) + if ( + !planned || + planned.kind !== selected.kind || + planned.id !== selected.id || + planned.version !== selected.version || + planned.previousVersion !== selected.sourcePreviousVersion || + planned.releaseTag !== selected.releaseTag + ) { + throw new Error( + `selection context ${selected.kind}/${selected.id} differs from selected version changes`, + ) + } + } +} + function releaseReference(value, label) { if ( !value || @@ -225,6 +288,32 @@ async function verifyReleaseAsset(catalogDirectory, catalogRealPath, tag, asset) } } +async function verifyExactReleaseDirectory(catalogDirectory, catalogRealPath, release) { + const expectedNames = new Set(release.assets.map((asset) => asset.name)) + if (expectedNames.size !== release.assets.length) { + throw new Error(`release-plan ${release.tag} contains duplicate asset names`) + } + const releaseDirectory = path.join(catalogDirectory, "releases", release.tag) + const state = await fs.lstat(releaseDirectory).catch((cause) => { + if (cause?.code === "ENOENT") return undefined + throw cause + }) + if (!state?.isDirectory() || state.isSymbolicLink()) { + throw new Error(`release-plan ${release.tag} has no exact local Release directory`) + } + const real = await fs.realpath(releaseDirectory) + if (!real.startsWith(`${catalogRealPath}${path.sep}`)) { + throw new Error(`release-plan ${release.tag} directory escapes the catalog directory`) + } + const entries = await fs.readdir(real, { withFileTypes: true }) + if ( + entries.length !== expectedNames.size || + entries.some((entry) => !entry.isFile() || !expectedNames.has(entry.name)) + ) { + throw new Error(`release-plan ${release.tag} directory contents differ from its exact assets`) + } +} + async function readExactLocalRelease(catalogDirectory, catalogRealPath, url, expected) { const { tag, name } = parseReleaseUrl(url, "Registry") const relativePath = `releases/${tag}/${name}` @@ -278,15 +367,34 @@ async function readExactCatalogFile( export async function verifyMarketplaceOutput( catalogDirectory, - { selectedVersions } = {}, + { selectedVersions, selectionContext } = {}, ) { - const [descriptor, registryV2, registryV1, showcaseV2, releasePlan] = await Promise.all([ + const [ + descriptorValue, + registryV2Value, + registryV1Value, + showcaseV2Value, + showcaseV1Value, + releasePlan, + storedContext, + ] = await Promise.all([ readJson(path.join(catalogDirectory, "marketplace.json"), "Marketplace descriptor"), readJson(path.join(catalogDirectory, "registry-v2.json"), "Registry v2"), readJson(path.join(catalogDirectory, "registry-v1.json"), "Registry v1"), readJson(path.join(catalogDirectory, "showcase-v2.json"), "Showcase v2"), + readJson(path.join(catalogDirectory, "showcase-v1.json"), "Showcase v1"), readJson(path.join(catalogDirectory, "release-plan.json"), "release-plan"), + selectionContext === undefined + ? readOptionalJson( + path.join(catalogDirectory, "selection-context.json"), + "selection context", + ) + : undefined, ]) + const descriptor = descriptorValue + const registryV2 = registryV2Value + const registryV1 = registryV1Value + const showcaseV2 = showcaseV2Value if ( !Array.isArray(registryV2.packages) || descriptor?.schema !== "convax.marketplace/1" || @@ -319,8 +427,43 @@ export async function verifyMarketplaceOutput( "v1 Plugin/Skill versions differ from Registry v2", ) - const packageTags = new Set(registryV2.packages.map(expectedReleaseTag)) + const parsedDescriptor = parseMarketplaceDescriptor(descriptor) + const parsedRegistryV2 = parseRegistryV2(registryV2) + const parsedRegistryV1 = parseRegistryV1(registryV1) + const parsedShowcaseV2 = parseShowcaseV2( + showcaseV2, + parsedRegistryV2, + parsedDescriptor, + ) + const parsedShowcaseV1 = parseShowcaseV1( + showcaseV1Value, + parsedRegistryV1, + parsedDescriptor, + ) + const expectedV1 = projectRegistryV1(parsedRegistryV2, parsedRegistryV1.revision) + if (canonicalJson(parsedRegistryV1) !== canonicalJson(expectedV1)) { + throw new Error("Registry v1 differs from the exact Registry v2 projection") + } + const publishedPackageTags = selectedPackageTags(registryV2.packages, selectedVersions) + const contextValue = selectionContext ?? storedContext + if (contextValue === undefined) { + if (selectedVersions !== undefined) { + throw new Error("selected publication requires a production selection context") + } + } else { + const context = parseMarketplaceSelectionContext(contextValue, parsedDescriptor) + assertSelectionPlanMatchesContext(selectedVersions, context) + assertSelectiveMarketplaceClosure({ + context, + descriptor: parsedDescriptor, + registry: parsedRegistryV2, + showcase: parsedShowcaseV2, + registryV1: parsedRegistryV1, + showcaseV1: parsedShowcaseV1, + }) + } + const metadataTag = `registry-v2-${registryV2.revision}` const admittedPlanTags = new Set([...publishedPackageTags, metadataTag]) const assetsByUrl = new Map() @@ -344,6 +487,7 @@ export async function verifyMarketplaceOutput( assetsByUrl.set(asset.url, asset) releaseAssets += 1 } + await verifyExactReleaseDirectory(catalogDirectory, catalogRealPath, entry) } if ( actualTags.size !== admittedPlanTags.size || @@ -388,6 +532,7 @@ export async function verifyMarketplaceOutput( ["site/marketplace.json", "marketplace.json", "descriptor"], ["site/registry/v1/index.json", "registry-v1.json", "registry v1"], ["site/registry/v2/index.json", "registry-v2.json", "registry v2"], + ["site/showcase/v1/index.json", "showcase-v1.json", "showcase v1"], ["site/showcase/v2/index.json", "showcase-v2.json", "showcase v2"], ]) { const [siteBytes, flatBytes] = await Promise.all([ @@ -409,19 +554,39 @@ export async function verifyMarketplaceOutput( } } - for (const reference of collectReleaseReferences(registryV2)) { + const selectedIdentities = selectedVersions === undefined + ? undefined + : new Set(selectedVersions.map(packageIdentity)) + const verifyPlannedReference = async (reference) => { await readExactLocalRelease(catalogDirectory, catalogRealPath, reference.url, reference) - const { tag } = parseReleaseUrl(reference.url, "Registry") - if (!publishedPackageTags.has(tag)) continue const asset = assetsByUrl.get(reference.url) if (!asset) { throw new Error(`${reference.url} is absent from the release-plan`) } if (asset.size !== reference.size || asset.sha256 !== reference.sha256) { - throw new Error(`${reference.url} Registry metadata differs from release-plan bytes`) + throw new Error(`${reference.url} metadata differs from release-plan bytes`) + } + } + for (const entry of registryV2.packages) { + const requiresLocalRelease = selectedIdentities === undefined || + selectedIdentities.has(packageIdentity(entry)) + for (const reference of collectReleaseReferences(entry)) { + if (!requiresLocalRelease) continue + await verifyPlannedReference(reference) + } + } + for (const reference of collectReleaseReferences(parsedShowcaseV2, "Showcase v2")) { + await verifyPlannedReference(reference) + } + for (const entry of parsedShowcaseV1.packages) { + if (selectedIdentities !== undefined && !selectedIdentities.has(packageIdentity(entry))) { + continue + } + for (const reference of collectReleaseReferences(entry, "Showcase v1")) { + await verifyPlannedReference(reference) } } - for (const tag of packageTags) { + for (const tag of publishedPackageTags) { const releaseDirectory = path.join(catalogDirectory, "releases", tag) const state = await fs.lstat(releaseDirectory).catch((cause) => { if (cause?.code === "ENOENT") return undefined diff --git a/tooling/verify-product-lock-input.mjs b/tooling/verify-product-lock-input.mjs index eb46f39..062d78a 100644 --- a/tooling/verify-product-lock-input.mjs +++ b/tooling/verify-product-lock-input.mjs @@ -2,6 +2,20 @@ import { createHash } from "node:crypto" import { constants as fsConstants } from "node:fs" import { promises as fs } from "node:fs" import path from "node:path" +import { + canonicalJson, + parseMarketplaceDescriptor, + parseRegistryV1, + parseRegistryV2, + parseShowcaseV1, + parseShowcaseV2, + projectRegistryV1, +} from "@convax/marketplace" +import { + assertSelectiveMarketplaceClosure, + packageIdentity, + parseMarketplaceSelectionContext, +} from "@convax/marketplace-kit" const releaseBase = "https://github.com/microvoid/convax-plugins/releases/download/" const digestPattern = /^[a-f0-9]{64}$/ @@ -36,6 +50,79 @@ function exactKeys(value, keys, label) { } } +function inheritedSelection({ + context, + descriptor, + registry, + registryV1, + selectedVersions, + showcase, + showcaseV1, +}) { + if (context === undefined) { + if (selectedVersions !== undefined) { + throw new Error("selected product lock requires a production selection context") + } + return new Set() + } + if (!Array.isArray(selectedVersions) || selectedVersions.length === 0) { + throw new Error("selection context requires selected version changes") + } + const parsedContext = parseMarketplaceSelectionContext(context, descriptor) + const selectedPackages = selectedVersions.map((entry) => { + exactKeys( + entry, + [ + "id", + "kind", + "releaseTag", + "version", + ...(entry?.previousVersion === undefined ? [] : ["previousVersion"]), + ], + "selected version change", + ) + if ( + typeof entry.kind !== "string" || + typeof entry.id !== "string" || + typeof entry.version !== "string" || + typeof entry.releaseTag !== "string" || + (entry.previousVersion !== undefined && typeof entry.previousVersion !== "string") + ) { + throw new Error("selected version change is incomplete") + } + return { + kind: entry.kind, + id: entry.id, + version: entry.version, + ...(entry.previousVersion === undefined + ? {} + : { sourcePreviousVersion: entry.previousVersion }), + releaseTag: entry.releaseTag, + } + }) + const contextSelections = parsedContext.selectedPackages.map((entry) => ({ + kind: entry.kind, + id: entry.id, + version: entry.version, + ...(entry.sourcePreviousVersion === undefined + ? {} + : { sourcePreviousVersion: entry.sourcePreviousVersion }), + releaseTag: entry.releaseTag, + })) + if (canonicalJson(selectedPackages) !== canonicalJson(contextSelections)) { + throw new Error("selection context differs from selected version changes") + } + return assertSelectiveMarketplaceClosure({ + context: parsedContext, + descriptor, + registry, + registryV1, + showcase, + showcaseV1, + }) + .inheritedIdentities +} + function parseReleaseUrl(url, label) { if (typeof url !== "string" || !url.startsWith(releaseBase)) { throw new Error(`${label} must use the Official immutable Release origin`) @@ -155,6 +242,25 @@ async function readStableFile( } } +async function readOptionalStableJson( + root, + rootRealPath, + relativePath, + label, + maximumSize, +) { + const absolute = path.join(root, ...relativePath.split("/")) + const state = await fs.lstat(absolute).catch((cause) => { + if (cause?.code === "ENOENT") return undefined + throw cause + }) + if (state === undefined) return undefined + return parseJson( + await readStableFile(root, rootRealPath, relativePath, label, maximumSize), + label, + ) +} + function releasePlanIndex(plan, area) { if ( plan?.schema !== "convax.release-plan/1" || @@ -205,7 +311,7 @@ async function verifyLockedArtifact({ releaseAssets, label, expected, - allowExistingRelease = false, + inherited = false, }) { exactKeys(lock, ["path", "url"], label) const relativePath = validateRelativePath(lock.path, `${label}.path`) @@ -213,11 +319,19 @@ async function verifyLockedArtifact({ throw new Error(`${label} differs from Registry v2`) } const { tag, name } = parseReleaseUrl(lock.url, `${label}.url`) - if (relativePath !== `${area}/releases/${tag}/${name}`) { + const releaseAsset = releaseAssets.get(lock.url) + if (inherited) { + if (!expected) throw new Error(`${label} cannot inherit bytes without Registry metadata`) + if (relativePath !== `${area}/inherited/${expected.sha256}/${name}`) { + throw new Error(`${label} inherited path differs from its Registry digest and basename`) + } + if (releaseAsset) { + throw new Error(`${label} is inherited but also appears in the ${area} release-plan`) + } + } else if (relativePath !== `${area}/releases/${tag}/${name}`) { throw new Error(`${label} path and URL identify different immutable bytes`) } - const releaseAsset = releaseAssets.get(lock.url) - if (!releaseAsset && (!allowExistingRelease || !expected)) { + if (!inherited && !releaseAsset) { throw new Error(`${label} is absent from the ${area} release-plan`) } if (releaseAsset) { @@ -243,12 +357,12 @@ async function verifyLockedArtifact({ label, maximumSize, ) - const admitted = releaseAsset ?? expected + const admitted = inherited ? expected : releaseAsset if (bytes.length !== admitted.size || sha256(bytes) !== admitted.sha256) { throw new Error( - releaseAsset - ? `${label} bytes differ from the immutable release-plan` - : `${label} bytes differ from Registry v2`, + inherited + ? `${label} bytes differ from Registry v2` + : `${label} bytes differ from the immutable release-plan`, ) } return bytes @@ -270,7 +384,10 @@ function registryArtifact(entry, label) { return delivery } -export async function verifyProductLockInput(inputFile) { +export async function verifyProductLockInput( + inputFile, + { selectedVersions, selectionContext } = {}, +) { const lockPath = path.resolve(inputFile) const root = path.dirname(lockPath) const rootRealPath = await fs.realpath(root) @@ -312,7 +429,16 @@ export async function verifyProductLockInput(inputFile) { throw new Error("product lock must contain exactly one ffmpeg-tools preinstall") } - const [catalogPlan, builtinPlan, registry] = await Promise.all([ + const [ + catalogPlan, + builtinPlan, + registryValue, + showcaseValue, + registryV1Value, + showcaseV1Value, + descriptorValue, + storedContext, + ] = await Promise.all([ readStableFile( root, rootRealPath, @@ -334,18 +460,68 @@ export async function verifyProductLockInput(inputFile) { "Registry v2", metadataLimit, ).then((bytes) => parseJson(bytes, "Registry v2")), + readStableFile( + root, + rootRealPath, + "catalog/showcase-v2.json", + "Showcase v2", + metadataLimit, + ).then((bytes) => parseJson(bytes, "Showcase v2")), + readStableFile( + root, + rootRealPath, + "catalog/registry-v1.json", + "Registry v1", + metadataLimit, + ).then((bytes) => parseJson(bytes, "Registry v1")), + readStableFile( + root, + rootRealPath, + "catalog/showcase-v1.json", + "Showcase v1", + metadataLimit, + ).then((bytes) => parseJson(bytes, "Showcase v1")), + readStableFile( + root, + rootRealPath, + "catalog/marketplace.json", + "Marketplace descriptor", + metadataLimit, + ).then((bytes) => parseJson(bytes, "Marketplace descriptor")), + selectionContext === undefined + ? readOptionalStableJson( + root, + rootRealPath, + "catalog/selection-context.json", + "selection context", + metadataLimit, + ) + : undefined, ]) + const descriptor = parseMarketplaceDescriptor(descriptorValue) + const registry = parseRegistryV2(registryValue) + const showcase = parseShowcaseV2(showcaseValue, registry, descriptor) + const registryV1 = parseRegistryV1(registryV1Value) + const showcaseV1 = parseShowcaseV1(showcaseV1Value, registryV1, descriptor) const catalogAssets = releasePlanIndex(catalogPlan, "catalog") const builtinAssets = releasePlanIndex(builtinPlan, "builtin") + const projectedRegistryV1 = projectRegistryV1(registry, registryV1.revision) if ( - registry?.schema !== "convax.registry/2" || registry.marketplaceId !== "convax-official" || - typeof registry.revision !== "string" || - !digestPattern.test(registry.revision) || - !Array.isArray(registry.packages) + registryV1.sequence !== registry.sequence || + canonicalJson(registryV1.packages) !== canonicalJson(projectedRegistryV1.packages) ) { - throw new Error("Registry v2 is not the strict Official registry") - } + throw new Error("Registry v2/v1 are not the strict Official projections") + } + const inheritedIdentities = inheritedSelection({ + context: selectionContext ?? storedContext, + descriptor, + registry, + registryV1, + selectedVersions, + showcase, + showcaseV1, + }) const builtinBundle = await verifyLockedArtifact({ root, @@ -411,16 +587,6 @@ export async function verifyProductLockInput(inputFile) { throw new Error(`Official ${slot} Release differs from flat ${name}`) } } - const descriptor = parseJson( - await readStableFile( - root, - rootRealPath, - "catalog/marketplace.json", - "Marketplace descriptor", - metadataLimit, - ), - "Marketplace descriptor", - ) if ( descriptor?.id !== "convax-official" || descriptor.registry?.v2?.url !== "https://microvoid.github.io/convax-plugins/registry/v2/index.json" || @@ -471,7 +637,7 @@ export async function verifyProductLockInput(inputFile) { releaseAssets: catalogAssets, label: "preinstalled Plugin artifact", expected: registryArtifact(plugin, "ffmpeg-tools"), - allowExistingRelease: true, + inherited: inheritedIdentities.has(packageIdentity(plugin)), }) const declaredOwnedSkills = plugin.manifest?.contributes?.skills @@ -493,7 +659,7 @@ export async function verifyProductLockInput(inputFile) { releaseAssets: catalogAssets, label: "preinstalled owned Skill artifact", expected: registryArtifact(ownedSkill, "ffmpeg-canvas"), - allowExistingRelease: true, + inherited: inheritedIdentities.has(packageIdentity(ownedSkill)), }) const lockedCompanion = preinstalled.companions[0] @@ -515,7 +681,7 @@ export async function verifyProductLockInput(inputFile) { releaseAssets: catalogAssets, label: "preinstalled companion artifact", expected: companionTarget.artifact, - allowExistingRelease: true, + inherited: inheritedIdentities.has(packageIdentity(plugin)), }) return { @@ -527,8 +693,15 @@ export async function verifyProductLockInput(inputFile) { async function main(argv) { if (argv.length > 1) throw new Error("Usage: verify-product-lock-input [input-file]") + const selectedVersions = process.env.CONVAX_MARKETPLACE_CHANGED + ? JSON.parse(await fs.readFile( + path.resolve(process.env.CONVAX_MARKETPLACE_CHANGED), + "utf8", + )) + : undefined const result = await verifyProductLockInput( path.resolve(argv[0] ?? "dist/product-lock-input.json"), + { selectedVersions }, ) console.log( `Verified ${result.builtinReservations} Builtin reservation, ` + diff --git a/tooling/workspaces.test.js b/tooling/workspaces.test.js index 31c7463..c47c2bd 100644 --- a/tooling/workspaces.test.js +++ b/tooling/workspaces.test.js @@ -70,13 +70,15 @@ describe("Bun workspace ownership", () => { expect(Object.keys(lock.workspaces)).toContain("packages/skills/ffmpeg-canvas") }) - test("dogfoods one exact public Kit and documents the third-party scaffold path", async () => { + test("dogfoods exact public Marketplace packages and documents the third-party scaffold path", async () => { const [rootPackage, readme, readmeZh] = await Promise.all([ readJson(path.join(root, "package.json")), fs.readFile(path.join(root, "README.md"), "utf8"), fs.readFile(path.join(root, "README.zh-CN.md"), "utf8"), ]) - expect(rootPackage.devDependencies["@convax/marketplace-kit"]).toBe("0.1.0") + expect(rootPackage.devDependencies["@convax/marketplace"]).toBe("0.1.1") + expect(rootPackage.devDependencies["@convax/marketplace"]).not.toContain("file:") + expect(rootPackage.devDependencies["@convax/marketplace-kit"]).toBe("0.1.1") expect(rootPackage.devDependencies["@convax/marketplace-kit"]).not.toContain("file:") expect(rootPackage.scripts["marketplace:check"]).toBe("convax-marketplace check .") expect(rootPackage.scripts["marketplace:build-index"]).toContain("official-marketplace-build.mjs")