From f55edffc9e88924f9a4c95f41c4d0bc9b72422f8 Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Tue, 4 Aug 2026 13:19:22 -0500 Subject: [PATCH 1/3] Distinguish compilation provenance digests Compilation artifacts record both the exact submitted Head digest and the canonical Foundation Plan digest. These values legitimately differ after normalization, so validate each at its actual boundary while keeping artifact bytes bound to succeeded status metadata. Exercise the distinction in unit, CLI, and packed-runtime tests. --- README.md | 6 ++++-- scripts/smoke-package.js | 5 ++++- src/compilation-artifact.js | 21 +++++++++++++++++++-- test/compilation-artifact.test.js | 18 ++++++++++++++---- test/compilation.test.js | 11 ++++++----- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 38211bf..17da0e9 100644 --- a/README.md +++ b/README.md @@ -193,8 +193,10 @@ firstdraft compilation download 01900000-0000-7000-8000-000000000001 --output .. The command validates the UUID and output path before network access, makes one status `GET`, requires `succeeded`, and makes one artifact `GET`. It never starts work or polls. Historical artifact validation uses -the retained `compilation.head_source_sha256`, not the current local Plan or ETag; both artifact -`head_source_sha256` and `foundation_plan.sha256` must equal that retained Head. +the retained `compilation.head_source_sha256`, not the current local Plan or ETag, to pin the artifact's exact +`head_source_sha256`. The artifact's canonical `foundation_plan.sha256` may differ because it identifies the +normalized Compiler input. It is validated as a SHA-256 digest inside the exact artifact bytes authenticated by +the status response's `artifact.sha256`; it is not equated to the submitted Head digest. Before materialization, the CLI verifies the artifact media type, declared and actual byte sizes, strong digest ETag, exact-byte SHA-256, canonical UTF-8 JSON envelope, provenance, metadata-only manifest digest, portable paths, diff --git a/scripts/smoke-package.js b/scripts/smoke-package.js index fcde686..518998d 100644 --- a/scripts/smoke-package.js +++ b/scripts/smoke-package.js @@ -327,6 +327,9 @@ async function exercisePackedCompilation(projectDirectory) { path.join(projectDirectory, ".firstdraft", "foundation-plan.json"), ); const headSha256 = sha256(plan); + const foundationPlanSha256 = sha256( + Buffer.from("canonical Foundation Plan snapshot"), + ); const statusPath = `/v1/projects/${projectId}/compilations/${compilationId}`; const artifactPath = `${statusPath}/artifact`; const compilerRelease = "foundation-plan-rails/compiler-scalar-2026-08"; @@ -360,7 +363,7 @@ async function exercisePackedCompilation(projectDirectory) { head_source_sha256: headSha256, foundation_plan: { format: "firstdraft.foundation-plan.sketch/0.19", - sha256: headSha256, + sha256: foundationPlanSha256, }, analysis: { id: analysisId, diff --git a/src/compilation-artifact.js b/src/compilation-artifact.js index 2e4d77f..a550479 100644 --- a/src/compilation-artifact.js +++ b/src/compilation-artifact.js @@ -64,6 +64,19 @@ export class CompilationArtifactInvalidError extends Error {} export class CompilationMaterializationError extends Error {} export class CompilationOutputPathError extends Error {} +/** + * @typedef {object} ValidatedCompilationProvenance + * @property {string} compilation_id + * @property {string} project_id + * @property {number} graph_version + * @property {string} head_source_sha256 + * @property {{format: string, sha256: string}} foundation_plan + * @property {{id: string, release: string}} analysis + * @property {string} compiler_release + * @property {{id: string, profile: string}} target + * @property {{repository: string, revision: string, sha256: string}} core + */ + /** * @typedef {object} CompilationArtifactExpectations * @property {string} projectId @@ -88,7 +101,7 @@ export class CompilationOutputPathError extends Error {} /** * @typedef {object} ValidatedCompilationArtifact * @property {string} manifest_sha256 - * @property {Record} provenance + * @property {ValidatedCompilationProvenance} provenance * @property {ValidatedArtifactFile[]} files */ @@ -236,8 +249,11 @@ export function materializeCompilationArtifact(artifact, target) { /** * @param {unknown} value * @param {CompilationArtifactExpectations} expected + * @returns {ValidatedCompilationProvenance} */ function parseProvenance(value, expected) { + // The outer artifact digest authenticates the canonical Plan digest, which + // need not equal the digest of the exact submitted Head bytes. if ( !hasExactKeys(value, PROVENANCE_KEYS) || value.compilation_id !== expected.compilationId || @@ -246,7 +262,8 @@ function parseProvenance(value, expected) { value.head_source_sha256 !== expected.headSourceSha256 || !hasExactKeys(value.foundation_plan, FOUNDATION_PLAN_KEYS) || value.foundation_plan.format !== FOUNDATION_PLAN_FORMAT || - value.foundation_plan.sha256 !== expected.headSourceSha256 || + typeof value.foundation_plan.sha256 !== "string" || + !SHA256_PATTERN.test(value.foundation_plan.sha256) || !hasExactKeys(value.analysis, ANALYSIS_KEYS) || value.analysis.id !== expected.analysisRunId || !isRelease(value.analysis.release) || diff --git a/test/compilation-artifact.test.js b/test/compilation-artifact.test.js index 8aa645a..8a66248 100644 --- a/test/compilation-artifact.test.js +++ b/test/compilation-artifact.test.js @@ -28,6 +28,7 @@ const COMPILATION_ID = "01900000-0000-7000-8000-000000000802"; const ANALYSIS_ID = "01900000-0000-7000-8000-000000000803"; const SUBJECT_ID = "01900000-0000-7000-8000-000000000804"; const HEAD_SHA256 = "1".repeat(64); +const FOUNDATION_PLAN_SHA256 = "5".repeat(64); const COMPILER_RELEASE = "foundation-plan-rails/compiler-scalar-2026-08"; const TARGET = { id: "rails", profile: "rails-sketch/2026-08" }; const EXPECTED = { @@ -44,6 +45,15 @@ test("parses canonical binary-safe artifact bytes and materializes an exact tree assert.equal(MAX_ARTIFACT_BYTES, 16 * 1024 * 1024); const fixture = artifactFixture(); const artifact = parseCompilationArtifact(fixture.source, EXPECTED); + assert.equal(artifact.provenance.head_source_sha256, HEAD_SHA256); + assert.equal( + artifact.provenance.foundation_plan.sha256, + FOUNDATION_PLAN_SHA256, + ); + assert.notEqual( + artifact.provenance.head_source_sha256, + artifact.provenance.foundation_plan.sha256, + ); const parent = temporaryDirectory(context); const target = resolveOutputTarget({ cwd: parent, @@ -134,7 +144,7 @@ test("rejects noncanonical, duplicate-key, additive, and non-UTF-8 envelopes", ( } }); -test("pins every available provenance identity", () => { +test("pins external provenance identities and validates nested metadata", () => { /** @type {[string, string | number][]} */ const cases = [ ["compilation_id", "01900000-0000-7000-8000-000000000899"], @@ -159,7 +169,7 @@ test("pins every available provenance identity", () => { provenance: { foundation_plan: { format: "firstdraft.foundation-plan.sketch/0.18", - sha256: HEAD_SHA256, + sha256: FOUNDATION_PLAN_SHA256, }, }, }), @@ -167,7 +177,7 @@ test("pins every available provenance identity", () => { provenance: { foundation_plan: { format: FOUNDATION_PLAN_FORMAT, - sha256: "2".repeat(64), + sha256: "not-a-sha256", }, }, }), @@ -364,7 +374,7 @@ function artifactFixture(changes = {}) { head_source_sha256: HEAD_SHA256, foundation_plan: { format: FOUNDATION_PLAN_FORMAT, - sha256: HEAD_SHA256, + sha256: FOUNDATION_PLAN_SHA256, }, analysis: { id: ANALYSIS_ID, diff --git a/test/compilation.test.js b/test/compilation.test.js index ef7d8dd..2e08092 100644 --- a/test/compilation.test.js +++ b/test/compilation.test.js @@ -23,6 +23,7 @@ const COMPILATION_ID = "01900000-0000-7000-8000-000000003002"; const ANALYSIS_ID = "01900000-0000-7000-8000-000000003004"; const API_TOKEN = `fd_${"b".repeat(43)}`; const RETAINED_HEAD = "1".repeat(64); +const CANONICAL_PLAN = "4".repeat(64); const LOCAL_HEAD = "9".repeat(64); const CREATED_AT = "2026-08-04T12:00:00.000000Z"; const STARTED_AT = "2026-08-04T12:00:01.000000Z"; @@ -142,7 +143,7 @@ test("compilation status has a bounded wait and validates exact response shapes" } }); -test("compilation download reads retained status and artifact once without starting work", async (context) => { +test("compilation download distinguishes Head and Plan provenance without starting work", async (context) => { const cwd = remoteDirectory(context); const fixture = artifactFixture(); const status = compilationBody("succeeded", { artifact: fixture }); @@ -198,7 +199,7 @@ test("download requires succeeded status and validates historical Head provenanc assert.equal(queuedCalls.length, 1); assert.equal(existsSync(queuedOutput), false); - const mismatched = artifactFixture({ foundationPlanSha256: LOCAL_HEAD }); + const mismatched = artifactFixture({ headSourceSha256: LOCAL_HEAD }); const mismatchOutput = path.join(cwd, "mismatch-output"); const mismatch = await invoke( ["compilation", "download", COMPILATION_ID, "--output", mismatchOutput], @@ -381,7 +382,7 @@ function compilationBody(status, changes = {}) { }; } -/** @param {{foundationPlanSha256?: string}} [changes] */ +/** @param {{headSourceSha256?: string}} [changes] */ function artifactFixture(changes = {}) { const contents = Buffer.from("Movie Catalog\n"); const file = { @@ -409,10 +410,10 @@ function artifactFixture(changes = {}) { compilation_id: COMPILATION_ID, project_id: PROJECT_ID, graph_version: 7, - head_source_sha256: RETAINED_HEAD, + head_source_sha256: changes.headSourceSha256 ?? RETAINED_HEAD, foundation_plan: { format: FOUNDATION_PLAN_FORMAT, - sha256: changes.foundationPlanSha256 ?? RETAINED_HEAD, + sha256: CANONICAL_PLAN, }, analysis: { id: ANALYSIS_ID, From 6a9c8e2437bb1c31d360a5f9acd03c39c2dbbbd8 Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Tue, 4 Aug 2026 16:45:02 -0500 Subject: [PATCH 2/3] Make runtime digest reproducible Commit the exact length-delimited hashing recipe used by external evidence. Normalized relative-path ordering keeps the same runtime identity reproducible across supported platforms. --- README.md | 9 +++++++ scripts/runtime-digest.js | 50 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 scripts/runtime-digest.js diff --git a/README.md b/README.md index 17da0e9..120af62 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,13 @@ npm run check npm run pack:check ``` +To reproduce the length-delimited SHA-256 used by external evidence to identify the packaged JavaScript runtime +inputs (`package.json`, `bin/firstdraft.js`, and every `.js` file under `src/`), run from the repository root: + +```sh +node scripts/runtime-digest.js +``` + ## Start a Foundation Plan From the project that the Plan describes: @@ -156,6 +163,8 @@ structured output; no Compilation or Publication is requested. Only a `valid` analysis proceeds to the internal GitHub Publication lifecycle. Invoking `plan compile` is the authorization to request that lifecycle. Immediately before its conditional mutation, the CLI re-reads the local Plan and requires its exact bytes to match the accepted Head, so bytes changed after analysis cannot be published. +It extracts the accepted source SHA-256 from the saved ETag, hashes the current local bytes, and then sends that +complete ETag in `If-Match`. The command writes no progress output to stdout. Success is exactly one validated JSON object containing the retained `project`, `compilation`, and `publication`, including the private GitHub repository URL. diff --git a/scripts/runtime-digest.js b/scripts/runtime-digest.js new file mode 100644 index 0000000..18b6f93 --- /dev/null +++ b/scripts/runtime-digest.js @@ -0,0 +1,50 @@ +import { createHash } from "node:crypto"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const repository = fileURLToPath(new URL("..", import.meta.url)); +const inputs = listJavaScriptFiles(path.join(repository, "src")) + .concat([ + path.join(repository, "bin", "firstdraft.js"), + path.join(repository, "package.json"), + ]) + .map((file) => ({ + file, + relativePath: path.relative(repository, file).split(path.sep).join("/"), + })) + .sort((left, right) => + left.relativePath < right.relativePath + ? -1 + : left.relativePath > right.relativePath + ? 1 + : 0, + ); +const digest = createHash("sha256"); + +for (const { file, relativePath } of inputs) { + const source = readFileSync(file); + const relativePathLength = Buffer.alloc(4); + relativePathLength.writeUInt32BE(Buffer.byteLength(relativePath)); + const sourceLength = Buffer.alloc(8); + sourceLength.writeBigUInt64BE(BigInt(source.byteLength)); + digest.update(relativePathLength); + digest.update(relativePath); + digest.update(sourceLength); + digest.update(source); +} + +process.stdout.write(`${digest.digest("hex")}\n`); + +/** @param {string} directory @returns {string[]} */ +function listJavaScriptFiles(directory) { + return readdirSync(directory, { withFileTypes: true }) + .flatMap((entry) => { + const entryPath = path.join(directory, entry.name); + + if (entry.isDirectory()) return listJavaScriptFiles(entryPath); + + return entry.isFile() && entry.name.endsWith(".js") ? [entryPath] : []; + }) + .sort(); +} From ccdc2df04672490d2cffb7e60954d07a82cc7146 Mon Sep 17 00:00:00 2001 From: Raghu Betina Date: Tue, 4 Aug 2026 16:44:56 -0500 Subject: [PATCH 3/3] Explain Unicode transliteration map Record why normalization alone cannot preserve these letters before the ASCII identifier filter. This keeps the explicit map from looking arbitrary. --- src/application-identity.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/application-identity.js b/src/application-identity.js index c5b759b..8907b3f 100644 --- a/src/application-identity.js +++ b/src/application-identity.js @@ -5,6 +5,7 @@ export const MAX_DERIVED_APPLICATION_KEY_BYTES = 63; const DIGEST_HEX_LENGTH = 12; const APPLICATION_KEY_PATTERN = /^[a-z][a-z0-9_]*$/; const MARK_PATTERN = /\p{Mark}/u; +// NFKD does not decompose these Latin letters, so map them before ASCII filtering. const LATIN_TRANSLITERATIONS = new Map([ ["Æ", "ae"], ["æ", "ae"],