From 6370366d41f775daf5aad02a69b73207557cfd28 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 14:44:17 +0200 Subject: [PATCH 01/10] Retry stapling of notarized macOS pkg on transient failure xcrun stapler staple can fail right after notarytool submit --wait reports success, because Apple's notarization ticket can take a moment to propagate before it is fetchable. Retry a few times with a delay instead of failing the whole release build immediately. --- package/src/macos/installer.ts | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/package/src/macos/installer.ts b/package/src/macos/installer.ts index a14a00fe77a..4e1f35e262d 100644 --- a/package/src/macos/installer.ts +++ b/package/src/macos/installer.ts @@ -420,8 +420,30 @@ async function waitForNotaryStatus( } async function stapleNotary(input: string) { - await runCmd( - "xcrun", - ["stapler", "staple", input], - ); + // Apple's stapling ticket can take a few seconds to propagate after + // notarytool reports success, so an immediate staple attempt can fail + // with a transient error even though notarization succeeded. Retry + // with a short delay before giving up. + const maxAttempts = 5; + const retryDelayMs = 15000; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + await runCmd( + "xcrun", + ["stapler", "staple", input], + ); + return; + } catch (e) { + if (attempt === maxAttempts) { + throw e; + } + warning( + `Stapling failed (attempt ${attempt}/${maxAttempts}), retrying in ${ + retryDelayMs / 1000 + }s...`, + ); + sleepSync(retryDelayMs); + } + } } From 6efb7e953dfb1d1186f5aeb0173fe8fd0bb5b595 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 15:41:43 +0200 Subject: [PATCH 02/10] Use existing withRetry helper for pkg stapling retry src/core/retry.ts already provides the attempt/backoff pattern used elsewhere (e.g. src/core/download.ts); reuse it instead of a hand-rolled loop plus the sync sleep polyfill. --- package/src/macos/installer.ts | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/package/src/macos/installer.ts b/package/src/macos/installer.ts index 4e1f35e262d..c46c00fc873 100644 --- a/package/src/macos/installer.ts +++ b/package/src/macos/installer.ts @@ -18,6 +18,7 @@ import { Configuration } from "../common/config.ts"; import { runCmd } from "../util/cmd.ts"; import { getEnv } from "../util/utils.ts"; import { makeTarball } from "../util/tar.ts"; +import { withRetry } from "../../../src/core/retry.ts"; // Packaging specific configuration // (Some things are global others may be platform specific) @@ -424,26 +425,14 @@ async function stapleNotary(input: string) { // notarytool reports success, so an immediate staple attempt can fail // with a transient error even though notarization succeeded. Retry // with a short delay before giving up. - const maxAttempts = 5; - const retryDelayMs = 15000; - - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - await runCmd( - "xcrun", - ["stapler", "staple", input], - ); - return; - } catch (e) { - if (attempt === maxAttempts) { - throw e; - } - warning( - `Stapling failed (attempt ${attempt}/${maxAttempts}), retrying in ${ - retryDelayMs / 1000 - }s...`, - ); - sleepSync(retryDelayMs); - } - } + await withRetry(async () => { + await runCmd( + "xcrun", + ["stapler", "staple", input], + ); + }, { + attempts: 5, + minWait: 15000, + maxWait: 15000, + }); } From a6ea77ff21269787d38570c8e8c5c51e7c687f5a Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 15:54:42 +0200 Subject: [PATCH 03/10] Log stdout on command failure, not just stderr xcrun stapler (and potentially other tools invoked here) can write its actual failure reason to stdout rather than stderr. Without this, a failing command only surfaces an empty error message in CI logs. --- package/src/util/cmd.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package/src/util/cmd.ts b/package/src/util/cmd.ts index 0ce30974905..8bdb68a5e10 100644 --- a/package/src/util/cmd.ts +++ b/package/src/util/cmd.ts @@ -37,6 +37,10 @@ export async function runCmd( const code = output.code; info(`Status ${code}`); if (code !== 0) { + // Some tools (e.g. xcrun stapler) write their actual failure reason to + // stdout rather than stderr; without this, that reason is only visible + // at debug level and failures show an empty error message. + error(stdout); error(stderr); throw Error(`Command ${[runCmd, ...args]} failed.`); } From 6726ca78e36268e0819770541fcd6b6e37ab6aa7 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 16:14:39 +0200 Subject: [PATCH 04/10] Extend staple retry window and gate it to the known transient error The retry loop wasn't giving the notarization ticket enough time to become queryable in Apple's CloudKit-backed notary database: both a production run and a dry-run dispatch exhausted the previous ~80s retry window with the identical failure every attempt: CloudKit query for ... failed due to "Record not found". Could not find base64 encoded ticket in response for ... That message was only visible once runCmd started including stdout in its thrown error. Widen the retry window (6 attempts, 20-30s spacing, ~2.5min worst case) and only retry on that specific transient signature so a real signing/notarization failure still surfaces immediately instead of being retried for minutes. --- package/src/macos/installer.ts | 20 +++++++++++++------- package/src/util/cmd.ts | 4 +++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/package/src/macos/installer.ts b/package/src/macos/installer.ts index c46c00fc873..647767c4ac5 100644 --- a/package/src/macos/installer.ts +++ b/package/src/macos/installer.ts @@ -421,18 +421,24 @@ async function waitForNotaryStatus( } async function stapleNotary(input: string) { - // Apple's stapling ticket can take a few seconds to propagate after - // notarytool reports success, so an immediate staple attempt can fail - // with a transient error even though notarization succeeded. Retry - // with a short delay before giving up. + // Apple's notarization ticket can take a while to become queryable in + // Apple's CloudKit-backed notary database after notarytool reports + // success, so an immediate staple attempt can fail with: + // CloudKit query for ... failed due to "Record not found". + // Could not find base64 encoded ticket in response for ... + // Retry only on that known transient signature; anything else is a + // real signing/notarization problem and should fail immediately. await withRetry(async () => { await runCmd( "xcrun", ["stapler", "staple", input], ); }, { - attempts: 5, - minWait: 15000, - maxWait: 15000, + attempts: 6, + minWait: 20000, + maxWait: 30000, + retry: (err: Error) => + err.message.includes("Record not found") || + err.message.includes("CloudKit"), }); } diff --git a/package/src/util/cmd.ts b/package/src/util/cmd.ts index 8bdb68a5e10..e738b16e58b 100644 --- a/package/src/util/cmd.ts +++ b/package/src/util/cmd.ts @@ -42,7 +42,9 @@ export async function runCmd( // at debug level and failures show an empty error message. error(stdout); error(stderr); - throw Error(`Command ${[runCmd, ...args]} failed.`); + throw Error( + `Command ${[runCmd, ...args]} failed.\n${stdout}\n${stderr}`, + ); } return { From 80c4a2f06fca09bc836ee9b9affd7b103c7b61ff Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 16:21:14 +0200 Subject: [PATCH 05/10] Fix off-by-one in staple attempts and narrow retry predicate withRetry's attempts option counts retries after the first call, so attempts: 6 was actually running 7 total invocations / ~3min worst case instead of the intended 6 / ~2.5min. Also require both the CloudKit and "Record not found" signature fragments (not either alone) so an unrelated CloudKit error still fails fast instead of being retried for minutes. --- package/src/macos/installer.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/package/src/macos/installer.ts b/package/src/macos/installer.ts index 647767c4ac5..e135f76e582 100644 --- a/package/src/macos/installer.ts +++ b/package/src/macos/installer.ts @@ -434,11 +434,13 @@ async function stapleNotary(input: string) { ["stapler", "staple", input], ); }, { - attempts: 6, + // withRetry's `attempts` counts retries after the first call (see + // src/core/retry.ts), so 5 here yields 6 total staple invocations. + attempts: 5, minWait: 20000, maxWait: 30000, retry: (err: Error) => - err.message.includes("Record not found") || - err.message.includes("CloudKit"), + err.message.includes("CloudKit") && + err.message.includes("Record not found"), }); } From b7a0505a7e92f40243e51abb1724ebc580477510 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 19:25:32 +0200 Subject: [PATCH 06/10] Verify notarization is Accepted and make macOS staple non-fatal A bounded staple retry alone cannot fix this: every post-binary-bump CI run exhausted the retry window with the same CloudKit "Record not found" error and never recovered, and the evidence could not rule out the ticket simply never being issued. notarizeAndWait only trusted notarytool's exit code and scraped an id from stdout -- it never checked the terminal Accepted/Invalid status, which is the signal that actually distinguishes a propagation delay from a rejected submission, and that status never appeared in our CI logs at all. Parse and require status: Accepted before stapling (dumping the developer log on rejection), and always echo notarytool's output so the verdict is visible. Stapling only buys offline Gatekeeper validation -- a notarized-but-unstapled package still validates online on first launch -- so once the package is verified notarized, a persistent CloudKit propagation delay is downgraded to a warning and the release proceeds instead of failing. Any non-propagation staple error still throws. --- package/src/macos/installer.ts | 138 +++++++++++++++++------ package/src/macos/notarize-status.ts | 32 ++++++ tests/unit/macos-notarize-status.test.ts | 63 +++++++++++ 3 files changed, 199 insertions(+), 34 deletions(-) create mode 100644 package/src/macos/notarize-status.ts create mode 100644 tests/unit/macos-notarize-status.test.ts diff --git a/package/src/macos/installer.ts b/package/src/macos/installer.ts index e135f76e582..cf3c166eed7 100644 --- a/package/src/macos/installer.ts +++ b/package/src/macos/installer.ts @@ -19,6 +19,7 @@ import { runCmd } from "../util/cmd.ts"; import { getEnv } from "../util/utils.ts"; import { makeTarball } from "../util/tar.ts"; import { withRetry } from "../../../src/core/retry.ts"; +import { parseNotarizationResult } from "./notarize-status.ts"; // Packaging specific configuration // (Some things are global others may be platform specific) @@ -254,14 +255,13 @@ export async function makeInstallerMac(config: Configuration) { const password = getEnv("QUARTO_APPLE_CONNECT_PW", ""); const teamId = getEnv("QUARTO_APPLE_CONNECT_TEAMID", ""); if (username.length > 0 && password.length > 0) { - const requestId = await notarizeAndWait( + await notarizeAndWait( packagePath, username, password, - teamId + teamId, ); - // Staple the notary to the package await stapleNotary(packagePath); @@ -345,16 +345,66 @@ async function notarizeAndWait( ], ); - if (result.status.success) { - const match = result.stdout.match(/id: (.*)/); - if (match) { - const id = match[1]; - return id; - } else { - throw new Error("Notarization Failed to return an Id:\n" + result.stdout); + // Always surface notarytool's own output (submission id + terminal status). + // runCmd only logs stdout at debug level, so without this the Accepted / + // Invalid verdict never appears in CI logs. + info(result.stdout); + + if (!result.status.success) { + throw new Error("Notarization Failed\n" + result.stdout + result.stderr); + } + + const { id, status } = parseNotarizationResult(result.stdout); + if (!id) { + throw new Error("Notarization Failed to return an Id:\n" + result.stdout); + } + + // `notarytool submit --wait` can exit 0 even when the service rejects the + // submission; the trustworthy signal is the terminal status. Only a genuinely + // notarized package should reach the (offline-only) stapling step below. + if (status !== "Accepted") { + await dumpNotarizationLog(id, username, password, teamId); + throw new Error( + `Notarization was not accepted (status: ${status ?? "unknown"}) for ${input}`, + ); + } + + return id; +} + +// Best-effort dump of the full developer log for a submission, so a rejected +// notarization shows its reasons in CI. Never masks the caller's own error. +async function dumpNotarizationLog( + id: string, + username: string, + password: string, + teamId: string, +) { + try { + const log = await runCmd( + "xcrun", + [ + "notarytool", + "log", + id, + "--apple-id", + username, + "--password", + password, + "--team-id", + teamId, + ], + ); + info(log.stdout); + if (log.stderr) { + error(log.stderr); } - } else { - throw new Error("Notarization Failed\n" + result.stderr); + } catch (err) { + warning( + `Could not fetch notarization log for ${id}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); } } @@ -420,27 +470,47 @@ async function waitForNotaryStatus( return notaryResult; } +// Apple's notarization ticket can take a while to become queryable in Apple's +// CloudKit-backed notary database after notarytool reports success, so an +// immediate staple attempt can fail with: +// CloudKit query for ... failed due to "Record not found". +// Could not find base64 encoded ticket in response for ... +// This delay has no published bound; in CI it has exceeded a multi-minute +// retry window and never recovered within the build. +const isCloudKitPropagation = (err: Error) => + err.message.includes("CloudKit") && + err.message.includes("Record not found"); + async function stapleNotary(input: string) { - // Apple's notarization ticket can take a while to become queryable in - // Apple's CloudKit-backed notary database after notarytool reports - // success, so an immediate staple attempt can fail with: - // CloudKit query for ... failed due to "Record not found". - // Could not find base64 encoded ticket in response for ... - // Retry only on that known transient signature; anything else is a - // real signing/notarization problem and should fail immediately. - await withRetry(async () => { - await runCmd( - "xcrun", - ["stapler", "staple", input], - ); - }, { - // withRetry's `attempts` counts retries after the first call (see - // src/core/retry.ts), so 5 here yields 6 total staple invocations. - attempts: 5, - minWait: 20000, - maxWait: 30000, - retry: (err: Error) => - err.message.includes("CloudKit") && - err.message.includes("Record not found"), - }); + // Stapling only enables *offline* Gatekeeper validation; a notarized but + // unstapled package still validates on first (online) launch. Since the + // package is already verified Accepted by notarizeAndWait, a persistent + // CloudKit propagation delay must not fail the whole release: retry a bounded + // number of times to catch the common short delay, then continue with a + // warning. Any other staple failure is a real problem and still throws. + try { + await withRetry(async () => { + await runCmd( + "xcrun", + ["stapler", "staple", input], + ); + }, { + // withRetry's `attempts` counts retries after the first call (see + // src/core/retry.ts), so 3 here yields 4 total staple invocations. + attempts: 3, + minWait: 20000, + maxWait: 30000, + retry: isCloudKitPropagation, + }); + } catch (err) { + if (err instanceof Error && isCloudKitPropagation(err)) { + warning( + `Could not staple the notarization ticket to ${input} after retries ` + + `(Apple's ticket is not yet queryable). The package is notarized and ` + + `validates online on first launch; continuing without an offline staple.`, + ); + } else { + throw err; + } + } } diff --git a/package/src/macos/notarize-status.ts b/package/src/macos/notarize-status.ts new file mode 100644 index 00000000000..7c05cb602ab --- /dev/null +++ b/package/src/macos/notarize-status.ts @@ -0,0 +1,32 @@ +/* + * notarize-status.ts + * + * Copyright (C) 2020-2025 Posit Software, PBC + * + */ + +export interface NotarizationResult { + id?: string; + status?: string; +} + +// Parses the stdout of `xcrun notarytool submit --wait`. +// +// The output carries several `id:` lines (Submission ID, uploaded-file id, +// processing-complete id) and a terminal `status:` line inside the final +// "Processing complete" block. We take the first `id:` (the submission id, +// which is what `xcrun notarytool log ` expects) and the terminal +// `status:` (Accepted / Invalid / Rejected). +export function parseNotarizationResult(stdout: string): NotarizationResult { + // First `id:` line is the submission id (also what `notarytool log` wants). + const idMatch = stdout.match(/^\s*id:\s*(\S+)/m); + // Terminal `status:` line lives in the final "Processing complete" block. + // (`Current status:` lines are streamed progress, not the verdict, and don't + // match this anchored `status:` prefix.) Take the last one to be safe. + const statusMatches = [...stdout.matchAll(/^\s*status:\s*(\S+)/gm)]; + const lastStatus = statusMatches.at(-1); + return { + id: idMatch?.[1], + status: lastStatus?.[1], + }; +} diff --git a/tests/unit/macos-notarize-status.test.ts b/tests/unit/macos-notarize-status.test.ts new file mode 100644 index 00000000000..6ab25cc72a9 --- /dev/null +++ b/tests/unit/macos-notarize-status.test.ts @@ -0,0 +1,63 @@ +/* + * macos-notarize-status.test.ts + * + * Tests the parser that extracts the submission id and terminal status from + * `xcrun notarytool submit --wait` output. macOS release builds gate stapling + * (and, since PR #14718, whether the release proceeds at all) on the notary + * service returning `status: Accepted` -- a check the build previously skipped, + * trusting only the process exit code. + * + * Inputs are hand-written samples of notarytool's own output (the contract + * Apple hands us), so a change to Quarto's own build logic is what these guard, + * not Apple's service. + * + * Copyright (C) 2020-2025 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assertEquals } from "testing/asserts"; +import { parseNotarizationResult } from "../../package/src/macos/notarize-status.ts"; + +const acceptedOutput = `Conducting pre-submission checks for quarto-1.10.17-macos.pkg and initiating connection to the Apple notary service... +Submission ID received + id: 2efe2717-52ef-43a5-96dc-0797e4ca1041 +Successfully uploaded file + id: 2efe2717-52ef-43a5-96dc-0797e4ca1041 + path: /Users/runner/work/quarto-cli/quarto-1.10.17-macos.pkg +Waiting for processing to complete. +Current status: Accepted.......... +Processing complete + id: 2efe2717-52ef-43a5-96dc-0797e4ca1041 + status: Accepted +`; + +const invalidOutput = `Conducting pre-submission checks for quarto-1.10.17-macos.pkg and initiating connection to the Apple notary service... +Submission ID received + id: 9a9b9c9d-1111-2222-3333-444455556666 +Successfully uploaded file + id: 9a9b9c9d-1111-2222-3333-444455556666 + path: /Users/runner/work/quarto-cli/quarto-1.10.17-macos.pkg +Waiting for processing to complete. +Current status: Invalid.......... +Processing complete + id: 9a9b9c9d-1111-2222-3333-444455556666 + status: Invalid +`; + +unitTest("parseNotarizationResult - accepted submission", async () => { + const result = parseNotarizationResult(acceptedOutput); + assertEquals(result.id, "2efe2717-52ef-43a5-96dc-0797e4ca1041"); + assertEquals(result.status, "Accepted"); +}); + +unitTest("parseNotarizationResult - invalid submission", async () => { + const result = parseNotarizationResult(invalidOutput); + assertEquals(result.id, "9a9b9c9d-1111-2222-3333-444455556666"); + assertEquals(result.status, "Invalid"); +}); + +unitTest("parseNotarizationResult - no id or status when absent", async () => { + const result = parseNotarizationResult("some unrelated output\n"); + assertEquals(result.id, undefined); + assertEquals(result.status, undefined); +}); From 29d8f1bbc8dfb01e8e28424290e3fba429565bc7 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 19:48:37 +0200 Subject: [PATCH 07/10] Sign dart-sass sass.snapshot to fix macOS notarization Dart Sass 1.101.0 (bundled via #14664) switched its macOS AOT snapshot, dart-sass/src/sass.snapshot, from an ELF blob to a native Mach-O. Apple's notary ignores non-native files but requires every Mach-O in the payload to be signed, so the never-signed snapshot flipped notarization to Invalid ("Archive contains critical validation errors") and no ticket was issued. The subsequent "CloudKit ... Record not found" from stapler was a symptom of the missing ticket, not a propagation race; the earlier exit-code-only notarization check masked the rejection entirely. Sign the snapshot alongside the sass wrapper (no entitlements: it is loaded by the already-entitled dart VM, so it needs only a valid signature and secure timestamp). The staple retry is kept as narrow insurance against a genuine, rare CloudKit timing gap but no longer swallows failures: with notarization now verified Accepted before stapling, a persistent staple failure is a real anomaly and fails the build. Document the format-flip incident and add a pre-merge step to the dependency upgrade checklist: bundled-binary bumps must be validated by dispatching create-release.yml on the branch, since normal CI never builds, signs, or notarizes the installers. --- dev-docs/upgrade-dependencies.md | 11 +++++ llm-docs/code-signing-installers.md | 17 +++++++- package/src/macos/installer.ts | 62 ++++++++++++----------------- 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/dev-docs/upgrade-dependencies.md b/dev-docs/upgrade-dependencies.md index ad8567ee6ba..555817f9bfc 100644 --- a/dev-docs/upgrade-dependencies.md +++ b/dev-docs/upgrade-dependencies.md @@ -2,6 +2,17 @@ Change version numbers in `./configuration` to correspond to new versions. Update hardcoded version strings in `src/command/check/check.ts` (`versionConstraints` array, ~line 249) so that they match the new versions in `configuration`. The `configuration` file warns about this in a comment. +## Verify installer signing & notarization before merging (bundled binaries) + +Regular CI does **not** build, sign, or notarize the installers — that only happens in `create-release.yml`. A bundled binary bump (Deno, Pandoc, Dart Sass, Typst, esbuild, …) can change what the platform code-signing/notarization steps must cover, so a bump that passes normal CI can still break the macOS/Windows release build. **Before merging any PR that bumps a bundled binary**, dispatch the release workflow on the PR branch without publishing and confirm the installer jobs pass: + +```bash +gh workflow run create-release.yml --repo quarto-dev/quarto-cli --ref -f publish-release=false +# then watch make-installer-mac and make-installer-win in the resulting run +``` + +Why this bites (real incident, #14664): Dart Sass 1.101.0 changed its macOS AOT snapshot (`dart-sass/src/sass.snapshot`) container from ELF to Mach-O. Apple's notary ignores non-native files but requires every Mach-O signed, so the never-signed snapshot flipped notarization from Accepted to Invalid — invisible to normal CI, only caught at release time. When a bump does require a new signing entry, add it in `package/src/macos/installer.ts` / the Windows `sign-files` paths; see `llm-docs/code-signing-installers.md`. + ## Upgrade deno ### Upgrade standard library diff --git a/llm-docs/code-signing-installers.md b/llm-docs/code-signing-installers.md index 97e6e9b926b..642fe81b487 100644 --- a/llm-docs/code-signing-installers.md +++ b/llm-docs/code-signing-installers.md @@ -82,7 +82,7 @@ Sequence: 1. **Decide whether to sign** — reads `QUARTO_APPLE_APP_DEV_ID`. Empty → skip everything below with a warning. 2. **Sign individual binaries** with `codesign` (via `signCode()` at `package/src/macos/installer.ts:293`). Two lists: - **With entitlements** (`entitlements.plist`): per-arch `deno`, `dart-sass/src/dart`, `esbuild`, `pandoc`, `typst`, optional `typst-gather`, optional `deno_dom/libplugin.dylib`. Entitlements grant JIT (`com.apple.security.cs.allow-jit`) and unsigned-executable-memory (`com.apple.security.cs.allow-unsigned-executable-memory`) — required by Deno's V8 and by the other JIT-using tools. - - **Without entitlements**: shell wrappers and JS bundle — `quarto`, `quarto.js`, `dart-sass/sass`. + - **Without entitlements**: shell wrappers, JS bundle, and the Dart Sass AOT snapshot — `quarto`, `quarto.js`, `dart-sass/sass`, `dart-sass/src/sass.snapshot`. The snapshot is a Mach-O loaded by the (entitled) `dart` VM, not its own process, so it needs a valid signature + secure timestamp but no entitlements. See the notarization-Invalid troubleshooting below for why it must be signed. - `codesign` invocation: `-s --timestamp --options=runtime --force --deep --verbose=4` (`+ --entitlements ` for the first list). `--options=runtime` opts into Hardened Runtime, a notarization prerequisite. 3. **Build the tarball** (`quarto--macos.tar.gz`) from the signed `pkg-working` tree. The tarball ships unwrapped binaries with the signatures embedded — no `.pkg`, no notarization, but each Mach-O is signed individually. 4. **Build the core `.pkg`** with `pkgbuild` (identifier `org.rstudio.quarto`), then sign it via `productsign --sign --timestamp` (function `signPackage()` at `package/src/macos/installer.ts:282`). Renames signed output back over the original. Note: the `pkgbuild` invocation passes `--install-location` twice (`/Library/Quarto` then `/Applications/quarto`); CLI argv semantics mean the later flag wins, so the effective install location is `/Applications/quarto`. The earlier `/Library/Quarto` value is dead and should be cleaned up next time the file is touched. @@ -121,6 +121,21 @@ If the Account Holder is unreachable (departed staff, etc.): Determining which type the current signing team uses: inspect any past released `.pkg` from a non-CI environment (CI logs mask the Team ID portion of cert Common Names). On a Mac, `pkgutil --check-signature quarto-X.Y.Z-macos.pkg` shows the full chain. On Linux/WSL, the `.pkg` is a xar archive — its TOC contains base64-encoded X.509 certs that can be extracted with Python's stdlib and parsed with `openssl x509`. The cert subject's `O = ...` field is a company name for org enrollment, a person name for individual enrollment. +### Troubleshooting: notarization Invalid "Archive contains critical validation errors" + +Symptom: `notarizeAndWait()` throws `Notarization was not accepted (status: Invalid)`, and the dumped `xcrun notarytool log ` lists per-file `issues` such as: + +```text +"The binary is not signed with a valid Developer ID certificate." +"The signature does not include a secure timestamp." +``` + +Root cause: the named file is a Mach-O binary inside the bundle that no `codesign` call covered. Every native Mach-O in the payload must be individually signed (with `--timestamp`); Apple's notary rejects the whole submission if any is unsigned. Fix: add the file to `signWithEntitlements` (if it JITs / is a directly-executed hardened-runtime process) or `signWithoutEntitlements` (data-like code loaded by another signed process) in `package/src/macos/installer.ts`, then re-run the release build. + +This is easy to miss because a bundled binary can *change format across a version bump*. Concrete case (#14664): Dart Sass `src/sass.snapshot` shipped as an **ELF** blob through 1.87.0 — Apple's notary ignored it (not native code), so it was never signed and notarization passed for years. Dart Sass 1.101.0 switched the macOS AOT snapshot to a native **Mach-O** (`cf fa ed fe`), which the notary then required signed; the never-signed snapshot flipped the result to Invalid. Diagnosing this depended on the terminal-status check in `notarizeAndWait()` — `notarytool submit --wait` had exited 0, so an exit-code-only check would (and previously did) mask the rejection and proceed to `stapler staple`, whose downstream "CloudKit … Record not found" error is a misleading symptom of the missing ticket, not a propagation race. + +Prevention: dispatch `create-release.yml` (publish-release=false) on the branch before merging any bundled-binary bump — see `dev-docs/upgrade-dependencies.md`. + ### Historical: `waitForNotaryStatus` / `altool` (candidate for removal) `waitForNotaryStatus()` at `package/src/macos/installer.ts:360` is **dead code** — defined and never called. Kept here so we have enough context to decide whether to delete it next time the file is touched. diff --git a/package/src/macos/installer.ts b/package/src/macos/installer.ts index cf3c166eed7..79effdbeb66 100644 --- a/package/src/macos/installer.ts +++ b/package/src/macos/installer.ts @@ -104,6 +104,11 @@ export async function makeInstallerMac(config: Configuration) { "dart", )); signWithoutEntitlements.push(join(config.directoryInfo.pkgWorking.bin, "tools", arch, "dart-sass", "sass")); + // Dart Sass 1.101.0 ships src/sass.snapshot as a Mach-O binary that + // Apple's notary service requires signed; the dart VM that loads it + // carries the runtime entitlements, so the snapshot itself needs only a + // valid Developer ID signature + secure timestamp. + signWithoutEntitlements.push(join(config.directoryInfo.pkgWorking.bin, "tools", arch, "dart-sass", "src", "sass.snapshot")); signWithEntitlements.push(join(config.directoryInfo.pkgWorking.bin, "tools", arch, "esbuild")); signWithEntitlements.push(join(config.directoryInfo.pkgWorking.bin, "tools", arch, "pandoc")); @@ -470,47 +475,32 @@ async function waitForNotaryStatus( return notaryResult; } -// Apple's notarization ticket can take a while to become queryable in Apple's -// CloudKit-backed notary database after notarytool reports success, so an -// immediate staple attempt can fail with: +// Once notarizeAndWait confirms the submission is Accepted, the ticket exists +// and stapling normally succeeds immediately. Rarely, the ticket can take a +// moment to become queryable in Apple's CloudKit-backed notary database, so an +// immediate staple fails with: // CloudKit query for ... failed due to "Record not found". // Could not find base64 encoded ticket in response for ... -// This delay has no published bound; in CI it has exceeded a multi-minute -// retry window and never recovered within the build. +// A short bounded retry absorbs that rare timing gap. Anything else - or a +// persistent "Record not found" that outlasts the retry - is a real problem and +// must fail the build (a rejected submission never reaches here; its status is +// verified in notarizeAndWait). const isCloudKitPropagation = (err: Error) => err.message.includes("CloudKit") && err.message.includes("Record not found"); async function stapleNotary(input: string) { - // Stapling only enables *offline* Gatekeeper validation; a notarized but - // unstapled package still validates on first (online) launch. Since the - // package is already verified Accepted by notarizeAndWait, a persistent - // CloudKit propagation delay must not fail the whole release: retry a bounded - // number of times to catch the common short delay, then continue with a - // warning. Any other staple failure is a real problem and still throws. - try { - await withRetry(async () => { - await runCmd( - "xcrun", - ["stapler", "staple", input], - ); - }, { - // withRetry's `attempts` counts retries after the first call (see - // src/core/retry.ts), so 3 here yields 4 total staple invocations. - attempts: 3, - minWait: 20000, - maxWait: 30000, - retry: isCloudKitPropagation, - }); - } catch (err) { - if (err instanceof Error && isCloudKitPropagation(err)) { - warning( - `Could not staple the notarization ticket to ${input} after retries ` + - `(Apple's ticket is not yet queryable). The package is notarized and ` + - `validates online on first launch; continuing without an offline staple.`, - ); - } else { - throw err; - } - } + await withRetry(async () => { + await runCmd( + "xcrun", + ["stapler", "staple", input], + ); + }, { + // withRetry's `attempts` counts retries after the first call (see + // src/core/retry.ts), so 3 here yields 4 total staple invocations. + attempts: 3, + minWait: 20000, + maxWait: 30000, + retry: isCloudKitPropagation, + }); } From 95811ecf301a694d8ade90f83d94a5049c61f66a Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 20:02:51 +0200 Subject: [PATCH 08/10] Correct copyright year to 2026 on new files --- package/src/macos/notarize-status.ts | 2 +- tests/unit/macos-notarize-status.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package/src/macos/notarize-status.ts b/package/src/macos/notarize-status.ts index 7c05cb602ab..88b8dbb9ff3 100644 --- a/package/src/macos/notarize-status.ts +++ b/package/src/macos/notarize-status.ts @@ -1,7 +1,7 @@ /* * notarize-status.ts * - * Copyright (C) 2020-2025 Posit Software, PBC + * Copyright (C) 2026 Posit Software, PBC * */ diff --git a/tests/unit/macos-notarize-status.test.ts b/tests/unit/macos-notarize-status.test.ts index 6ab25cc72a9..d2d88cae243 100644 --- a/tests/unit/macos-notarize-status.test.ts +++ b/tests/unit/macos-notarize-status.test.ts @@ -11,7 +11,7 @@ * Apple hands us), so a change to Quarto's own build logic is what these guard, * not Apple's service. * - * Copyright (C) 2020-2025 Posit Software, PBC + * Copyright (C) 2026 Posit Software, PBC */ import { unitTest } from "../test.ts"; From 9ad7e95db14b3a4ad430b51a761532dec012af23 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 20:04:23 +0200 Subject: [PATCH 09/10] Add rule: new source files use single current-year copyright --- .claude/rules/copyright.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .claude/rules/copyright.md diff --git a/.claude/rules/copyright.md b/.claude/rules/copyright.md new file mode 100644 index 00000000000..0ac0e0ac80f --- /dev/null +++ b/.claude/rules/copyright.md @@ -0,0 +1,10 @@ +--- +paths: + - "**/*.ts" + - "**/*.js" + - "**/*.lua" +--- + +# Copyright headers + +New source files get a single current-year header — `Copyright (C) Posit Software, PBC` (e.g. `2026`), not a back-dated `2020-` range. Leave existing files' headers alone unless you're already substantially editing them. From d090a8f48d302c561541b9db67c2d68753e971cb Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Jul 2026 20:05:19 +0200 Subject: [PATCH 10/10] Copyright rule: read current year from system, don't guess --- .claude/rules/copyright.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.claude/rules/copyright.md b/.claude/rules/copyright.md index 0ac0e0ac80f..b69669d75ea 100644 --- a/.claude/rules/copyright.md +++ b/.claude/rules/copyright.md @@ -8,3 +8,5 @@ paths: # Copyright headers New source files get a single current-year header — `Copyright (C) Posit Software, PBC` (e.g. `2026`), not a back-dated `2020-` range. Leave existing files' headers alone unless you're already substantially editing them. + +Don't guess the year — read it from the system (`date +%Y`, or the current date stated at session start). Copying a header from an existing file carries a stale year.