From a523b615c177622f1132cef2d5b481be4bb84bba Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 16:46:35 -0500 Subject: [PATCH 01/11] feat(runtime): ship precompiled macOS artifacts - Add a versioned macOS arm64 runtime pack with deterministic feature selection and artifact verification. - Link LLVM helper objects against release-built runtime and vendor inputs without compiling user-machine C. - Wire package publishing, cache validation, documentation, and full-gate coverage for the new path. --- .github/workflows/ci.yml | 6 + .github/workflows/release.yml | 10 +- .gitignore | 2 + AGENTS.md | 11 +- CHANGELOG.md | 1 + README.md | 4 +- RELEASING.md | 20 +- docs/src/app/cli/page.mdx | 12 +- docs/src/app/how-it-works/page.mdx | 10 +- docs/src/app/native-objects/page.mdx | 7 +- packages/cli/README.md | 2 +- packages/cli/scripts/warm-cache.mjs | 8 +- packages/cli/src/bootstrap.ts | 13 +- packages/cli/test/runtime-pack.test.ts | 54 ++ packages/compiler/package.json | 3 +- packages/compiler/src/backend/llvm/emitter.ts | 6 +- .../compiler/src/backend/runtime-pack.test.ts | 168 ++++++ packages/compiler/src/backend/runtime-pack.ts | 486 ++++++++++++++++++ packages/compiler/src/index.ts | 92 +++- packages/runtime-darwin-arm64/package.json | 26 + .../runtime-pack-matrix.mjs | 112 ++++ .../runtime-darwin-arm64/scripts/build.mjs | 189 +++++++ .../runtime-darwin-arm64/scripts/verify.mjs | 26 + packages/runtime/package.json | 2 +- pnpm-lock.yaml | 5 + scripts/surface-manifest.mjs | 2 +- scripts/sync-versions.mjs | 2 +- tests/harness/surface-manifest.test.ts | 3 +- 28 files changed, 1222 insertions(+), 60 deletions(-) create mode 100644 packages/cli/test/runtime-pack.test.ts create mode 100644 packages/compiler/src/backend/runtime-pack.test.ts create mode 100644 packages/compiler/src/backend/runtime-pack.ts create mode 100644 packages/runtime-darwin-arm64/package.json create mode 100644 packages/runtime-darwin-arm64/runtime-pack-matrix.mjs create mode 100644 packages/runtime-darwin-arm64/scripts/build.mjs create mode 100644 packages/runtime-darwin-arm64/scripts/verify.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2dd3d5475..c56c60648 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,7 @@ jobs: - run: brew install llvm@22 - run: pnpm install --frozen-lockfile - run: pnpm --filter @scriptc/llvm-darwin-arm64 build:native + - run: pnpm --filter @scriptc/runtime-darwin-arm64 build:native - run: pnpm build # Separate vitest invocations because the shard axes must not mix: a file # lands in exactly ONE --shard slice, so an env-sharded file behind @@ -135,6 +136,7 @@ jobs: - run: brew install llvm@22 - run: pnpm install --frozen-lockfile - run: pnpm --filter @scriptc/llvm-darwin-arm64 build:native + - run: pnpm --filter @scriptc/runtime-darwin-arm64 build:native - run: pnpm build - name: No-clang assembly/object contract if: matrix.shard == 1 @@ -144,7 +146,9 @@ jobs: run: >- pnpm test packages/compiler/test/native-codegen-integration.test.ts + packages/compiler/src/backend/runtime-pack.test.ts packages/cli/test/native-link-info.test.ts + packages/cli/test/runtime-pack.test.ts tests/harness/native-object-example.test.ts - name: LLVM-tier helper object differential (${{ matrix.shard }}/3) env: @@ -160,11 +164,13 @@ jobs: if: matrix.shard == 1 run: | pnpm --dir packages/runtime pack --pack-destination "$RUNNER_TEMP" --silent + pnpm --dir packages/runtime-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/compiler pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/cli pack --pack-destination "$RUNNER_TEMP" --silent PREFIX="$RUNNER_TEMP/installed-scriptc" npm install --prefix "$PREFIX" --ignore-scripts \ "$RUNNER_TEMP/scriptc-runtime-$(node -p "require('./packages/runtime/package.json').version").tgz" \ + "$RUNNER_TEMP/scriptc-runtime-darwin-arm64-$(node -p "require('./packages/runtime-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-llvm-darwin-arm64-$(node -p "require('./packages/llvm-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-compiler-$(node -p "require('./packages/compiler/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-$(node -p "require('./packages/cli/package.json').version").tgz" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d71d70b0b..abcaf23ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,9 +79,9 @@ jobs: # Publishing uses npm trusted publishing (OIDC): the job's id-token # permission lets npm mint short-lived credentials, so no npm token - # secret exists anywhere in this repo. All four packages — - # @scriptc/runtime, @scriptc/llvm-darwin-arm64, @scriptc/compiler, - # and scriptc — must each be + # secret exists anywhere in this repo. All five packages — + # @scriptc/runtime, @scriptc/runtime-darwin-arm64, + # @scriptc/llvm-darwin-arm64, @scriptc/compiler, and scriptc — must each be # configured on npmjs.com with a GitHub Actions trusted publisher # pointing at repository vercel-labs/scriptc, workflow release.yml, # environment Release. A package missing that configuration fails @@ -92,12 +92,13 @@ jobs: run: | pnpm install --frozen-lockfile pnpm --filter @scriptc/llvm-darwin-arm64 build:native + pnpm --filter @scriptc/runtime-darwin-arm64 build:native pnpm -r build - name: Check version sync run: | VERSION="${{ needs.check-release.outputs.version }}" - for pkg in packages/runtime packages/llvm-darwin-arm64 packages/compiler packages/cli; do + for pkg in packages/runtime packages/runtime-darwin-arm64 packages/llvm-darwin-arm64 packages/compiler packages/cli; do V=$(node -p "require('./$pkg/package.json').version") if [ "$V" != "$VERSION" ]; then echo "Version mismatch: $pkg is $V, expected $VERSION" @@ -149,6 +150,7 @@ jobs: } publish_dir packages/runtime + publish_dir packages/runtime-darwin-arm64 publish_dir packages/llvm-darwin-arm64 "$HELPER_TARBALL" publish_dir packages/compiler publish_dir packages/cli diff --git a/.gitignore b/.gitignore index efe9e4d57..bf1db443b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ node_modules/ !tests/fixtures/node-types/node_modules/ dist/ /packages/llvm-darwin-arm64/bin/ +/packages/runtime-darwin-arm64/artifacts/ +/packages/runtime-darwin-arm64/runtime-pack.json !tests/fixtures/fetch/node_modules/eventsource-parser/dist/ !tests/fixtures/npm/node_modules/*/dist/ !tests/fixtures/npm/workspace/*/dist/ diff --git a/AGENTS.md b/AGENTS.md index 49ac17e3e..e3f7a9974 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,11 +9,12 @@ pnpm install && pnpm -r build # build the workspace pnpm test:sandbox # full gate: ~4m custom image, ~9m cold managed fallback ``` -The ordinary workspace build does not rebuild the packaged macOS LLVM helper. -When changing native assembly/object emission, install CMake, Ninja, and -Homebrew `llvm@22`, then run -`pnpm --filter @scriptc/llvm-darwin-arm64 build:native` explicitly. The macOS -full test suite also needs that generated helper. +The ordinary workspace build does not rebuild packaged macOS native artifacts. +When changing native assembly/object emission or runtime-pack selection, +install CMake, Ninja, and Homebrew `llvm@22`, then run +`pnpm --filter @scriptc/llvm-darwin-arm64 build:native` and +`pnpm --filter @scriptc/runtime-darwin-arm64 build:native` explicitly. The +macOS full test suite also needs those generated artifacts. Use focused local tests while iterating, then use `pnpm test:sandbox` whenever a full validation gate is required. It loads Sandbox configuration from the diff --git a/CHANGELOG.md b/CHANGELOG.md index c5eeb553c..5be612cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to scriptc will be documented in this file. ### Features +- **macOS arm64 executables use release-built runtime packs.** LLVM-tier builds now emit the program object through the bundled helper and link feature-selected, hashed runtime/vendor artifacts without compiling C on the user's machine. Explicit C, LLVM fallback, and sanitizer builds retain the external C-toolchain path. - **Builds can stop at typed IR, readable C, or textual LLVM IR.** `scriptc build --emit=ir|c|llvm` writes one primary source artifact with stable default suffixes and requires only Node—no external compiler, archiver, linker, or executable cache. `--emit=exe` remains the default, and executable builds retain the former additive `--emit-ir` flag for one release with a deprecation warning; library mode keeps its additive `--emit-ir` option. diff --git a/README.md b/README.md index 1219ce3ce..5914125a2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # scriptc -scriptc compiles TypeScript and JavaScript to typed IR, readable C, textual LLVM IR, native assembly and objects, native executables, and WebAssembly modules. It uses the TypeScript compiler for parsing and type checking. Source outputs require only Node; macOS 15+ arm64 assembly/object output uses scriptc's bundled LLVM helper; executable builds currently use clang to compile/link the runtime. +scriptc compiles TypeScript and JavaScript to typed IR, readable C, textual LLVM IR, native assembly and objects, native executables, and WebAssembly modules. It uses the TypeScript compiler for parsing and type checking. Source outputs require only Node. On macOS 15+ arm64, ordinary LLVM-tier executables use scriptc's bundled helper and precompiled runtime pack; clang is only the platform linker driver and does not compile program or runtime C. Static builds include a small native runtime, but no Node or JavaScript engine. Code that cannot compile statically is reported as a diagnostic. For npm packages and `any`-typed code, `--dynamic` embeds [quickjs-ng](https://github.com/quickjs-ng/quickjs) explicitly. @@ -8,7 +8,7 @@ scriptc is experimental and targets macOS, Linux, Windows, and WebAssembly via W ## Installation -The compiler requires Node.js 24 or newer. `--emit=ir|c|llvm` needs only Node. On macOS 15+ arm64, `--emit=asm|obj` additionally uses the optional platform helper installed with scriptc, but needs no compiler, archiver, linker, or SDK. Executable builds still require clang and the platform SDK. The executables it produces do not require Node. +The compiler requires Node.js 24 or newer. `--emit=ir|c|llvm` needs only Node. On macOS 15+ arm64, `--emit=asm|obj` additionally uses the optional platform helper installed with scriptc, but needs no compiler, archiver, linker, or SDK. Executable builds need a platform linker driver and SDK; explicit C builds, LLVM fallbacks, and `--sanitize` additionally need a C compiler. The executables it produces do not require Node. ```console $ npm install -g scriptc diff --git a/RELEASING.md b/RELEASING.md index 8a1cffa36..6a7606917 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,6 +1,6 @@ # Releasing -Releases are manual, single-commit affairs. The maintainer controls the changelog voice and format. The four npm packages — `@scriptc/runtime`, `@scriptc/llvm-darwin-arm64`, `@scriptc/compiler`, and `scriptc` — always publish together at the same version. +Releases are manual, single-commit affairs. The maintainer controls the changelog voice and format. The five npm packages — `@scriptc/runtime`, `@scriptc/runtime-darwin-arm64`, `@scriptc/llvm-darwin-arm64`, `@scriptc/compiler`, and `scriptc` — always publish together at the same version. To prepare a release: @@ -12,18 +12,20 @@ To prepare a release: 6. With Zig on `PATH`, run `SCRIPTC_CROSS=1 pnpm exec vitest run tests/harness/library-cross.test.ts` and require the cross-target library conformance lane to pass 7. Commit to `main` -CI (`.github/workflows/release.yml`) compares the version in `packages/cli/package.json` to what `scriptc` has on npm. If it differs, it builds the workspace, verifies all four package versions match (a mismatch fails with a hint to run `scripts/sync-versions.mjs`), and publishes to npm in dependency order — `@scriptc/runtime`, `@scriptc/llvm-darwin-arm64`, `@scriptc/compiler`, then `scriptc` — so each package's dependencies are resolvable the moment it lands. After the publish succeeds, a separate job creates the git tag `v` and the GitHub release with the marked changelog entry as its body, and attaches `surface-manifest.json` — the machine-readable listing of the surface the static tier compiles at that version (stable per-entry ids, so two releases diff mechanically; see `packages/compiler/src/coverage/surface-manifest.ts` for the schema). The job regenerates the manifest from the tree and fails on any byte difference from the committed file before attaching, so the asset is always the manifest of the code being released. The same file ships inside the `@scriptc/compiler` package as `@scriptc/compiler/surface-manifest.json`. +CI (`.github/workflows/release.yml`) compares the version in `packages/cli/package.json` to what `scriptc` has on npm. If it differs, it builds the workspace, verifies all five package versions match (a mismatch fails with a hint to run `scripts/sync-versions.mjs`), and publishes to npm in dependency order — `@scriptc/runtime`, `@scriptc/runtime-darwin-arm64`, `@scriptc/llvm-darwin-arm64`, `@scriptc/compiler`, then `scriptc` — so each package's dependencies are resolvable the moment it lands. After the publish succeeds, a separate job creates the git tag `v` and the GitHub release with the marked changelog entry as its body, and attaches `surface-manifest.json` — the machine-readable listing of the surface the static tier compiles at that version (stable per-entry ids, so two releases diff mechanically; see `packages/compiler/src/coverage/surface-manifest.ts` for the schema). The job regenerates the manifest from the tree and fails on any byte difference from the committed file before attaching, so the asset is always the manifest of the code being released. The same file ships inside the `@scriptc/compiler` package as `@scriptc/compiler/surface-manifest.json`. The release job runs on macOS arm64, builds and strips the pinned LLVM helper, -and publishes its constrained platform package before `@scriptc/compiler`. -Executable/runtime compilation still uses the user's local clang; the helper -owns only assembly/object code generation. The npm package's best-effort -postinstall warms runtime, TLS, and engine caches against that exact local -toolchain. The GitHub release remains a tag, release notes, and the manifest -asset; the npm publish never waits on the GitHub release. +then builds the matching precompiled runtime pack before publishing both +constrained platform packages ahead of `@scriptc/compiler`. Ordinary LLVM-tier +executables use the helper for the program object and the platform pack for +runtime objects; the user's toolchain performs only the final platform link. +Explicit C builds, LLVM refusals, and `--sanitize` retain the external C +toolchain path. npm postinstall skips local runtime-cache compilation when the +platform pack is available. The GitHub release remains a tag, release notes, and the +manifest asset; the npm publish never waits on the GitHub release. Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. -Each of the four packages must have a GitHub Actions trusted publisher for +Each of the five packages must have a GitHub Actions trusted publisher for `release.yml` and the `Release` environment. A missing configuration fails before upload. Re-runs skip package versions already present on npm, so a partially published release can be resumed safely. diff --git a/docs/src/app/cli/page.mdx b/docs/src/app/cli/page.mdx index 0dfbcdf3e..2c7a72849 100644 --- a/docs/src/app/cli/page.mdx +++ b/docs/src/app/cli/page.mdx @@ -59,8 +59,10 @@ object, performs no link, and prints a versioned JSON recipe with the target, libraries. It never reports private scriptc cache paths. See Native Program Objects for complete C-driver and direct-linker examples. ---emit=exe is the default and retains the existing executable -behavior. +--emit=exe is the default. On macOS 15+ arm64, LLVM-tier builds +emit the program object through the helper and link release-built runtime +objects; explicit C, LLVM fallback, and sanitizer builds retain runtime C +compilation. ## scriptc run @@ -79,7 +81,7 @@ Analyzes the program without producing a binary and reports, statement by statem ## scriptc cache warm -Prebuilds the release runtime objects and native TLS/dynamic-engine archives against the currently selected compiler, SDK, and target. npm installations run this best-effort automatically; use the explicit command when preparing a container image, CI runner, or installation whose lifecycle scripts were disabled. Pass one or more of `runtime`, `tls`, and `dynamic` to seed only those families. Warming applies to persistently cached native executable targets; WASI and mobile library targets report a target-level error. It also reports an error when mutable toolchain inputs have disabled persistent caching, rather than doing disposable work. The entries use the ordinary strict cache identities, and later builds still revalidate their compiler and dependency inputs. +Prebuilds release runtime objects and native TLS/dynamic-engine archives for targets that still compile runtime C locally. macOS arm64 installations already carry the release-built runtime pack and skip automatic warming. Use this command when preparing another supported target's container image or CI runner. Pass one or more of `runtime`, `tls`, and `dynamic` to seed only those families. ## Options @@ -100,7 +102,7 @@ Prebuilds the release runtime objects and native TLS/dynamic-engine archives aga
Bind signature-only TypeScript declarations to native C ABI symbols and link the manifest's archive, object, and system-library inputs. See Native FFI.
--backend <c|llvm>
-
Code generator: llvm (default — emits LLVM IR text, compiled by the same clang) or c (the readable debugging backend). Unset, a native build can fall back to C when the program is outside the LLVM tier. The production wasm32-wasi target never falls back: a missing LLVM lowering is SC3001. Use --backend c explicitly only when inspecting generated C; on WASI that inspection lane accepts async-free programs only and reports SC3001 for coroutine-dependent surfaces.
+
Code generator: llvm (default) or c (the readable debugging backend). On macOS arm64, LLVM-tier executable code generation uses the bundled helper and precompiled runtime pack before the platform link. Unset, a native build can fall back to C when the program is outside the LLVM tier. The production wasm32-wasi target never falls back: a missing LLVM lowering is SC3001.
--npm-static <pkg[,pkg…]|auto>
EXPERIMENTAL. Compile the named npm packages' shipped JS statically as program modules instead of embedding them for the engine (repeatable; auto opts in every eligible direct import). A package the preflight refuses falls back to the island with a coverage-report note. See npm Dependencies for maturity notes.
@@ -154,7 +156,7 @@ fib-linux: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), dynamically ## Backends -The default backend emits textual LLVM IR, compiled by the same clang that links the runtime. On native targets, a program outside that tier is never miscompiled—the build falls back to the C backend transparently and says so in one stderr line. The production wasm32-wasi target uses LLVM's 32-bit ABI path and never falls back; an LLVM coverage gap is a build diagnostic. Dynamic npm embedding is LLVM surface on every target. +The default backend emits textual LLVM IR. On macOS arm64 it is lowered to an object by the bundled helper and linked with the precompiled runtime pack; other executable targets retain their existing toolchain path. A program outside the LLVM tier is never miscompiled—the native build falls back to the C backend transparently and says so in one stderr line. The production wasm32-wasi target uses LLVM's 32-bit ABI path and never falls back; an LLVM coverage gap is a build diagnostic. Dynamic npm embedding is LLVM surface on every target. The C backend is a debugging aid: deliberately readable, source-line-annotated output with differential tests against LLVM wherever the two overlap. Pin it when you want to inspect what your program became: diff --git a/docs/src/app/how-it-works/page.mdx b/docs/src/app/how-it-works/page.mdx index 6e000acce..8b9207395 100644 --- a/docs/src/app/how-it-works/page.mdx +++ b/docs/src/app/how-it-works/page.mdx @@ -4,15 +4,15 @@ ``` TypeScript ──tsc: parse + typecheck──▶ lowering ──▶ typed IR ──▶ LLVM IR ──scriptc LLVM helper──▶ assembly/object - │ │ └──clang + runtime/SDK──▶ executable - │ └─────▶ C ────────────────clang───────┘ + │ │ └──precompiled runtime + linker──▶ executable + │ └─────▶ C ────────────────C compiler/linker─────┘ └── serialized IR ``` 1. **Frontend** — the real TypeScript compiler parses and type-checks your program against `es2025` (plus `@types/node` when your project has it), honoring your `tsconfig.json` for checker strictness. The frontend then lowers the checked AST into a typed intermediate representation, using tsc's own type and narrowing answers to drive every decision. A construct with no lowering is a precise diagnostic at this stage — never a miscompile later. 2. **Typed IR** — the only interface between the ends: a validated, serializable representation (`--emit=ir` writes it as JSON and stops). Types are concrete here; generics have been monomorphized, unions are tagged values, closures have explicit captures. 3. **Backends** — `--emit=c` writes readable C and stops; `--emit=llvm` writes textual LLVM IR and stops. Neither source-output command discovers or invokes a native toolchain. On macOS 15+ arm64, `--emit=asm|obj` sends LLVM IR to a version-matched out-of-process helper linked to LLVM 22; it needs no clang or linker and emits macOS 14-targeted artifacts. Executable builds default to LLVM and can fall back to C on a native program outside the LLVM tier (one stderr note; `--backend llvm` pins it and fails with a diagnostic instead). The production wasm32-wasi target never falls back. -4. **Link** — the runtime is a C library of link-gated feature units: binaries pay only for what they use. A hello-world links nothing but libSystem; a regex-using program links the regex engine; an `http` server links the net stack. +4. **Link** — on macOS arm64 the release packages contain one precompiled object per runtime feature unit plus QuickJS, libregexp, zlib, and mbedTLS archives. A hashed manifest maps IR feature gates to an ordered typed link plan, so binaries still pay only for what they use. The user needs a platform linker and SDK, but ordinary LLVM-tier builds compile no C. AddressSanitizer, explicit C builds, and LLVM refusals keep the external C-toolchain path. Program objects define main and leave their selected scr_* runtime functions undefined. The @@ -84,6 +84,10 @@ Where matching Node byte-for-byte is impossible or deliberately not the goal (ti packages/runtime The C runtime: refcounted values with a cycle collector, fibers and the event loop, the server stack, JS-exact number formatting, the island glue. + + packages/runtime-darwin-arm64 + The release-built runtime object pack and its target, ABI, feature, hash, system-library, compiler, and license manifest. + packages/cli scriptc build | run | coverage. diff --git a/docs/src/app/native-objects/page.mdx b/docs/src/app/native-objects/page.mdx index 979bfa9e2..9f57fd262 100644 --- a/docs/src/app/native-objects/page.mdx +++ b/docs/src/app/native-objects/page.mdx @@ -39,9 +39,10 @@ The `scriptc.native-link-info.v1` document reports: Paths inside each source set are relative to `runtime_pack.root`. FFI library paths are the manifest-resolved absolute inputs. No path points into scriptc's -private build cache. The source pack requires a C compiler; the final link -requires the macOS SDK and linker. Precompiled runtime packs are not shipped -yet. +private build cache. This external recipe remains source-based for transparency +and custom-toolchain embedding. Ordinary scriptc executables instead consume +the installed, hashed `@scriptc/runtime-darwin-arm64` object pack and require +only the final macOS SDK and linker. ## C compiler as linker driver diff --git a/packages/cli/README.md b/packages/cli/README.md index 3e5b87b35..d0ce5ffd2 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -22,7 +22,7 @@ $ scriptc build fib.ts -o fib && ./fib $ npm install -g scriptc ``` -Requires Node.js 24. Executable builds require clang on the PATH (Xcode Command Line Tools on macOS, `clang` package on Linux). `--emit=ir|c|llvm` requires only Node. On macOS 15+ arm64, `--emit=asm|obj` uses the matching optional `@scriptc/llvm-darwin-arm64` helper installed with scriptc and requires no external compiler, archiver, linker, or SDK. +Requires Node.js 24. Executable builds require a platform linker driver and SDK. On macOS 15+ arm64, LLVM-tier executables use the matching optional helper and precompiled runtime pack, so the driver only links; explicit C builds, LLVM fallbacks, and `--sanitize` still compile C. `--emit=ir|c|llvm` requires only Node, while `--emit=asm|obj` requires neither an external compiler nor a linker. Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. Library identity getters live in a tiny C translation unit, so build-id-only changes reuse the large compiled program object and compile only that small member before rearchiving. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. diff --git a/packages/cli/scripts/warm-cache.mjs b/packages/cli/scripts/warm-cache.mjs index 7f8be8436..39eab364d 100644 --- a/packages/cli/scripts/warm-cache.mjs +++ b/packages/cli/scripts/warm-cache.mjs @@ -8,7 +8,13 @@ import { fileURLToPath } from "node:url"; // images already manage cache warming explicitly. Published npm packages do // not contain src/, so only installed consumers take this best-effort path. const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); -if (!existsSync(join(packageRoot, "src")) && process.env["SCRIPTC_NO_CACHE"] !== "1") { +if ( + !existsSync(join(packageRoot, "src")) && + process.env["SCRIPTC_NO_CACHE"] !== "1" && + // The macOS arm64 install already carries immutable release/dev runtime + // artifacts. Do not compile a second toolchain-specific copy at install. + !(process.platform === "darwin" && process.arch === "arm64") +) { try { const { warmNativeCaches } = await import("@scriptc/compiler"); await warmNativeCaches(); diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index a0bb1e43f..7d4ea43a0 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -113,8 +113,17 @@ async function tryFastPath(): Promise { ...(optimization === "dev" ? { optimization: "dev" as const } : {}), npmStatic, ffiProfile: ffiPath === null ? null : { path: ffiPath, bytes: ffiBytes! }, - target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${arch}:driver-tu`, - compiler: [process.env["SCRIPTC_CC"] ?? "clang"], + target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${arch}:${ + process.platform === "darwin" && arch === "arm64" && + (process.env["SCRIPTC_TARGET"] ?? "") === "" && + backend !== "c" && !values.sanitize && + process.env["SCRIPTC_RUNTIME_PACK"] !== "0" && + process.env["SCRIPTC_FETCH_CURL"] !== "1" && + ((process.env["SCRIPTC_CC"] ?? "") === "" || process.env["SCRIPTC_CC"] === "clang") + ? "runtime-pack" + : "driver-tu" + }`, + compiler: [process.env["SCRIPTC_LINKER"] ?? process.env["SCRIPTC_CC"] ?? "clang"], nativeEnvironment, nodeVersion: process.version, }); diff --git a/packages/cli/test/runtime-pack.test.ts b/packages/cli/test/runtime-pack.test.ts new file mode 100644 index 000000000..5f9f5f242 --- /dev/null +++ b/packages/cli/test/runtime-pack.test.ts @@ -0,0 +1,54 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { release as osRelease, tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); +const supported = process.platform === "darwin" && process.arch === "arm64" && + Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24; +const dirs: string[] = []; + +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe.runIf(supported)("precompiled runtime executable builds", () => { + test("ordinary LLVM executables invoke the linker but no C compiler mode", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-cli-")); + dirs.push(dir); + const entry = join(dir, "main.ts"); + const output = join(dir, "program"); + const wrapper = join(dir, "linker"); + const log = join(dir, "linker.json"); + await writeFile(entry, 'console.log("precompiled runtime");\n'); + await writeFile(wrapper, [ + "#!/bin/sh", + `node -e 'require("fs").writeFileSync(process.argv[1], JSON.stringify(process.argv.slice(2)))' '${log}' \"$@\"`, + "exec clang \"$@\"", + "", + ].join("\n")); + await chmod(wrapper, 0o755); + await execFileAsync(process.execPath, [ + "--import", tsxLoader, cliEntry, "build", entry, "-o", output, + ], { + env: { + ...process.env, + SCRIPTC_NO_CACHE: "1", + SCRIPTC_LINKER: wrapper, + }, + }); + const args = JSON.parse(await readFile(log, "utf8")) as string[]; + expect(args).not.toContain("-c"); + expect(args.some((arg) => arg.endsWith(".c") || arg.endsWith(".ll"))).toBe(false); + expect(args.some((arg) => arg.includes("runtime-darwin-arm64/artifacts"))).toBe(true); + await expect(execFileAsync(output, [], { encoding: "utf8" })) + .resolves.toMatchObject({ stdout: "precompiled runtime\n" }); + }); +}); diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 789fda57b..7885daa40 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -37,6 +37,7 @@ "typescript5": "npm:typescript@5.9.3" }, "optionalDependencies": { - "@scriptc/llvm-darwin-arm64": "workspace:*" + "@scriptc/llvm-darwin-arm64": "workspace:*", + "@scriptc/runtime-darwin-arm64": "workspace:*" } } diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 36fbcfad8..c12119598 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,9 +1,9 @@ import { InternalCompilerError } from "../../errors.js"; /* IR → LLVM IR text (.ll). The LLVM backend consumes the SAME in-memory * IrModule the C backend does (never the JSON dump — see the -0 lesson in - * the survey) and produces a textual module that rides compileC's - * program-TU seat: clang compiles .ll on the exact command line that - * compiles the .c, linking the same scr_* runtime with the same C ABI. + * the survey). Its textual module is either lowered by scriptc's native + * helper and linked with a runtime pack or occupies the legacy compiler + * driver's program-TU seat. Both paths use the same scr_* C ABI. * * Phase 1 was the TRIVIAL TIER: f64/bool/string locals and params, the * scalar operator set, structured control flow, direct calls, interned diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts new file mode 100644 index 000000000..801be261f --- /dev/null +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -0,0 +1,168 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, test } from "vitest"; +import type { NativeLinkFeatures } from "./native-link-info.js"; +import { + effectiveRuntimeFeatures, + evaluateRuntimePredicate, + loadRuntimePack, + parseRuntimePackManifest, + type RuntimePackManifest, +} from "./runtime-pack.js"; +import { MACOS_ARM64_TARGET } from "./targets.js"; + +const BASE: NativeLinkFeatures = { + dynamic: false, + regex: false, + copying: false, + textDecoderLegacy: false, + fileHandle: false, + fetch: false, + netIsland: false, + zlib: false, + assert: false, + inspect: false, + dynInvoke: false, + dc: false, + dynAsync: false, + events: false, + emitter: false, + symbol: false, + searchParams: false, + qs: false, + parseArgs: false, + stream: false, + net: false, + http: false, + http2: false, + dgram: false, + watch: false, + foreignFfi: false, + nodeTest: false, + tls: false, + tlsCa: false, +}; + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-unit-")); + const packagePath = join(root, "package.json"); + await writeFile(packagePath, JSON.stringify({ + name: "@scriptc/runtime-darwin-arm64", + version: "0.0.35", + })); + const artifact = async (path: string, bytes: string) => { + const output = join(root, path); + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, bytes); + return { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + size: Buffer.byteLength(bytes), + }; + }; + const base = await artifact("artifacts/base.o", "base"); + const legacy = await artifact("artifacts/legacy.o", "legacy"); + const dynamic = await artifact("artifacts/dynamic.o", "dynamic"); + const regex = await artifact("artifacts/regex.a", "regex"); + const quickjs = await artifact("artifacts/qjs.a", "qjs"); + await writeFile(join(root, "license.txt"), "license"); + const units = [{ + source: "scr_bytes.c", + predicate: true, + variants: [ + { id: "default", when: {}, defines: [], ...base }, + { id: "legacy", when: { textDecoderLegacy: true }, defines: ["SCR_TEXT_DECODER_LEGACY"], ...legacy }, + { id: "dynamic", when: { dynamic: true }, defines: ["SCR_DYNAMIC"], ...dynamic }, + ], + }]; + const manifest: RuntimePackManifest = { + schema: "scriptc.runtime-pack.v1", + format: 1, + package: "@scriptc/runtime-darwin-arm64", + version: "0.0.35", + target: { + name: "macos-arm64", + llvm_triple: "arm64-apple-macosx14.0.0", + architecture: "arm64", + object_format: "macho", + minimum_os: "14.0", + }, + runtime_abi: { version: 1, marker: "scr_runtime_abi_v1" }, + compiler: { + command: "clang", + identity: "fixture clang", + target: "arm64-apple-macosx14.0.0", + }, + macros: { + executable: ["SCR_DYNAMIC", "SCR_TEXT_DECODER_LEGACY"], + excluded: ["SCR_LIB", "SCR_THREAD_INSTANCES", "SCR_RC_AUDIT"], + sanitizer: "external-toolchain-required", + }, + flavors: { + release: { optimization: "-O2", runtime_units: units }, + dev: { optimization: "-O0", runtime_units: units }, + }, + archives: [ + { id: "libregexp", predicate: { all: ["regex"], not: ["dynamic"] }, ...regex }, + { id: "quickjs", predicate: "dynamic", ...quickjs }, + ], + system_libraries: [{ name: "System", predicate: true }], + licenses: [{ path: "license.txt", license: "fixture" }], + }; + await writeFile(join(root, "runtime-pack.json"), JSON.stringify(manifest)); + return { root, packagePath, manifest }; +} + +describe("runtime pack manifests", () => { + test("feature implications and predicates are deterministic", () => { + const features = effectiveRuntimeFeatures({ ...BASE, dynamic: true, fetch: true }); + expect(features).toMatchObject({ + nativeFetch: true, + netIslandEffective: true, + netEffective: true, + httpEffective: true, + tlsEffective: true, + tlsCaEffective: true, + zlibEffective: true, + }); + expect(evaluateRuntimePredicate({ all: ["fetch"], not: ["regex"] }, features)).toBe(true); + expect(evaluateRuntimePredicate({ any: ["regex", "dynamic"] }, features)).toBe(true); + }); + + test("selection chooses the most-specific variant and feature archive", async () => { + const { packagePath } = await fixture(); + const resolver = () => packagePath; + const legacy = await loadRuntimePack({ + target: MACOS_ARM64_TARGET, + features: { ...BASE, textDecoderLegacy: true, regex: true }, + optimization: "release", + resolver, + }); + expect(legacy.runtimeObjects.map((path) => path.split("/").at(-1))).toEqual(["legacy.o"]); + expect(legacy.archives.map((path) => path.split("/").at(-1))).toEqual(["regex.a"]); + const dynamic = await loadRuntimePack({ + target: MACOS_ARM64_TARGET, + features: { ...BASE, dynamic: true, regex: true }, + optimization: "dev", + resolver, + }); + expect(dynamic.flavor).toBe("dev"); + expect(dynamic.runtimeObjects.map((path) => path.split("/").at(-1))).toEqual(["dynamic.o"]); + expect(dynamic.archives.map((path) => path.split("/").at(-1))).toEqual(["qjs.a"]); + }); + + test("malformed manifests and damaged artifacts fail before linking", async () => { + const { packagePath, manifest, root } = await fixture(); + expect(() => parseRuntimePackManifest({ ...manifest, format: 2 })).toThrow("malformed"); + await writeFile(join(root, "artifacts/base.o"), "damaged"); + await expect(loadRuntimePack({ + target: MACOS_ARM64_TARGET, + features: BASE, + optimization: "release", + resolver: () => packagePath, + })).rejects.toThrow("hash mismatch"); + expect(await readFile(packagePath, "utf8")).toContain("runtime-darwin-arm64"); + }); +}); diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts new file mode 100644 index 000000000..7ed59e785 --- /dev/null +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -0,0 +1,486 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +import { readFile, realpath, rename, rm, stat } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { compilerReleaseVersion } from "../library/sidecar.js"; +import { privateSiblingPath } from "./build-cache.js"; +import type { NativeLinkFeatures } from "./native-link-info.js"; +import { CcCompileError, subprocessFailureDetail, type NativeArtifactDependency } from "./native-toolchain.js"; +import { RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "./runtime-abi.js"; +import type { NativeTargetSpec } from "./targets.js"; + +const execFileAsync = promisify(execFile); +export const RUNTIME_PACK_SCHEMA = "scriptc.runtime-pack.v1" as const; +export const RUNTIME_PACK_FORMAT = 1 as const; + +export type RuntimePredicate = + | boolean + | string + | { all?: string[]; any?: string[]; not?: string[] }; + +interface RuntimePackArtifact { + path: string; + sha256: string; + size: number; +} + +interface RuntimePackVariant extends RuntimePackArtifact { + id: string; + when: Record; + defines: string[]; +} + +interface RuntimePackUnit { + source: string; + predicate: RuntimePredicate; + variants: RuntimePackVariant[]; +} + +interface RuntimePackArchive extends RuntimePackArtifact { + id: "quickjs" | "libregexp" | "zlib" | "mbedtls"; + predicate: RuntimePredicate; +} + +export interface RuntimePackManifest { + schema: typeof RUNTIME_PACK_SCHEMA; + format: typeof RUNTIME_PACK_FORMAT; + package: string; + version: string; + target: { + name: NativeTargetSpec["name"]; + llvm_triple: NativeTargetSpec["llvmTriple"]; + architecture: "arm64"; + object_format: NativeTargetSpec["objectFormat"]; + minimum_os: NativeTargetSpec["minimumOs"]; + }; + runtime_abi: { version: number; marker: string }; + compiler: { command: string; identity: string; target: string }; + macros: { + executable: string[]; + excluded: string[]; + sanitizer: "external-toolchain-required"; + }; + flavors: Record<"release" | "dev", { + optimization: "-O2" | "-O0"; + runtime_units: RuntimePackUnit[]; + }>; + archives: RuntimePackArchive[]; + system_libraries: { name: string; predicate: RuntimePredicate }[]; + licenses: { path: string; license: string }[]; +} + +export interface RuntimeFeatureSet extends NativeLinkFeatures { + nativeFetch: boolean; + netIslandEffective: boolean; + netEffective: boolean; + httpEffective: boolean; + tlsEffective: boolean; + tlsCaEffective: boolean; + zlibEffective: boolean; +} + +export interface RuntimePackSelection { + root: string; + manifestPath: string; + manifest: RuntimePackManifest; + flavor: "release" | "dev"; + features: RuntimeFeatureSet; + runtimeObjects: string[]; + archives: string[]; + systemLibraries: string[]; + dependencyPaths: string[]; +} + +export interface RuntimeLinkPlan { + target: NativeTargetSpec; + outputPath: string; + inputs: string[]; + systemLibraries: string[]; + driverFlags: string[]; + dependencyPaths: string[]; + runtimePack: RuntimePackSelection; +} + +export class RuntimePackError extends Error { + constructor(message: string, readonly code: "missing" | "invalid" | "unsupported") { + super(message); + this.name = "RuntimePackError"; + } +} + +function object(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : null; +} + +function validDigest(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-f]{64}$/.test(value); +} + +function validPredicate(value: unknown): value is RuntimePredicate { + if (typeof value === "boolean" || typeof value === "string") return true; + const item = object(value); + if (item === null) return false; + const keys = Object.keys(item); + if (keys.some((key) => key !== "all" && key !== "any" && key !== "not")) return false; + return keys.length > 0 && keys.every((key) => + Array.isArray(item[key]) && (item[key] as unknown[]).every((feature) => typeof feature === "string") + ); +} + +function validArtifact(value: unknown): value is RuntimePackArtifact { + const item = object(value); + return item !== null && typeof item.path === "string" && !item.path.startsWith("/") && + !item.path.split(/[\\/]/).includes("..") && validDigest(item.sha256) && + typeof item.size === "number" && Number.isInteger(item.size) && item.size >= 0; +} + +export function parseRuntimePackManifest(value: unknown): RuntimePackManifest { + const manifest = object(value); + const target = object(manifest?.target); + const abi = object(manifest?.runtime_abi); + const compiler = object(manifest?.compiler); + const macros = object(manifest?.macros); + const flavors = object(manifest?.flavors); + const validFlavor = (value: unknown, optimization: string): boolean => { + const flavor = object(value); + return flavor?.optimization === optimization && Array.isArray(flavor.runtime_units) && + flavor.runtime_units.every((raw) => { + const unit = object(raw); + return typeof unit?.source === "string" && validPredicate(unit.predicate) && + Array.isArray(unit.variants) && unit.variants.length > 0 && unit.variants.every((variantRaw) => { + const variant = object(variantRaw); + const when = object(variant?.when); + return validArtifact(variantRaw) && typeof variant?.id === "string" && when !== null && + Object.values(when).every((entry) => typeof entry === "boolean") && + Array.isArray(variant.defines) && variant.defines.every((entry) => typeof entry === "string"); + }); + }); + }; + if ( + manifest?.schema !== RUNTIME_PACK_SCHEMA || manifest.format !== RUNTIME_PACK_FORMAT || + typeof manifest.package !== "string" || typeof manifest.version !== "string" || + target?.name !== "macos-arm64" || target.llvm_triple !== "arm64-apple-macosx14.0.0" || + target.architecture !== "arm64" || target.object_format !== "macho" || target.minimum_os !== "14.0" || + abi?.version !== RUNTIME_ABI_VERSION || abi.marker !== RUNTIME_ABI_MARKER || + typeof compiler?.command !== "string" || typeof compiler.identity !== "string" || + compiler.target !== target.llvm_triple || + !Array.isArray(macros?.executable) || !macros.executable.every((entry) => typeof entry === "string") || + !Array.isArray(macros.excluded) || !macros.excluded.every((entry) => typeof entry === "string") || + macros.sanitizer !== "external-toolchain-required" || + flavors === null || !validFlavor(flavors.release, "-O2") || !validFlavor(flavors.dev, "-O0") || + !Array.isArray(manifest.archives) || !manifest.archives.every((raw) => { + const archive = object(raw); + return validArtifact(raw) && typeof archive?.id === "string" && validPredicate(archive.predicate); + }) || + !Array.isArray(manifest.system_libraries) || !manifest.system_libraries.every((raw) => { + const library = object(raw); + return typeof library?.name === "string" && validPredicate(library.predicate); + }) || + !Array.isArray(manifest.licenses) || !manifest.licenses.every((raw) => { + const license = object(raw); + return typeof license?.path === "string" && typeof license.license === "string"; + }) + ) throw new RuntimePackError("installed runtime-pack.json is malformed or incompatible", "invalid"); + return manifest as unknown as RuntimePackManifest; +} + +export function effectiveRuntimeFeatures( + features: NativeLinkFeatures, + env: NodeJS.ProcessEnv = process.env, +): RuntimeFeatureSet { + const curlFetch = features.dynamic && features.fetch && env["SCRIPTC_FETCH_CURL"] === "1"; + if (curlFetch) { + throw new RuntimePackError( + "SCRIPTC_FETCH_CURL=1 is an external developer-toolchain comparison mode and is not available with precompiled runtime packs", + "unsupported", + ); + } + const nativeFetch = features.fetch; + const netIslandEffective = features.dynamic && (features.netIsland || nativeFetch); + const netEffective = features.net || nativeFetch || netIslandEffective; + const httpEffective = features.http || nativeFetch || netIslandEffective; + const tlsEffective = features.tls || nativeFetch || netIslandEffective; + const tlsCaEffective = features.tlsCa || tlsEffective; + return { + ...features, + nativeFetch, + netIslandEffective, + netEffective, + httpEffective, + tlsEffective, + tlsCaEffective, + zlibEffective: features.zlib || nativeFetch, + }; +} + +export function evaluateRuntimePredicate( + predicate: RuntimePredicate, + features: object, +): boolean { + const values = features as Record; + if (typeof predicate === "boolean") return predicate; + if (typeof predicate === "string") return values[predicate] === true; + return (predicate.all?.every((name) => values[name] === true) ?? true) && + (predicate.any?.some((name) => values[name] === true) ?? true) && + (predicate.not?.every((name) => values[name] !== true) ?? true); +} + +function selectVariant(unit: RuntimePackUnit, features: RuntimeFeatureSet): RuntimePackVariant { + const matches = unit.variants.filter((variant) => + Object.entries(variant.when).every(([name, expected]) => features[name as keyof RuntimeFeatureSet] === expected) + ); + matches.sort((a, b) => Object.keys(b.when).length - Object.keys(a.when).length || a.id.localeCompare(b.id)); + const selected = matches[0]; + if (selected === undefined) { + throw new RuntimePackError(`runtime pack has no variant for ${unit.source}`, "invalid"); + } + return selected; +} + +async function verifyArtifact(root: string, artifact: RuntimePackArtifact): Promise { + const path = join(root, artifact.path); + let bytes: Buffer; + try { + bytes = await readFile(path); + } catch { + throw new RuntimePackError(`runtime pack artifact is missing: ${artifact.path}`, "invalid"); + } + if (bytes.length !== artifact.size || createHash("sha256").update(bytes).digest("hex") !== artifact.sha256) { + throw new RuntimePackError(`runtime pack artifact hash mismatch: ${artifact.path}`, "invalid"); + } + return path; +} + +export async function loadRuntimePack(options: { + target: NativeTargetSpec; + features: NativeLinkFeatures; + optimization: "release" | "dev"; + env?: NodeJS.ProcessEnv; + resolver?: (specifier: string) => string; +}): Promise { + const packageName = options.target.name === "macos-arm64" + ? "@scriptc/runtime-darwin-arm64" + : (() => { throw new RuntimePackError(`no runtime pack supports ${options.target.name}`, "unsupported"); })(); + const resolvePackageJson = options.resolver ?? ((specifier: string) => createRequire(import.meta.url).resolve(specifier)); + let packagePath: string; + try { + packagePath = resolvePackageJson(`${packageName}/package.json`); + } catch { + throw new RuntimePackError( + `precompiled runtime package ${packageName} is not installed; reinstall scriptc with optional dependencies enabled for macOS arm64`, + "missing", + ); + } + const root = dirname(packagePath); + const manifestPath = join(root, "runtime-pack.json"); + let packageManifest: { name?: string; version?: string }; + let manifest: RuntimePackManifest; + try { + [packageManifest, manifest] = await Promise.all([ + readFile(packagePath, "utf8").then((text) => JSON.parse(text)), + readFile(manifestPath, "utf8").then((text) => parseRuntimePackManifest(JSON.parse(text))), + ]); + } catch (error) { + if (error instanceof RuntimePackError) throw error; + throw new RuntimePackError(`could not read ${packageName}/runtime-pack.json`, "invalid"); + } + if ( + packageManifest.name !== packageName || manifest.package !== packageName || + packageManifest.version !== compilerReleaseVersion() || manifest.version !== compilerReleaseVersion() + ) { + throw new RuntimePackError( + `runtime pack version mismatch: expected ${packageName}@${compilerReleaseVersion()}, found ${packageManifest.name}@${packageManifest.version}`, + "invalid", + ); + } + if ( + manifest.target.name !== options.target.name || + manifest.target.llvm_triple !== options.target.llvmTriple || + manifest.target.object_format !== options.target.objectFormat || + manifest.target.minimum_os !== options.target.minimumOs + ) throw new RuntimePackError(`runtime pack does not support target ${options.target.name}`, "invalid"); + const features = effectiveRuntimeFeatures(options.features, options.env); + const flavor = options.optimization; + const selectedUnits = manifest.flavors[flavor].runtime_units + .filter((unit) => evaluateRuntimePredicate(unit.predicate, features)); + const selectedVariants = selectedUnits.map((unit) => selectVariant(unit, features)); + const selectedArchives = manifest.archives + .filter((archive) => evaluateRuntimePredicate(archive.predicate, features)); + const [runtimeObjects, archives] = await Promise.all([ + Promise.all(selectedVariants.map((artifact) => verifyArtifact(root, artifact))), + Promise.all(selectedArchives.map((artifact) => verifyArtifact(root, artifact))), + ]); + await Promise.all(manifest.licenses.map((license) => readFile(join(root, license.path)))).catch(() => { + throw new RuntimePackError("runtime pack license payload is incomplete", "invalid"); + }); + return { + root, + manifestPath, + manifest, + flavor, + features, + runtimeObjects, + archives, + systemLibraries: manifest.system_libraries + .filter((entry) => evaluateRuntimePredicate(entry.predicate, features)) + .map((entry) => entry.name), + dependencyPaths: [manifestPath, ...runtimeObjects, ...archives], + }; +} + +export async function createRuntimeLinkPlan(options: { + target: NativeTargetSpec; + programObject: string; + outPath: string; + features: NativeLinkFeatures; + ffi: FfiProfile | null; + optimization: "release" | "dev"; + env?: NodeJS.ProcessEnv; + resolver?: (specifier: string) => string; +}): Promise { + const runtimePack = await loadRuntimePack(options); + return { + target: options.target, + outputPath: options.outPath, + inputs: [ + options.programObject, + ...(options.ffi?.libraries ?? []), + ...runtimePack.runtimeObjects, + ...runtimePack.archives, + ], + systemLibraries: [...new Set([ + ...(options.ffi?.systemLibraries ?? []), + ...runtimePack.systemLibraries, + ])], + driverFlags: [ + "-target", options.target.llvmTriple, "-pthread", + ...(runtimePack.features.dynamic ? ["-Wl,-dead_strip"] : []), + ], + dependencyPaths: [ + ...runtimePack.dependencyPaths, + ...(options.ffi?.libraries ?? []), + ], + runtimePack, + }; +} + +async function snapshotDependencies(paths: readonly string[]): Promise { + const { lstat } = await import("node:fs/promises"); + return Promise.all([...new Set(paths.map((path) => resolve(path)))].sort().map(async (path) => { + const info = await lstat(path); + const kind = info.isFile() ? "file" : info.isDirectory() ? "directory" : "symlink"; + const dependency: NativeArtifactDependency = { + path, + kind, + dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), + mtimeMs: Number(info.mtimeMs), ctimeMs: Number(info.ctimeMs), + }; + if (kind === "symlink") { + const targetPath = await realpath(path); + const target = await stat(path); + const targetKind = target.isFile() ? "file" : target.isDirectory() ? "directory" : null; + if (targetKind === null) throw new Error(`unsupported linker dependency: ${path}`); + dependency.targetPath = targetPath; + dependency.targetKind = targetKind; + dependency.targetDev = Number(target.dev); + dependency.targetIno = Number(target.ino); + dependency.targetSize = Number(target.size); + dependency.targetMtimeMs = Number(target.mtimeMs); + dependency.targetCtimeMs = Number(target.ctimeMs); + } + return dependency; + })); +} + +async function resolveExecutable(command: string, env: NodeJS.ProcessEnv = process.env): Promise { + if (command.includes("/") || command.includes("\\")) return resolve(command); + const path = env["PATH"] ?? ""; + const { access } = await import("node:fs/promises"); + const { delimiter } = await import("node:path"); + for (const directory of path.split(delimiter)) { + if (directory === "") continue; + const candidate = join(directory, command); + try { + await access(candidate); + return await realpath(candidate).catch(() => candidate); + } catch { + continue; + } + } + return command; +} + +async function linkToolchainDependencies(linker: string): Promise { + if (process.platform !== "darwin") return [linker]; + const sdkRoot = process.env["SDKROOT"] ?? + (await execFileAsync("xcrun", ["--sdk", "macosx", "--show-sdk-path"])).stdout.trim(); + const platformLinker = (await execFileAsync("xcrun", ["--sdk", "macosx", "--find", "ld"])).stdout.trim(); + const compilerRuntime = (await execFileAsync(linker, ["-print-file-name=libclang_rt.osx.a"])) + .stdout.trim(); + const candidates = [ + linker, + platformLinker, + compilerRuntime, + join(sdkRoot, "SDKSettings.json"), + join(sdkRoot, "usr", "lib", "libSystem.tbd"), + join(sdkRoot, "usr", "lib", "libm.tbd"), + join(sdkRoot, "usr", "lib", "libz.tbd"), + ]; + const present = await Promise.all(candidates.map(async (path) => + await stat(path).then(() => path, () => null))); + return present.filter((path): path is string => path !== null); +} + +export async function linkRuntimePackExecutable( + plan: RuntimeLinkPlan, + options: { + linker?: string; + onArtifactReady?: (artifact: { dependencies: NativeArtifactDependency[] }) => Promise; + } = {}, +): Promise { + const linker = options.linker ?? process.env["SCRIPTC_LINKER"] ?? "clang"; + const linkerPath = await resolveExecutable(linker); + const privateOut = privateSiblingPath(plan.outputPath, "runtime-pack-link"); + const args = [ + ...plan.driverFlags, + ...plan.inputs, + ...plan.systemLibraries.map((name) => `-l${name}`), + "-o", privateOut, + ]; + try { + await execFileAsync(linker, args); + const output = await stat(privateOut); + if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable"); + await rename(privateOut, plan.outputPath).catch(async () => { + await rm(plan.outputPath, { force: true }); + await rename(privateOut, plan.outputPath); + }); + if (options.onArtifactReady !== undefined) { + // A complete executable cache entry is published only when the driver, + // platform linker, compiler runtime, selected SDK stubs/settings, pack, + // and FFI inputs all have a replayable metadata proof. Failure to prove + // any ambient input keeps a correct executable but no complete cache. + const dependencies = await linkToolchainDependencies(linkerPath) + .then((toolchain) => snapshotDependencies([...toolchain, ...plan.dependencyPaths])) + .catch(() => null); + if (dependencies !== null) { + await options.onArtifactReady({ dependencies }).catch(() => undefined); + } + } + } catch (error) { + if (error instanceof CcCompileError) throw error; + const detail = subprocessFailureDetail(error); + throw new CcCompileError( + linker, + detail, + `${linker} failed linking ${basename(plan.outputPath)} from the precompiled runtime pack.\n${detail}`, + ); + } finally { + await rm(privateOut, { force: true }).catch(() => undefined); + } +} diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 3356cd324..f5ba5c94f 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -9,6 +9,7 @@ import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen import { privateSiblingPath } from "./backend/build-cache.js"; import { nativeCodegenTarget, nativeCodegenTargetRefusal } from "./backend/targets.js"; import { createNativeLinkInfo, type NativeLinkInfo } from "./backend/native-link-info.js"; +import { createRuntimeLinkPlan, linkRuntimePackExecutable, RuntimePackError } from "./backend/runtime-pack.js"; import { splitLlvmLibraryProgram, splitLlvmProgram } from "./backend/llvm/split.js"; import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "./backend/library-identity-markers.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, nativeCodegenDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; @@ -183,9 +184,10 @@ export interface CompileBaseOptions { * changes. */ dynamic?: boolean; /** Code generator for the program TU. Unset (the release default): the - * LLVM backend emits LLVM IR text (.ll) that rides the SAME clang - * command line in the program-TU seat, and a program outside the LLVM - * tier falls back to the debugging C backend transparently — the IR is + * LLVM backend emits LLVM IR text (.ll). Supported macOS arm64 builds send + * it through the bundled helper and link a precompiled runtime pack; other + * targets retain their established compiler-driver path. A program outside + * the LLVM tier falls back to the debugging C backend transparently — the IR is * backend-agnostic, so only the emit retries; CompileResult records the * lane (`backend`, plus `llvmRefusal` when the fallback engaged). ONLY a * tier refusal (LlvmUnsupportedError) falls back — every real diagnostic @@ -222,11 +224,8 @@ export interface CompileBaseOptions { * historical compile() API, whose omitted output kind means executable. */ export interface CompileOptions extends CompileBaseOptions { outputKind?: "exe"; - /** Internal validation lane: executable builds still use the existing - * linker/runtime recipe, but the program LLVM TU is compiled to an object - * by the bundled helper first. Not a CLI contract; Phase 2 corpus tests use - * it to compare helper and clang objects under identical link inputs. - * Requires backend explicitly set to llvm. */ + /** Internal validation lane retained for helper-object artifact tests. + * Supported ordinary LLVM executable builds select this path automatically. */ nativeProgramObject?: boolean; } @@ -1054,6 +1053,28 @@ async function compileExecutableNative( onArtifactReady?: NonNullable[0]["onArtifactReady"]>, ): Promise { const programIsObject = /\.(?:o|obj)$/.test(cPath); + const runtimePackTarget = programIsObject && !sanitize && process.env["SCRIPTC_RUNTIME_PACK"] !== "0" + ? nativeCodegenTarget() + : null; + if (runtimePackTarget !== null) { + const plan = await createRuntimeLinkPlan({ + target: runtimePackTarget, + programObject: cPath, + outPath, + features, + ffi, + optimization: features.optimization ?? "release", + }); + await linkRuntimePackExecutable(plan, { + // A caller-selected linker can be a mutable wrapper with hidden inputs. + // Keep that path honest by relinking; the default resolved driver plus + // selected immutable pack/FFI files carries a replayable proof. + ...(onArtifactReady === undefined || process.env["SCRIPTC_LINKER"] !== undefined + ? {} + : { onArtifactReady }), + }); + return; + } const effectiveProgramSplit = programSplit ?? (!programIsObject && features.optimization === "dev" && features.backend === "llvm" && !sanitize @@ -1153,6 +1174,23 @@ async function emitNativeProgramObject( } } +function usesPrecompiledRuntimePack( + opts: CompileRequestOptions, + backend: "c" | "llvm", +): boolean { + if ( + backend !== "llvm" || opts.sanitize === true || + process.env["SCRIPTC_RUNTIME_PACK"] === "0" || + process.env["SCRIPTC_FETCH_CURL"] === "1" + ) return false; + const cc = process.env["SCRIPTC_CC"] ?? ""; + return (cc === "" || cc === "clang") && nativeCodegenTarget() !== null; +} + +function runtimePackDiagnostic(error: RuntimePackError, entryPath: string): ScrDiagnostic { + return nativeCodegenDiag(error.code === "unsupported" ? "SC3002" : "SC3003", error.message, entryPath); +} + async function compileTracked( entryPath: string, opts: CompileRequestOptions, @@ -1283,8 +1321,16 @@ async function compileTracked( opts.ffiProfilePath === undefined || ffiProfileBytes === null ? null : { path: opts.ffiProfilePath, bytes: ffiProfileBytes }, - target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}:${opts.nativeProgramObject === true ? "helper-object" : "driver-tu"}`, - compiler: [process.env["SCRIPTC_CC"] ?? "clang"], + target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}:${ + opts.nativeProgramObject === true || ( + opts.backend !== "c" && opts.sanitize !== true && + process.env["SCRIPTC_RUNTIME_PACK"] !== "0" && + process.env["SCRIPTC_FETCH_CURL"] !== "1" && + ((process.env["SCRIPTC_CC"] ?? "") === "" || process.env["SCRIPTC_CC"] === "clang") && + nativeCodegenTarget() !== null + ) ? "runtime-pack" : "driver-tu" + }`, + compiler: [process.env["SCRIPTC_LINKER"] ?? process.env["SCRIPTC_CC"] ?? "clang"], nativeEnvironment: await executableNativeEnvironmentFingerprint(), nodeVersion: process.version, implementation: implementation.digest, @@ -1331,7 +1377,9 @@ async function compileTracked( } let nativeInputPath = earlyHit.cPath; let nativeProgramObject: { linkPath: string; artifactPath: string } | null = null; - if (opts.nativeProgramObject === true) { + const useRuntimePack = opts.nativeProgramObject === true || + usesPrecompiledRuntimePack(opts, earlyHit.native.backend); + if (useRuntimePack) { if (earlyHit.native.backend !== "llvm") { throw new InternalCompilerError( "native program-object cache hit restored a non-LLVM translation unit", @@ -1361,7 +1409,7 @@ async function compileTracked( opts.sanitize ?? false, ffi, null, - async ({ dependencies }) => { + opts.nativeProgramObject === true ? undefined : async ({ dependencies }) => { await publishEarlyExecutableCache(cacheRoot, executableCacheOptions, { ...earlyHit, executableRestored: true, @@ -1370,10 +1418,13 @@ async function compileTracked( }); }, ); - if (nativeProgramObject !== null) { + if (nativeProgramObject !== null && opts.nativeProgramObject === true) { await rename(nativeProgramObject.linkPath, nativeProgramObject.artifactPath); } } catch (err) { + if (err instanceof RuntimePackError) { + return { ok: false, diagnostics: [runtimePackDiagnostic(err, entryPath)], sourceTexts: new Map() }; + } if (ffi !== null && err instanceof CcCompileError) { return { ok: false, @@ -1596,7 +1647,9 @@ async function compileTracked( const ll = emitLlvmModule(lowered.module!, { pointerBits: buildPlatform === "wasi" ? 32 : 64, wasi: buildPlatform === "wasi", - runtimeAbiMarker: opts.nativeProgramObject === true, + runtimeAbiMarker: + opts.nativeProgramObject === true || + usesPrecompiledRuntimePack(opts, "llvm"), }); cPath = defaultSourcePaths.llvm; await writeFile(cPath, ll); @@ -1649,7 +1702,9 @@ async function compileTracked( let publishedExecutable = false; let nativeProgramObject: { linkPath: string; artifactPath: string } | null = null; try { - if (opts.nativeProgramObject === true) { + const useRuntimePack = opts.nativeProgramObject === true || + usesPrecompiledRuntimePack(opts, backend); + if (useRuntimePack) { if (backend !== "llvm" || llvmSource === null) { throw new InternalCompilerError("native program-object validation requires the LLVM backend"); } @@ -1671,7 +1726,7 @@ async function compileTracked( opts.sanitize ?? false, ffi, programSplit, - async ({ dependencies }) => { + opts.nativeProgramObject === true ? undefined : async ({ dependencies }) => { await publishEarlyExecutableCache(cacheRoot, executableCacheOptions, { cPath, native: nativeFeatures, @@ -1683,10 +1738,13 @@ async function compileTracked( publishedExecutable = true; }, ); - if (nativeProgramObject !== null) { + if (nativeProgramObject !== null && opts.nativeProgramObject === true) { await rename(nativeProgramObject.linkPath, nativeProgramObject.artifactPath); } } catch (err) { + if (err instanceof RuntimePackError) { + return { ok: false, diagnostics: [runtimePackDiagnostic(err, entryPath)], sourceTexts }; + } if (ffi !== null && err instanceof CcCompileError) { return { ok: false, diff --git a/packages/runtime-darwin-arm64/package.json b/packages/runtime-darwin-arm64/package.json new file mode 100644 index 000000000..f55dafa47 --- /dev/null +++ b/packages/runtime-darwin-arm64/package.json @@ -0,0 +1,26 @@ +{ + "name": "@scriptc/runtime-darwin-arm64", + "version": "0.0.35", + "description": "Precompiled scriptc runtime pack for macOS arm64", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/runtime-darwin-arm64" + }, + "os": ["darwin"], + "cpu": ["arm64"], + "files": [ + "artifacts", + "runtime-pack.json" + ], + "scripts": { + "build": "true", + "build:native": "node scripts/build.mjs", + "prepack": "node scripts/verify.mjs" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs new file mode 100644 index 000000000..cf4c7cbca --- /dev/null +++ b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs @@ -0,0 +1,112 @@ +/** + * The executable runtime-pack matrix. This is the single source of truth for + * both release compilation and the generated feature predicates consumed by + * the compiler. A predicate is intentionally data, not JavaScript, so the + * installed compiler can validate and evaluate it deterministically. + */ + +const any = (...features) => ({ any: features }); +const all = (...features) => ({ all: features }); + +const BASE_RUNTIME_SOURCES = [ + "scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", + "scr_bytes_io.c", "scr_map.c", "scr_closure.c", "scr_ffi.c", + "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", + "scr_console.c", "scr_lib.c", "scr_path.c", "scr_url.c", "scr_json.c", + "scr_async.c", "scr_child.c", "scr_cycle.c", +]; + +const optional = [ + ["scr_copying.c", "copying"], + ["scr_file_handle.c", "fileHandle"], + ["scr_regex.c", "regex"], + ["scr_assert.c", any("assert", "regex", "symbol")], + ["scr_inspect.c", "inspect"], + ["scr_dyn_invoke.c", any("dynInvoke", "nativeFetch")], + ["scr_dc.c", "dc"], + ["scr_async_dyn.c", any("dynAsync", "dynInvoke", "dc", "nativeFetch")], + ["scr_zlib.c", "zlib"], + ["scr_zlib_island.c", all("zlib", "dynamic")], + ["scr_events.c", "events"], + ["scr_readline.c", "events"], + ["scr_events_emitter.c", "emitter"], + ["scr_dyn_handle.c", any("emitter", "netEffective")], + ["scr_symbol.c", "symbol"], + ["scr_url_params.c", "searchParams"], + ["scr_qs.c", "qs"], + ["scr_util.c", "parseArgs"], + ["scr_stream.c", "stream"], + ["scr_loop_kqueue.c", any("netEffective", "dgram")], + ["scr_loop_epoll.c", any("netEffective", "dgram")], + ["scr_loop_wsapoll.c", any("netEffective", "dgram")], + ["scr_net.c", "netEffective"], + ["scr_http.c", "httpEffective"], + ["scr_http2.c", "http2"], + ["scr_dgram.c", "dgram"], + ["scr_watch.c", "watch"], + ["scr_ffi_queue.c", "foreignFfi"], + ["scr_test.c", "nodeTest"], + ["scr_tls_ca.c", "tlsCaEffective"], + ["scr_tls.c", "tlsEffective"], + ["scr_fetch.c", "nativeFetch"], + ["scr_island.c", "dynamic"], + ["scr_web.c", "dynamic"], + ["scr_inspect_island.c", all("dynamic", "inspect")], + ["scr_net_island.c", "netIslandEffective"], +]; + +function variantsFor(source) { + const dynamicOnly = new Set([ + "scr_zlib_island.c", "scr_island.c", "scr_web.c", + "scr_inspect_island.c", "scr_net_island.c", + ]); + if (dynamicOnly.has(source)) { + return [{ id: "dynamic", when: { dynamic: true }, defines: ["SCR_DYNAMIC"] }]; + } + const variants = [{ id: "default", when: {}, defines: [] }]; + if (source === "scr_bytes.c") { + variants.push({ + id: "text-decoder-legacy", + when: { textDecoderLegacy: true }, + defines: ["SCR_TEXT_DECODER_LEGACY"], + }); + } + variants.push({ id: "dynamic", when: { dynamic: true }, defines: ["SCR_DYNAMIC"] }); + if (source === "scr_bytes.c") { + variants.push({ + id: "dynamic-text-decoder-legacy", + when: { dynamic: true, textDecoderLegacy: true }, + defines: ["SCR_DYNAMIC", "SCR_TEXT_DECODER_LEGACY"], + }); + } + return variants; +} + +export const RUNTIME_PACK_MATRIX = { + schema: "scriptc.runtime-pack-matrix.v1", + target: { + name: "macos-arm64", + llvm_triple: "arm64-apple-macosx14.0.0", + architecture: "arm64", + object_format: "macho", + minimum_os: "14.0", + }, + flavors: { + release: { optimization: "-O2" }, + dev: { optimization: "-O0" }, + }, + runtime_units: [ + ...BASE_RUNTIME_SOURCES.map((source) => ({ source, predicate: true })), + ...optional.map(([source, predicate]) => ({ source, predicate })), + ].map((unit) => ({ ...unit, variants: variantsFor(unit.source) })), + archives: [ + { id: "quickjs", predicate: "dynamic" }, + { id: "libregexp", predicate: { all: ["regex"], not: ["dynamic"] } }, + { id: "zlib", predicate: "zlibEffective" }, + { id: "mbedtls", predicate: "tlsEffective" }, + ], + system_libraries: [ + { name: "System", predicate: true }, + { name: "m", predicate: "dynamic" }, + ], +}; diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs new file mode 100644 index 000000000..a07807751 --- /dev/null +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -0,0 +1,189 @@ +#!/usr/bin/env node +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { availableParallelism } from "node:os"; +import { + copyFile, mkdir, readFile, readdir, rename, rm, stat, writeFile, +} from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { RUNTIME_PACK_MATRIX } from "../runtime-pack-matrix.mjs"; + +const run = promisify(execFile); +const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); +const repoRoot = fileURLToPath(new URL("../../..", import.meta.url)); +const runtimeRoot = join(repoRoot, "packages/runtime"); +const runtimeSrc = join(runtimeRoot, "src"); +const vendorRoot = join(runtimeRoot, "vendor"); +const outputRoot = join(packageRoot, "artifacts"); +const manifestPath = join(packageRoot, "runtime-pack.json"); + +if (process.platform !== "darwin" || process.arch !== "arm64") { + process.stdout.write("@scriptc/runtime-darwin-arm64: skipped on this host\n"); + process.exit(0); +} + +const packageManifest = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf8")); +const compiler = process.env.CC ?? "clang"; +const archiver = process.env.AR ?? "ar"; +const compilerVersion = (await run(compiler, ["--version"])).stdout.split("\n", 1)[0].trim(); +const commonFlags = [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, + "-std=c11", "-pthread", "-fno-math-errno", "-fno-strict-aliasing", + "-Wno-deprecated-declarations", "-I", runtimeSrc, +]; +const quickjs = join(vendorRoot, "quickjs-ng"); +const zlib = join(vendorRoot, "zlib"); +const mbedtls = join(vendorRoot, "mbedtls"); +const QJS_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"]; +const LRE_SOURCES = ["libregexp.c", "libunicode.c"]; +const ZLIB_SOURCES = [ + "adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", + "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c", +]; + +async function sha256(path) { + return createHash("sha256").update(await readFile(path)).digest("hex"); +} + +async function compile(source, output, flags) { + await mkdir(dirname(output), { recursive: true }); + await run(compiler, [...flags, "-c", source, "-o", output]); +} + +async function parallel(items, task) { + const width = Math.max(1, Math.min(8, availableParallelism())); + for (let i = 0; i < items.length; i += width) { + await Promise.all(items.slice(i, i + width).map(task)); + } +} + +async function archive(id, sources, sourceRoot, flags) { + const root = join(outputRoot, "vendor", id); + const objectRoot = join(root, "objects"); + await parallel(sources, async (source) => { + await compile(join(sourceRoot, source), join(objectRoot, source.replace(/\.c$/, ".o")), flags); + }); + const output = join(root, `libscriptc-${id}.a`); + const objects = sources.map((source) => join(objectRoot, source.replace(/\.c$/, ".o"))); + await run(archiver, ["rcs", output, ...objects]); + await rm(objectRoot, { recursive: true, force: true }); + return { + id, + path: output.slice(packageRoot.length + 1), + sha256: await sha256(output), + size: (await stat(output)).size, + }; +} + +const privateRoot = `${outputRoot}.tmp-${process.pid}`; +await rm(privateRoot, { recursive: true, force: true }); +await rm(outputRoot, { recursive: true, force: true }); +await mkdir(privateRoot, { recursive: true }); +// Build under the final path spelling so object debug metadata remains stable; +// artifacts contain no source checkout paths after stripping debug information. +await rename(privateRoot, outputRoot); + +try { + const flavors = {}; + for (const [flavor, flavorSpec] of Object.entries(RUNTIME_PACK_MATRIX.flavors)) { + const units = []; + for (const unit of RUNTIME_PACK_MATRIX.runtime_units) { + const variants = []; + for (const variant of unit.variants) { + const variantName = variant.id === "default" ? "default" : variant.id; + const output = join(outputRoot, flavor, "runtime", variantName, unit.source.replace(/\.c$/, ".o")); + const includeFlags = [ + ...(unit.source === "scr_regex.c" || variant.defines.includes("SCR_DYNAMIC") + ? ["-I", quickjs] + : []), + ...(unit.source === "scr_tls.c" ? ["-I", join(mbedtls, "include")] : []), + ...(unit.source === "scr_zlib.c" || unit.source === "scr_fetch.c" + ? ["-I", zlib] + : []), + ]; + await compile(join(runtimeSrc, unit.source), output, [ + ...commonFlags, flavorSpec.optimization, + ...variant.defines.map((define) => `-D${define}`), + ...includeFlags, + ]); + variants.push({ + id: variant.id, + when: variant.when, + defines: variant.defines, + path: output.slice(packageRoot.length + 1), + sha256: await sha256(output), + size: (await stat(output)).size, + }); + } + units.push({ source: unit.source, predicate: unit.predicate, variants }); + } + flavors[flavor] = { optimization: flavorSpec.optimization, runtime_units: units }; + } + + const mbedtlsSources = (await readdir(join(mbedtls, "library"))) + .filter((name) => !name.startsWith(".") && name.endsWith(".c")) + .sort(); + const archives = [ + await archive("quickjs", QJS_SOURCES, quickjs, [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=gnu11", + "-fvisibility=hidden", "-funsigned-char", "-DQUICKJS_NG_BUILD", + "-D_GNU_SOURCE", "-DNDEBUG", "-Os", "-I", quickjs, + ]), + await archive("libregexp", LRE_SOURCES, quickjs, [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", "-I", quickjs, + ]), + await archive("zlib", ZLIB_SOURCES, zlib, [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", "-I", zlib, + ]), + await archive("mbedtls", mbedtlsSources, join(mbedtls, "library"), [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", + "-I", join(mbedtls, "include"), "-I", join(mbedtls, "library"), + ]), + ]; + const archiveSpecs = new Map(RUNTIME_PACK_MATRIX.archives.map((entry) => [entry.id, entry])); + const licensed = [ + [join(runtimeRoot, "LICENSE"), "artifacts/licenses/scriptc-runtime.txt", "Apache-2.0"], + [join(quickjs, "LICENSE"), "artifacts/licenses/quickjs-ng.txt", "MIT"], + [join(vendorRoot, "ryu", "LICENSE-Boost"), "artifacts/licenses/ryu.txt", "BSL-1.0"], + [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], + [join(mbedtls, "LICENSE"), "artifacts/licenses/mbedtls.txt", "Apache-2.0"], + ]; + await Promise.all(licensed.map(async ([source, destination]) => { + const output = join(packageRoot, destination); + await mkdir(dirname(output), { recursive: true }); + await copyFile(source, output); + })); + const manifest = { + schema: "scriptc.runtime-pack.v1", + format: 1, + package: packageManifest.name, + version: packageManifest.version, + target: RUNTIME_PACK_MATRIX.target, + runtime_abi: { version: 1, marker: "scr_runtime_abi_v1" }, + compiler: { + command: compiler, + identity: compilerVersion, + target: RUNTIME_PACK_MATRIX.target.llvm_triple, + }, + macros: { + executable: ["SCR_DYNAMIC", "SCR_TEXT_DECODER_LEGACY"], + excluded: ["SCR_LIB", "SCR_THREAD_INSTANCES", "SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], + sanitizer: "external-toolchain-required", + }, + flavors, + archives: archives.map((entry) => ({ ...entry, predicate: archiveSpecs.get(entry.id).predicate })), + system_libraries: RUNTIME_PACK_MATRIX.system_libraries, + licenses: licensed.map(([, path, license]) => ({ path, license })), + }; + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + process.stdout.write( + `built ${packageManifest.name}@${packageManifest.version}: ` + + `${Object.keys(flavors).length} flavors, ${archives.length} vendor archives\n`, + ); +} catch (error) { + await rm(outputRoot, { recursive: true, force: true }); + await rm(manifestPath, { force: true }); + throw error; +} diff --git a/packages/runtime-darwin-arm64/scripts/verify.mjs b/packages/runtime-darwin-arm64/scripts/verify.mjs new file mode 100644 index 000000000..617f470dc --- /dev/null +++ b/packages/runtime-darwin-arm64/scripts/verify.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { access, readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const manifest = JSON.parse(await readFile(join(root, "runtime-pack.json"), "utf8")); +const packageManifest = JSON.parse(await readFile(join(root, "package.json"), "utf8")); +if ( + manifest.schema !== "scriptc.runtime-pack.v1" || manifest.format !== 1 || + manifest.package !== packageManifest.name || manifest.version !== packageManifest.version +) throw new Error("runtime pack identity does not match package.json"); +const artifacts = [ + ...Object.values(manifest.flavors).flatMap((flavor) => + flavor.runtime_units.flatMap((unit) => unit.variants)), + ...manifest.archives, +]; +for (const artifact of artifacts) { + const path = join(root, artifact.path); + const bytes = await readFile(path); + const digest = createHash("sha256").update(bytes).digest("hex"); + if (digest !== artifact.sha256) throw new Error(`runtime pack hash mismatch: ${artifact.path}`); +} +for (const license of manifest.licenses) await access(join(root, license.path)); +process.stdout.write(`verified ${manifest.package}@${manifest.version}: ${artifacts.length} artifacts\n`); diff --git a/packages/runtime/package.json b/packages/runtime/package.json index bbcf509d1..582e0a2e0 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,7 +1,7 @@ { "name": "@scriptc/runtime", "version": "0.0.35", - "description": "scriptc native runtime — C sources, compiled into every scriptc binary", + "description": "scriptc native runtime sources and vendored dependencies", "license": "Apache-2.0", "homepage": "https://scriptc.dev", "repository": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a374bcb96..2b0718654 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,11 +54,16 @@ importers: '@scriptc/llvm-darwin-arm64': specifier: workspace:* version: link:../llvm-darwin-arm64 + '@scriptc/runtime-darwin-arm64': + specifier: workspace:* + version: link:../runtime-darwin-arm64 packages/llvm-darwin-arm64: {} packages/runtime: {} + packages/runtime-darwin-arm64: {} + packages: '@bytecodealliance/preview2-shim@0.17.6': diff --git a/scripts/surface-manifest.mjs b/scripts/surface-manifest.mjs index 12af45687..e48cea28d 100644 --- a/scripts/surface-manifest.mjs +++ b/scripts/surface-manifest.mjs @@ -28,7 +28,7 @@ const root = fileURLToPath(new URL("..", import.meta.url)); const readJson = (path) => JSON.parse(readFileSync(root + path, "utf8")); const version = readJson("packages/cli/package.json").version; -for (const pkg of ["runtime", "llvm-darwin-arm64", "compiler"]) { +for (const pkg of ["runtime", "runtime-darwin-arm64", "llvm-darwin-arm64", "compiler"]) { const v = readJson(`packages/${pkg}/package.json`).version; if (v !== version) { console.error( diff --git a/scripts/sync-versions.mjs b/scripts/sync-versions.mjs index 2d9dcfa29..85d859e93 100644 --- a/scripts/sync-versions.mjs +++ b/scripts/sync-versions.mjs @@ -16,7 +16,7 @@ if (typeof version !== "string" || version.length === 0) { process.exit(1); } -for (const pkg of ["runtime", "compiler", "llvm-darwin-arm64"]) { +for (const pkg of ["runtime", "runtime-darwin-arm64", "compiler", "llvm-darwin-arm64"]) { const path = manifest(pkg); const json = read(path); if (json.version === version) { diff --git a/tests/harness/surface-manifest.test.ts b/tests/harness/surface-manifest.test.ts index 6866e6933..a8a7f6cdf 100644 --- a/tests/harness/surface-manifest.test.ts +++ b/tests/harness/surface-manifest.test.ts @@ -63,10 +63,11 @@ describe("surface manifest generation", () => { test("the version spine is the exact published version string", () => { const parsed = JSON.parse(committed) as SurfaceManifest; expect(parsed.compilerVersion).toBe(releaseVersion); - // The four packages publish in lockstep; a drifted stamp would make + // The five packages publish in lockstep; a drifted stamp would make // the spine ambiguous for a version pin. expect(readJson("packages/compiler/package.json").version).toBe(releaseVersion); expect(readJson("packages/runtime/package.json").version).toBe(releaseVersion); + expect(readJson("packages/runtime-darwin-arm64/package.json").version).toBe(releaseVersion); expect(readJson("packages/llvm-darwin-arm64/package.json").version).toBe(releaseVersion); }); From 712a8bb5fb7c062065d4d12b72db498a1ef05b9d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 17:13:39 -0500 Subject: [PATCH 02/11] fix(runtime): harden precompiled artifact handling --- docs/src/app/cli/page.mdx | 2 +- packages/cli/scripts/runtime-pack-host.mjs | 10 ++++ packages/cli/scripts/warm-cache.mjs | 7 ++- packages/cli/test/runtime-pack-host.test.ts | 11 ++++ packages/compiler/src/index.ts | 9 +-- .../test/native-codegen-integration.test.ts | 59 +++++++++++++++++++ .../runtime-darwin-arm64/scripts/build.mjs | 8 ++- .../runtime-darwin-arm64/scripts/verify.mjs | 26 +++++++- 8 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 packages/cli/scripts/runtime-pack-host.mjs create mode 100644 packages/cli/test/runtime-pack-host.test.ts diff --git a/docs/src/app/cli/page.mdx b/docs/src/app/cli/page.mdx index 2c7a72849..a022b936e 100644 --- a/docs/src/app/cli/page.mdx +++ b/docs/src/app/cli/page.mdx @@ -81,7 +81,7 @@ Analyzes the program without producing a binary and reports, statement by statem ## scriptc cache warm -Prebuilds release runtime objects and native TLS/dynamic-engine archives for targets that still compile runtime C locally. macOS arm64 installations already carry the release-built runtime pack and skip automatic warming. Use this command when preparing another supported target's container image or CI runner. Pass one or more of `runtime`, `tls`, and `dynamic` to seed only those families. +Prebuilds release runtime objects and native TLS/dynamic-engine archives for targets that still compile runtime C locally. macOS 15+ arm64 installations already carry the release-built runtime pack and skip automatic warming; older macOS hosts retain warming for the source-toolchain path. Use this command when preparing another supported target's container image or CI runner. Pass one or more of `runtime`, `tls`, and `dynamic` to seed only those families. ## Options diff --git a/packages/cli/scripts/runtime-pack-host.mjs b/packages/cli/scripts/runtime-pack-host.mjs new file mode 100644 index 000000000..dc18cfdc0 --- /dev/null +++ b/packages/cli/scripts/runtime-pack-host.mjs @@ -0,0 +1,10 @@ +import { release } from "node:os"; + +export function hostSupportsRuntimePack( + platform = process.platform, + architecture = process.arch, + hostRelease = release(), +) { + const darwinMajor = Number.parseInt(hostRelease.split(".", 1)[0] ?? "", 10); + return platform === "darwin" && architecture === "arm64" && darwinMajor >= 24; +} diff --git a/packages/cli/scripts/warm-cache.mjs b/packages/cli/scripts/warm-cache.mjs index 39eab364d..5ed13d0fb 100644 --- a/packages/cli/scripts/warm-cache.mjs +++ b/packages/cli/scripts/warm-cache.mjs @@ -3,6 +3,7 @@ import { existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { hostSupportsRuntimePack } from "./runtime-pack-host.mjs"; // Workspace installs should stay cheap and deterministic: repository test // images already manage cache warming explicitly. Published npm packages do @@ -11,9 +12,9 @@ const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); if ( !existsSync(join(packageRoot, "src")) && process.env["SCRIPTC_NO_CACHE"] !== "1" && - // The macOS arm64 install already carries immutable release/dev runtime - // artifacts. Do not compile a second toolchain-specific copy at install. - !(process.platform === "darwin" && process.arch === "arm64") + // A supported macOS arm64 install already carries immutable release/dev + // runtime artifacts. Older macOS hosts retain the source-toolchain path. + !hostSupportsRuntimePack() ) { try { const { warmNativeCaches } = await import("@scriptc/compiler"); diff --git a/packages/cli/test/runtime-pack-host.test.ts b/packages/cli/test/runtime-pack-host.test.ts new file mode 100644 index 000000000..76f5904ba --- /dev/null +++ b/packages/cli/test/runtime-pack-host.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, test } from "vitest"; +import { hostSupportsRuntimePack } from "../scripts/runtime-pack-host.mjs"; + +describe("runtime-pack host support", () => { + test("requires macOS 15 or newer on arm64", () => { + expect(hostSupportsRuntimePack("darwin", "arm64", "24.0.0")).toBe(true); + expect(hostSupportsRuntimePack("darwin", "arm64", "23.6.0")).toBe(false); + expect(hostSupportsRuntimePack("darwin", "x64", "24.0.0")).toBe(false); + expect(hostSupportsRuntimePack("linux", "arm64", "24.0.0")).toBe(false); + }); +}); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index f5ba5c94f..7b7e167a8 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1066,10 +1066,11 @@ async function compileExecutableNative( optimization: features.optimization ?? "release", }); await linkRuntimePackExecutable(plan, { - // A caller-selected linker can be a mutable wrapper with hidden inputs. - // Keep that path honest by relinking; the default resolved driver plus - // selected immutable pack/FFI files carries a replayable proof. - ...(onArtifactReady === undefined || process.env["SCRIPTC_LINKER"] !== undefined + // A caller-selected linker can be a mutable wrapper with hidden inputs, + // while an FFI profile can name thin archives or ambient -l libraries + // whose transitive files are not represented by the top-level paths. + // Keep both postures honest by performing their final link every time. + ...(onArtifactReady === undefined || process.env["SCRIPTC_LINKER"] !== undefined || ffi !== null ? {} : { onArtifactReady }), }); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index c083f3873..0bd312467 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -252,6 +252,65 @@ describe.runIf(supported)("LLVM native helper integration", () => { } }); + test("runtime-pack FFI system libraries are relinked after an in-place rebuild", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-ffi-cache-")); + dirs.push(dir); + const entry = join(dir, "main.ts"); + const profile = join(dir, "ffi.json"); + const source = join(dir, "probe.c"); + const object = join(dir, "probe.o"); + const library = join(dir, "libscriptc_cache_probe.a"); + const cache = join(dir, "cache"); + const output = join(dir, "program"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldLibraryPath = process.env["LIBRARY_PATH"]; + const rebuildLibrary = async (value: number) => { + await writeFile(source, `double scriptc_cache_probe(void) { return ${value}; }\n`); + await execFileAsync("clang", ["-c", source, "-o", object]); + await execFileAsync("ar", ["rcs", library, object]); + }; + try { + await writeFile(entry, [ + "declare function nativeValue(): number;", + "console.log(nativeValue());", + "", + ].join("\n")); + await writeFile(profile, JSON.stringify({ + ffi_format: 1, + functions: [{ + name: "nativeValue", + symbol: "scriptc_cache_probe", + params: [], + returns: "f64", + }], + libraries: [], + system_libraries: ["scriptc_cache_probe"], + })); + process.env["SCRIPTC_CACHE_DIR"] = cache; + delete process.env["SCRIPTC_NO_CACHE"]; + process.env["LIBRARY_PATH"] = dir; + const options = { outDir: dir, outPath: output, backend: "llvm" as const, ffiProfilePath: profile }; + + await rebuildLibrary(1); + const first = await compile(entry, options); + if (!first.ok) throw new Error(first.diagnostics.map((d) => d.message).join("\n")); + expect((await execFileAsync(output, [], { encoding: "utf8" })).stdout.trim()).toBe("1"); + + await rebuildLibrary(2); + const second = await compile(entry, options); + if (!second.ok) throw new Error(second.diagnostics.map((d) => d.message).join("\n")); + expect((await execFileAsync(output, [], { encoding: "utf8" })).stdout.trim()).toBe("2"); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldLibraryPath === undefined) delete process.env["LIBRARY_PATH"]; + else process.env["LIBRARY_PATH"] = oldLibraryPath; + } + }); + test("object emission preserves outbound FFI declarations as native C ABI references", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-helper-ffi-")); dirs.push(dir); diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index a07807751..080bbbfde 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -18,6 +18,7 @@ const runtimeSrc = join(runtimeRoot, "src"); const vendorRoot = join(runtimeRoot, "vendor"); const outputRoot = join(packageRoot, "artifacts"); const manifestPath = join(packageRoot, "runtime-pack.json"); +const sourcePathFlags = [`-ffile-prefix-map=${repoRoot}=.`]; if (process.platform !== "darwin" || process.arch !== "arm64") { process.stdout.write("@scriptc/runtime-darwin-arm64: skipped on this host\n"); @@ -49,7 +50,7 @@ async function sha256(path) { async function compile(source, output, flags) { await mkdir(dirname(output), { recursive: true }); - await run(compiler, [...flags, "-c", source, "-o", output]); + await run(compiler, [...sourcePathFlags, ...flags, "-c", source, "-o", output]); } async function parallel(items, task) { @@ -81,8 +82,9 @@ const privateRoot = `${outputRoot}.tmp-${process.pid}`; await rm(privateRoot, { recursive: true, force: true }); await rm(outputRoot, { recursive: true, force: true }); await mkdir(privateRoot, { recursive: true }); -// Build under the final path spelling so object debug metadata remains stable; -// artifacts contain no source checkout paths after stripping debug information. +// Build under the final path spelling so object metadata remains stable. The +// compiler prefix map above also keeps __FILE__ and any debug paths independent +// of the producer's checkout location. await rename(privateRoot, outputRoot); try { diff --git a/packages/runtime-darwin-arm64/scripts/verify.mjs b/packages/runtime-darwin-arm64/scripts/verify.mjs index 617f470dc..53f92afd7 100644 --- a/packages/runtime-darwin-arm64/scripts/verify.mjs +++ b/packages/runtime-darwin-arm64/scripts/verify.mjs @@ -16,11 +16,35 @@ const artifacts = [ flavor.runtime_units.flatMap((unit) => unit.variants)), ...manifest.archives, ]; + +function printableStrings(bytes) { + const strings = []; + let start = -1; + for (let i = 0; i <= bytes.length; i++) { + const byte = bytes[i]; + if (byte !== undefined && byte >= 0x20 && byte <= 0x7e) { + if (start === -1) start = i; + continue; + } + if (start !== -1 && i - start >= 4) strings.push(bytes.subarray(start, i).toString("ascii")); + start = -1; + } + return strings; +} + for (const artifact of artifacts) { const path = join(root, artifact.path); const bytes = await readFile(path); const digest = createHash("sha256").update(bytes).digest("hex"); - if (digest !== artifact.sha256) throw new Error(`runtime pack hash mismatch: ${artifact.path}`); + if (bytes.length !== artifact.size || digest !== artifact.sha256) { + throw new Error(`runtime pack hash mismatch: ${artifact.path}`); + } + const checkoutPath = printableStrings(bytes).find((value) => + value.startsWith("/") && value.includes("/packages/runtime/") + ); + if (checkoutPath !== undefined) { + throw new Error(`runtime pack contains an absolute source path in ${artifact.path}: ${checkoutPath}`); + } } for (const license of manifest.licenses) await access(join(root, license.path)); process.stdout.write(`verified ${manifest.package}@${manifest.version}: ${artifacts.length} artifacts\n`); From d20a839ded4c063cec803520e4a42d2736ded2d7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 17:54:52 -0500 Subject: [PATCH 03/11] fix(runtime): make precompiled artifacts reproducible --- packages/cli/scripts/runtime-pack-host.d.mts | 5 ++++ packages/cli/src/bootstrap.ts | 3 +- .../runtime-darwin-arm64/scripts/archive.mjs | 12 ++++++++ .../runtime-darwin-arm64/scripts/build.mjs | 3 +- .../runtime-darwin-arm64/test/archive.test.ts | 29 +++++++++++++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 packages/cli/scripts/runtime-pack-host.d.mts create mode 100644 packages/runtime-darwin-arm64/scripts/archive.mjs create mode 100644 packages/runtime-darwin-arm64/test/archive.test.ts diff --git a/packages/cli/scripts/runtime-pack-host.d.mts b/packages/cli/scripts/runtime-pack-host.d.mts new file mode 100644 index 000000000..1a7b53d29 --- /dev/null +++ b/packages/cli/scripts/runtime-pack-host.d.mts @@ -0,0 +1,5 @@ +export function hostSupportsRuntimePack( + platform?: NodeJS.Platform, + architecture?: string, + hostRelease?: string, +): boolean; diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 7d4ea43a0..5daf7d8b4 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -7,6 +7,7 @@ import { arch } from "node:process"; import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { hostSupportsRuntimePack } from "../scripts/runtime-pack-host.mjs"; import { CLI_OPTIONS, USAGE } from "./usage.js"; // Node 24 can persist V8's compiled module bytecode. scriptc's CLI imports @@ -114,7 +115,7 @@ async function tryFastPath(): Promise { npmStatic, ffiProfile: ffiPath === null ? null : { path: ffiPath, bytes: ffiBytes! }, target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${arch}:${ - process.platform === "darwin" && arch === "arm64" && + hostSupportsRuntimePack(process.platform, arch) && (process.env["SCRIPTC_TARGET"] ?? "") === "" && backend !== "c" && !values.sanitize && process.env["SCRIPTC_RUNTIME_PACK"] !== "0" && diff --git a/packages/runtime-darwin-arm64/scripts/archive.mjs b/packages/runtime-darwin-arm64/scripts/archive.mjs new file mode 100644 index 000000000..41c1c8b16 --- /dev/null +++ b/packages/runtime-darwin-arm64/scripts/archive.mjs @@ -0,0 +1,12 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const run = promisify(execFile); + +export async function createDeterministicArchive(archiver, output, objects) { + await run(archiver, ["rcs", output, ...objects], { + // Apple/BSD ar otherwise copies each freshly compiled object's timestamp + // into the archive, changing the pack hashes on every release build. + env: { ...process.env, ZERO_AR_DATE: "1" }, + }); +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 080bbbfde..be3d5cdc1 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -9,6 +9,7 @@ import { basename, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { RUNTIME_PACK_MATRIX } from "../runtime-pack-matrix.mjs"; +import { createDeterministicArchive } from "./archive.mjs"; const run = promisify(execFile); const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); @@ -68,7 +69,7 @@ async function archive(id, sources, sourceRoot, flags) { }); const output = join(root, `libscriptc-${id}.a`); const objects = sources.map((source) => join(objectRoot, source.replace(/\.c$/, ".o"))); - await run(archiver, ["rcs", output, ...objects]); + await createDeterministicArchive(archiver, output, objects); await rm(objectRoot, { recursive: true, force: true }); return { id, diff --git a/packages/runtime-darwin-arm64/test/archive.test.ts b/packages/runtime-darwin-arm64/test/archive.test.ts new file mode 100644 index 000000000..70b0b1fab --- /dev/null +++ b/packages/runtime-darwin-arm64/test/archive.test.ts @@ -0,0 +1,29 @@ +import { mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { createDeterministicArchive } from "../scripts/archive.mjs"; + +const dirs: string[] = []; + +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe.runIf(process.platform === "darwin")("runtime-pack archives", () => { + test("ignore input timestamps", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-archive-")); + dirs.push(dir); + const member = join(dir, "member.o"); + const first = join(dir, "first.a"); + const second = join(dir, "second.a"); + await writeFile(member, "runtime-pack archive member\n"); + + await utimes(member, new Date(1_000), new Date(1_000)); + await createDeterministicArchive("ar", first, [member]); + await utimes(member, new Date(2_000), new Date(2_000)); + await createDeterministicArchive("ar", second, [member]); + + expect(await readFile(second)).toEqual(await readFile(first)); + }); +}); From fb82e89ee78cc9ecc51d7f59f271cf14c42c454e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 18:24:45 -0500 Subject: [PATCH 04/11] fix(runtime): reject opaque linkers from executable cache --- .../compiler/src/backend/native-toolchain.ts | 2 +- packages/compiler/src/index.ts | 20 +++-- .../test/native-codegen-integration.test.ts | 79 ++++++++++++++++++- 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index d66d2db7e..489ba1946 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -2111,7 +2111,7 @@ function isAppleSystemClangHandoff( * can safely represent that behavior. Accept direct Clang/Zig drivers (plus * Apple's system shim) and conservatively keep wrapper-driven builds on the * uncached path. */ -async function compilerDriverSupportsPersistentCache( +export async function compilerDriverSupportsPersistentCache( driver: Pick, environmentFingerprint: string, ): Promise { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 7b7e167a8..93d9fe2fb 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -2,7 +2,7 @@ import { InternalCompilerError } from "./errors.js"; import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; -import { buildCacheRoot, CcCompileError, clearCcCaches, compileC, compileLibArchive, configuredTargetPlatform, executableNativeEnvironmentFingerprint, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/native-toolchain.js"; +import { buildCacheRoot, CcCompileError, clearCcCaches, compileC, compileLibArchive, compilerDriverSupportsPersistentCache, configuredTargetPlatform, executableNativeEnvironmentFingerprint, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform, toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "./backend/native-toolchain.js"; import { emitCModule } from "./backend/c/c-emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen.js"; @@ -1065,14 +1065,20 @@ async function compileExecutableNative( ffi, optimization: features.optimization ?? "release", }); + const cacheableLinker = + onArtifactReady !== undefined && process.env["SCRIPTC_LINKER"] === undefined && ffi === null && + toolchainEnvironmentCachePolicy().completeArtifacts && + await compilerDriverSupportsPersistentCache( + resolveCc(), + toolchainEnvironmentFingerprint(), + ); await linkRuntimePackExecutable(plan, { // A caller-selected linker can be a mutable wrapper with hidden inputs, - // while an FFI profile can name thin archives or ambient -l libraries - // whose transitive files are not represented by the top-level paths. - // Keep both postures honest by performing their final link every time. - ...(onArtifactReady === undefined || process.env["SCRIPTC_LINKER"] !== undefined || ffi !== null - ? {} - : { onArtifactReady }), + // and a PATH-selected `clang` can be one too. FFI profiles and mutable + // linker search environments likewise name transitive files that the + // top-level dependency snapshot cannot prove. Only a direct driver in a + // stable link environment may publish a reusable final executable. + ...(cacheableLinker ? { onArtifactReady } : {}), }); return; } diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 0bd312467..5797dc4a3 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -1,6 +1,6 @@ import { execFile, spawn } from "node:child_process"; import { createRequire } from "node:module"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { release as osRelease, tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; @@ -311,6 +311,83 @@ describe.runIf(supported)("LLVM native helper integration", () => { } }); + test("runtime-pack PATH linker wrappers cannot restore hidden link inputs", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-linker-cache-")); + dirs.push(dir); + const entry = join(dir, "main.ts"); + const wrapper = join(dir, "clang"); + const firstSource = join(dir, "first.c"); + const secondSource = join(dir, "second.c"); + const firstObject = join(dir, "first.o"); + const secondObject = join(dir, "second.o"); + const cache = join(dir, "cache"); + const output = join(dir, "program"); + const oldPath = process.env["PATH"]; + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldLinkInput = process.env["SCRIPTC_TEST_LINK_INPUT"]; + try { + await Promise.all([ + writeFile(entry, 'console.log("program");\n'), + writeFile(firstSource, [ + "#include ", + '__attribute__((constructor)) static void marker(void) { write(1, "first\\n", 6); }', + "", + ].join("\n")), + writeFile(secondSource, [ + "#include ", + '__attribute__((constructor)) static void marker(void) { write(1, "second\\n", 7); }', + "", + ].join("\n")), + writeFile(wrapper, [ + "#!/bin/sh", + 'for arg in "$@"; do', + ' if [ "$arg" = "-c" ]; then exec /usr/bin/clang "$@"; fi', + "done", + 'has_output=""', + 'for arg in "$@"; do [ "$arg" = "-o" ] && has_output=1; done', + 'if [ -n "$has_output" ] && [ -n "$SCRIPTC_TEST_LINK_INPUT" ]; then', + ' exec /usr/bin/clang "$@" "$SCRIPTC_TEST_LINK_INPUT"', + "fi", + 'exec /usr/bin/clang "$@"', + "", + ].join("\n")), + ]); + await chmod(wrapper, 0o755); + await Promise.all([ + execFileAsync("/usr/bin/clang", [ + "-target", MACOS_ARM64_TARGET.llvmTriple, "-c", firstSource, "-o", firstObject, + ]), + execFileAsync("/usr/bin/clang", [ + "-target", MACOS_ARM64_TARGET.llvmTriple, "-c", secondSource, "-o", secondObject, + ]), + ]); + process.env["PATH"] = `${dir}:${oldPath ?? "/usr/bin:/bin"}`; + process.env["SCRIPTC_CACHE_DIR"] = cache; + delete process.env["SCRIPTC_NO_CACHE"]; + const options = { outDir: dir, outPath: output, backend: "llvm" as const }; + + process.env["SCRIPTC_TEST_LINK_INPUT"] = firstObject; + const first = await compile(entry, options); + if (!first.ok) throw new Error(first.diagnostics.map((d) => d.message).join("\n")); + expect((await execFileAsync(output, [], { encoding: "utf8" })).stdout).toBe("first\nprogram\n"); + + process.env["SCRIPTC_TEST_LINK_INPUT"] = secondObject; + const second = await compile(entry, options); + if (!second.ok) throw new Error(second.diagnostics.map((d) => d.message).join("\n")); + expect((await execFileAsync(output, [], { encoding: "utf8" })).stdout).toBe("second\nprogram\n"); + } finally { + if (oldPath === undefined) delete process.env["PATH"]; + else process.env["PATH"] = oldPath; + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldLinkInput === undefined) delete process.env["SCRIPTC_TEST_LINK_INPUT"]; + else process.env["SCRIPTC_TEST_LINK_INPUT"] = oldLinkInput; + } + }); + test("object emission preserves outbound FFI declarations as native C ABI references", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-helper-ffi-")); dirs.push(dir); From 29d6318be0379df26705dd2784c148d5344e0c3c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 19:02:48 -0500 Subject: [PATCH 05/11] fix(runtime): normalize precompiled archive metadata --- RELEASING.md | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 7 ++- .../runtime-darwin-arm64/scripts/archive.mjs | 45 ++++++++++++++++++- .../runtime-darwin-arm64/test/archive.test.ts | 30 +++++++++++-- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 6a7606917..dfd348ff3 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,7 +5,7 @@ Releases are manual, single-commit affairs. The maintainer controls the changelo To prepare a release: 1. Bump the version in `packages/cli/package.json` -2. Run `node scripts/sync-versions.mjs` to stamp the same version into `packages/runtime`, `packages/llvm-darwin-arm64`, and `packages/compiler`, then `pnpm manifest` to restamp `packages/compiler/surface-manifest.json` with the new version, and commit both (the test suite's staleness guard fails on a version drift) +2. Run `node scripts/sync-versions.mjs` to stamp the same version into `packages/runtime`, `packages/runtime-darwin-arm64`, `packages/llvm-darwin-arm64`, and `packages/compiler`, then `pnpm manifest` to restamp `packages/compiler/surface-manifest.json` with the new version, and commit both (the test suite's staleness guard fails on a version drift) 3. Fold the `## Unreleased` section of `CHANGELOG.md` into a new `## ` entry (newest first, below `## Unreleased`), and leave `## Unreleased` empty for the next cycle 4. Wrap the new entry in `` and `` markers; this marked block is also the GitHub release body 5. Remove the `` and `` markers from the previous release entry; only the latest release should have markers diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 801be261f..d9e03dd31 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { describe, expect, test } from "vitest"; +import { compilerReleaseVersion } from "../library/sidecar.js"; import type { NativeLinkFeatures } from "./native-link-info.js"; import { effectiveRuntimeFeatures, @@ -13,6 +14,8 @@ import { } from "./runtime-pack.js"; import { MACOS_ARM64_TARGET } from "./targets.js"; +const VERSION = compilerReleaseVersion(); + const BASE: NativeLinkFeatures = { dynamic: false, regex: false, @@ -50,7 +53,7 @@ async function fixture() { const packagePath = join(root, "package.json"); await writeFile(packagePath, JSON.stringify({ name: "@scriptc/runtime-darwin-arm64", - version: "0.0.35", + version: VERSION, })); const artifact = async (path: string, bytes: string) => { const output = join(root, path); @@ -81,7 +84,7 @@ async function fixture() { schema: "scriptc.runtime-pack.v1", format: 1, package: "@scriptc/runtime-darwin-arm64", - version: "0.0.35", + version: VERSION, target: { name: "macos-arm64", llvm_triple: "arm64-apple-macosx14.0.0", diff --git a/packages/runtime-darwin-arm64/scripts/archive.mjs b/packages/runtime-darwin-arm64/scripts/archive.mjs index 41c1c8b16..ded5b9fed 100644 --- a/packages/runtime-darwin-arm64/scripts/archive.mjs +++ b/packages/runtime-darwin-arm64/scripts/archive.mjs @@ -1,12 +1,53 @@ import { execFile } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; import { promisify } from "node:util"; const run = promisify(execFile); +const GLOBAL_HEADER = "!\n"; +const MEMBER_HEADER_SIZE = 60; + +function replaceField(bytes, offset, width, value) { + bytes.write(value.padEnd(width, " "), offset, width, "ascii"); +} + +async function normalizeMetadata(output) { + const bytes = await readFile(output); + if (bytes.subarray(0, GLOBAL_HEADER.length).toString("ascii") !== GLOBAL_HEADER) { + throw new Error(`archiver produced an invalid archive: ${output}`); + } + let offset = GLOBAL_HEADER.length; + while (offset < bytes.length) { + const headerEnd = offset + MEMBER_HEADER_SIZE; + if ( + headerEnd > bytes.length || + bytes.subarray(offset + 58, headerEnd).toString("ascii") !== "`\n" + ) throw new Error(`archiver produced a malformed member header: ${output}`); + const sizeText = bytes.subarray(offset + 48, offset + 58).toString("ascii").trim(); + if (!/^\d+$/.test(sizeText)) { + throw new Error(`archiver produced a malformed member size: ${output}`); + } + const size = Number(sizeText); + if (!Number.isSafeInteger(size) || headerEnd + size > bytes.length) { + throw new Error(`archiver produced an invalid member size: ${output}`); + } + // ar stores these values in fixed-width ASCII fields. They do not affect + // member offsets or the symbol table, so normalizing them after indexing + // works with Apple/BSD ar as well as archivers that implement a D mode. + replaceField(bytes, offset + 16, 12, "0"); // timestamp + replaceField(bytes, offset + 28, 6, "0"); // uid + replaceField(bytes, offset + 34, 6, "0"); // gid + replaceField(bytes, offset + 40, 8, "100644"); // mode + offset = headerEnd + size + (size % 2); + } + if (offset !== bytes.length) throw new Error(`archiver produced a truncated archive: ${output}`); + await writeFile(output, bytes); +} export async function createDeterministicArchive(archiver, output, objects) { await run(archiver, ["rcs", output, ...objects], { - // Apple/BSD ar otherwise copies each freshly compiled object's timestamp - // into the archive, changing the pack hashes on every release build. + // Ask Apple/BSD ar to omit timestamps up front, then normalize every + // variable member-header field below for cross-account reproducibility. env: { ...process.env, ZERO_AR_DATE: "1" }, }); + await normalizeMetadata(output); } diff --git a/packages/runtime-darwin-arm64/test/archive.test.ts b/packages/runtime-darwin-arm64/test/archive.test.ts index 70b0b1fab..7e5e4feac 100644 --- a/packages/runtime-darwin-arm64/test/archive.test.ts +++ b/packages/runtime-darwin-arm64/test/archive.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; @@ -6,12 +6,28 @@ import { createDeterministicArchive } from "../scripts/archive.mjs"; const dirs: string[] = []; +function memberMetadata(bytes: Buffer) { + const entries: { timestamp: string; uid: string; gid: string; mode: string }[] = []; + let offset = Buffer.byteLength("!\n"); + while (offset < bytes.length) { + const size = Number(bytes.subarray(offset + 48, offset + 58).toString("ascii").trim()); + entries.push({ + timestamp: bytes.subarray(offset + 16, offset + 28).toString("ascii").trim(), + uid: bytes.subarray(offset + 28, offset + 34).toString("ascii").trim(), + gid: bytes.subarray(offset + 34, offset + 40).toString("ascii").trim(), + mode: bytes.subarray(offset + 40, offset + 48).toString("ascii").trim(), + }); + offset += 60 + size + (size % 2); + } + return entries; +} + afterEach(async () => { await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); describe.runIf(process.platform === "darwin")("runtime-pack archives", () => { - test("ignore input timestamps", async () => { + test("normalize timestamps, ownership, and modes", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-archive-")); dirs.push(dir); const member = join(dir, "member.o"); @@ -19,11 +35,19 @@ describe.runIf(process.platform === "darwin")("runtime-pack archives", () => { const second = join(dir, "second.a"); await writeFile(member, "runtime-pack archive member\n"); + await chmod(member, 0o600); await utimes(member, new Date(1_000), new Date(1_000)); await createDeterministicArchive("ar", first, [member]); await utimes(member, new Date(2_000), new Date(2_000)); await createDeterministicArchive("ar", second, [member]); - expect(await readFile(second)).toEqual(await readFile(first)); + const firstBytes = await readFile(first); + expect(await readFile(second)).toEqual(firstBytes); + const metadata = memberMetadata(firstBytes); + expect(metadata.length).toBeGreaterThanOrEqual(1); + expect(metadata.every((entry) => + entry.timestamp === "0" && entry.uid === "0" && entry.gid === "0" && + entry.mode === "100644" + )).toBe(true); }); }); From 7e655e7bb18fdd5188fd6ed6d826d9406372e0f6 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 19:39:56 -0500 Subject: [PATCH 06/11] fix(runtime): harden precompiled artifact caching --- .gitignore | 1 + .../src/backend/native-codegen.test.ts | 6 +- .../compiler/src/backend/native-codegen.ts | 15 +- packages/compiler/src/backend/runtime-pack.ts | 2 + packages/compiler/src/index.ts | 26 +- .../test/native-codegen-integration.test.ts | 59 ++- .../scripts/build-state.mjs | 102 ++++++ .../runtime-darwin-arm64/scripts/build.mjs | 336 ++++++++++-------- .../runtime-darwin-arm64/test/archive.test.ts | 59 ++- 9 files changed, 438 insertions(+), 168 deletions(-) create mode 100644 packages/runtime-darwin-arm64/scripts/build-state.mjs diff --git a/.gitignore b/.gitignore index bf1db443b..34734916d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ dist/ /packages/llvm-darwin-arm64/bin/ /packages/runtime-darwin-arm64/artifacts/ /packages/runtime-darwin-arm64/runtime-pack.json +/packages/runtime-darwin-arm64/.runtime-pack-* !tests/fixtures/fetch/node_modules/eventsource-parser/dist/ !tests/fixtures/npm/node_modules/*/dist/ !tests/fixtures/npm/workspace/*/dist/ diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index b16d758a3..c4a7c7d45 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -76,14 +76,16 @@ test("resolves a package helper, emits atomically, and caches by all native inpu const pkg = await fakePackage(); const first = join(pkg.root, "first.o"); const second = join(pkg.root, "second.o"); - await emitNativeArtifact(request(pkg.root, pkg.packageJson, first)); - await emitNativeArtifact(request(pkg.root, pkg.packageJson, second)); + const firstArtifact = await emitNativeArtifact(request(pkg.root, pkg.packageJson, first)); + const secondArtifact = await emitNativeArtifact(request(pkg.root, pkg.packageJson, second)); expect(await readFile(first, "utf8")).toContain("define i32 @answer"); expect(await readFile(second)).toEqual(await readFile(first)); const expectedMode = 0o666 & ~process.umask(); expect((await stat(first)).mode & 0o777).toBe(expectedMode); expect((await stat(second)).mode & 0o777).toBe(expectedMode); expect((await readFile(pkg.log, "utf8")).trim().split("\n")).toHaveLength(1); + expect(firstArtifact.dependencyPaths).toEqual([pkg.packageJson, pkg.bin]); + expect(secondArtifact.dependencyPaths).toEqual(firstArtifact.dependencyPaths); }); test("cache publication failures do not discard a valid requested artifact", async () => { diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 0f72dbe1e..d998a8877 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -50,10 +50,16 @@ interface HelperIdentity { } interface ResolvedHelper { + packageJsonPath: string; binaryPath: string; identity: HelperIdentity; } +export interface NativeCodegenArtifact { + /** Installed package inputs whose identity selected and produced the artifact. */ + dependencyPaths: string[]; +} + const resolvedHelperCache = new Map>(); export interface NativeCodegenOptions { @@ -206,6 +212,7 @@ async function resolveHelper( ); } return { + packageJsonPath, binaryPath, identity: { packageName: target.helperPackage, @@ -262,7 +269,7 @@ async function installVerifiedCache(source: string, destination: string): Promis } } -export async function emitNativeArtifact(options: NativeCodegenOptions): Promise { +export async function emitNativeArtifact(options: NativeCodegenOptions): Promise { const target = options.target ?? nativeCodegenTarget(); if (target === null) { throw new NativeCodegenError( @@ -287,8 +294,11 @@ export async function emitNativeArtifact(options: NativeCodegenOptions): Promise ? null : join(root, "native-codegen-v1", key.slice(0, 2), `${key}.${options.outputKind === "obj" ? "o" : "s"}`); await mkdir(dirname(options.outputPath), { recursive: true }); + const artifact = { + dependencyPaths: [helper.packageJsonPath, helper.binaryPath], + } satisfies NativeCodegenArtifact; if (cached !== null && await validCachedFile(cached) && - await installVerifiedCache(cached, options.outputPath)) return; + await installVerifiedCache(cached, options.outputPath)) return artifact; const stage = privateSiblingPath(options.outputPath, `native-${options.outputKind}`); const input = privateSiblingPath(options.outputPath, "native-input"); @@ -320,6 +330,7 @@ export async function emitNativeArtifact(options: NativeCodegenOptions): Promise if (cached !== null) await publishCachedFile(stage, cached).catch(() => undefined); await rename(stage, options.outputPath); await pruneBuildCache(root); + return artifact; } finally { await Promise.all([ rm(stage, { force: true }).catch(() => undefined), diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts index 7ed59e785..7891b42c3 100644 --- a/packages/compiler/src/backend/runtime-pack.ts +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -340,6 +340,7 @@ export async function createRuntimeLinkPlan(options: { features: NativeLinkFeatures; ffi: FfiProfile | null; optimization: "release" | "dev"; + programObjectDependencies?: readonly string[]; env?: NodeJS.ProcessEnv; resolver?: (specifier: string) => string; }): Promise { @@ -363,6 +364,7 @@ export async function createRuntimeLinkPlan(options: { ], dependencyPaths: [ ...runtimePack.dependencyPaths, + ...(options.programObjectDependencies ?? []), ...(options.ffi?.libraries ?? []), ], runtimePack, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 93d9fe2fb..11a72c92b 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1050,6 +1050,7 @@ async function compileExecutableNative( sanitize: boolean, ffi: FfiProfile | null, programSplit: ReturnType = null, + programObjectDependencies: readonly string[] = [], onArtifactReady?: NonNullable[0]["onArtifactReady"]>, ): Promise { const programIsObject = /\.(?:o|obj)$/.test(cPath); @@ -1064,6 +1065,7 @@ async function compileExecutableNative( features, ffi, optimization: features.optimization ?? "release", + programObjectDependencies, }); const cacheableLinker = onArtifactReady !== undefined && process.env["SCRIPTC_LINKER"] === undefined && ffi === null && @@ -1158,7 +1160,7 @@ async function emitNativeProgramObject( entryPath: string, opts: CompileRequestOptions, llvm: string, -): Promise<{ linkPath: string; artifactPath: string }> { +): Promise<{ linkPath: string; artifactPath: string; dependencyPaths: string[] }> { const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const artifactPath = join(opts.outDir, `${stem}.helper.o`); // compileExecutableNative recognizes object inputs by suffix. The random @@ -1166,7 +1168,7 @@ async function emitNativeProgramObject( // it rather than attempting to compile it as source. const linkPath = `${privateSiblingPath(artifactPath, "native-program-object")}.o`; try { - await emitNativeArtifact({ + const artifact = await emitNativeArtifact({ outputPath: linkPath, llvm, outputKind: "obj", @@ -1174,7 +1176,7 @@ async function emitNativeProgramObject( optimization: opts.optimization === "dev" ? "0" : "2", ...(opts.sanitize === undefined ? {} : { sanitize: opts.sanitize }), }); - return { linkPath, artifactPath }; + return { linkPath, artifactPath, dependencyPaths: artifact.dependencyPaths }; } catch (error) { await rm(linkPath, { force: true }).catch(() => undefined); throw error; @@ -1329,7 +1331,9 @@ async function compileTracked( ? null : { path: opts.ffiProfilePath, bytes: ffiProfileBytes }, target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}:${ - opts.nativeProgramObject === true || ( + opts.nativeProgramObject === true + ? "helper-object" + : ( opts.backend !== "c" && opts.sanitize !== true && process.env["SCRIPTC_RUNTIME_PACK"] !== "0" && process.env["SCRIPTC_FETCH_CURL"] !== "1" && @@ -1383,7 +1387,11 @@ async function compileTracked( }; } let nativeInputPath = earlyHit.cPath; - let nativeProgramObject: { linkPath: string; artifactPath: string } | null = null; + let nativeProgramObject: { + linkPath: string; + artifactPath: string; + dependencyPaths: string[]; + } | null = null; const useRuntimePack = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, earlyHit.native.backend); if (useRuntimePack) { @@ -1416,6 +1424,7 @@ async function compileTracked( opts.sanitize ?? false, ffi, null, + nativeProgramObject?.dependencyPaths, opts.nativeProgramObject === true ? undefined : async ({ dependencies }) => { await publishEarlyExecutableCache(cacheRoot, executableCacheOptions, { ...earlyHit, @@ -1707,7 +1716,11 @@ async function compileTracked( } const executableCacheOptions = earlyCacheOptions; let publishedExecutable = false; - let nativeProgramObject: { linkPath: string; artifactPath: string } | null = null; + let nativeProgramObject: { + linkPath: string; + artifactPath: string; + dependencyPaths: string[]; + } | null = null; try { const useRuntimePack = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, backend); @@ -1733,6 +1746,7 @@ async function compileTracked( opts.sanitize ?? false, ffi, programSplit, + nativeProgramObject?.dependencyPaths, opts.nativeProgramObject === true ? undefined : async ({ dependencies }) => { await publishEarlyExecutableCache(cacheRoot, executableCacheOptions, { cPath, diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 5797dc4a3..347d090c3 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -1,6 +1,6 @@ import { execFile, spawn } from "node:child_process"; import { createRequire } from "node:module"; -import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { release as osRelease, tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; @@ -252,6 +252,63 @@ describe.runIf(supported)("LLVM native helper integration", () => { } }); + test("helper-object validation does not reuse an ordinary runtime-pack executable entry", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-helper-cache-identity-")); + dirs.push(dir); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = join(dir, "cache"); + delete process.env["SCRIPTC_NO_CACHE"]; + const entry = join(repoRoot, "tests/corpus/001-hello.ts"); + const options = { outDir: dir, outPath: join(dir, "program"), backend: "llvm" as const }; + const ordinary = await compile(entry, options); + if (!ordinary.ok) throw new Error(ordinary.diagnostics.map((d) => d.message).join("\n")); + + const validation = await compile(entry, { ...options, nativeProgramObject: true }); + if (!validation.ok) throw new Error(validation.diagnostics.map((d) => d.message).join("\n")); + expect((await stat(join(dir, "001-hello.helper.o"))).size).toBeGreaterThan(0); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } + }); + + test("runtime-pack executable proofs include the helper package and binary", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-helper-cache-proof-")); + dirs.push(dir); + const cache = join(dir, "cache"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cache; + delete process.env["SCRIPTC_NO_CACHE"]; + const result = await compile(join(repoRoot, "tests/corpus/001-hello.ts"), { + outDir: dir, + outPath: join(dir, "program"), + backend: "llvm", + }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + + const stampName = (await readdir(join(cache, "early-exe"), { recursive: true })) + .find((path) => path.endsWith("stamp.json")); + expect(stampName).toBeDefined(); + const stamp = JSON.parse(await readFile(join(cache, "early-exe", stampName!), "utf8")) as { + nativeDependencies: { path: string }[]; + }; + const dependencies = stamp.nativeDependencies.map((entry) => entry.path); + expect(dependencies).toContain(helperPackage); + expect(dependencies).toContain(join(dirname(helperPackage), "bin", "scriptc-llvm-codegen")); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } + }); + test("runtime-pack FFI system libraries are relinked after an in-place rebuild", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-ffi-cache-")); dirs.push(dir); diff --git a/packages/runtime-darwin-arm64/scripts/build-state.mjs b/packages/runtime-darwin-arm64/scripts/build-state.mjs new file mode 100644 index 000000000..697f379a2 --- /dev/null +++ b/packages/runtime-darwin-arm64/scripts/build-state.mjs @@ -0,0 +1,102 @@ +import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; + +const wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function lockIsActive(lockPath) { + try { + const owner = JSON.parse(await readFile(`${lockPath}/owner.json`, "utf8")); + if (!Number.isInteger(owner.pid) || owner.pid <= 0) return true; + try { + process.kill(owner.pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } + } catch { + // mkdir() wins before owner.json is written. Treat a fresh owner-less + // directory as active, but recover one left behind by a crashed builder. + const info = await stat(lockPath).catch(() => null); + return info !== null && Date.now() - info.mtimeMs < 5_000; + } +} + +export async function withBuildLock( + lockPath, + task, + { retryMilliseconds = 50, timeoutMilliseconds = 10 * 60_000 } = {}, +) { + const started = Date.now(); + for (;;) { + try { + await mkdir(lockPath); + await writeFile(`${lockPath}/owner.json`, JSON.stringify({ pid: process.pid })); + break; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + if (!(await lockIsActive(lockPath))) { + const abandoned = `${lockPath}.abandoned-${process.pid}-${Math.random().toString(36).slice(2)}`; + try { + await rename(lockPath, abandoned); + await rm(abandoned, { recursive: true, force: true }); + } catch (takeoverError) { + if (takeoverError?.code !== "ENOENT") throw takeoverError; + } + continue; + } + if (Date.now() - started >= timeoutMilliseconds) { + throw new Error(`timed out waiting for runtime-pack build lock: ${lockPath}`); + } + await wait(retryMilliseconds); + } + } + try { + return await task(); + } finally { + await rm(lockPath, { recursive: true, force: true }); + } +} + +async function moveAside(path, backup) { + try { + await rename(path, backup); + return true; + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } +} + +export async function installRuntimePack({ + outputRoot, + manifestPath, + stagedOutputRoot, + stagedManifestPath, + backupRoot, + backupManifestPath, +}) { + let outputBackedUp = false; + let manifestBackedUp = false; + let outputInstalled = false; + let manifestInstalled = false; + try { + outputBackedUp = await moveAside(outputRoot, backupRoot); + manifestBackedUp = await moveAside(manifestPath, backupManifestPath); + await rename(stagedOutputRoot, outputRoot); + outputInstalled = true; + await rename(stagedManifestPath, manifestPath); + manifestInstalled = true; + } catch (error) { + if (manifestInstalled) await rm(manifestPath, { force: true }).catch(() => undefined); + if (outputInstalled) await rm(outputRoot, { recursive: true, force: true }).catch(() => undefined); + if (manifestBackedUp) await rename(backupManifestPath, manifestPath).catch(() => undefined); + if (outputBackedUp) await rename(backupRoot, outputRoot).catch(() => undefined); + throw error; + } + // Backup cleanup is outside the transactional install. Once both staged + // outputs are live, a cleanup failure must not remove them or attempt a + // rollback from a backup that may already have been deleted. + await Promise.all([ + rm(backupRoot, { recursive: true, force: true }).catch(() => undefined), + rm(backupManifestPath, { force: true }).catch(() => undefined), + ]); +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index be3d5cdc1..338bb96f4 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -3,13 +3,14 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { availableParallelism } from "node:os"; import { - copyFile, mkdir, readFile, readdir, rename, rm, stat, writeFile, + copyFile, mkdir, readFile, readdir, rm, stat, writeFile, } from "node:fs/promises"; -import { basename, dirname, join } from "node:path"; +import { basename, dirname, join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { RUNTIME_PACK_MATRIX } from "../runtime-pack-matrix.mjs"; import { createDeterministicArchive } from "./archive.mjs"; +import { installRuntimePack, withBuildLock } from "./build-state.mjs"; const run = promisify(execFile); const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); @@ -19,174 +20,197 @@ const runtimeSrc = join(runtimeRoot, "src"); const vendorRoot = join(runtimeRoot, "vendor"); const outputRoot = join(packageRoot, "artifacts"); const manifestPath = join(packageRoot, "runtime-pack.json"); -const sourcePathFlags = [`-ffile-prefix-map=${repoRoot}=.`]; if (process.platform !== "darwin" || process.arch !== "arm64") { process.stdout.write("@scriptc/runtime-darwin-arm64: skipped on this host\n"); process.exit(0); } -const packageManifest = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf8")); -const compiler = process.env.CC ?? "clang"; -const archiver = process.env.AR ?? "ar"; -const compilerVersion = (await run(compiler, ["--version"])).stdout.split("\n", 1)[0].trim(); -const commonFlags = [ - "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, - "-std=c11", "-pthread", "-fno-math-errno", "-fno-strict-aliasing", - "-Wno-deprecated-declarations", "-I", runtimeSrc, -]; -const quickjs = join(vendorRoot, "quickjs-ng"); -const zlib = join(vendorRoot, "zlib"); -const mbedtls = join(vendorRoot, "mbedtls"); -const QJS_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"]; -const LRE_SOURCES = ["libregexp.c", "libunicode.c"]; -const ZLIB_SOURCES = [ - "adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", - "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c", -]; +async function build() { + const buildRoot = join( + packageRoot, + `.runtime-pack-build-${process.pid}-${Math.random().toString(36).slice(2)}`, + ); + const stagedOutputRoot = join(buildRoot, "artifacts"); + const stagedManifestPath = join(buildRoot, "runtime-pack.json"); + const artifactPath = (path) => + ["artifacts", ...relative(stagedOutputRoot, path).split(sep)].join("/"); + const sourcePathFlags = [ + `-ffile-prefix-map=${buildRoot}=${packageRoot}`, + `-ffile-prefix-map=${repoRoot}=.`, + ]; -async function sha256(path) { - return createHash("sha256").update(await readFile(path)).digest("hex"); -} + const packageManifest = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf8")); + const compiler = process.env.CC ?? "clang"; + const archiver = process.env.AR ?? "ar"; + const compilerVersion = (await run(compiler, ["--version"])).stdout.split("\n", 1)[0].trim(); + const commonFlags = [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, + "-std=c11", "-pthread", "-fno-math-errno", "-fno-strict-aliasing", + "-Wno-deprecated-declarations", "-I", runtimeSrc, + ]; + const quickjs = join(vendorRoot, "quickjs-ng"); + const zlib = join(vendorRoot, "zlib"); + const mbedtls = join(vendorRoot, "mbedtls"); + const QJS_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"]; + const LRE_SOURCES = ["libregexp.c", "libunicode.c"]; + const ZLIB_SOURCES = [ + "adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", + "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c", + ]; -async function compile(source, output, flags) { - await mkdir(dirname(output), { recursive: true }); - await run(compiler, [...sourcePathFlags, ...flags, "-c", source, "-o", output]); -} + async function sha256(path) { + return createHash("sha256").update(await readFile(path)).digest("hex"); + } -async function parallel(items, task) { - const width = Math.max(1, Math.min(8, availableParallelism())); - for (let i = 0; i < items.length; i += width) { - await Promise.all(items.slice(i, i + width).map(task)); + async function compile(source, output, flags) { + await mkdir(dirname(output), { recursive: true }); + await run(compiler, [...sourcePathFlags, ...flags, "-c", source, "-o", output]); } -} -async function archive(id, sources, sourceRoot, flags) { - const root = join(outputRoot, "vendor", id); - const objectRoot = join(root, "objects"); - await parallel(sources, async (source) => { - await compile(join(sourceRoot, source), join(objectRoot, source.replace(/\.c$/, ".o")), flags); - }); - const output = join(root, `libscriptc-${id}.a`); - const objects = sources.map((source) => join(objectRoot, source.replace(/\.c$/, ".o"))); - await createDeterministicArchive(archiver, output, objects); - await rm(objectRoot, { recursive: true, force: true }); - return { - id, - path: output.slice(packageRoot.length + 1), - sha256: await sha256(output), - size: (await stat(output)).size, - }; -} + async function parallel(items, task) { + const width = Math.max(1, Math.min(8, availableParallelism())); + for (let i = 0; i < items.length; i += width) { + await Promise.all(items.slice(i, i + width).map(task)); + } + } + + async function archive(id, sources, sourceRoot, flags) { + const root = join(stagedOutputRoot, "vendor", id); + const objectRoot = join(root, "objects"); + await parallel(sources, async (source) => { + await compile(join(sourceRoot, source), join(objectRoot, source.replace(/\.c$/, ".o")), flags); + }); + const output = join(root, `libscriptc-${id}.a`); + const objects = sources.map((source) => join(objectRoot, source.replace(/\.c$/, ".o"))); + await createDeterministicArchive(archiver, output, objects); + await rm(objectRoot, { recursive: true, force: true }); + return { + id, + path: artifactPath(output), + sha256: await sha256(output), + size: (await stat(output)).size, + }; + } -const privateRoot = `${outputRoot}.tmp-${process.pid}`; -await rm(privateRoot, { recursive: true, force: true }); -await rm(outputRoot, { recursive: true, force: true }); -await mkdir(privateRoot, { recursive: true }); -// Build under the final path spelling so object metadata remains stable. The -// compiler prefix map above also keeps __FILE__ and any debug paths independent -// of the producer's checkout location. -await rename(privateRoot, outputRoot); + await rm(buildRoot, { recursive: true, force: true }); + await mkdir(stagedOutputRoot, { recursive: true }); -try { - const flavors = {}; - for (const [flavor, flavorSpec] of Object.entries(RUNTIME_PACK_MATRIX.flavors)) { - const units = []; - for (const unit of RUNTIME_PACK_MATRIX.runtime_units) { - const variants = []; - for (const variant of unit.variants) { - const variantName = variant.id === "default" ? "default" : variant.id; - const output = join(outputRoot, flavor, "runtime", variantName, unit.source.replace(/\.c$/, ".o")); - const includeFlags = [ - ...(unit.source === "scr_regex.c" || variant.defines.includes("SCR_DYNAMIC") - ? ["-I", quickjs] - : []), - ...(unit.source === "scr_tls.c" ? ["-I", join(mbedtls, "include")] : []), - ...(unit.source === "scr_zlib.c" || unit.source === "scr_fetch.c" - ? ["-I", zlib] - : []), - ]; - await compile(join(runtimeSrc, unit.source), output, [ - ...commonFlags, flavorSpec.optimization, - ...variant.defines.map((define) => `-D${define}`), - ...includeFlags, - ]); - variants.push({ - id: variant.id, - when: variant.when, - defines: variant.defines, - path: output.slice(packageRoot.length + 1), - sha256: await sha256(output), - size: (await stat(output)).size, - }); + try { + const flavors = {}; + for (const [flavor, flavorSpec] of Object.entries(RUNTIME_PACK_MATRIX.flavors)) { + const units = []; + for (const unit of RUNTIME_PACK_MATRIX.runtime_units) { + const variants = []; + for (const variant of unit.variants) { + const variantName = variant.id === "default" ? "default" : variant.id; + const output = join( + stagedOutputRoot, + flavor, + "runtime", + variantName, + unit.source.replace(/\.c$/, ".o"), + ); + const includeFlags = [ + ...(unit.source === "scr_regex.c" || variant.defines.includes("SCR_DYNAMIC") + ? ["-I", quickjs] + : []), + ...(unit.source === "scr_tls.c" ? ["-I", join(mbedtls, "include")] : []), + ...(unit.source === "scr_zlib.c" || unit.source === "scr_fetch.c" + ? ["-I", zlib] + : []), + ]; + await compile(join(runtimeSrc, unit.source), output, [ + ...commonFlags, flavorSpec.optimization, + ...variant.defines.map((define) => `-D${define}`), + ...includeFlags, + ]); + variants.push({ + id: variant.id, + when: variant.when, + defines: variant.defines, + path: artifactPath(output), + sha256: await sha256(output), + size: (await stat(output)).size, + }); + } + units.push({ source: unit.source, predicate: unit.predicate, variants }); } - units.push({ source: unit.source, predicate: unit.predicate, variants }); + flavors[flavor] = { optimization: flavorSpec.optimization, runtime_units: units }; } - flavors[flavor] = { optimization: flavorSpec.optimization, runtime_units: units }; - } - const mbedtlsSources = (await readdir(join(mbedtls, "library"))) - .filter((name) => !name.startsWith(".") && name.endsWith(".c")) - .sort(); - const archives = [ - await archive("quickjs", QJS_SOURCES, quickjs, [ - "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=gnu11", - "-fvisibility=hidden", "-funsigned-char", "-DQUICKJS_NG_BUILD", - "-D_GNU_SOURCE", "-DNDEBUG", "-Os", "-I", quickjs, - ]), - await archive("libregexp", LRE_SOURCES, quickjs, [ - "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", "-I", quickjs, - ]), - await archive("zlib", ZLIB_SOURCES, zlib, [ - "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", "-I", zlib, - ]), - await archive("mbedtls", mbedtlsSources, join(mbedtls, "library"), [ - "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", - "-I", join(mbedtls, "include"), "-I", join(mbedtls, "library"), - ]), - ]; - const archiveSpecs = new Map(RUNTIME_PACK_MATRIX.archives.map((entry) => [entry.id, entry])); - const licensed = [ - [join(runtimeRoot, "LICENSE"), "artifacts/licenses/scriptc-runtime.txt", "Apache-2.0"], - [join(quickjs, "LICENSE"), "artifacts/licenses/quickjs-ng.txt", "MIT"], - [join(vendorRoot, "ryu", "LICENSE-Boost"), "artifacts/licenses/ryu.txt", "BSL-1.0"], - [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], - [join(mbedtls, "LICENSE"), "artifacts/licenses/mbedtls.txt", "Apache-2.0"], - ]; - await Promise.all(licensed.map(async ([source, destination]) => { - const output = join(packageRoot, destination); - await mkdir(dirname(output), { recursive: true }); - await copyFile(source, output); - })); - const manifest = { - schema: "scriptc.runtime-pack.v1", - format: 1, - package: packageManifest.name, - version: packageManifest.version, - target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 1, marker: "scr_runtime_abi_v1" }, - compiler: { - command: compiler, - identity: compilerVersion, - target: RUNTIME_PACK_MATRIX.target.llvm_triple, - }, - macros: { - executable: ["SCR_DYNAMIC", "SCR_TEXT_DECODER_LEGACY"], - excluded: ["SCR_LIB", "SCR_THREAD_INSTANCES", "SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], - sanitizer: "external-toolchain-required", - }, - flavors, - archives: archives.map((entry) => ({ ...entry, predicate: archiveSpecs.get(entry.id).predicate })), - system_libraries: RUNTIME_PACK_MATRIX.system_libraries, - licenses: licensed.map(([, path, license]) => ({ path, license })), - }; - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - process.stdout.write( - `built ${packageManifest.name}@${packageManifest.version}: ` + - `${Object.keys(flavors).length} flavors, ${archives.length} vendor archives\n`, - ); -} catch (error) { - await rm(outputRoot, { recursive: true, force: true }); - await rm(manifestPath, { force: true }); - throw error; + const mbedtlsSources = (await readdir(join(mbedtls, "library"))) + .filter((name) => !name.startsWith(".") && name.endsWith(".c")) + .sort(); + const archives = [ + await archive("quickjs", QJS_SOURCES, quickjs, [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=gnu11", + "-fvisibility=hidden", "-funsigned-char", "-DQUICKJS_NG_BUILD", + "-D_GNU_SOURCE", "-DNDEBUG", "-Os", "-I", quickjs, + ]), + await archive("libregexp", LRE_SOURCES, quickjs, [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", "-I", quickjs, + ]), + await archive("zlib", ZLIB_SOURCES, zlib, [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", "-I", zlib, + ]), + await archive("mbedtls", mbedtlsSources, join(mbedtls, "library"), [ + "-target", RUNTIME_PACK_MATRIX.target.llvm_triple, "-std=c11", "-Os", + "-I", join(mbedtls, "include"), "-I", join(mbedtls, "library"), + ]), + ]; + const archiveSpecs = new Map(RUNTIME_PACK_MATRIX.archives.map((entry) => [entry.id, entry])); + const licensed = [ + [join(runtimeRoot, "LICENSE"), "artifacts/licenses/scriptc-runtime.txt", "Apache-2.0"], + [join(quickjs, "LICENSE"), "artifacts/licenses/quickjs-ng.txt", "MIT"], + [join(vendorRoot, "ryu", "LICENSE-Boost"), "artifacts/licenses/ryu.txt", "BSL-1.0"], + [join(zlib, "LICENSE"), "artifacts/licenses/zlib.txt", "Zlib"], + [join(mbedtls, "LICENSE"), "artifacts/licenses/mbedtls.txt", "Apache-2.0"], + ]; + await Promise.all(licensed.map(async ([source, destination]) => { + const output = join(buildRoot, destination); + await mkdir(dirname(output), { recursive: true }); + await copyFile(source, output); + })); + const manifest = { + schema: "scriptc.runtime-pack.v1", + format: 1, + package: packageManifest.name, + version: packageManifest.version, + target: RUNTIME_PACK_MATRIX.target, + runtime_abi: { version: 1, marker: "scr_runtime_abi_v1" }, + compiler: { + command: compiler, + identity: compilerVersion, + target: RUNTIME_PACK_MATRIX.target.llvm_triple, + }, + macros: { + executable: ["SCR_DYNAMIC", "SCR_TEXT_DECODER_LEGACY"], + excluded: ["SCR_LIB", "SCR_THREAD_INSTANCES", "SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], + sanitizer: "external-toolchain-required", + }, + flavors, + archives: archives.map((entry) => ({ ...entry, predicate: archiveSpecs.get(entry.id).predicate })), + system_libraries: RUNTIME_PACK_MATRIX.system_libraries, + licenses: licensed.map(([, path, license]) => ({ path, license })), + }; + await writeFile(stagedManifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + const backupSuffix = `${process.pid}-${Math.random().toString(36).slice(2)}`; + await installRuntimePack({ + outputRoot, + manifestPath, + stagedOutputRoot, + stagedManifestPath, + backupRoot: join(packageRoot, `.runtime-pack-artifacts-backup-${backupSuffix}`), + backupManifestPath: join(packageRoot, `.runtime-pack-manifest-backup-${backupSuffix}`), + }); + process.stdout.write( + `built ${packageManifest.name}@${packageManifest.version}: ` + + `${Object.keys(flavors).length} flavors, ${archives.length} vendor archives\n`, + ); + } finally { + await rm(buildRoot, { recursive: true, force: true }); + } } + +await withBuildLock(join(packageRoot, ".runtime-pack-build.lock"), build); diff --git a/packages/runtime-darwin-arm64/test/archive.test.ts b/packages/runtime-darwin-arm64/test/archive.test.ts index 7e5e4feac..07cf73453 100644 --- a/packages/runtime-darwin-arm64/test/archive.test.ts +++ b/packages/runtime-darwin-arm64/test/archive.test.ts @@ -1,8 +1,9 @@ -import { chmod, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, rm, stat, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; import { createDeterministicArchive } from "../scripts/archive.mjs"; +import { installRuntimePack, withBuildLock } from "../scripts/build-state.mjs"; const dirs: string[] = []; @@ -27,6 +28,62 @@ afterEach(async () => { }); describe.runIf(process.platform === "darwin")("runtime-pack archives", () => { + test("serializes concurrent pack builders", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-build-lock-")); + dirs.push(dir); + const lock = join(dir, "build.lock"); + const events: string[] = []; + let releaseFirst!: () => void; + const firstMayFinish = new Promise((resolve) => { releaseFirst = resolve; }); + let firstStarted!: () => void; + const firstDidStart = new Promise((resolve) => { firstStarted = resolve; }); + const first = withBuildLock(lock, async () => { + events.push("first:start"); + firstStarted(); + await firstMayFinish; + events.push("first:end"); + }, { retryMilliseconds: 5 }); + await firstDidStart; + const second = withBuildLock(lock, async () => { + events.push("second:start"); + events.push("second:end"); + }, { retryMilliseconds: 5 }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(events).toEqual(["first:start"]); + releaseFirst(); + await Promise.all([first, second]); + expect(events).toEqual(["first:start", "first:end", "second:start", "second:end"]); + }); + + test("installs a staged pack over the previous complete pair", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-build-install-")); + dirs.push(dir); + const outputRoot = join(dir, "artifacts"); + const manifestPath = join(dir, "runtime-pack.json"); + const stagedOutputRoot = join(dir, "stage", "artifacts"); + const stagedManifestPath = join(dir, "stage", "runtime-pack.json"); + await Promise.all([mkdir(outputRoot), mkdir(stagedOutputRoot, { recursive: true })]); + await Promise.all([ + writeFile(join(outputRoot, "runtime.o"), "old object"), + writeFile(manifestPath, "old manifest"), + writeFile(join(stagedOutputRoot, "runtime.o"), "new object"), + writeFile(stagedManifestPath, "new manifest"), + ]); + + await installRuntimePack({ + outputRoot, + manifestPath, + stagedOutputRoot, + stagedManifestPath, + backupRoot: join(dir, "artifacts.backup"), + backupManifestPath: join(dir, "manifest.backup"), + }); + + expect(await readFile(join(outputRoot, "runtime.o"), "utf8")).toBe("new object"); + expect(await readFile(manifestPath, "utf8")).toBe("new manifest"); + expect(await stat(join(dir, "artifacts.backup")).then(() => true, () => false)).toBe(false); + }); + test("normalize timestamps, ownership, and modes", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-archive-")); dirs.push(dir); From 35ab325ede6e7413a44b65e65d37529b64be943c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 20:50:14 -0500 Subject: [PATCH 07/11] fix(runtime): close executable cache link race --- .../compiler/src/backend/runtime-pack.test.ts | 53 ++++++++++++++++++- packages/compiler/src/backend/runtime-pack.ts | 32 +++++++---- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index d9e03dd31..4557ddf41 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -1,13 +1,15 @@ import { createHash } from "node:crypto"; -import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { describe, expect, test } from "vitest"; import { compilerReleaseVersion } from "../library/sidecar.js"; import type { NativeLinkFeatures } from "./native-link-info.js"; import { + createRuntimeLinkPlan, effectiveRuntimeFeatures, evaluateRuntimePredicate, + linkRuntimePackExecutable, loadRuntimePack, parseRuntimePackManifest, type RuntimePackManifest, @@ -168,4 +170,53 @@ describe("runtime pack manifests", () => { })).rejects.toThrow("hash mismatch"); expect(await readFile(packagePath, "utf8")).toContain("runtime-darwin-arm64"); }); + + test.runIf(process.platform === "darwin")( + "does not publish an executable cache proof when a dependency changes during linking", + async () => { + const { root, packagePath } = await fixture(); + const programObject = join(root, "program.o"); + const dependency = join(root, "link-dependency.a"); + const output = join(root, "program"); + const linker = join(root, "linker.mjs"); + await Promise.all([ + writeFile(programObject, "program object"), + writeFile(dependency, "before link"), + writeFile(linker, [ + "#!/usr/bin/env node", + 'import { writeFileSync } from "node:fs";', + `const dependency = ${JSON.stringify(dependency)};`, + 'if (process.argv[2]?.startsWith("-print-file-name=")) {', + ' process.stdout.write(`${dependency}\\n`);', + " process.exit(0);", + "}", + 'const outputIndex = process.argv.indexOf("-o");', + 'if (outputIndex < 0) process.exit(2);', + 'writeFileSync(dependency, "changed during link");', + 'writeFileSync(process.argv[outputIndex + 1], "linked executable");', + "", + ].join("\n")), + ]); + await chmod(linker, 0o755); + const plan = await createRuntimeLinkPlan({ + target: MACOS_ARM64_TARGET, + programObject, + outPath: output, + features: BASE, + ffi: null, + optimization: "release", + programObjectDependencies: [dependency], + resolver: () => packagePath, + }); + let published = false; + + await linkRuntimePackExecutable(plan, { + linker, + onArtifactReady: async () => { published = true; }, + }); + + expect(await readFile(output, "utf8")).toBe("linked executable"); + expect(published).toBe(false); + }, + ); }); diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts index 7891b42c3..7162d8c2a 100644 --- a/packages/compiler/src/backend/runtime-pack.ts +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -8,7 +8,12 @@ import type { FfiProfile } from "../ffi/ffi-manifest.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; import { privateSiblingPath } from "./build-cache.js"; import type { NativeLinkFeatures } from "./native-link-info.js"; -import { CcCompileError, subprocessFailureDetail, type NativeArtifactDependency } from "./native-toolchain.js"; +import { + CcCompileError, + nativeArtifactDependenciesStillMatch, + subprocessFailureDetail, + type NativeArtifactDependency, +} from "./native-toolchain.js"; import { RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "./runtime-abi.js"; import type { NativeTargetSpec } from "./targets.js"; @@ -454,6 +459,14 @@ export async function linkRuntimePackExecutable( ...plan.systemLibraries.map((name) => `-l${name}`), "-o", privateOut, ]; + // Snapshot every cache-bearing input before the linker can consume it. A + // package update or SDK/toolchain replacement during the subprocess must + // not let output built from one state ride a proof captured from the next. + const preLinkDependencies = options.onArtifactReady === undefined + ? null + : await linkToolchainDependencies(linkerPath) + .then((toolchain) => snapshotDependencies([...toolchain, ...plan.dependencyPaths])) + .catch(() => null); try { await execFileAsync(linker, args); const output = await stat(privateOut); @@ -462,17 +475,16 @@ export async function linkRuntimePackExecutable( await rm(plan.outputPath, { force: true }); await rename(privateOut, plan.outputPath); }); - if (options.onArtifactReady !== undefined) { + if ( + options.onArtifactReady !== undefined && preLinkDependencies !== null && + await nativeArtifactDependenciesStillMatch(preLinkDependencies).catch(() => false) + ) { // A complete executable cache entry is published only when the driver, // platform linker, compiler runtime, selected SDK stubs/settings, pack, - // and FFI inputs all have a replayable metadata proof. Failure to prove - // any ambient input keeps a correct executable but no complete cache. - const dependencies = await linkToolchainDependencies(linkerPath) - .then((toolchain) => snapshotDependencies([...toolchain, ...plan.dependencyPaths])) - .catch(() => null); - if (dependencies !== null) { - await options.onArtifactReady({ dependencies }).catch(() => undefined); - } + // and FFI inputs all remained unchanged across the link. Failure to + // prove any ambient input keeps a correct executable but no complete + // cache. + await options.onArtifactReady({ dependencies: preLinkDependencies }).catch(() => undefined); } } catch (error) { if (error instanceof CcCompileError) throw error; From b61f85db47e15417d56a31a89de638b03168ca92 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 21:49:10 -0500 Subject: [PATCH 08/11] fix(runtime): stage verified pack artifacts --- packages/cli/test/runtime-pack.test.ts | 5 +- .../compiler/src/backend/runtime-pack.test.ts | 72 ++++++++++- packages/compiler/src/backend/runtime-pack.ts | 122 +++++++++++++++--- 3 files changed, 179 insertions(+), 20 deletions(-) diff --git a/packages/cli/test/runtime-pack.test.ts b/packages/cli/test/runtime-pack.test.ts index 5f9f5f242..1db698c85 100644 --- a/packages/cli/test/runtime-pack.test.ts +++ b/packages/cli/test/runtime-pack.test.ts @@ -47,7 +47,10 @@ describe.runIf(supported)("precompiled runtime executable builds", () => { const args = JSON.parse(await readFile(log, "utf8")) as string[]; expect(args).not.toContain("-c"); expect(args.some((arg) => arg.endsWith(".c") || arg.endsWith(".ll"))).toBe(false); - expect(args.some((arg) => arg.includes("runtime-darwin-arm64/artifacts"))).toBe(true); + expect(args.some((arg) => + arg.includes("scriptc-runtime-pack-link-") && arg.includes("/artifacts/") + )).toBe(true); + expect(args.some((arg) => arg.includes("runtime-darwin-arm64/artifacts"))).toBe(false); await expect(execFileAsync(output, [], { encoding: "utf8" })) .resolves.toMatchObject({ stdout: "precompiled runtime\n" }); }); diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 4557ddf41..316c7b281 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, mkdir, readFile, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { describe, expect, test } from "vitest"; @@ -171,6 +171,76 @@ describe("runtime pack manifests", () => { expect(await readFile(packagePath, "utf8")).toContain("runtime-darwin-arm64"); }); + test("rejects a selected artifact replaced before private link staging", async () => { + const { root, packagePath } = await fixture(); + const programObject = join(root, "program.o"); + const output = join(root, "program"); + const linker = join(root, "linker.mjs"); + await Promise.all([ + writeFile(programObject, "program object"), + writeFile(linker, [ + "#!/usr/bin/env node", + 'import { writeFileSync } from "node:fs";', + 'const outputIndex = process.argv.indexOf("-o");', + 'writeFileSync(process.argv[outputIndex + 1], "linked executable");', + "", + ].join("\n")), + ]); + await chmod(linker, 0o755); + const plan = await createRuntimeLinkPlan({ + target: MACOS_ARM64_TARGET, + programObject, + outPath: output, + features: BASE, + ffi: null, + optimization: "release", + resolver: () => packagePath, + }); + await writeFile(join(root, "artifacts/base.o"), "tampered"); + + await expect(linkRuntimePackExecutable(plan, { linker })).rejects.toThrow( + "runtime pack changed after artifact selection", + ); + expect(await stat(output).then(() => true, () => false)).toBe(false); + }); + + test("links a private verified copy when the installed artifact changes during linking", async () => { + const { root, packagePath } = await fixture(); + const programObject = join(root, "program.o"); + const runtimeObject = join(root, "artifacts/base.o"); + const output = join(root, "program"); + const linker = join(root, "linker.mjs"); + await Promise.all([ + writeFile(programObject, "program object"), + writeFile(linker, [ + "#!/usr/bin/env node", + 'import { readFileSync, writeFileSync } from "node:fs";', + `const installed = ${JSON.stringify(runtimeObject)};`, + 'const outputIndex = process.argv.indexOf("-o");', + 'const staged = process.argv.find((arg) => arg.endsWith("/artifacts/base.o"));', + 'if (staged === undefined || staged === installed) process.exit(2);', + 'writeFileSync(installed, "tampered");', + 'writeFileSync(process.argv[outputIndex + 1], readFileSync(staged));', + "", + ].join("\n")), + ]); + await chmod(linker, 0o755); + const plan = await createRuntimeLinkPlan({ + target: MACOS_ARM64_TARGET, + programObject, + outPath: output, + features: BASE, + ffi: null, + optimization: "release", + resolver: () => packagePath, + }); + + await linkRuntimePackExecutable(plan, { linker }); + + expect(await readFile(output, "utf8")).toBe("base"); + expect(await readFile(runtimeObject, "utf8")).toBe("tampered"); + }); + test.runIf(process.platform === "darwin")( "does not publish an executable cache proof when a dependency changes during linking", async () => { diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts index 7162d8c2a..f3638ac19 100644 --- a/packages/compiler/src/backend/runtime-pack.ts +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -1,7 +1,8 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { createRequire } from "node:module"; -import { readFile, realpath, rename, rm, stat } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; import type { FfiProfile } from "../ffi/ffi-manifest.js"; @@ -97,6 +98,10 @@ export interface RuntimePackSelection { archives: string[]; systemLibraries: string[]; dependencyPaths: string[]; + /** Exact installed inputs observed while the selected artifacts were verified. */ + sourceDependencies: NativeArtifactDependency[]; + selectedRuntimeArtifacts: RuntimePackArtifact[]; + selectedArchiveArtifacts: RuntimePackArtifact[]; } export interface RuntimeLinkPlan { @@ -261,6 +266,45 @@ async function verifyArtifact(root: string, artifact: RuntimePackArtifact): Prom return path; } +async function stageRuntimePackArtifacts(selection: RuntimePackSelection): Promise<{ + root: string; + replacements: Map; +}> { + if (!(await nativeArtifactDependenciesStillMatch(selection.sourceDependencies).catch(() => false))) { + throw new RuntimePackError("runtime pack changed after artifact selection", "invalid"); + } + const stageRoot = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-link-")); + try { + const replacements = new Map(); + await Promise.all([ + ...selection.selectedRuntimeArtifacts, + ...selection.selectedArchiveArtifacts, + ].map(async (artifact) => { + const source = join(selection.root, artifact.path); + const destination = join(stageRoot, artifact.path); + const bytes = await readFile(source).catch(() => { + throw new RuntimePackError(`runtime pack artifact is missing: ${artifact.path}`, "invalid"); + }); + if ( + bytes.length !== artifact.size || + createHash("sha256").update(bytes).digest("hex") !== artifact.sha256 + ) { + throw new RuntimePackError(`runtime pack artifact hash mismatch: ${artifact.path}`, "invalid"); + } + await mkdir(dirname(destination), { recursive: true }); + await writeFile(destination, bytes, { flag: "wx", mode: 0o400 }); + replacements.set(source, destination); + })); + if (!(await nativeArtifactDependenciesStillMatch(selection.sourceDependencies).catch(() => false))) { + throw new RuntimePackError("runtime pack changed while staging verified artifacts", "invalid"); + } + return { root: stageRoot, replacements }; + } catch (error) { + await rm(stageRoot, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + export async function loadRuntimePack(options: { target: NativeTargetSpec; features: NativeLinkFeatures; @@ -283,6 +327,12 @@ export async function loadRuntimePack(options: { } const root = dirname(packagePath); const manifestPath = join(root, "runtime-pack.json"); + let identityDependencies: NativeArtifactDependency[]; + try { + identityDependencies = await snapshotDependencies([packagePath, manifestPath]); + } catch { + throw new RuntimePackError(`could not read ${packageName}/runtime-pack.json`, "invalid"); + } let packageManifest: { name?: string; version?: string }; let manifest: RuntimePackManifest; try { @@ -316,6 +366,16 @@ export async function loadRuntimePack(options: { const selectedVariants = selectedUnits.map((unit) => selectVariant(unit, features)); const selectedArchives = manifest.archives .filter((archive) => evaluateRuntimePredicate(archive.predicate, features)); + const selectedArtifactPaths = [ + ...selectedVariants, + ...selectedArchives, + ].map((artifact) => join(root, artifact.path)); + let artifactDependencies: NativeArtifactDependency[]; + try { + artifactDependencies = await snapshotDependencies(selectedArtifactPaths); + } catch { + throw new RuntimePackError("runtime pack artifact set changed during selection", "invalid"); + } const [runtimeObjects, archives] = await Promise.all([ Promise.all(selectedVariants.map((artifact) => verifyArtifact(root, artifact))), Promise.all(selectedArchives.map((artifact) => verifyArtifact(root, artifact))), @@ -323,6 +383,10 @@ export async function loadRuntimePack(options: { await Promise.all(manifest.licenses.map((license) => readFile(join(root, license.path)))).catch(() => { throw new RuntimePackError("runtime pack license payload is incomplete", "invalid"); }); + const sourceDependencies = [...identityDependencies, ...artifactDependencies]; + if (!(await nativeArtifactDependenciesStillMatch(sourceDependencies).catch(() => false))) { + throw new RuntimePackError("runtime pack changed while verifying selected artifacts", "invalid"); + } return { root, manifestPath, @@ -334,7 +398,10 @@ export async function loadRuntimePack(options: { systemLibraries: manifest.system_libraries .filter((entry) => evaluateRuntimePredicate(entry.predicate, features)) .map((entry) => entry.name), - dependencyPaths: [manifestPath, ...runtimeObjects, ...archives], + dependencyPaths: [packagePath, manifestPath, ...runtimeObjects, ...archives], + sourceDependencies, + selectedRuntimeArtifacts: selectedVariants, + selectedArchiveArtifacts: selectedArchives, }; } @@ -453,21 +520,35 @@ export async function linkRuntimePackExecutable( const linker = options.linker ?? process.env["SCRIPTC_LINKER"] ?? "clang"; const linkerPath = await resolveExecutable(linker); const privateOut = privateSiblingPath(plan.outputPath, "runtime-pack-link"); - const args = [ - ...plan.driverFlags, - ...plan.inputs, - ...plan.systemLibraries.map((name) => `-l${name}`), - "-o", privateOut, - ]; - // Snapshot every cache-bearing input before the linker can consume it. A - // package update or SDK/toolchain replacement during the subprocess must - // not let output built from one state ride a proof captured from the next. - const preLinkDependencies = options.onArtifactReady === undefined - ? null - : await linkToolchainDependencies(linkerPath) - .then((toolchain) => snapshotDependencies([...toolchain, ...plan.dependencyPaths])) - .catch(() => null); + let stagedRoot: string | null = null; try { + const staged = await stageRuntimePackArtifacts(plan.runtimePack); + stagedRoot = staged.root; + const args = [ + ...plan.driverFlags, + ...plan.inputs.map((input) => staged.replacements.get(input) ?? input), + ...plan.systemLibraries.map((name) => `-l${name}`), + "-o", privateOut, + ]; + // The pack dependency snapshots bracket both verification passes and the + // private staging copy. Snapshot the remaining cache-bearing inputs before + // the linker consumes them, then require the complete set to remain stable + // through publication. + const packDependencyPaths = new Set( + plan.runtimePack.sourceDependencies.map((dependency) => resolve(dependency.path)), + ); + const additionalDependencyPaths = plan.dependencyPaths.filter( + (path) => !packDependencyPaths.has(resolve(path)), + ); + const preLinkDependencies = options.onArtifactReady === undefined || + !(await nativeArtifactDependenciesStillMatch(plan.runtimePack.sourceDependencies).catch(() => false)) + ? null + : await linkToolchainDependencies(linkerPath) + .then(async (toolchain) => [ + ...plan.runtimePack.sourceDependencies, + ...await snapshotDependencies([...toolchain, ...additionalDependencyPaths]), + ]) + .catch(() => null); await execFileAsync(linker, args); const output = await stat(privateOut); if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable"); @@ -487,7 +568,7 @@ export async function linkRuntimePackExecutable( await options.onArtifactReady({ dependencies: preLinkDependencies }).catch(() => undefined); } } catch (error) { - if (error instanceof CcCompileError) throw error; + if (error instanceof CcCompileError || error instanceof RuntimePackError) throw error; const detail = subprocessFailureDetail(error); throw new CcCompileError( linker, @@ -495,6 +576,11 @@ export async function linkRuntimePackExecutable( `${linker} failed linking ${basename(plan.outputPath)} from the precompiled runtime pack.\n${detail}`, ); } finally { - await rm(privateOut, { force: true }).catch(() => undefined); + await Promise.all([ + rm(privateOut, { force: true }).catch(() => undefined), + stagedRoot === null + ? Promise.resolve() + : rm(stagedRoot, { recursive: true, force: true }).catch(() => undefined), + ]); } } From a3a05876512c18ff7124b9af535290bfce0d962b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 27 Aug 2026 22:44:25 -0500 Subject: [PATCH 09/11] fix(runtime): bracket helper cache inputs --- README.md | 9 ++-- .../src/backend/native-codegen.test.ts | 23 +++++++++- .../compiler/src/backend/native-codegen.ts | 41 ++++++++++++++--- .../compiler/src/backend/native-toolchain.ts | 9 ++++ .../compiler/src/backend/runtime-pack.test.ts | 44 ++++++++++++++++++- packages/compiler/src/backend/runtime-pack.ts | 28 +++++++----- packages/compiler/src/index.ts | 16 +++---- 7 files changed, 138 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5914125a2..d96b14c58 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,11 @@ $ pnpm test:sandbox ``` The normal workspace build needs no local LLVM installation. To rebuild the -optional macOS arm64 assembly/object helper, install CMake, Ninja, and -Homebrew `llvm@22`, then run -`pnpm --filter @scriptc/llvm-darwin-arm64 build:native`. The macOS full test -suite also uses that generated helper. +optional macOS arm64 native artifacts, install CMake, Ninja, and Homebrew +`llvm@22`, then run +`pnpm --filter @scriptc/llvm-darwin-arm64 build:native` and +`pnpm --filter @scriptc/runtime-darwin-arm64 build:native`. The macOS full test +suite also uses those generated artifacts. `pnpm test:sandbox` loads `.env.local`, preflights Vercel authentication and project access, and uses the managed `vercel/sandbox/universal` image by diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index c4a7c7d45..64186cbfa 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; import { emitNativeArtifact, NativeCodegenError } from "./native-codegen.js"; +import { nativeArtifactDependenciesStillMatch } from "./native-toolchain.js"; import { MACOS_ARM64_TARGET } from "./targets.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; @@ -17,6 +18,7 @@ async function fakePackage(options: { emitFailure?: boolean; emptyOutput?: boolean; missingOutput?: boolean; + changePackageDuringEmit?: boolean; } = {}) { const root = await mkdtemp(join(tmpdir(), "scriptc-native-helper-test-")); dirs.push(root); @@ -48,6 +50,7 @@ while [ "$#" -gt 0 ]; do if [ "$1" = --input ]; then input="$2"; shift 2; continue; fi shift done +${options.changePackageDuringEmit === true ? `printf '\\n' >> '${packageJson}'` : ""} ${options.emitFailure === true ? "printf '%s\\n' '{\"ok\":false,\"code\":\"verification_failed\",\"message\":\"bad module\"}' >&2; exit 1" : options.emptyOutput === true @@ -84,8 +87,24 @@ test("resolves a package helper, emits atomically, and caches by all native inpu expect((await stat(first)).mode & 0o777).toBe(expectedMode); expect((await stat(second)).mode & 0o777).toBe(expectedMode); expect((await readFile(pkg.log, "utf8")).trim().split("\n")).toHaveLength(1); - expect(firstArtifact.dependencyPaths).toEqual([pkg.packageJson, pkg.bin]); - expect(secondArtifact.dependencyPaths).toEqual(firstArtifact.dependencyPaths); + expect(firstArtifact.dependencies.map((dependency) => dependency.path)).toEqual([ + pkg.bin, + pkg.packageJson, + ].sort()); + expect(secondArtifact.dependencies).toEqual(firstArtifact.dependencies); +}); + +test("returns the pre-emission helper snapshot when its package changes during emission", async () => { + const pkg = await fakePackage({ changePackageDuringEmit: true }); + + const artifact = await emitNativeArtifact(request(pkg.root, pkg.packageJson)); + + expect(await readFile(join(pkg.root, "program.o"), "utf8")).toContain("define i32 @answer"); + expect(await nativeArtifactDependenciesStillMatch(artifact.dependencies)).toBe(false); + expect(await stat(join(pkg.root, "cache", "native-codegen-v1")).then( + () => true, + () => false, + )).toBe(false); }); test("cache publication failures do not discard a valid requested artifact", async () => { diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index d998a8877..cadab0e14 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -5,7 +5,14 @@ import { createRequire } from "node:module"; import { access, chmod, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; -import { buildCacheRoot, prepareBuildCacheRoot, pruneBuildCache } from "./native-toolchain.js"; +import { + buildCacheRoot, + nativeArtifactDependenciesStillMatch, + prepareBuildCacheRoot, + pruneBuildCache, + snapshotNativeArtifactDependencies, + type NativeArtifactDependency, +} from "./native-toolchain.js"; import { copyValidCachedFile, privateSiblingPath, @@ -53,11 +60,12 @@ interface ResolvedHelper { packageJsonPath: string; binaryPath: string; identity: HelperIdentity; + dependencies: NativeArtifactDependency[]; } export interface NativeCodegenArtifact { - /** Installed package inputs whose identity selected and produced the artifact. */ - dependencyPaths: string[]; + /** Exact installed inputs observed before the helper identity and emission. */ + dependencies: NativeArtifactDependency[]; } const resolvedHelperCache = new Map>(); @@ -187,7 +195,17 @@ async function resolveHelper( "unusable_binary", ); } - const cacheKey = `${binaryPath}\0${binaryStat.size}\0${binaryStat.mtimeMs}`; + let dependencies: NativeArtifactDependency[]; + try { + dependencies = await snapshotNativeArtifactDependencies([packageJsonPath, binaryPath]); + } catch { + throw new NativeCodegenError( + "SC3003", + `LLVM native helper package ${target.helperPackage} changed while its inputs were being inspected; retry the build or reinstall scriptc`, + "helper_changed", + ); + } + const cacheKey = JSON.stringify(dependencies); const load = async (): Promise => { let stdout: string; let binary: Buffer; @@ -211,9 +229,17 @@ async function resolveHelper( "invalid_version_response", ); } + if (!(await nativeArtifactDependenciesStillMatch(dependencies).catch(() => false))) { + throw new NativeCodegenError( + "SC3003", + `LLVM native helper package ${target.helperPackage} changed during its identity check; retry the build or reinstall scriptc`, + "helper_changed", + ); + } return { packageJsonPath, binaryPath, + dependencies, identity: { packageName: target.helperPackage, binaryDigest: createHash("sha256").update(binary).digest("hex"), @@ -295,7 +321,7 @@ export async function emitNativeArtifact(options: NativeCodegenOptions): Promise : join(root, "native-codegen-v1", key.slice(0, 2), `${key}.${options.outputKind === "obj" ? "o" : "s"}`); await mkdir(dirname(options.outputPath), { recursive: true }); const artifact = { - dependencyPaths: [helper.packageJsonPath, helper.binaryPath], + dependencies: helper.dependencies, } satisfies NativeCodegenArtifact; if (cached !== null && await validCachedFile(cached) && await installVerifiedCache(cached, options.outputPath)) return artifact; @@ -327,7 +353,10 @@ export async function emitNativeArtifact(options: NativeCodegenOptions): Promise // Cache publication is an optimization boundary. The helper has already // produced a valid caller artifact, so a read-only/full cache must not // discard it or turn an otherwise successful build into an exception. - if (cached !== null) await publishCachedFile(stage, cached).catch(() => undefined); + if ( + cached !== null && + await nativeArtifactDependenciesStillMatch(helper.dependencies).catch(() => false) + ) await publishCachedFile(stage, cached).catch(() => undefined); await rename(stage, options.outputPath); await pruneBuildCache(root); return artifact; diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index 489ba1946..ed1ead16a 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -3472,6 +3472,15 @@ async function snapshotLocalArtifactDependencies( ); } +/** Capture exact filesystem identities for inputs produced outside the C + * toolchain but consumed by its cache proofs. Callers carry this snapshot + * forward so later stages can prove the same inputs remained installed. */ +export async function snapshotNativeArtifactDependencies( + dependencyPaths: readonly string[], +): Promise { + return snapshotLocalArtifactDependencies(dependencyPaths); +} + export async function nativeArtifactDependenciesStillMatch( dependencies: readonly NativeArtifactDependency[], ): Promise { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 316c7b281..989e60f1d 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { describe, expect, test } from "vitest"; import { compilerReleaseVersion } from "../library/sidecar.js"; +import { snapshotNativeArtifactDependencies } from "./native-toolchain.js"; import type { NativeLinkFeatures } from "./native-link-info.js"; import { createRuntimeLinkPlan, @@ -241,6 +242,47 @@ describe("runtime pack manifests", () => { expect(await readFile(runtimeObject, "utf8")).toBe("tampered"); }); + test("does not publish a cache proof from a stale program-object dependency snapshot", async () => { + const { root, packagePath } = await fixture(); + const programObject = join(root, "program.o"); + const helper = join(root, "helper"); + const output = join(root, "program"); + const linker = join(root, "linker.mjs"); + await Promise.all([ + writeFile(programObject, "program object"), + writeFile(helper, "helper before emission"), + writeFile(linker, [ + "#!/usr/bin/env node", + 'import { writeFileSync } from "node:fs";', + 'const outputIndex = process.argv.indexOf("-o");', + 'writeFileSync(process.argv[outputIndex + 1], "linked executable");', + "", + ].join("\n")), + ]); + await chmod(linker, 0o755); + const helperDependencies = await snapshotNativeArtifactDependencies([helper]); + await writeFile(helper, "helper replaced during emission"); + const plan = await createRuntimeLinkPlan({ + target: MACOS_ARM64_TARGET, + programObject, + outPath: output, + features: BASE, + ffi: null, + optimization: "release", + programObjectDependencies: helperDependencies, + resolver: () => packagePath, + }); + let published = false; + + await linkRuntimePackExecutable(plan, { + linker, + onArtifactReady: async () => { published = true; }, + }); + + expect(await readFile(output, "utf8")).toBe("linked executable"); + expect(published).toBe(false); + }); + test.runIf(process.platform === "darwin")( "does not publish an executable cache proof when a dependency changes during linking", async () => { @@ -275,7 +317,7 @@ describe("runtime pack manifests", () => { features: BASE, ffi: null, optimization: "release", - programObjectDependencies: [dependency], + programObjectDependencies: await snapshotNativeArtifactDependencies([dependency]), resolver: () => packagePath, }); let published = false; diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts index f3638ac19..a6645a3fa 100644 --- a/packages/compiler/src/backend/runtime-pack.ts +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -111,6 +111,8 @@ export interface RuntimeLinkPlan { systemLibraries: string[]; driverFlags: string[]; dependencyPaths: string[]; + /** Inputs already snapshotted by the stage that produced the program object. */ + programObjectDependencies: NativeArtifactDependency[]; runtimePack: RuntimePackSelection; } @@ -412,7 +414,7 @@ export async function createRuntimeLinkPlan(options: { features: NativeLinkFeatures; ffi: FfiProfile | null; optimization: "release" | "dev"; - programObjectDependencies?: readonly string[]; + programObjectDependencies?: readonly NativeArtifactDependency[]; env?: NodeJS.ProcessEnv; resolver?: (specifier: string) => string; }): Promise { @@ -436,9 +438,9 @@ export async function createRuntimeLinkPlan(options: { ], dependencyPaths: [ ...runtimePack.dependencyPaths, - ...(options.programObjectDependencies ?? []), ...(options.ffi?.libraries ?? []), ], + programObjectDependencies: [...(options.programObjectDependencies ?? [])], runtimePack, }; } @@ -530,22 +532,26 @@ export async function linkRuntimePackExecutable( ...plan.systemLibraries.map((name) => `-l${name}`), "-o", privateOut, ]; - // The pack dependency snapshots bracket both verification passes and the - // private staging copy. Snapshot the remaining cache-bearing inputs before - // the linker consumes them, then require the complete set to remain stable - // through publication. - const packDependencyPaths = new Set( - plan.runtimePack.sourceDependencies.map((dependency) => resolve(dependency.path)), + // The pack snapshots bracket both verification passes and private staging; + // the program-object snapshot begins before helper emission. Snapshot the + // remaining cache-bearing inputs before the linker consumes them, then + // require the complete set to remain stable through publication. + const inheritedDependencies = [ + ...plan.runtimePack.sourceDependencies, + ...plan.programObjectDependencies, + ]; + const inheritedDependencyPaths = new Set( + inheritedDependencies.map((dependency) => resolve(dependency.path)), ); const additionalDependencyPaths = plan.dependencyPaths.filter( - (path) => !packDependencyPaths.has(resolve(path)), + (path) => !inheritedDependencyPaths.has(resolve(path)), ); const preLinkDependencies = options.onArtifactReady === undefined || - !(await nativeArtifactDependenciesStillMatch(plan.runtimePack.sourceDependencies).catch(() => false)) + !(await nativeArtifactDependenciesStillMatch(inheritedDependencies).catch(() => false)) ? null : await linkToolchainDependencies(linkerPath) .then(async (toolchain) => [ - ...plan.runtimePack.sourceDependencies, + ...inheritedDependencies, ...await snapshotDependencies([...toolchain, ...additionalDependencyPaths]), ]) .catch(() => null); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 11a72c92b..3c3890f9e 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -2,7 +2,7 @@ import { InternalCompilerError } from "./errors.js"; import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; -import { buildCacheRoot, CcCompileError, clearCcCaches, compileC, compileLibArchive, compilerDriverSupportsPersistentCache, configuredTargetPlatform, executableNativeEnvironmentFingerprint, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform, toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "./backend/native-toolchain.js"; +import { buildCacheRoot, CcCompileError, clearCcCaches, compileC, compileLibArchive, compilerDriverSupportsPersistentCache, configuredTargetPlatform, executableNativeEnvironmentFingerprint, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform, toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint, type NativeArtifactDependency } from "./backend/native-toolchain.js"; import { emitCModule } from "./backend/c/c-emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen.js"; @@ -1050,7 +1050,7 @@ async function compileExecutableNative( sanitize: boolean, ffi: FfiProfile | null, programSplit: ReturnType = null, - programObjectDependencies: readonly string[] = [], + programObjectDependencies: readonly NativeArtifactDependency[] = [], onArtifactReady?: NonNullable[0]["onArtifactReady"]>, ): Promise { const programIsObject = /\.(?:o|obj)$/.test(cPath); @@ -1160,7 +1160,7 @@ async function emitNativeProgramObject( entryPath: string, opts: CompileRequestOptions, llvm: string, -): Promise<{ linkPath: string; artifactPath: string; dependencyPaths: string[] }> { +): Promise<{ linkPath: string; artifactPath: string; dependencies: NativeArtifactDependency[] }> { const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const artifactPath = join(opts.outDir, `${stem}.helper.o`); // compileExecutableNative recognizes object inputs by suffix. The random @@ -1176,7 +1176,7 @@ async function emitNativeProgramObject( optimization: opts.optimization === "dev" ? "0" : "2", ...(opts.sanitize === undefined ? {} : { sanitize: opts.sanitize }), }); - return { linkPath, artifactPath, dependencyPaths: artifact.dependencyPaths }; + return { linkPath, artifactPath, dependencies: artifact.dependencies }; } catch (error) { await rm(linkPath, { force: true }).catch(() => undefined); throw error; @@ -1390,7 +1390,7 @@ async function compileTracked( let nativeProgramObject: { linkPath: string; artifactPath: string; - dependencyPaths: string[]; + dependencies: NativeArtifactDependency[]; } | null = null; const useRuntimePack = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, earlyHit.native.backend); @@ -1424,7 +1424,7 @@ async function compileTracked( opts.sanitize ?? false, ffi, null, - nativeProgramObject?.dependencyPaths, + nativeProgramObject?.dependencies, opts.nativeProgramObject === true ? undefined : async ({ dependencies }) => { await publishEarlyExecutableCache(cacheRoot, executableCacheOptions, { ...earlyHit, @@ -1719,7 +1719,7 @@ async function compileTracked( let nativeProgramObject: { linkPath: string; artifactPath: string; - dependencyPaths: string[]; + dependencies: NativeArtifactDependency[]; } | null = null; try { const useRuntimePack = opts.nativeProgramObject === true || @@ -1746,7 +1746,7 @@ async function compileTracked( opts.sanitize ?? false, ffi, programSplit, - nativeProgramObject?.dependencyPaths, + nativeProgramObject?.dependencies, opts.nativeProgramObject === true ? undefined : async ({ dependencies }) => { await publishEarlyExecutableCache(cacheRoot, executableCacheOptions, { cPath, From 214bb31e9af57646adfc5fef631dbdbabbdc99c7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 28 Aug 2026 07:22:09 -0500 Subject: [PATCH 10/11] fix(runtime): preserve executable link identity --- packages/cli/test/runtime-pack.test.ts | 22 ++++++++--- .../compiler/src/backend/runtime-pack.test.ts | 37 ++++++++++++++++++- packages/compiler/src/backend/runtime-pack.ts | 11 ++++-- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/packages/cli/test/runtime-pack.test.ts b/packages/cli/test/runtime-pack.test.ts index 1db698c85..8bd62ef2c 100644 --- a/packages/cli/test/runtime-pack.test.ts +++ b/packages/cli/test/runtime-pack.test.ts @@ -2,7 +2,7 @@ import { execFile } from "node:child_process"; import { createRequire } from "node:module"; import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { release as osRelease, tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { promisify } from "node:util"; import { afterEach, describe, expect, test } from "vitest"; @@ -35,13 +35,17 @@ describe.runIf(supported)("precompiled runtime executable builds", () => { "", ].join("\n")); await chmod(wrapper, 0o755); - await execFileAsync(process.execPath, [ + const cliArgs = [ "--import", tsxLoader, cliEntry, "build", entry, "-o", output, - ], { + ]; + const env = { + ...process.env, + SCRIPTC_NO_CACHE: "1", + SCRIPTC_LINKER: wrapper, + }; + await execFileAsync(process.execPath, cliArgs, { env: { - ...process.env, - SCRIPTC_NO_CACHE: "1", - SCRIPTC_LINKER: wrapper, + ...env, }, }); const args = JSON.parse(await readFile(log, "utf8")) as string[]; @@ -53,5 +57,11 @@ describe.runIf(supported)("precompiled runtime executable builds", () => { expect(args.some((arg) => arg.includes("runtime-darwin-arm64/artifacts"))).toBe(false); await expect(execFileAsync(output, [], { encoding: "utf8" })) .resolves.toMatchObject({ stdout: "precompiled runtime\n" }); + const firstExecutable = await readFile(output); + const signature = await execFileAsync("codesign", ["-dvvv", output], { encoding: "utf8" }); + expect(signature.stderr).toContain(`Identifier=${basename(output)}`); + + await execFileAsync(process.execPath, cliArgs, { env }); + expect(await readFile(output)).toEqual(firstExecutable); }); }); diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 989e60f1d..245bc39cc 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { chmod, mkdtemp, mkdir, readFile, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { describe, expect, test } from "vitest"; import { compilerReleaseVersion } from "../library/sidecar.js"; import { snapshotNativeArtifactDependencies } from "./native-toolchain.js"; @@ -242,6 +242,41 @@ describe("runtime pack manifests", () => { expect(await readFile(runtimeObject, "utf8")).toBe("tampered"); }); + test("preserves the requested basename in the private linker output path", async () => { + const { root, packagePath } = await fixture(); + const programObject = join(root, "program.o"); + const output = join(root, "requested-name"); + const linker = join(root, "linker.mjs"); + await Promise.all([ + writeFile(programObject, "program object"), + writeFile(linker, [ + "#!/usr/bin/env node", + 'import { writeFileSync } from "node:fs";', + 'const outputIndex = process.argv.indexOf("-o");', + 'const output = process.argv[outputIndex + 1];', + 'writeFileSync(output, JSON.stringify(output));', + "", + ].join("\n")), + ]); + await chmod(linker, 0o755); + const plan = await createRuntimeLinkPlan({ + target: MACOS_ARM64_TARGET, + programObject, + outPath: output, + features: BASE, + ffi: null, + optimization: "release", + resolver: () => packagePath, + }); + + await linkRuntimePackExecutable(plan, { linker }); + + const privateOutput = JSON.parse(await readFile(output, "utf8")) as string; + expect(privateOutput).not.toBe(output); + expect(basename(privateOutput)).toBe(basename(output)); + expect(dirname(dirname(privateOutput))).toBe(dirname(output)); + }); + test("does not publish a cache proof from a stale program-object dependency snapshot", async () => { const { root, packagePath } = await fixture(); const programObject = join(root, "program.o"); diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts index a6645a3fa..45802c14b 100644 --- a/packages/compiler/src/backend/runtime-pack.ts +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -7,7 +7,6 @@ import { basename, dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; import type { FfiProfile } from "../ffi/ffi-manifest.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; -import { privateSiblingPath } from "./build-cache.js"; import type { NativeLinkFeatures } from "./native-link-info.js"; import { CcCompileError, @@ -521,7 +520,13 @@ export async function linkRuntimePackExecutable( ): Promise { const linker = options.linker ?? process.env["SCRIPTC_LINKER"] ?? "clang"; const linkerPath = await resolveExecutable(linker); - const privateOut = privateSiblingPath(plan.outputPath, "runtime-pack-link"); + // ld64 derives an ad-hoc signature identifier from the output basename. + // Keep that basename caller-visible while a private sibling directory gives + // the link its own inode and preserves an atomic same-filesystem install. + const privateOutRoot = await mkdtemp( + join(dirname(plan.outputPath), ".scriptc-runtime-pack-link-"), + ); + const privateOut = join(privateOutRoot, basename(plan.outputPath)); let stagedRoot: string | null = null; try { const staged = await stageRuntimePackArtifacts(plan.runtimePack); @@ -583,7 +588,7 @@ export async function linkRuntimePackExecutable( ); } finally { await Promise.all([ - rm(privateOut, { force: true }).catch(() => undefined), + rm(privateOutRoot, { recursive: true, force: true }).catch(() => undefined), stagedRoot === null ? Promise.resolve() : rm(stagedRoot, { recursive: true, force: true }).catch(() => undefined), From faec06ac2f6ce7c5bbaa3a8063b2c28c14e6918f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 28 Aug 2026 07:52:23 -0500 Subject: [PATCH 11/11] fix(runtime): trace selected linker dependencies --- .../compiler/src/backend/native-toolchain.ts | 16 +++ .../compiler/src/backend/runtime-pack.test.ts | 105 +++++++++++++++++- packages/compiler/src/backend/runtime-pack.ts | 46 +------- 3 files changed, 121 insertions(+), 46 deletions(-) diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index ed1ead16a..de1d42da2 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -3210,6 +3210,22 @@ function implicitLinkerFingerprint( ); } +/** Resolve the exact files consumed by one compiler-driver link invocation. + * Runtime packs reuse the native toolchain's strict dry-run plus real linker + * trace so a PATH-selected Clang carries its own linker, SDK, compiler + * runtime, and injected inputs into the executable cache proof. */ +export async function nativeLinkerDependencyPaths( + linker: string, + linkArgs: readonly string[], +): Promise { + const fingerprint = await implicitLinkerFingerprint( + { argv: [linker], targetArgs: [], target: null }, + toolchainEnvironmentFingerprint(), + linkArgs, + ); + return fingerprintDependencyPaths(fingerprint); +} + let ccacheMemo: Promise | null = null; /** Reset process observations whose validity is bounded to one public native diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 245bc39cc..cc5aede2b 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -318,7 +318,85 @@ describe("runtime pack manifests", () => { expect(published).toBe(false); }); - test.runIf(process.platform === "darwin")( + test( + "cache proofs follow the selected driver to its linker, SDK, and compiler runtime", + async () => { + const { root, packagePath } = await fixture(); + const programObject = join(root, "program.o"); + const output = join(root, "program"); + const driver = join(root, "clang.mjs"); + const platformLinker = join(root, "toolchain", "ld"); + const sdkSettings = join(root, "driver-sdk", "SDKSettings.json"); + const systemStub = join(root, "driver-sdk", "usr", "lib", "libSystem.tbd"); + const compilerRuntime = join(root, "toolchain", "libclang_rt.osx.a"); + await Promise.all([ + mkdir(dirname(platformLinker), { recursive: true }), + mkdir(dirname(systemStub), { recursive: true }), + writeFile(programObject, "program object"), + ]); + await Promise.all([ + writeFile(platformLinker, "selected platform linker"), + writeFile(sdkSettings, "selected SDK settings"), + writeFile(systemStub, "selected System stub"), + writeFile(compilerRuntime, "selected compiler runtime"), + writeFile(driver, [ + "#!/usr/bin/env node", + 'import { writeFileSync } from "node:fs";', + `const dependencies = ${JSON.stringify([ + platformLinker, + sdkSettings, + systemStub, + compilerRuntime, + ])};`, + 'const args = process.argv.slice(2);', + 'const outputIndex = args.indexOf("-o");', + 'if (args.includes("-print-prog-name=ld")) {', + ` process.stdout.write(${JSON.stringify(`${platformLinker}\n`)});`, + " process.exit(0);", + "}", + 'if (args.includes("-###")) {', + ' process.stderr.write(`${dependencies.map(JSON.stringify).join(" ")}\\n`);', + " process.exit(0);", + "}", + 'if (args.includes("-Wl,-t")) {', + ' process.stdout.write(`${dependencies.join("\\n")}\\n`);', + ' writeFileSync(args[outputIndex + 1], "link trace output");', + " process.exit(0);", + "}", + 'writeFileSync(args[outputIndex + 1], args.includes("-c") ? "probe object" : "linked executable");', + "", + ].join("\n")), + ]); + await chmod(driver, 0o755); + const plan = await createRuntimeLinkPlan({ + target: MACOS_ARM64_TARGET, + programObject, + outPath: output, + features: BASE, + ffi: null, + optimization: "release", + resolver: () => packagePath, + }); + let dependencyPaths: string[] = []; + + await linkRuntimePackExecutable(plan, { + linker: driver, + onArtifactReady: async ({ dependencies }) => { + dependencyPaths = dependencies.map((dependency) => dependency.path); + }, + }); + + expect(await readFile(output, "utf8")).toBe("linked executable"); + expect(dependencyPaths).toEqual(expect.arrayContaining([ + platformLinker, + sdkSettings, + systemStub, + compilerRuntime, + ])); + }, + ); + + test( "does not publish an executable cache proof when a dependency changes during linking", async () => { const { root, packagePath } = await fixture(); @@ -326,21 +404,38 @@ describe("runtime pack manifests", () => { const dependency = join(root, "link-dependency.a"); const output = join(root, "program"); const linker = join(root, "linker.mjs"); + const platformLinker = join(root, "ld"); await Promise.all([ writeFile(programObject, "program object"), writeFile(dependency, "before link"), + writeFile(platformLinker, "selected platform linker"), writeFile(linker, [ "#!/usr/bin/env node", 'import { writeFileSync } from "node:fs";', `const dependency = ${JSON.stringify(dependency)};`, - 'if (process.argv[2]?.startsWith("-print-file-name=")) {', - ' process.stdout.write(`${dependency}\\n`);', + `const platformLinker = ${JSON.stringify(platformLinker)};`, + 'const args = process.argv.slice(2);', + 'const outputIndex = args.indexOf("-o");', + 'if (args.includes("-print-prog-name=ld")) {', + ' process.stdout.write(`${platformLinker}\\n`);', + " process.exit(0);", + "}", + 'if (args.includes("-###")) {', + ' process.stderr.write(`${JSON.stringify(platformLinker)} ${JSON.stringify(dependency)}\\n`);', + " process.exit(0);", + "}", + 'if (args.includes("-Wl,-t")) {', + ' process.stdout.write(`${platformLinker}\\n${dependency}\\n`);', + ' writeFileSync(args[outputIndex + 1], "link trace output");', " process.exit(0);", "}", - 'const outputIndex = process.argv.indexOf("-o");', 'if (outputIndex < 0) process.exit(2);', + 'if (args.includes("-c")) {', + ' writeFileSync(args[outputIndex + 1], "probe object");', + " process.exit(0);", + "}", 'writeFileSync(dependency, "changed during link");', - 'writeFileSync(process.argv[outputIndex + 1], "linked executable");', + 'writeFileSync(args[outputIndex + 1], "linked executable");', "", ].join("\n")), ]); diff --git a/packages/compiler/src/backend/runtime-pack.ts b/packages/compiler/src/backend/runtime-pack.ts index 45802c14b..2df1f270d 100644 --- a/packages/compiler/src/backend/runtime-pack.ts +++ b/packages/compiler/src/backend/runtime-pack.ts @@ -11,6 +11,7 @@ import type { NativeLinkFeatures } from "./native-link-info.js"; import { CcCompileError, nativeArtifactDependenciesStillMatch, + nativeLinkerDependencyPaths, subprocessFailureDetail, type NativeArtifactDependency, } from "./native-toolchain.js"; @@ -472,45 +473,6 @@ async function snapshotDependencies(paths: readonly string[]): Promise { - if (command.includes("/") || command.includes("\\")) return resolve(command); - const path = env["PATH"] ?? ""; - const { access } = await import("node:fs/promises"); - const { delimiter } = await import("node:path"); - for (const directory of path.split(delimiter)) { - if (directory === "") continue; - const candidate = join(directory, command); - try { - await access(candidate); - return await realpath(candidate).catch(() => candidate); - } catch { - continue; - } - } - return command; -} - -async function linkToolchainDependencies(linker: string): Promise { - if (process.platform !== "darwin") return [linker]; - const sdkRoot = process.env["SDKROOT"] ?? - (await execFileAsync("xcrun", ["--sdk", "macosx", "--show-sdk-path"])).stdout.trim(); - const platformLinker = (await execFileAsync("xcrun", ["--sdk", "macosx", "--find", "ld"])).stdout.trim(); - const compilerRuntime = (await execFileAsync(linker, ["-print-file-name=libclang_rt.osx.a"])) - .stdout.trim(); - const candidates = [ - linker, - platformLinker, - compilerRuntime, - join(sdkRoot, "SDKSettings.json"), - join(sdkRoot, "usr", "lib", "libSystem.tbd"), - join(sdkRoot, "usr", "lib", "libm.tbd"), - join(sdkRoot, "usr", "lib", "libz.tbd"), - ]; - const present = await Promise.all(candidates.map(async (path) => - await stat(path).then(() => path, () => null))); - return present.filter((path): path is string => path !== null); -} - export async function linkRuntimePackExecutable( plan: RuntimeLinkPlan, options: { @@ -519,7 +481,6 @@ export async function linkRuntimePackExecutable( } = {}, ): Promise { const linker = options.linker ?? process.env["SCRIPTC_LINKER"] ?? "clang"; - const linkerPath = await resolveExecutable(linker); // ld64 derives an ad-hoc signature identifier from the output basename. // Keep that basename caller-visible while a private sibling directory gives // the link its own inode and preserves an atomic same-filesystem install. @@ -554,7 +515,10 @@ export async function linkRuntimePackExecutable( const preLinkDependencies = options.onArtifactReady === undefined || !(await nativeArtifactDependenciesStillMatch(inheritedDependencies).catch(() => false)) ? null - : await linkToolchainDependencies(linkerPath) + : await nativeLinkerDependencyPaths(linker, [ + ...plan.driverFlags, + ...plan.systemLibraries.map((name) => `-l${name}`), + ]) .then(async (toolchain) => [ ...inheritedDependencies, ...await snapshotDependencies([...toolchain, ...additionalDependencyPaths]),