diff --git a/common/changes/@microsoft/rush/resolver-cache-binary-format_2026-09-29-00-00.json b/common/changes/@microsoft/rush/resolver-cache-binary-format_2026-09-29-00-00.json new file mode 100644 index 00000000000..996f7ff0d66 --- /dev/null +++ b/common/changes/@microsoft/rush/resolver-cache-binary-format_2026-09-29-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "comment": "Add the `useProjectDependencyGraph` experiment. When enabled, `rush install` writes a `dependency-graph.bin` file into each project's `.rush/temp` folder containing the slice of the workspace dependency graph reachable from that project, and that file replaces `shrinkwrap-deps.json` when computing whether a project's dependencies have changed.", + "type": "none", + "packageName": "@microsoft/rush" + } + ], + "packageName": "@microsoft/rush" +} diff --git a/common/changes/@rushstack/resolver-cache/resolver-cache-binary-format_2026-09-29-00-00.json b/common/changes/@rushstack/resolver-cache/resolver-cache-binary-format_2026-09-29-00-00.json new file mode 100644 index 00000000000..36e370a0c98 --- /dev/null +++ b/common/changes/@rushstack/resolver-cache/resolver-cache-binary-format_2026-09-29-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "comment": "Initial release. Provides the `RRC1` binary resolver cache format, lockfile-derived Merkle hashing of dependency contexts, per-project graph slicing, and a parser that transparently accepts either the binary format or the legacy monolithic JSON cache.", + "type": "minor", + "packageName": "@rushstack/resolver-cache" + } + ], + "packageName": "@rushstack/resolver-cache" +} diff --git a/common/changes/@rushstack/webpack-workspace-resolve-plugin/resolver-cache-binary-format_2026-09-29-00-00.json b/common/changes/@rushstack/webpack-workspace-resolve-plugin/resolver-cache-binary-format_2026-09-29-00-00.json new file mode 100644 index 00000000000..a2c81befa53 --- /dev/null +++ b/common/changes/@rushstack/webpack-workspace-resolve-plugin/resolver-cache-binary-format_2026-09-29-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "comment": "Add `loadResolverCacheAsync` and `loadResolverCache`, which read the first available resolver cache file from a list of candidates, transparently accepting either the new binary format or the legacy monolithic JSON cache. The `ISerializedResolveContext` and `IResolverCacheFile` types are now re-exported from the new `@rushstack/resolver-cache` package.", + "type": "minor", + "packageName": "@rushstack/webpack-workspace-resolve-plugin" + } + ], + "packageName": "@rushstack/webpack-workspace-resolve-plugin" +} diff --git a/libraries/resolver-cache/.npmignore b/libraries/resolver-cache/.npmignore new file mode 100644 index 00000000000..f7a40e10213 --- /dev/null +++ b/libraries/resolver-cache/.npmignore @@ -0,0 +1,36 @@ +# THIS IS A STANDARD TEMPLATE FOR .npmignore FILES IN THIS REPO. + +# Ignore all files by default, to avoid accidentally publishing unintended files. +* + +# Use negative patterns to bring back the specific things we want to publish. +!/bin/** +!/lib/** +!/lib-*/** +!/dist/** +!/includes/** + +!CHANGELOG.md +!CHANGELOG.json +!heft-plugin.json +!rush-plugin-manifest.json +!ThirdPartyNotice.txt + +# Ignore certain patterns that should not get published. +/dist/*.stats.* +/lib/**/test/ +/lib-*/**/test/ +*.test.js +*.test.[cm]js +*.test.d.ts +*.test.d.[cm]ts + +# NOTE: These don't need to be specified, because NPM includes them automatically. +# +# package.json +# README.md +# LICENSE + +# --------------------------------------------------------------------------- +# DO NOT MODIFY ABOVE THIS LINE! Add any project-specific overrides below. +# --------------------------------------------------------------------------- diff --git a/libraries/resolver-cache/LICENSE b/libraries/resolver-cache/LICENSE new file mode 100644 index 00000000000..ad73d857028 --- /dev/null +++ b/libraries/resolver-cache/LICENSE @@ -0,0 +1,24 @@ +@rushstack/lookup-by-path + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/libraries/resolver-cache/README.md b/libraries/resolver-cache/README.md new file mode 100644 index 00000000000..360d7860eff --- /dev/null +++ b/libraries/resolver-cache/README.md @@ -0,0 +1,50 @@ +# @rushstack/resolver-cache + +A dedicated binary format for the Rush resolver cache, plus helpers for hashing and slicing the +dependency graph that it describes. + +This package intentionally has no runtime dependencies, so that a consumer such as a Webpack plugin +can decode a cache without taking a dependency on `@microsoft/rush-lib`. + +## The format + +The format is optimized for a single linear decode; full decode is cheap enough that random access +is not worth the extra complexity. + +- **Strings** are stored once, in a lexicographically sorted, front-coded table. Each entry is a + `[prefixIndexDelta, suffixLengthInCharacters]` pair, where the prefix is the complete value of the + entry `prefixIndexDelta` positions earlier. The table is closed under branch-point prefixes, so a + suitable prefix always exists. All suffixes share one UTF-8 blob, which the decoder decodes exactly + once and then indexes with `substring`. +- **Suffix lengths are measured in UTF-16 code units**, not bytes and not code points. Byte lengths + would force a separate decode per entry; code points would disagree with `substring`. An encoder + written in another language must match this definition. +- **Integers** are LEB128 varints, and are stored as deltas against a nearby value (the previous + dependency key, or the ordinal of the context being decoded) so that they almost always fit in a + single byte. + +## Context identity + +A context is identified solely by its root path. Contexts that share a package name and version but +differ in root path are distinct and must never be merged. In particular, a PNPM injected dependency +resolves its own dependencies within the consuming subspace, so it has a different dependency graph +than the workspace project it was copied from, and the same project may be injected more than once +with different peer resolutions. + +## Hashes + +`computeContextHashes` produces a Merkle hash for each context. The preimage is purely +lockfile-derived: the root path and name of the context, its dependency keys, and the hashes of the +contexts those keys resolve to. File contents, build outputs, and timestamps are excluded, because +Rush tracks those separately through the build graph. The hashes are therefore computable from a +checkout that has never been installed. + +Contexts that participate in a dependency cycle are condensed into a strongly connected component +and hashed as a unit. + +## Links + +- [CHANGELOG.md](https://github.com/microsoft/rushstack/blob/main/libraries/resolver-cache/CHANGELOG.md) - Find + out what's new in the latest version + +`@rushstack/resolver-cache` is part of the [Rush Stack](https://rushstack.io/) family of projects. diff --git a/libraries/resolver-cache/config/api-extractor.json b/libraries/resolver-cache/config/api-extractor.json new file mode 100644 index 00000000000..3dbb76c0e6f --- /dev/null +++ b/libraries/resolver-cache/config/api-extractor.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "extends": "local-node-rig/profiles/default/config/api-extractor-base.json" +} diff --git a/libraries/resolver-cache/config/jest.config.json b/libraries/resolver-cache/config/jest.config.json new file mode 100644 index 00000000000..d1749681d90 --- /dev/null +++ b/libraries/resolver-cache/config/jest.config.json @@ -0,0 +1,3 @@ +{ + "extends": "local-node-rig/profiles/default/config/jest.config.json" +} diff --git a/libraries/resolver-cache/config/rig.json b/libraries/resolver-cache/config/rig.json new file mode 100644 index 00000000000..165ffb001f5 --- /dev/null +++ b/libraries/resolver-cache/config/rig.json @@ -0,0 +1,7 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + + "rigPackageName": "local-node-rig" +} diff --git a/libraries/resolver-cache/eslint.config.js b/libraries/resolver-cache/eslint.config.js new file mode 100644 index 00000000000..87132f43292 --- /dev/null +++ b/libraries/resolver-cache/eslint.config.js @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +const nodeProfile = require('local-node-rig/profiles/default/includes/eslint/flat/profile/node'); +const friendlyLocalsMixin = require('local-node-rig/profiles/default/includes/eslint/flat/mixins/friendly-locals'); +const tsdocMixin = require('local-node-rig/profiles/default/includes/eslint/flat/mixins/tsdoc'); + +module.exports = [ + ...nodeProfile, + ...friendlyLocalsMixin, + ...tsdocMixin, + { + files: ['**/*.ts', '**/*.tsx'], + languageOptions: { + parserOptions: { + tsconfigRootDir: __dirname + } + } + } +]; diff --git a/libraries/resolver-cache/package.json b/libraries/resolver-cache/package.json new file mode 100644 index 00000000000..6dab8975cbd --- /dev/null +++ b/libraries/resolver-cache/package.json @@ -0,0 +1,67 @@ +{ + "name": "@rushstack/resolver-cache", + "version": "0.1.0", + "description": "Codec and resolution helpers for the Rush resolver cache binary format.", + "main": "./lib-commonjs/index.js", + "module": "./lib-esm/index.js", + "types": "./dist/resolver-cache.d.ts", + "exports": { + ".": { + "types": "./dist/resolver-cache.d.ts", + "node": "./lib-commonjs/index.js", + "import": "./lib-esm/index.js", + "require": "./lib-commonjs/index.js" + }, + "./lib/*": { + "types": "./lib-dts/*.d.ts", + "node": "./lib-commonjs/*.js", + "import": "./lib-esm/*.js", + "require": "./lib-commonjs/*.js" + }, + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "lib/*": [ + "lib-dts/*" + ] + } + }, + "keywords": [ + "rush", + "resolver", + "cache", + "binary", + "lockfile" + ], + "license": "MIT", + "repository": { + "url": "https://github.com/microsoft/rushstack.git", + "type": "git", + "directory": "libraries/resolver-cache" + }, + "engines": { + "node": ">=20.9.0" + }, + "scripts": { + "build": "heft build --clean", + "_phase:build": "heft run --only build -- --clean", + "_phase:pack": "rush-pnpm pack", + "_phase:test": "heft run --only test -- --clean" + }, + "devDependencies": { + "@rushstack/heft": "workspace:*", + "@types/node": "20.17.19", + "eslint": "~9.37.0", + "local-node-rig": "workspace:*" + }, + "peerDependencies": { + "@types/node": "*" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + }, + "sideEffects": false +} diff --git a/libraries/resolver-cache/src/BinaryReader.ts b/libraries/resolver-cache/src/BinaryReader.ts new file mode 100644 index 00000000000..998dab2d4fd --- /dev/null +++ b/libraries/resolver-cache/src/BinaryReader.ts @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * A forward-only cursor over a buffer produced by `BinaryWriter`. + * + * @internal + */ +export class BinaryReader { + readonly #buffer: Uint8Array; + #offset: number; + + public constructor(buffer: Uint8Array) { + this.#buffer = buffer; + this.#offset = 0; + } + + /** + * The current read position. + */ + public get offset(): number { + return this.#offset; + } + + /** + * True if every byte has been consumed. + */ + public get atEnd(): boolean { + return this.#offset >= this.#buffer.length; + } + + #require(additionalBytes: number): void { + if (this.#offset + additionalBytes > this.#buffer.length) { + throw new Error( + `Unexpected end of buffer: needed ${additionalBytes} byte(s) at offset ${this.#offset} ` + + `but only ${this.#buffer.length - this.#offset} remain` + ); + } + } + + /** + * Reads a varint that describes how many items follow, rejecting values that could not possibly + * be satisfied by the remaining bytes. + * + * @remarks + * Decoders frequently preallocate storage from a count, so an implausible count read from a + * corrupt or hostile file would otherwise request an arbitrarily large allocation before the + * first missing byte is noticed. + * + * @param minimumBytesPerItem - The smallest number of bytes any single item can occupy. + */ + public readCount(minimumBytesPerItem: number): number { + const count: number = this.readVarint(); + const maximumCount: number = Math.floor( + (this.#buffer.length - this.#offset) / Math.max(minimumBytesPerItem, 1) + ); + if (count > maximumCount) { + throw new Error( + `Declared item count ${count} exceeds the ${maximumCount} item(s) that the remaining ` + + `${this.#buffer.length - this.#offset} byte(s) could contain` + ); + } + return count; + } + + /** + * Reads a single byte. + */ + public readUint8(): number { + this.#require(1); + return this.#buffer[this.#offset++]; + } + + /** + * Reads a 16-bit little-endian integer. + */ + public readUint16(): number { + this.#require(2); + const value: number = this.#buffer[this.#offset] | (this.#buffer[this.#offset + 1] << 8); + this.#offset += 2; + return value; + } + + /** + * Reads an unsigned LEB128 variable-length integer. + */ + public readVarint(): number { + let result: number = 0; + let scale: number = 1; + for (;;) { + const byte: number = this.readUint8(); + result += (byte & 0x7f) * scale; + if ((byte & 0x80) === 0) { + break; + } + scale *= 0x80; + if (!Number.isSafeInteger(result + scale)) { + throw new RangeError('Varint exceeds the safe integer range'); + } + } + return result; + } + + /** + * Reads a zigzag-encoded signed integer. + */ + public readSignedVarint(): number { + const raw: number = this.readVarint(); + return raw % 2 === 0 ? raw / 2 : -(raw + 1) / 2; + } + + /** + * Reads `byteLength` raw bytes. The result is a view over the original buffer, not a copy. + */ + public readBytes(byteLength: number): Uint8Array { + this.#require(byteLength); + const bytes: Uint8Array = this.#buffer.subarray(this.#offset, this.#offset + byteLength); + this.#offset += byteLength; + return bytes; + } + + /** + * Reads a varint byte length followed by that many bytes. + */ + public readLengthPrefixedBytes(): Uint8Array { + return this.readBytes(this.readVarint()); + } +} diff --git a/libraries/resolver-cache/src/BinaryWriter.ts b/libraries/resolver-cache/src/BinaryWriter.ts new file mode 100644 index 00000000000..ca52fc4b695 --- /dev/null +++ b/libraries/resolver-cache/src/BinaryWriter.ts @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * A growable buffer that supports appending LEB128 variable-length integers. + * + * @remarks + * The resolver cache format is designed for a single linear pass in both directions, so this + * writer deliberately offers no seek or patch operations. + * + * @internal + */ +export class BinaryWriter { + #buffer: Uint8Array; + #length: number; + + public constructor(initialCapacity: number = 1024) { + this.#buffer = new Uint8Array(initialCapacity); + this.#length = 0; + } + + /** + * The number of bytes written so far. + */ + public get length(): number { + return this.#length; + } + + #ensure(additionalBytes: number): void { + const required: number = this.#length + additionalBytes; + if (required <= this.#buffer.length) { + return; + } + + let capacity: number = this.#buffer.length * 2; + while (capacity < required) { + capacity *= 2; + } + + const replacement: Uint8Array = new Uint8Array(capacity); + replacement.set(this.#buffer.subarray(0, this.#length)); + this.#buffer = replacement; + } + + /** + * Appends a single byte. + */ + public writeUint8(value: number): void { + this.#ensure(1); + this.#buffer[this.#length++] = value & 0xff; + } + + /** + * Appends a 16-bit little-endian integer. + */ + public writeUint16(value: number): void { + this.#ensure(2); + this.#buffer[this.#length++] = value & 0xff; + this.#buffer[this.#length++] = (value >>> 8) & 0xff; + } + + /** + * Appends an unsigned LEB128 variable-length integer. + */ + public writeVarint(value: number): void { + if (!Number.isInteger(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) { + throw new RangeError(`Cannot encode ${value} as an unsigned varint`); + } + + let remaining: number = value; + // Values above 2^31 cannot use bitwise operators, so fall back to arithmetic. + while (remaining >= 0x80) { + this.writeUint8((remaining % 0x80) + 0x80); + remaining = Math.floor(remaining / 0x80); + } + this.writeUint8(remaining); + } + + /** + * Appends a signed integer using zigzag encoding, so that small magnitudes of either sign + * occupy a single byte. + */ + public writeSignedVarint(value: number): void { + if (!Number.isInteger(value)) { + throw new RangeError(`Cannot encode ${value} as a signed varint`); + } + this.writeVarint(value < 0 ? -2 * value - 1 : 2 * value); + } + + /** + * Appends raw bytes without a length prefix. + */ + public writeBytes(bytes: Uint8Array): void { + this.#ensure(bytes.length); + this.#buffer.set(bytes, this.#length); + this.#length += bytes.length; + } + + /** + * Appends a varint byte length followed by the bytes themselves. + */ + public writeLengthPrefixedBytes(bytes: Uint8Array): void { + this.writeVarint(bytes.length); + this.writeBytes(bytes); + } + + /** + * Returns a copy of the bytes written so far. + */ + public toUint8Array(): Uint8Array { + return this.#buffer.slice(0, this.#length); + } +} diff --git a/libraries/resolver-cache/src/ResolverCacheCodec.ts b/libraries/resolver-cache/src/ResolverCacheCodec.ts new file mode 100644 index 00000000000..87d64e2ae92 --- /dev/null +++ b/libraries/resolver-cache/src/ResolverCacheCodec.ts @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { BinaryReader } from './BinaryReader'; +import { BinaryWriter } from './BinaryWriter'; +import { readStringTable, StringTableBuilder, writeStringTable } from './StringTable'; +import { + ResolverCacheHashAlgorithm, + type IHashedResolverCacheFile, + type IResolverCacheFile, + type ISerializedResolveContext +} from './types'; + +/** + * The first four bytes of every resolver cache binary file. + * + * @beta + */ +export const RESOLVER_CACHE_MAGIC: Uint8Array = new Uint8Array([0x52, 0x52, 0x43, 0x01]); + +/** + * The format version understood by this package. + * + * @beta + */ +export const RESOLVER_CACHE_FORMAT_VERSION: 1 = 1; + +const FLAG_HAS_DIR_INFO: 1 = 1; +const FLAG_HAS_HASHES: 2 = 2; +const FLAG_SCOPED: 4 = 4; + +const HASH_BYTE_LENGTHS: Record = { + [ResolverCacheHashAlgorithm.None]: 0, + [ResolverCacheHashAlgorithm.Sha256]: 32 +}; + +/** + * Options for {@link encodeResolverCache}. + * + * @beta + */ +export interface IEncodeResolverCacheOptions { + /** + * The cache to encode. + */ + cache: IResolverCacheFile; + /** + * Optional raw digest bytes for each context, parallel to `cache.contexts`. + */ + hashes?: readonly Uint8Array[]; + /** + * The algorithm that produced `hashes`. Required when `hashes` is supplied. + */ + hashAlgorithm?: ResolverCacheHashAlgorithm; + /** + * True when this file contains only the slice of the graph visible to a single project, rather + * than the whole workspace. + */ + scoped?: boolean; +} + +/** + * Returns true if `buffer` begins with the resolver cache binary magic. + * + * @beta + */ +export function isResolverCacheBinary(buffer: Uint8Array): boolean { + if (buffer.length < RESOLVER_CACHE_MAGIC.length) { + return false; + } + for (let i: number = 0; i < RESOLVER_CACHE_MAGIC.length; ++i) { + if (buffer[i] !== RESOLVER_CACHE_MAGIC[i]) { + return false; + } + } + return true; +} + +function buildStringTable(contexts: readonly ISerializedResolveContext[]): StringTableBuilder { + const builder: StringTableBuilder = new StringTableBuilder(); + for (const context of contexts) { + builder.add(context.root); + builder.add(context.name); + if (context.deps) { + for (const key of Object.keys(context.deps)) { + builder.add(key); + } + } + if (context.dirInfoFiles) { + for (const file of context.dirInfoFiles) { + builder.add(file); + } + } + } + return builder; +} + +function writeContext( + writer: BinaryWriter, + builder: StringTableBuilder, + context: ISerializedResolveContext, + ordinal: number, + hasDirInfo: boolean +): void { + writer.writeVarint(builder.getIndex(context.root)); + writer.writeVarint(builder.getIndex(context.name)); + + const depEntries: [number, number][] = []; + if (context.deps) { + for (const [key, targetOrdinal] of Object.entries(context.deps)) { + depEntries.push([builder.getIndex(key), targetOrdinal]); + } + // Sorting by string index makes the key deltas monotonically increasing, which keeps them in + // the single-byte varint range for all but the most extreme graphs. + depEntries.sort((x: [number, number], y: [number, number]) => x[0] - y[0]); + } + + writer.writeVarint(depEntries.length); + let previousKeyIndex: number = 0; + for (const [keyIndex, targetOrdinal] of depEntries) { + writer.writeVarint(keyIndex - previousKeyIndex); + previousKeyIndex = keyIndex; + // Ordinals are assigned in sorted-root-path order, so dependencies are usually nearby. + writer.writeSignedVarint(targetOrdinal - ordinal); + } + + if (hasDirInfo) { + const dirInfoFiles: string[] = context.dirInfoFiles ?? []; + writer.writeVarint(dirInfoFiles.length); + const indices: number[] = dirInfoFiles.map((file: string) => builder.getIndex(file)).sort(compareNumbers); + let previousFileIndex: number = 0; + for (const fileIndex of indices) { + writer.writeVarint(fileIndex - previousFileIndex); + previousFileIndex = fileIndex; + } + } +} + +function compareNumbers(x: number, y: number): number { + return x - y; +} + +function readContext( + reader: BinaryReader, + strings: readonly string[], + ordinal: number, + hasDirInfo: boolean +): ISerializedResolveContext { + const root: string = strings[reader.readVarint()]; + const name: string = strings[reader.readVarint()]; + + const depCount: number = reader.readVarint(); + let deps: Record | undefined; + if (depCount > 0) { + deps = {}; + let keyIndex: number = 0; + for (let i: number = 0; i < depCount; ++i) { + keyIndex += reader.readVarint(); + deps[strings[keyIndex]] = ordinal + reader.readSignedVarint(); + } + } + + let dirInfoFiles: string[] | undefined; + if (hasDirInfo) { + const dirInfoCount: number = reader.readCount(1); + if (dirInfoCount > 0) { + dirInfoFiles = new Array(dirInfoCount); + let fileIndex: number = 0; + for (let i: number = 0; i < dirInfoCount; ++i) { + fileIndex += reader.readVarint(); + dirInfoFiles[i] = strings[fileIndex]; + } + } + } + + const context: ISerializedResolveContext = { root, name }; + if (deps) { + context.deps = deps; + } + if (dirInfoFiles) { + context.dirInfoFiles = dirInfoFiles; + } + return context; +} + +/** + * Encodes a resolver cache into the binary format. + * + * @beta + */ +export function encodeResolverCache(options: IEncodeResolverCacheOptions): Uint8Array { + const { cache, hashes, scoped } = options; + const { basePath, contexts } = cache; + + const hashAlgorithm: ResolverCacheHashAlgorithm = + options.hashAlgorithm ?? (hashes ? ResolverCacheHashAlgorithm.Sha256 : ResolverCacheHashAlgorithm.None); + const hashByteLength: number = HASH_BYTE_LENGTHS[hashAlgorithm]; + + if (hashes) { + if (hashAlgorithm === ResolverCacheHashAlgorithm.None) { + throw new Error('A hash algorithm must be specified when hashes are provided'); + } + if (hashes.length !== contexts.length) { + throw new Error( + `Expected ${contexts.length} hash(es) to match the context count, but received ${hashes.length}` + ); + } + for (const hash of hashes) { + if (hash.length !== hashByteLength) { + throw new Error(`Expected each hash to be ${hashByteLength} bytes, but received ${hash.length}`); + } + } + } + + const hasDirInfo: boolean = contexts.some( + (context: ISerializedResolveContext) => !!context.dirInfoFiles?.length + ); + + const builder: StringTableBuilder = buildStringTable(contexts); + const strings: readonly string[] = builder.finalize(); + + const writer: BinaryWriter = new BinaryWriter(64 * 1024); + writer.writeBytes(RESOLVER_CACHE_MAGIC); + writer.writeUint16(RESOLVER_CACHE_FORMAT_VERSION); + writer.writeUint16( + (hasDirInfo ? FLAG_HAS_DIR_INFO : 0) | (hashes ? FLAG_HAS_HASHES : 0) | (scoped ? FLAG_SCOPED : 0) + ); + writer.writeUint8(hashes ? hashAlgorithm : ResolverCacheHashAlgorithm.None); + + writer.writeLengthPrefixedBytes(new TextEncoder().encode(basePath)); + writeStringTable(writer, strings); + + writer.writeVarint(contexts.length); + for (let ordinal: number = 0; ordinal < contexts.length; ++ordinal) { + writeContext(writer, builder, contexts[ordinal], ordinal, hasDirInfo); + } + + if (hashes) { + for (const hash of hashes) { + writer.writeBytes(hash); + } + } + + return writer.toUint8Array(); +} + +/** + * Decodes a buffer produced by {@link encodeResolverCache}. + * + * @beta + */ +export function decodeResolverCache(buffer: Uint8Array): IHashedResolverCacheFile { + if (!isResolverCacheBinary(buffer)) { + throw new Error('The buffer is not a resolver cache binary file'); + } + + const reader: BinaryReader = new BinaryReader(buffer); + reader.readBytes(RESOLVER_CACHE_MAGIC.length); + + const formatVersion: number = reader.readUint16(); + if (formatVersion !== RESOLVER_CACHE_FORMAT_VERSION) { + throw new Error( + `Unsupported resolver cache format version ${formatVersion}; expected ${RESOLVER_CACHE_FORMAT_VERSION}` + ); + } + + const flags: number = reader.readUint16(); + const hasDirInfo: boolean = (flags & FLAG_HAS_DIR_INFO) !== 0; + const hasHashes: boolean = (flags & FLAG_HAS_HASHES) !== 0; + + const hashAlgorithm: ResolverCacheHashAlgorithm = reader.readUint8(); + const hashByteLength: number | undefined = HASH_BYTE_LENGTHS[hashAlgorithm]; + if (hashByteLength === undefined) { + throw new Error(`Unsupported resolver cache hash algorithm ${hashAlgorithm}`); + } + + const basePath: string = new TextDecoder('utf-8', { fatal: true }).decode( + reader.readLengthPrefixedBytes() + ); + const strings: string[] = readStringTable(reader); + + // Each context contributes at least three varint bytes (root, name, dependency count). + const contextCount: number = reader.readCount(3); + const contexts: ISerializedResolveContext[] = new Array(contextCount); + for (let ordinal: number = 0; ordinal < contextCount; ++ordinal) { + contexts[ordinal] = readContext(reader, strings, ordinal, hasDirInfo); + } + + const hashes: Uint8Array[] = []; + if (hasHashes) { + for (let i: number = 0; i < contextCount; ++i) { + hashes.push(reader.readBytes(hashByteLength)); + } + } + + return { + basePath, + contexts, + hashAlgorithm: hasHashes ? hashAlgorithm : ResolverCacheHashAlgorithm.None, + hashes + }; +} diff --git a/libraries/resolver-cache/src/StringTable.ts b/libraries/resolver-cache/src/StringTable.ts new file mode 100644 index 00000000000..a751791089e --- /dev/null +++ b/libraries/resolver-cache/src/StringTable.ts @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { BinaryReader } from './BinaryReader'; +import type { BinaryWriter } from './BinaryWriter'; + +const TEXT_ENCODER: TextEncoder = new TextEncoder(); +const TEXT_DECODER: TextDecoder = new TextDecoder('utf-8', { fatal: true }); + +/** + * A lexicographically sorted, front-coded table of strings. + * + * @remarks + * Each entry is stored as a `[prefixIndexDelta, suffixLengthInCharacters]` pair. The prefix is the + * complete value of the entry `prefixIndexDelta` positions earlier in the table; a delta of `0` + * means the entry has no prefix. All suffixes are concatenated into a single UTF-8 blob so that a + * decoder performs exactly one (SIMD-accelerated) UTF-8 decode and then indexes into the resulting + * string with `substring`. + * + * Suffix lengths are measured in UTF-16 code units (JavaScript `String.prototype.length`), NOT in + * bytes and NOT in Unicode code points. Byte lengths would force a separate decode per entry, and + * code points would disagree with `substring`. Encoders written in other languages must match this + * definition. + * + * @beta + */ +export interface IStringTable { + /** + * The decoded strings, in table order. + */ + readonly strings: readonly string[]; +} + +/** + * Returns the number of leading UTF-16 code units shared by `a` and `b`, never splitting a + * surrogate pair. + */ +function getCommonPrefixLength(a: string, b: string): number { + const limit: number = Math.min(a.length, b.length); + let length: number = 0; + while (length < limit && a.charCodeAt(length) === b.charCodeAt(length)) { + ++length; + } + + // Never end a prefix on a lone high surrogate; the suffix must independently be valid UTF-8. + if (length > 0) { + const lastCharCode: number = a.charCodeAt(length - 1); + if (lastCharCode >= 0xd800 && lastCharCode <= 0xdbff) { + --length; + } + } + + return length; +} + +/** + * Accumulates the set of strings that a resolver cache file needs, then assigns table indices. + * + * @beta + */ +export class StringTableBuilder { + readonly #strings: Set = new Set(); + #indices: Map | undefined = undefined; + + /** + * Records that `value` must appear in the table. + */ + public add(value: string): void { + if (this.#indices) { + throw new Error('Cannot add strings after the table has been finalized'); + } + this.#strings.add(value); + } + + /** + * Sorts the recorded strings, inserts the synthetic branch-point prefixes that front coding + * needs, and assigns indices. Subsequent calls return the same result. + */ + public finalize(): readonly string[] { + if (!this.#indices) { + const sorted: string[] = Array.from(this.#strings).sort(); + + // Front coding can only reference a complete earlier entry, so materialize the branch points + // of the implied trie. Without these, sibling paths that share a long directory prefix would + // each have to store that prefix in full. + const augmented: Set = new Set(sorted); + for (let i: number = 1; i < sorted.length; ++i) { + const commonLength: number = getCommonPrefixLength(sorted[i - 1], sorted[i]); + if (commonLength > 0) { + augmented.add(sorted[i - 1].slice(0, commonLength)); + } + } + + const finalOrder: string[] = Array.from(augmented).sort(); + const indices: Map = new Map(); + for (let i: number = 0; i < finalOrder.length; ++i) { + indices.set(finalOrder[i], i); + } + this.#indices = indices; + } + + return Array.from(this.#indices.keys()); + } + + /** + * Returns the table index of a previously added string. + */ + public getIndex(value: string): number { + if (!this.#indices) { + throw new Error('The table must be finalized before indices can be read'); + } + const index: number | undefined = this.#indices.get(value); + if (index === undefined) { + throw new Error(`The string ${JSON.stringify(value)} was not added to the string table`); + } + return index; + } +} + +/** + * Writes a finalized string table to `writer`. + * + * @beta + */ +export function writeStringTable(writer: BinaryWriter, strings: readonly string[]): void { + writer.writeVarint(strings.length); + + // A stack of table indices whose values are all prefixes of the string being encoded. Because the + // table is sorted and closed under branch-point prefixes, the top of the stack is always the + // longest available prefix. + const prefixStack: number[] = []; + const suffixes: Uint8Array[] = []; + let blobByteLength: number = 0; + + for (let i: number = 0; i < strings.length; ++i) { + const value: string = strings[i]; + + while (prefixStack.length > 0 && !value.startsWith(strings[prefixStack[prefixStack.length - 1]])) { + prefixStack.pop(); + } + + let prefixIndexDelta: number = 0; + let prefixLength: number = 0; + if (prefixStack.length > 0) { + const prefixIndex: number = prefixStack[prefixStack.length - 1]; + prefixIndexDelta = i - prefixIndex; + prefixLength = strings[prefixIndex].length; + } + + const suffix: string = value.slice(prefixLength); + writer.writeVarint(prefixIndexDelta); + writer.writeVarint(suffix.length); + + const encodedSuffix: Uint8Array = TEXT_ENCODER.encode(suffix); + suffixes.push(encodedSuffix); + blobByteLength += encodedSuffix.length; + + prefixStack.push(i); + } + + writer.writeVarint(blobByteLength); + for (const suffix of suffixes) { + writer.writeBytes(suffix); + } +} + +/** + * Reads a string table written by {@link writeStringTable}. + * + * @beta + */ +export function readStringTable(reader: BinaryReader): string[] { + // Each entry contributes at least two varint bytes to the header that follows. + const count: number = reader.readCount(2); + const prefixIndexDeltas: Uint32Array = new Uint32Array(count); + const suffixLengths: Uint32Array = new Uint32Array(count); + + for (let i: number = 0; i < count; ++i) { + prefixIndexDeltas[i] = reader.readVarint(); + suffixLengths[i] = reader.readVarint(); + } + + const blobByteLength: number = reader.readVarint(); + // A single decode of the whole blob; every entry below is a `substring` of the result. + const blob: string = TEXT_DECODER.decode(reader.readBytes(blobByteLength)); + + const strings: string[] = new Array(count); + let offset: number = 0; + for (let i: number = 0; i < count; ++i) { + const suffixLength: number = suffixLengths[i]; + const suffix: string = blob.substring(offset, offset + suffixLength); + offset += suffixLength; + + const prefixIndexDelta: number = prefixIndexDeltas[i]; + strings[i] = prefixIndexDelta === 0 ? suffix : strings[i - prefixIndexDelta] + suffix; + } + + if (offset !== blob.length) { + throw new Error( + `String table blob was not fully consumed: ${offset} of ${blob.length} characters were used` + ); + } + + return strings; +} diff --git a/libraries/resolver-cache/src/computeContextHashes.ts b/libraries/resolver-cache/src/computeContextHashes.ts new file mode 100644 index 00000000000..f3d696e923a --- /dev/null +++ b/libraries/resolver-cache/src/computeContextHashes.ts @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { IResolverCacheFile, ISerializedResolveContext } from './types'; + +/** + * A digest function, for example a wrapper around `node:crypto`. + * + * @remarks + * Supplying the digest keeps this package free of a runtime dependency on `node:crypto`, so that + * bundlers can include the decoder without pulling in Node built-ins. + * + * @beta + */ +export type HashFunction = (data: Uint8Array) => Uint8Array; + +const TEXT_ENCODER: TextEncoder = new TextEncoder(); + +/** + * Partitions the dependency graph into strongly connected components using an iterative Tarjan + * traversal, returning the components in reverse topological order so that every component is + * emitted only after all of the components it depends upon. + * + * @remarks + * Cycles are not hypothetical: real PNPM lockfiles contain mutually dependent packages, so a naive + * post-order Merkle recursion would either recurse forever or produce an order-dependent hash. + * The traversal is iterative because dependency chains can be deeper than the JavaScript stack. + */ +function findStronglyConnectedComponents(adjacency: readonly number[][]): number[][] { + const count: number = adjacency.length; + const index: Int32Array = new Int32Array(count).fill(-1); + const lowLink: Int32Array = new Int32Array(count); + const onStack: Uint8Array = new Uint8Array(count); + const tarjanStack: number[] = []; + const components: number[][] = []; + + let nextIndex: number = 0; + + for (let start: number = 0; start < count; ++start) { + if (index[start] !== -1) { + continue; + } + + // Each frame is [node, nextEdgeToVisit]. + const callStack: number[][] = [[start, 0]]; + index[start] = lowLink[start] = nextIndex++; + tarjanStack.push(start); + onStack[start] = 1; + + while (callStack.length > 0) { + const frame: number[] = callStack[callStack.length - 1]; + const node: number = frame[0]; + const edges: number[] = adjacency[node]; + + if (frame[1] < edges.length) { + const next: number = edges[frame[1]++]; + if (index[next] === -1) { + index[next] = lowLink[next] = nextIndex++; + tarjanStack.push(next); + onStack[next] = 1; + callStack.push([next, 0]); + } else if (onStack[next]) { + lowLink[node] = Math.min(lowLink[node], index[next]); + } + continue; + } + + callStack.pop(); + if (callStack.length > 0) { + const parent: number = callStack[callStack.length - 1][0]; + lowLink[parent] = Math.min(lowLink[parent], lowLink[node]); + } + + if (lowLink[node] === index[node]) { + const component: number[] = []; + for (;;) { + const member: number = tarjanStack.pop()!; + onStack[member] = 0; + component.push(member); + if (member === node) { + break; + } + } + components.push(component); + } + } + } + + return components; +} + +function toHex(bytes: Uint8Array): string { + let result: string = ''; + for (let i: number = 0; i < bytes.length; ++i) { + result += bytes[i].toString(16).padStart(2, '0'); + } + return result; +} + +/** + * Computes a Merkle hash for every context in the cache. + * + * @remarks + * The preimage of a context hash consists only of information derived from the lockfile: the root + * path and package name of the context, the dependency keys it declares, and the hashes of the + * contexts those keys resolve to. File contents, `package.json` contents, timestamps, and build + * outputs are deliberately excluded, because Rush already tracks those through the build graph. + * As a result these hashes can be computed from a checkout that has never been installed. + * + * Contexts that participate in a dependency cycle are condensed into a strongly connected + * component and hashed as a unit, so that every member of the cycle observes the same content. + * + * @param cache - The graph to hash + * @param hashFn - The digest function to use + * @returns Raw digest bytes for each context, parallel to `cache.contexts` + * + * @beta + */ +export function computeContextHashes(cache: IResolverCacheFile, hashFn: HashFunction): Uint8Array[] { + const { contexts } = cache; + const adjacency: number[][] = contexts.map((context: ISerializedResolveContext) => + context.deps ? Array.from(new Set(Object.values(context.deps))) : [] + ); + + for (const edges of adjacency) { + for (const target of edges) { + if (!Number.isInteger(target) || target < 0 || target >= contexts.length) { + throw new Error(`Dependency ordinal ${target} is out of range`); + } + } + } + + const hashes: (Uint8Array | undefined)[] = new Array(contexts.length); + + for (const component of findStronglyConnectedComponents(adjacency)) { + // Sort by root path so that the component digest does not depend on traversal order. + const members: number[] = component + .slice() + .sort((x: number, y: number) => (contexts[x].root < contexts[y].root ? -1 : 1)); + + const positionInComponent: Map = new Map(); + for (let i: number = 0; i < members.length; ++i) { + positionInComponent.set(members[i], i); + } + + const parts: string[] = []; + for (const member of members) { + const context: ISerializedResolveContext = contexts[member]; + parts.push(context.root, context.name); + + const deps: [string, number][] = Object.entries(context.deps ?? {}).sort( + (x: [string, number], y: [string, number]) => (x[0] < y[0] ? -1 : 1) + ); + for (const [key, target] of deps) { + const cyclePosition: number | undefined = positionInComponent.get(target); + if (cyclePosition !== undefined) { + // Refer to cycle members positionally; their hashes are not yet available, and using the + // position captures the shape of the cycle without introducing a circular dependency. + parts.push(key, `cycle:${cyclePosition}`); + } else { + const targetHash: Uint8Array | undefined = hashes[target]; + if (!targetHash) { + throw new Error( + 'Internal error: a dependency outside the current component has not been hashed yet' + ); + } + parts.push(key, toHex(targetHash)); + } + } + } + + const componentDigest: string = parts.join('\u0000'); + for (const member of members) { + hashes[member] = hashFn( + TEXT_ENCODER.encode(`${componentDigest}\u0000\u0000${contexts[member].root}`) + ); + } + } + + return hashes as Uint8Array[]; +} diff --git a/libraries/resolver-cache/src/index.ts b/libraries/resolver-cache/src/index.ts new file mode 100644 index 00000000000..fd7ec4be482 --- /dev/null +++ b/libraries/resolver-cache/src/index.ts @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * A dedicated binary format for the Rush resolver cache, along with helpers for hashing and + * slicing the dependency graph it describes. + * + * @remarks + * The format is optimized for a single linear decode. Strings are stored once, in a + * lexicographically sorted front-coded table whose suffixes share a single UTF-8 blob, and all + * integers are variable-length encoded as deltas against a nearby value. + * + * @packageDocumentation + */ + +export { + decodeResolverCache, + encodeResolverCache, + isResolverCacheBinary, + RESOLVER_CACHE_FORMAT_VERSION, + RESOLVER_CACHE_MAGIC, + type IEncodeResolverCacheOptions +} from './ResolverCacheCodec'; + +export { computeContextHashes, type HashFunction } from './computeContextHashes'; + +export { sliceResolverCache, type IResolverCacheSlice } from './sliceResolverCache'; + +export { parseResolverCache } from './parseResolverCache'; + +export { + ResolverCacheHashAlgorithm, + type IHashedResolverCacheFile, + type IResolverCacheFile, + type ISerializedResolveContext +} from './types'; diff --git a/libraries/resolver-cache/src/parseResolverCache.ts b/libraries/resolver-cache/src/parseResolverCache.ts new file mode 100644 index 00000000000..41ef22cf171 --- /dev/null +++ b/libraries/resolver-cache/src/parseResolverCache.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { decodeResolverCache, isResolverCacheBinary } from './ResolverCacheCodec'; +import { ResolverCacheHashAlgorithm, type IHashedResolverCacheFile, type IResolverCacheFile } from './types'; + +/** + * Parses a resolver cache from either the binary format or the legacy monolithic JSON format. + * + * @remarks + * The two formats are distinguished by the binary magic, not by file extension, so a consumer can + * attempt the scoped per-project binary file and transparently fall back to the workspace-wide JSON + * cache without knowing which one it was handed. + * + * @beta + */ +export function parseResolverCache(data: Uint8Array | string): IHashedResolverCacheFile { + if (typeof data !== 'string' && isResolverCacheBinary(data)) { + return decodeResolverCache(data); + } + + const text: string = typeof data === 'string' ? data : new TextDecoder('utf-8', { fatal: true }).decode(data); + const parsed: IResolverCacheFile = JSON.parse(text); + + if (typeof parsed?.basePath !== 'string' || !Array.isArray(parsed?.contexts)) { + throw new Error('The resolver cache JSON file is missing a "basePath" or "contexts" property'); + } + + return { + basePath: parsed.basePath, + contexts: parsed.contexts, + hashAlgorithm: ResolverCacheHashAlgorithm.None, + hashes: [] + }; +} diff --git a/libraries/resolver-cache/src/sliceResolverCache.ts b/libraries/resolver-cache/src/sliceResolverCache.ts new file mode 100644 index 00000000000..21723e8e94d --- /dev/null +++ b/libraries/resolver-cache/src/sliceResolverCache.ts @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { IResolverCacheFile, ISerializedResolveContext } from './types'; + +/** + * The result of {@link sliceResolverCache}. + * + * @beta + */ +export interface IResolverCacheSlice { + /** + * A cache containing only the contexts reachable from the requested roots, with all ordinals + * remapped to the new, smaller index space. + */ + cache: IResolverCacheFile; + /** + * The ordinal in the original cache that each context in the slice came from. + */ + originalOrdinals: number[]; +} + +/** + * Produces the slice of a resolver cache that is visible to a set of root contexts. + * + * @remarks + * Computing a slice is a reachability filter followed by an index remap; no resolution decisions + * are revisited. Reachability must follow every dependency edge, including edges that point at + * workspace projects, because a project can resolve modules through its workspace dependencies. + * + * A slice is strictly more precise than the whole-workspace cache: a project cannot resolve a + * package that is not reachable from it, so unrelated contexts are absent rather than merely + * unused. + * + * @beta + */ +export function sliceResolverCache( + cache: IResolverCacheFile, + rootOrdinals: Iterable +): IResolverCacheSlice { + const { basePath, contexts } = cache; + const reachable: Uint8Array = new Uint8Array(contexts.length); + const queue: number[] = []; + + for (const rootOrdinal of rootOrdinals) { + if (!Number.isInteger(rootOrdinal) || rootOrdinal < 0 || rootOrdinal >= contexts.length) { + throw new Error(`Root ordinal ${rootOrdinal} is out of range`); + } + if (!reachable[rootOrdinal]) { + reachable[rootOrdinal] = 1; + queue.push(rootOrdinal); + } + } + + while (queue.length > 0) { + const ordinal: number = queue.pop()!; + const deps: Record | undefined = contexts[ordinal].deps; + if (!deps) { + continue; + } + for (const target of Object.values(deps)) { + if (target < 0 || target >= contexts.length) { + throw new Error(`Dependency ordinal ${target} is out of range`); + } + if (!reachable[target]) { + reachable[target] = 1; + queue.push(target); + } + } + } + + // Preserve the relative order of the original cache so that ordinal deltas stay small. + const originalOrdinals: number[] = []; + const remapped: Int32Array = new Int32Array(contexts.length).fill(-1); + for (let ordinal: number = 0; ordinal < contexts.length; ++ordinal) { + if (reachable[ordinal]) { + remapped[ordinal] = originalOrdinals.length; + originalOrdinals.push(ordinal); + } + } + + const slicedContexts: ISerializedResolveContext[] = originalOrdinals.map((ordinal: number) => { + const context: ISerializedResolveContext = contexts[ordinal]; + const sliced: ISerializedResolveContext = { root: context.root, name: context.name }; + + if (context.deps) { + const deps: Record = {}; + for (const [key, target] of Object.entries(context.deps)) { + deps[key] = remapped[target]; + } + sliced.deps = deps; + } + + if (context.dirInfoFiles?.length) { + sliced.dirInfoFiles = context.dirInfoFiles; + } + + return sliced; + }); + + return { + cache: { basePath, contexts: slicedContexts }, + originalOrdinals + }; +} diff --git a/libraries/resolver-cache/src/test/ResolverCacheCodec.test.ts b/libraries/resolver-cache/src/test/ResolverCacheCodec.test.ts new file mode 100644 index 00000000000..160692ce0e5 --- /dev/null +++ b/libraries/resolver-cache/src/test/ResolverCacheCodec.test.ts @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { decodeResolverCache, encodeResolverCache, isResolverCacheBinary } from '../ResolverCacheCodec'; +import { parseResolverCache } from '../parseResolverCache'; +import { ResolverCacheHashAlgorithm, type IResolverCacheFile } from '../types'; + +const SAMPLE: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { + root: 'common/temp/default/node_modules/.pnpm/lodash@4.17.21/node_modules/lodash', + name: 'lodash' + }, + { + root: 'common/temp/default/node_modules/.pnpm/react@18.2.0/node_modules/react', + name: 'react', + deps: { lodash: 0 } + }, + { + root: 'libraries/example', + name: '@scope/example', + deps: { lodash: 0, react: 1 }, + dirInfoFiles: ['lib/esm/package.json'] + } + ] +}; + +describe('ResolverCacheCodec', () => { + it('round-trips a cache without hashes', () => { + const encoded: Uint8Array = encodeResolverCache({ cache: SAMPLE }); + + expect(isResolverCacheBinary(encoded)).toBe(true); + const decoded: IResolverCacheFile = decodeResolverCache(encoded); + expect(decoded.basePath).toEqual(SAMPLE.basePath); + expect(decoded.contexts).toEqual(SAMPLE.contexts); + }); + + it('round-trips a cache with hashes', () => { + const hashes: Uint8Array[] = SAMPLE.contexts.map((_context, index: number) => + new Uint8Array(32).fill(index + 1) + ); + + const decoded = decodeResolverCache(encodeResolverCache({ cache: SAMPLE, hashes, scoped: true })); + + expect(decoded.hashAlgorithm).toEqual(ResolverCacheHashAlgorithm.Sha256); + expect(decoded.hashes.map((hash: Uint8Array) => Array.from(hash))).toEqual( + hashes.map((hash: Uint8Array) => Array.from(hash)) + ); + }); + + it('is substantially smaller than the equivalent JSON', () => { + const encoded: Uint8Array = encodeResolverCache({ cache: SAMPLE }); + const json: Uint8Array = new TextEncoder().encode(JSON.stringify(SAMPLE)); + + expect(encoded.length).toBeLessThan(json.length); + }); + + it('preserves non-ASCII package names', () => { + const cache: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'packages/\u{1F600}a', name: '\u{1F600}a' }, + { root: 'packages/\u{1F601}b', name: '\u{1F601}b' }, + { root: 'packages/\u00e9', name: '\u00e9', deps: { '\u{1F600}a': 0 } } + ] + }; + + expect(decodeResolverCache(encodeResolverCache({ cache })).contexts).toEqual(cache.contexts); + }); + + it('does not merge distinct contexts that share a package name and version', () => { + // A PNPM injected dependency and the workspace project it was copied from share a name and + // version but resolve their own dependencies differently. Merging them would silently change + // module resolution. + const cache: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'common/temp/a/node_modules/.pnpm/typescript@4.9.5/node_modules/typescript', name: 'typescript' }, + { root: 'common/temp/a/node_modules/.pnpm/typescript@5.4.5/node_modules/typescript', name: 'typescript' }, + { root: 'eslint/eslint-config', name: '@rushstack/eslint-config', deps: { typescript: 0 } }, + { + root: 'common/temp/a/node_modules/.pnpm/file+eslint+eslint-config_typescript@5.4.5/node_modules/@rushstack/eslint-config', + name: '@rushstack/eslint-config', + deps: { typescript: 1 } + } + ] + }; + + const decoded: IResolverCacheFile = decodeResolverCache(encodeResolverCache({ cache })); + + expect(decoded.contexts).toHaveLength(4); + expect(decoded.contexts[2].deps).toEqual({ typescript: 0 }); + expect(decoded.contexts[3].deps).toEqual({ typescript: 1 }); + }); + + it('rejects a buffer with the wrong magic', () => { + expect(() => decodeResolverCache(new Uint8Array([1, 2, 3, 4]))).toThrowErrorMatchingInlineSnapshot( + `"The buffer is not a resolver cache binary file"` + ); + }); + + it('rejects a hash count that does not match the context count', () => { + expect(() => + encodeResolverCache({ cache: SAMPLE, hashes: [new Uint8Array(32)] }) + ).toThrowErrorMatchingInlineSnapshot( + `"Expected 3 hash(es) to match the context count, but received 1"` + ); + }); +}); + +describe('parseResolverCache', () => { + it('reads the binary format', () => { + const parsed = parseResolverCache(encodeResolverCache({ cache: SAMPLE })); + expect(parsed.contexts).toEqual(SAMPLE.contexts); + }); + + it('falls back to the legacy monolithic JSON format', () => { + const parsed = parseResolverCache(JSON.stringify(SAMPLE)); + expect(parsed.basePath).toEqual(SAMPLE.basePath); + expect(parsed.contexts).toEqual(SAMPLE.contexts); + expect(parsed.hashAlgorithm).toEqual(ResolverCacheHashAlgorithm.None); + }); + + it('falls back when handed JSON as bytes', () => { + const parsed = parseResolverCache(new TextEncoder().encode(JSON.stringify(SAMPLE))); + expect(parsed.contexts).toEqual(SAMPLE.contexts); + }); + + it('rejects JSON that is not a resolver cache', () => { + expect(() => parseResolverCache('{"hello":"world"}')).toThrowErrorMatchingInlineSnapshot( + `"The resolver cache JSON file is missing a \\"basePath\\" or \\"contexts\\" property"` + ); + }); +}); + +describe('decodeResolverCache corruption handling', () => { + it('rejects an implausible context count instead of preallocating', () => { + const encoded: Uint8Array = encodeResolverCache({ + cache: { basePath: '/repo/', contexts: [{ root: 'a', name: 'a' }] } + }); + + // Truncating the buffer leaves the declared context count larger than the remaining bytes + // could possibly describe, which is exactly the condition `readCount` exists to reject before + // any storage is preallocated from the declared count. + expect(() => decodeResolverCache(encoded.subarray(0, encoded.length - 1))).toThrow( + /Declared item count/ + ); + }); + + it('rejects a buffer that does not start with the magic', () => { + expect(() => decodeResolverCache(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toThrow( + 'not a resolver cache binary file' + ); + }); +}); diff --git a/libraries/resolver-cache/src/test/StringTable.test.ts b/libraries/resolver-cache/src/test/StringTable.test.ts new file mode 100644 index 00000000000..1de87b74f48 --- /dev/null +++ b/libraries/resolver-cache/src/test/StringTable.test.ts @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { BinaryReader } from '../BinaryReader'; +import { BinaryWriter } from '../BinaryWriter'; +import { readStringTable, StringTableBuilder, writeStringTable } from '../StringTable'; + +function roundTrip(values: readonly string[]): { strings: string[]; byteLength: number } { + const builder: StringTableBuilder = new StringTableBuilder(); + for (const value of values) { + builder.add(value); + } + const finalized: readonly string[] = builder.finalize(); + + const writer: BinaryWriter = new BinaryWriter(64); + writeStringTable(writer, finalized); + const encoded: Uint8Array = writer.toUint8Array(); + + return { strings: readStringTable(new BinaryReader(encoded)), byteLength: encoded.length }; +} + +describe('StringTable', () => { + it('round-trips a sorted table', () => { + const values: string[] = ['beta', 'alpha', 'alphabet', 'alpine']; + const { strings } = roundTrip(values); + + for (const value of values) { + expect(strings).toContain(value); + } + }); + + it('assigns stable indices for every added string', () => { + const builder: StringTableBuilder = new StringTableBuilder(); + const values: string[] = ['a/b/c', 'a/b/d', 'a/e']; + for (const value of values) { + builder.add(value); + } + const finalized: readonly string[] = builder.finalize(); + + for (const value of values) { + expect(finalized[builder.getIndex(value)]).toEqual(value); + } + }); + + it('shares long directory prefixes between siblings', () => { + const prefix: string = 'common/temp/default/node_modules/.pnpm/'; + const values: string[] = []; + for (let i: number = 0; i < 64; ++i) { + values.push(`${prefix}package-${i}@1.0.0/node_modules/package-${i}`); + } + + const { byteLength, strings } = roundTrip(values); + const naiveByteLength: number = values.reduce( + (total: number, value: string) => total + value.length + 2, + 0 + ); + + for (const value of values) { + expect(strings).toContain(value); + } + expect(byteLength).toBeLessThan(naiveByteLength / 2); + }); + + it('never splits a surrogate pair across a prefix boundary', () => { + // These two strings share the leading high surrogate of their first astral character, so a + // naive common-prefix computation would emit a lone surrogate that cannot be encoded as UTF-8. + const values: string[] = ['x\u{1F600}a', 'x\u{1F601}b']; + const { strings } = roundTrip(values); + + for (const value of values) { + expect(strings).toContain(value); + } + }); + + it('rejects reads of strings that were never added', () => { + const builder: StringTableBuilder = new StringTableBuilder(); + builder.add('present'); + builder.finalize(); + + expect(() => builder.getIndex('absent')).toThrowErrorMatchingInlineSnapshot( + `"The string \\"absent\\" was not added to the string table"` + ); + }); + + it('rejects additions after finalization', () => { + const builder: StringTableBuilder = new StringTableBuilder(); + builder.finalize(); + + expect(() => builder.add('late')).toThrowErrorMatchingInlineSnapshot( + `"Cannot add strings after the table has been finalized"` + ); + }); +}); + +describe('BinaryWriter', () => { + it('round-trips varints across byte-length boundaries', () => { + const values: number[] = [0, 1, 127, 128, 16383, 16384, 2 ** 31, Number.MAX_SAFE_INTEGER]; + const writer: BinaryWriter = new BinaryWriter(4); + for (const value of values) { + writer.writeVarint(value); + } + + const reader: BinaryReader = new BinaryReader(writer.toUint8Array()); + expect(values.map(() => reader.readVarint())).toEqual(values); + expect(reader.atEnd).toBe(true); + }); + + it('round-trips signed varints', () => { + const values: number[] = [0, -1, 1, -64, 64, -100000, 100000]; + const writer: BinaryWriter = new BinaryWriter(4); + for (const value of values) { + writer.writeSignedVarint(value); + } + + const reader: BinaryReader = new BinaryReader(writer.toUint8Array()); + expect(values.map(() => reader.readSignedVarint())).toEqual(values); + }); + + it('encodes small signed values in a single byte', () => { + const writer: BinaryWriter = new BinaryWriter(4); + writer.writeSignedVarint(-63); + writer.writeSignedVarint(63); + expect(writer.length).toEqual(2); + }); + + it('rejects negative unsigned varints', () => { + expect(() => new BinaryWriter().writeVarint(-1)).toThrowErrorMatchingInlineSnapshot( + `"Cannot encode -1 as an unsigned varint"` + ); + }); +}); + +describe('BinaryReader', () => { + it('reports truncated buffers', () => { + expect(() => new BinaryReader(new Uint8Array(1)).readBytes(4)).toThrowErrorMatchingInlineSnapshot( + `"Unexpected end of buffer: needed 4 byte(s) at offset 0 but only 1 remain"` + ); + }); +}); diff --git a/libraries/resolver-cache/src/test/computeContextHashes.test.ts b/libraries/resolver-cache/src/test/computeContextHashes.test.ts new file mode 100644 index 00000000000..575f1b06c35 --- /dev/null +++ b/libraries/resolver-cache/src/test/computeContextHashes.test.ts @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { createHash } from 'node:crypto'; + +import { computeContextHashes } from '../computeContextHashes'; +import { sliceResolverCache, type IResolverCacheSlice } from '../sliceResolverCache'; +import type { IResolverCacheFile } from '../types'; + +function sha256(data: Uint8Array): Uint8Array { + return new Uint8Array(createHash('sha256').update(data).digest()); +} + +function hashesOf(cache: IResolverCacheFile): string[] { + return computeContextHashes(cache, sha256).map((hash: Uint8Array) => Buffer.from(hash).toString('hex')); +} + +describe('computeContextHashes', () => { + it('produces a distinct hash per context', () => { + const cache: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a' }, + { root: 'b', name: 'b', deps: { a: 0 } }, + { root: 'c', name: 'c', deps: { a: 0, b: 1 } } + ] + }; + + const hashes: string[] = hashesOf(cache); + expect(new Set(hashes).size).toEqual(3); + expect(hashes[0]).toHaveLength(64); + }); + + it('propagates a change in a transitive dependency', () => { + const before: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: '.pnpm/leaf@1.0.0/node_modules/leaf', name: 'leaf' }, + { root: 'middle', name: 'middle', deps: { leaf: 0 } }, + { root: 'top', name: 'top', deps: { middle: 1 } } + ] + }; + const after: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: '.pnpm/leaf@1.0.1/node_modules/leaf', name: 'leaf' }, + { root: 'middle', name: 'middle', deps: { leaf: 0 } }, + { root: 'top', name: 'top', deps: { middle: 1 } } + ] + }; + + expect(hashesOf(after)[2]).not.toEqual(hashesOf(before)[2]); + }); + + it('is insensitive to the ordering of dependency keys', () => { + const forward: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a' }, + { root: 'b', name: 'b' }, + { root: 'c', name: 'c', deps: { a: 0, b: 1 } } + ] + }; + const reversed: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a' }, + { root: 'b', name: 'b' }, + { root: 'c', name: 'c', deps: { b: 1, a: 0 } } + ] + }; + + expect(hashesOf(reversed)).toEqual(hashesOf(forward)); + }); + + it('terminates on dependency cycles and hashes the cycle as a unit', () => { + const cache: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a', deps: { b: 1 } }, + { root: 'b', name: 'b', deps: { a: 0 } }, + { root: 'c', name: 'c', deps: { a: 0 } } + ] + }; + + const hashes: string[] = hashesOf(cache); + expect(new Set(hashes).size).toEqual(3); + }); + + it('detects a change inside a dependency cycle', () => { + const before: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a', deps: { b: 1 } }, + { root: 'b@1.0.0', name: 'b', deps: { a: 0 } } + ] + }; + const after: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a', deps: { b: 1 } }, + { root: 'b@1.0.1', name: 'b', deps: { a: 0 } } + ] + }; + + expect(hashesOf(after)[0]).not.toEqual(hashesOf(before)[0]); + }); + + it('handles chains deeper than the call stack', () => { + const contexts = [{ root: 'n0', name: 'n0' }]; + for (let i: number = 1; i < 50000; ++i) { + contexts.push({ root: `n${i}`, name: `n${i}`, deps: { [`n${i - 1}`]: i - 1 } } as never); + } + + expect(() => computeContextHashes({ basePath: '/repo/', contexts }, sha256)).not.toThrow(); + }); + + it('gives injected copies a different hash than their source project', () => { + // Same package name, different resolution: the injected copy sees typescript 5.4.5 while the + // workspace project sees 4.9.5. + const cache: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: '.pnpm/typescript@4.9.5/node_modules/typescript', name: 'typescript' }, + { root: '.pnpm/typescript@5.4.5/node_modules/typescript', name: 'typescript' }, + { root: 'eslint/eslint-config', name: 'config', deps: { typescript: 0 } }, + { root: '.pnpm/file+eslint+eslint-config/node_modules/config', name: 'config', deps: { typescript: 1 } } + ] + }; + + const hashes: string[] = hashesOf(cache); + expect(hashes[2]).not.toEqual(hashes[3]); + }); + + it('rejects out-of-range dependency ordinals', () => { + expect(() => + computeContextHashes( + { basePath: '/repo/', contexts: [{ root: 'a', name: 'a', deps: { b: 7 } }] }, + sha256 + ) + ).toThrowErrorMatchingInlineSnapshot(`"Dependency ordinal 7 is out of range"`); + }); +}); + +describe('sliceResolverCache', () => { + const cache: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'shared', name: 'shared' }, + { root: 'unrelated', name: 'unrelated' }, + { root: 'lib', name: 'lib', deps: { shared: 0 } }, + { root: 'app', name: 'app', deps: { lib: 2 } } + ] + }; + + it('keeps only the reachable contexts and remaps ordinals', () => { + const { cache: sliced, originalOrdinals }: IResolverCacheSlice = sliceResolverCache(cache, [3]); + + expect(sliced.contexts.map((context) => context.root)).toEqual(['shared', 'lib', 'app']); + expect(originalOrdinals).toEqual([0, 2, 3]); + expect(sliced.contexts[2].deps).toEqual({ lib: 1 }); + expect(sliced.contexts[1].deps).toEqual({ shared: 0 }); + }); + + it('preserves the hash of every retained context', () => { + const fullHashes: string[] = hashesOf(cache); + const { cache: sliced, originalOrdinals }: IResolverCacheSlice = sliceResolverCache(cache, [3]); + const slicedHashes: string[] = hashesOf(sliced); + + expect(slicedHashes).toEqual(originalOrdinals.map((ordinal: number) => fullHashes[ordinal])); + }); + + it('tolerates cycles', () => { + const cyclic: IResolverCacheFile = { + basePath: '/repo/', + contexts: [ + { root: 'a', name: 'a', deps: { b: 1 } }, + { root: 'b', name: 'b', deps: { a: 0 } }, + { root: 'c', name: 'c' } + ] + }; + + expect(sliceResolverCache(cyclic, [0]).cache.contexts).toHaveLength(2); + }); + + it('rejects an out-of-range root', () => { + expect(() => sliceResolverCache(cache, [9])).toThrowErrorMatchingInlineSnapshot( + `"Root ordinal 9 is out of range"` + ); + }); +}); diff --git a/libraries/resolver-cache/src/types.ts b/libraries/resolver-cache/src/types.ts new file mode 100644 index 00000000000..db0ffe53f21 --- /dev/null +++ b/libraries/resolver-cache/src/types.ts @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * Information about a local or installed npm package. + * + * @remarks + * This is the same shape as the legacy monolithic JSON resolver cache, so that consumers can treat + * the binary and JSON representations interchangeably. + * + * @beta + */ +export interface ISerializedResolveContext { + /** + * The path to the root folder of this context, relative to {@link IResolverCacheFile.basePath}. + * This path is normalized to use `/` as the separator and should not end with a trailing `/`. + * + * @remarks + * This path is the sole identity of a context. Two contexts that share a package name and version + * but differ in root path are distinct and must never be merged; in particular, a PNPM injected + * dependency resolves its own dependencies within the consuming subspace and therefore has a + * different dependency graph than the workspace project it was copied from. + */ + root: string; + /** + * The name of this package. Used to inject a self-reference into the dependency map. + */ + name: string; + /** + * Map of declared dependencies (if any) to the ordinal of the corresponding context. + */ + deps?: Record; + /** + * Set of relative paths to nested `package.json` files within this context. + * These paths are normalized to use `/` as the separator and should not begin with a leading `./`. + */ + dirInfoFiles?: string[]; +} + +/** + * The deserialized form of a resolver cache, whether it was read from the binary format or from the + * legacy monolithic JSON file. + * + * @beta + */ +export interface IResolverCacheFile { + /** + * The base path. All paths in context entries are prefixed by this path. + */ + basePath: string; + /** + * The ordered list of all contexts in the cache. + */ + contexts: ISerializedResolveContext[]; +} + +/** + * Identifies the digest algorithm used for the per-context Merkle hashes. + * + * @beta + */ +export const enum ResolverCacheHashAlgorithm { + /** + * The file carries no per-context hashes. + */ + None = 0, + /** + * SHA-256, stored as raw digest bytes (never hex, base64, or truncated). + */ + Sha256 = 1 +} + +/** + * A resolver cache that additionally carries the lockfile-derived Merkle hash of each context. + * + * @remarks + * The hash preimage is purely lockfile-derived: a context's own root path and name, plus the hashes + * of the contexts it resolves its dependencies to. It deliberately excludes file contents, build + * outputs, and timestamps, which Rush tracks separately through the build graph. This keeps the + * hash computable from a bare checkout that has never been installed. + * + * @beta + */ +export interface IHashedResolverCacheFile extends IResolverCacheFile { + /** + * The algorithm used to produce {@link IHashedResolverCacheFile.hashes}. + */ + hashAlgorithm: ResolverCacheHashAlgorithm; + /** + * Raw digest bytes for each context, parallel to + * {@link IResolverCacheFile.contexts}. Empty when the file carries no hashes. + */ + hashes: readonly Uint8Array[]; +} diff --git a/libraries/resolver-cache/tsconfig.json b/libraries/resolver-cache/tsconfig.json new file mode 100644 index 00000000000..dac21d04081 --- /dev/null +++ b/libraries/resolver-cache/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./node_modules/local-node-rig/profiles/default/tsconfig-base.json" +} diff --git a/libraries/rush-lib/package.json b/libraries/rush-lib/package.json index 31ff05552d7..5f98d563752 100644 --- a/libraries/rush-lib/package.json +++ b/libraries/rush-lib/package.json @@ -50,6 +50,7 @@ "@rushstack/npm-check-fork": "workspace:*", "@rushstack/package-deps-hash": "workspace:*", "@rushstack/package-extractor": "workspace:*", + "@rushstack/resolver-cache": "workspace:*", "@rushstack/rig-package": "workspace:*", "@rushstack/rush-pnpm-kit-v10": "workspace:*", "@rushstack/rush-pnpm-kit-v8": "workspace:*", diff --git a/libraries/rush-lib/src/api/ExperimentsConfiguration.ts b/libraries/rush-lib/src/api/ExperimentsConfiguration.ts index 4c179069759..6b866f7049d 100644 --- a/libraries/rush-lib/src/api/ExperimentsConfiguration.ts +++ b/libraries/rush-lib/src/api/ExperimentsConfiguration.ts @@ -178,6 +178,21 @@ export interface IExperimentsJson { */ provideNpmrcCredentialsViaEnvironment?: boolean; + /** + * (UNDER DEVELOPMENT) If true, during installation Rush writes a `dependency-graph.bin` file into + * each project's `.rush/temp` folder. The file contains the slice of the workspace dependency + * graph that is reachable from that project, encoded in the binary resolver cache format, along + * with a lockfile-derived Merkle hash for every context in the slice. + * + * @remarks + * When this experiment is enabled, the dependency graph file replaces `shrinkwrap-deps.json` when + * determining whether a project's dependencies have changed. The hashes it contains are derived + * purely from the lockfile; project file contents remain the responsibility of the build graph. + * + * Only supported when using PNPM workspaces. + */ + useProjectDependencyGraph?: boolean; + /** * If true, Rush may use the experimental Rush reporter system. If omitted or false, * Rush preserves the legacy reporting behavior. diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index 960c313db37..e4ebd99eb6c 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -377,8 +377,23 @@ export class ProjectChangeAnalyzer { const additionalRelativePathsToHash: string[] = []; const globalAdditionalFiles: string[] = []; if (rushConfiguration.isPnpm) { + // When the `useProjectDependencyGraph` experiment is enabled, the scoped dependency graph + // file supersedes `shrinkwrap-deps.json`. It covers strictly more of the lockfile, since it + // carries a Merkle hash of every context the project can resolve rather than a flattened + // list of specifiers, so change detection cannot become less sensitive. + const useProjectDependencyGraph: boolean = + !!rushConfiguration.experimentsConfiguration.configuration.useProjectDependencyGraph; + const { getProjectDependencyGraphFilePathForProject } = useProjectDependencyGraph + ? await import( + /* webpackChunkName: 'ProjectDependencyGraphFile' */ + './pnpm/ProjectDependencyGraphFile' + ) + : { getProjectDependencyGraphFilePathForProject: undefined }; + await Async.forEachAsync(rushConfiguration.projects, async (project: RushConfigurationProject) => { - const projectShrinkwrapFilePath: string = BaseProjectShrinkwrapFile.getFilePathForProject(project); + const projectShrinkwrapFilePath: string = + getProjectDependencyGraphFilePathForProject?.(project) ?? + BaseProjectShrinkwrapFile.getFilePathForProject(project); if (!(await FileSystem.existsAsync(projectShrinkwrapFilePath))) { if (rushConfiguration.subspacesFeatureEnabled) { return; diff --git a/libraries/rush-lib/src/logic/RushConstants.ts b/libraries/rush-lib/src/logic/RushConstants.ts index 2bf85db23db..69591726294 100644 --- a/libraries/rush-lib/src/logic/RushConstants.ts +++ b/libraries/rush-lib/src/logic/RushConstants.ts @@ -267,6 +267,14 @@ export class RushConstants { */ public static readonly projectShrinkwrapFilename: 'shrinkwrap-deps.json' = 'shrinkwrap-deps.json'; + /** + * The name of the file to drop in project-folder/.rush/temp/ containing the slice of the workspace + * dependency graph that is visible to the project, in the binary resolver cache format. When the + * `useProjectDependencyGraph` experiment is enabled, this file replaces `shrinkwrap-deps.json` + * for the purpose of detecting dependency changes. + */ + public static readonly projectDependencyGraphFilename: 'dependency-graph.bin' = 'dependency-graph.bin'; + /** * The value of the "commandKind" property for a bulk command in command-line.json */ diff --git a/libraries/rush-lib/src/logic/installManager/WorkspaceInstallManager.ts b/libraries/rush-lib/src/logic/installManager/WorkspaceInstallManager.ts index b3bb82a1f65..cec39e856b6 100644 --- a/libraries/rush-lib/src/logic/installManager/WorkspaceInstallManager.ts +++ b/libraries/rush-lib/src/logic/installManager/WorkspaceInstallManager.ts @@ -668,6 +668,36 @@ export class WorkspaceInstallManager extends BaseInstallManager { console.log(''); } + /** + * Writes (or removes) the per-project dependency graph files that back the + * `useProjectDependencyGraph` experiment. + */ + private async _updateProjectDependencyGraphFilesAsync( + subspace: Subspace, + tempShrinkwrapFile: BaseShrinkwrapFile + ): Promise { + const { + updateProjectDependencyGraphFilesAsync, + deleteProjectDependencyGraphFilesAsync + }: typeof import('../pnpm/ProjectDependencyGraphFile') = await import( + /* webpackChunkName: 'ProjectDependencyGraphFile' */ + '../pnpm/ProjectDependencyGraphFile' + ); + + const { useProjectDependencyGraph } = this.rushConfiguration.experimentsConfiguration.configuration; + if (!useProjectDependencyGraph || !this.rushConfiguration.isPnpm) { + // Leave no stale artifact behind if the experiment is turned off again. + await deleteProjectDependencyGraphFilesAsync(subspace); + return; + } + + await updateProjectDependencyGraphFilesAsync({ + rushConfiguration: this.rushConfiguration, + subspace, + shrinkwrapFile: tempShrinkwrapFile as PnpmShrinkwrapFile + }); + } + protected async postInstallAsync(subspace: Subspace): Promise { // Grab the temp shrinkwrap, as this was the most recently completed install. It may also be // more up-to-date than the checked-in shrinkwrap since filtered installs are not written back. @@ -688,6 +718,8 @@ export class WorkspaceInstallManager extends BaseInstallManager { }, { concurrency: 10 } ); + + await this._updateProjectDependencyGraphFilesAsync(subspace, tempShrinkwrapFile); } else if (this.rushConfiguration.isPnpm && this.rushConfiguration.pnpmOptions?.useWorkspaces) { // If we're in PNPM workspace mode and PNPM didn't create a shrinkwrap file, // there are no dependencies. Generate empty shrinkwrap files for all projects. diff --git a/libraries/rush-lib/src/logic/pnpm/PnpmDependencyGraph.ts b/libraries/rush-lib/src/logic/pnpm/PnpmDependencyGraph.ts new file mode 100644 index 00000000000..d95f68aee53 --- /dev/null +++ b/libraries/rush-lib/src/logic/pnpm/PnpmDependencyGraph.ts @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'node:path'; + +import { Import } from '@rushstack/node-core-library'; +import type { IResolverCacheFile, ISerializedResolveContext } from '@rushstack/resolver-cache'; + +import { + ShrinkwrapFileMajorVersion, + type IPnpmShrinkwrapDependencyYaml, + type IPnpmShrinkwrapImporterYaml, + type IPnpmVersionSpecifier, + type PnpmShrinkwrapFile +} from './PnpmShrinkwrapFile'; + +const pnpmKitV8: typeof import('@rushstack/rush-pnpm-kit-v8') = Import.lazy( + '@rushstack/rush-pnpm-kit-v8', + require +); +const pnpmKitV9: typeof import('@rushstack/rush-pnpm-kit-v9') = Import.lazy( + '@rushstack/rush-pnpm-kit-v9', + require +); +const pnpmKitV10: typeof import('@rushstack/rush-pnpm-kit-v10') = Import.lazy( + '@rushstack/rush-pnpm-kit-v10', + require +); + +const IS_WINDOWS: boolean = process.platform === 'win32'; + +/** + * Options for {@link buildPnpmResolverCache}. + */ +export interface IBuildPnpmResolverCacheOptions { + /** + * The lockfile to read the dependency graph from. + */ + shrinkwrapFile: PnpmShrinkwrapFile; + /** + * The absolute, slash-normalized folder that contains the lockfile. Importer keys and virtual + * store paths are resolved relative to this folder. + */ + lockfileFolder: string; + /** + * The major version of PNPM that produced the lockfile. This selects the virtual store naming + * scheme, which differs between PNPM 8, 9, and 10. + */ + pnpmMajorVersion: number; + /** + * The absolute, slash-normalized repository root, including a trailing slash. Context root paths + * are stored relative to this folder so that the resulting file is identical regardless of where + * the repository happens to be cloned. + */ + basePath: string; + /** + * Maps importer keys to the `name` field of the corresponding project's `package.json`. Importers + * that are absent fall back to their importer key. + */ + importerNames?: ReadonlyMap; +} + +/** + * A resolver cache built from a lockfile, along with a lookup from context root path to ordinal. + */ +export interface IPnpmResolverCache { + /** + * The graph itself, with contexts in sorted root path order. + */ + cache: IResolverCacheFile; + /** + * Maps the root path of each context to its ordinal in `cache.contexts`. + */ + ordinalByRoot: ReadonlyMap; +} + +interface IRawContext { + root: string; + name: string; + isProject: boolean; + dependencies: [string, IPnpmVersionSpecifier][]; +} + +function getVersionSpecifierString(specifier: IPnpmVersionSpecifier): string { + return typeof specifier === 'string' ? specifier : specifier.version; +} + +function createDepPathToFilename(pnpmMajorVersion: number): (depPath: string) => string { + if (pnpmMajorVersion >= 10) { + // The maximum virtual store directory name length defaults to 60 on Windows and 120 elsewhere. + return (depPath: string) => + pnpmKitV10.dependencyPath.depPathToFilename(depPath, IS_WINDOWS ? 60 : 120); + } + if (pnpmMajorVersion >= 9) { + return (depPath: string) => pnpmKitV9.dependencyPath.depPathToFilename(depPath, 120); + } + return (depPath: string) => pnpmKitV8.dependencyPath.depPathToFilename(depPath); +} + +/** + * Extracts the package name from a lockfile dependency path, e.g. `/@scope/name@1.0.0(peer@2.0.0)`. + */ +function getPackageNameFromKey(key: string): string { + const offset: number = key.startsWith('/') ? 1 : 0; + const versionSeparatorIndex: number = key.indexOf('@', offset + 1); + if (versionSeparatorIndex < 0) { + throw new Error(`Unable to determine the package name for lockfile key ${JSON.stringify(key)}`); + } + return key.slice(offset, versionSeparatorIndex); +} + +/** + * Collects the declared dependencies of an importer or package entry. + * + * @remarks + * `peerDependencies` are deliberately excluded. They are a constraint to be satisfied by the + * package manager rather than an edge in the resolved graph; PNPM records the resolution it chose + * in the peer suffix of the dependency path, which is already part of the context identity. + */ +function collectDependencies( + entry: IPnpmShrinkwrapImporterYaml | IPnpmShrinkwrapDependencyYaml +): [string, IPnpmVersionSpecifier][] { + const dependencies: [string, IPnpmVersionSpecifier][] = []; + for (const collection of [entry.dependencies, entry.optionalDependencies]) { + if (collection) { + for (const [name, specifier] of Object.entries(collection)) { + dependencies.push([name, specifier as IPnpmVersionSpecifier]); + } + } + } + return dependencies; +} + +/** + * Builds a resolver cache describing every context in a PNPM lockfile. + * + * @remarks + * A context is identified solely by its root path on disk. PNPM injected dependencies therefore + * produce contexts that are distinct from the workspace projects they were copied from: an injected + * copy lives under the consuming subspace's virtual store and resolves its own dependencies to + * other injected copies, so its dependency graph genuinely differs from that of the source project. + * The same project can even be injected more than once within a single subspace when consumers + * require different peer resolutions. Merging contexts by package name and version would silently + * change module resolution and must never be done. + */ +export function buildPnpmResolverCache(options: IBuildPnpmResolverCacheOptions): IPnpmResolverCache { + const { shrinkwrapFile, lockfileFolder, pnpmMajorVersion, basePath, importerNames } = options; + if (!basePath.endsWith('/')) { + throw new Error('The basePath must end with a trailing slash'); + } + const depPathToFilename: (depPath: string) => string = createDepPathToFilename(pnpmMajorVersion); + const isV6: boolean = shrinkwrapFile.shrinkwrapFileMajorVersion === ShrinkwrapFileMajorVersion.V6; + + const rawContexts: Map = new Map(); + + function getPackageRoot(key: string, name?: string): string { + const packageName: string = name ?? getPackageNameFromKey(key); + return `${lockfileFolder}/node_modules/.pnpm/${depPathToFilename(key)}/node_modules/${packageName}`; + } + + for (const [importerKey, importer] of shrinkwrapFile.importers) { + const root: string = path.posix.normalize(path.posix.join(lockfileFolder, importerKey)); + rawContexts.set(root, { + root, + name: importerNames?.get(importerKey) ?? importerKey, + isProject: true, + dependencies: collectDependencies(importer) + }); + } + + for (const [packageKey, packageEntry] of shrinkwrapFile.packages) { + const name: string = packageEntry.name ?? getPackageNameFromKey(packageKey); + const root: string = getPackageRoot(packageKey, name); + // A root path collision would mean two different resolutions share a folder, which PNPM does + // not produce; if it ever happened, silently overwriting would corrupt resolution. + if (!rawContexts.has(root)) { + rawContexts.set(root, { + root, + name, + isProject: false, + dependencies: collectDependencies(packageEntry) + }); + } + } + + function resolveDependencyRoot( + owner: IRawContext, + dependencyName: string, + specifier: IPnpmVersionSpecifier + ): string { + const version: string = getVersionSpecifierString(specifier); + + if (version.startsWith('link:')) { + // A `link:` dependency is a symlink relative to the folder that declared it. + const base: string = owner.isProject ? owner.root : lockfileFolder; + return path.posix.normalize(path.posix.join(base, version.slice('link:'.length))); + } + + if (version.startsWith('file:')) { + // An injected dependency is materialized inside this lockfile's virtual store, so it is a + // context in its own right rather than a reference to the source project. + const key: string = shrinkwrapFile.packages.has(version) + ? version + : buildDependencyKey(dependencyName, version); + return getPackageRoot(key, dependencyName); + } + + const key: string = shrinkwrapFile.packages.has(version) + ? version + : buildDependencyKey(dependencyName, version); + return getPackageRoot(key, dependencyName); + } + + function buildDependencyKey(name: string, version: string): string { + return isV6 ? `/${name}@${version}` : `${name}@${version}`; + } + + // Sorting by root path gives the string table long shared prefixes and keeps dependency ordinals + // close to the ordinal of the context that references them, so their deltas stay small. + const sortedRoots: string[] = Array.from(rawContexts.keys()).sort(); + const ordinalByRoot: Map = new Map(); + for (let i: number = 0; i < sortedRoots.length; ++i) { + ordinalByRoot.set(sortedRoots[i], i); + } + + const contexts: ISerializedResolveContext[] = sortedRoots.map((root: string) => { + const rawContext: IRawContext = rawContexts.get(root)!; + // Store paths relative to the repository root so that the encoded file does not depend on the + // absolute location of the clone. + const relativeRoot: string = root.startsWith(basePath) ? root.slice(basePath.length) : root; + const context: ISerializedResolveContext = { root: relativeRoot, name: rawContext.name }; + + let deps: Record | undefined; + for (const [dependencyName, specifier] of rawContext.dependencies) { + const targetRoot: string = resolveDependencyRoot(rawContext, dependencyName, specifier); + const targetOrdinal: number | undefined = ordinalByRoot.get(targetRoot); + if (targetOrdinal !== undefined) { + deps ??= {}; + deps[dependencyName] = targetOrdinal; + } + } + if (deps) { + context.deps = deps; + } + + return context; + }); + + return { cache: { basePath, contexts }, ordinalByRoot }; +} diff --git a/libraries/rush-lib/src/logic/pnpm/ProjectDependencyGraphFile.ts b/libraries/rush-lib/src/logic/pnpm/ProjectDependencyGraphFile.ts new file mode 100644 index 00000000000..3e1b3c6fae7 --- /dev/null +++ b/libraries/rush-lib/src/logic/pnpm/ProjectDependencyGraphFile.ts @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as crypto from 'node:crypto'; +import * as path from 'node:path'; + +import { Async, FileSystem, Path } from '@rushstack/node-core-library'; +import { + computeContextHashes, + encodeResolverCache, + sliceResolverCache, + type IResolverCacheSlice +} from '@rushstack/resolver-cache'; + +import type { RushConfiguration } from '../../api/RushConfiguration'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { Subspace } from '../../api/Subspace'; +import { RushConstants } from '../RushConstants'; +import { buildPnpmResolverCache, type IPnpmResolverCache } from './PnpmDependencyGraph'; +import type { PnpmShrinkwrapFile } from './PnpmShrinkwrapFile'; + +/** + * Options for {@link updateProjectDependencyGraphFilesAsync}. + */ +export interface IUpdateProjectDependencyGraphFilesOptions { + rushConfiguration: RushConfiguration; + subspace: Subspace; + shrinkwrapFile: PnpmShrinkwrapFile; +} + +function sha256(data: Uint8Array): Uint8Array { + return new Uint8Array(crypto.createHash('sha256').update(data).digest()); +} + +/** + * Gets the fully-qualified path to the `/.rush/temp/dependency-graph.bin` file for the + * specified project. + */ +export function getProjectDependencyGraphFilePathForProject(project: RushConfigurationProject): string { + return `${project.projectRushTempFolder}/${RushConstants.projectDependencyGraphFilename}`; +} + +/** + * Writes the scoped dependency graph file for every project in a subspace. + * + * @remarks + * Each project receives only the slice of the workspace graph that is reachable from it. This is + * stricter than a single workspace-wide file: a project's file cannot even describe a package that + * the project is unable to resolve, so an accidental dependency cannot hide behind an unrelated + * entry. Computing a slice is a reachability filter followed by an index remap, so all of the + * resolution work is shared across the whole subspace. + * + * The per-context hashes stored in the file are purely lockfile-derived. Project file contents are + * deliberately out of scope; Rush detects those changes through the build graph instead. + */ +export async function updateProjectDependencyGraphFilesAsync( + options: IUpdateProjectDependencyGraphFilesOptions +): Promise { + const { rushConfiguration, subspace, shrinkwrapFile } = options; + + const projects: ReadonlyArray = subspace.getProjects(); + if (projects.length === 0) { + return; + } + + const lockfileFolder: string = Path.convertToSlashes(subspace.getSubspaceTempFolderPath()); + const basePath: string = `${Path.convertToSlashes(rushConfiguration.rushJsonFolder)}/`; + + const importerNames: Map = new Map(); + const importerKeyByProject: Map = new Map(); + for (const project of projects) { + const importerKey: string = shrinkwrapFile.getImporterKeyByPath(lockfileFolder, project.projectFolder); + importerNames.set(importerKey, project.packageName); + importerKeyByProject.set(project, importerKey); + } + + const { cache, ordinalByRoot }: IPnpmResolverCache = buildPnpmResolverCache({ + shrinkwrapFile, + lockfileFolder, + pnpmMajorVersion: parseInt(rushConfiguration.packageManagerToolVersion, 10) || 8, + basePath, + importerNames + }); + + const hashes: Uint8Array[] = computeContextHashes(cache, sha256); + + await Async.forEachAsync( + projects, + async (project: RushConfigurationProject) => { + const filePath: string = getProjectDependencyGraphFilePathForProject(project); + const importerKey: string = importerKeyByProject.get(project)!; + const projectRoot: string = path.posix.normalize(path.posix.join(lockfileFolder, importerKey)); + const ordinal: number | undefined = ordinalByRoot.get(projectRoot); + + if (ordinal === undefined) { + // The project is not present in the lockfile, e.g. it declares no dependencies. + await FileSystem.deleteFileAsync(filePath, { throwIfNotExists: false }); + return; + } + + const slice: IResolverCacheSlice = sliceResolverCache(cache, [ordinal]); + const encoded: Uint8Array = encodeResolverCache({ + cache: slice.cache, + hashes: slice.originalOrdinals.map((originalOrdinal: number) => hashes[originalOrdinal]), + scoped: true + }); + + await FileSystem.writeFileAsync(filePath, Buffer.from(encoded), { ensureFolderExists: true }); + }, + { concurrency: 10 } + ); +} + +/** + * Deletes the dependency graph files for every project in a subspace. + */ +export async function deleteProjectDependencyGraphFilesAsync(subspace: Subspace): Promise { + await Async.forEachAsync( + subspace.getProjects(), + async (project: RushConfigurationProject) => { + await FileSystem.deleteFileAsync(getProjectDependencyGraphFilePathForProject(project), { + throwIfNotExists: false + }); + }, + { concurrency: 10 } + ); +} diff --git a/libraries/rush-lib/src/schemas/experiments.schema.json b/libraries/rush-lib/src/schemas/experiments.schema.json index de8925a12ce..eeb8833b9b4 100644 --- a/libraries/rush-lib/src/schemas/experiments.schema.json +++ b/libraries/rush-lib/src/schemas/experiments.schema.json @@ -102,6 +102,10 @@ "description": "If true, when using PNPM 10.34.2 through 10.x or PNPM 11.5.3 through versions earlier than 11.6.0, Rush resolves the \"${VAR}\" tokens that appear in credentials and registry URLs in the .npmrc file, instead of relying on PNPM to expand them. Credentials are passed to PNPM using \"npm_config_*\" environment variables and are not written to the generated .npmrc file. PNPM 11.6.0 and newer support URL-scoped \"pnpm_config_//...\" environment variables, which should instead be supplied directly by CI so the trusted environment binds each credential to its registry. Dynamic registry and proxy settings must likewise come from trusted user, global, CLI, or environment configuration.", "type": "boolean" }, + "useProjectDependencyGraph": { + "description": "(UNDER DEVELOPMENT) If true, during installation Rush writes a \"dependency-graph.bin\" file into each project's \".rush/temp\" folder, containing the slice of the workspace dependency graph that is reachable from that project along with a lockfile-derived Merkle hash for every context. This file replaces \"shrinkwrap-deps.json\" when computing whether a project's dependencies have changed. Only supported when using PNPM workspaces.", + "type": "boolean" + }, "useRushReporter": { "description": "If true, Rush may use the experimental Rush reporter system. If omitted or false, Rush preserves the legacy reporting behavior.", "type": "boolean" diff --git a/rush.json b/rush.json index 3169d424bd8..7e52dae94f0 100644 --- a/rush.json +++ b/rush.json @@ -1314,6 +1314,12 @@ "reviewCategory": "libraries", "shouldPublish": true }, + { + "packageName": "@rushstack/resolver-cache", + "projectFolder": "libraries/resolver-cache", + "reviewCategory": "libraries", + "shouldPublish": true + }, { "packageName": "@rushstack/rig-package", "projectFolder": "libraries/rig-package", diff --git a/webpack/webpack-workspace-resolve-plugin/package.json b/webpack/webpack-workspace-resolve-plugin/package.json index 074ca23a0a6..50fb7761db7 100644 --- a/webpack/webpack-workspace-resolve-plugin/package.json +++ b/webpack/webpack-workspace-resolve-plugin/package.json @@ -47,7 +47,8 @@ "@types/node": "*" }, "dependencies": { - "@rushstack/lookup-by-path": "workspace:*" + "@rushstack/lookup-by-path": "workspace:*", + "@rushstack/resolver-cache": "workspace:*" }, "devDependencies": { "@rushstack/heft": "workspace:*", diff --git a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts index 671e43de4a8..83827db0d8f 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/WorkspaceLayoutCache.ts @@ -4,48 +4,7 @@ import { sep as directorySeparator } from 'node:path'; import { LookupByPath, type IPrefixMatch } from '@rushstack/lookup-by-path'; - -/** - * Information about a local or installed npm package. - * @beta - */ -export interface ISerializedResolveContext { - /** - * The path to the root folder of this context. - * This path is normalized to use `/` as the separator and should not end with a trailing `/`. - */ - root: string; - /** - * The name of this package. Used to inject a self-reference into the dependency map. - */ - name: string; - /** - * Map of declared dependencies (if any) to the ordinal of the corresponding context. - */ - deps?: Record; - /** - * Set of relative paths to nested `package.json` files within this context. - * These paths are normalized to use `/` as the separator and should not begin with a leading `./`. - */ - dirInfoFiles?: string[]; -} - -/** - * The serialized form of the cache file. This file is expected to be generated by a separate tool from - * information known to the package manager. Namely, the dependency relationships between packages, and - * all the `package.json` files in the workspace (installed or local). - * @beta - */ -export interface IResolverCacheFile { - /** - * The base path. All paths in context entries are prefixed by this path. - */ - basePath: string; - /** - * The ordered list of all contexts in the cache - */ - contexts: ISerializedResolveContext[]; -} +import type { IResolverCacheFile, ISerializedResolveContext } from '@rushstack/resolver-cache'; /** * A context for resolving dependencies in a workspace. diff --git a/webpack/webpack-workspace-resolve-plugin/src/index.ts b/webpack/webpack-workspace-resolve-plugin/src/index.ts index 61866c3a162..9a3272bb505 100644 --- a/webpack/webpack-workspace-resolve-plugin/src/index.ts +++ b/webpack/webpack-workspace-resolve-plugin/src/index.ts @@ -6,7 +6,14 @@ export { WorkspaceLayoutCache, type IPathNormalizationFunction, type IWorkspaceLayoutCacheOptions, - type IResolveContext, - type ISerializedResolveContext, - type IResolverCacheFile + type IResolveContext } from './WorkspaceLayoutCache'; +export { + loadResolverCacheAsync, + loadResolverCache, + type ILoadResolverCacheOptions +} from './loadResolverCache'; + +// Re-exported so that consumers of this plugin do not need to take a direct dependency on +// `@rushstack/resolver-cache` merely to describe the cache data they pass in. +export type { ISerializedResolveContext, IResolverCacheFile } from '@rushstack/resolver-cache'; diff --git a/webpack/webpack-workspace-resolve-plugin/src/loadResolverCache.ts b/webpack/webpack-workspace-resolve-plugin/src/loadResolverCache.ts new file mode 100644 index 00000000000..24fb4f1428b --- /dev/null +++ b/webpack/webpack-workspace-resolve-plugin/src/loadResolverCache.ts @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { readFile, readFileSync } from 'node:fs'; +import { promisify } from 'node:util'; + +import { parseResolverCache, type IResolverCacheFile } from '@rushstack/resolver-cache'; + +const readFileAsync: (path: string) => Promise = promisify(readFile) as ( + path: string +) => Promise; + +/** + * Options for {@link loadResolverCacheAsync} and {@link loadResolverCache}. + * + * @beta + */ +export interface ILoadResolverCacheOptions { + /** + * Paths to candidate cache files, in order of preference. + * + * @remarks + * The first file that exists is used. Each file may be in either the binary format or the legacy + * monolithic JSON format; the two are distinguished by content, not by file extension. The usual + * configuration is to list the scoped per-project binary file first and the workspace-wide JSON + * cache second, so that the JSON cache acts as a fallback for workspaces that have not enabled + * the newer format. + */ + filePaths: readonly string[]; +} + +function isFileNotFound(error: unknown): boolean { + const code: unknown = (error as NodeJS.ErrnoException | undefined)?.code; + return code === 'ENOENT' || code === 'ENOTDIR' || code === 'EISDIR'; +} + +/** + * Loads the first available resolver cache file, preferring the binary format and falling back to + * the legacy monolithic JSON cache. + * + * @beta + */ +export async function loadResolverCacheAsync( + options: ILoadResolverCacheOptions +): Promise { + const { filePaths } = options; + + for (const filePath of filePaths) { + let data: Buffer; + try { + data = await readFileAsync(filePath); + } catch (error) { + if (isFileNotFound(error)) { + continue; + } + throw error; + } + + return parseResolverCache(data); + } + + throw new Error(`Unable to find a resolver cache file. Tried: ${filePaths.join(', ')}`); +} + +/** + * The synchronous form of {@link loadResolverCacheAsync}. + * + * @beta + */ +export function loadResolverCache(options: ILoadResolverCacheOptions): IResolverCacheFile { + const { filePaths } = options; + + for (const filePath of filePaths) { + let data: Buffer; + try { + data = readFileSync(filePath); + } catch (error) { + if (isFileNotFound(error)) { + continue; + } + throw error; + } + + return parseResolverCache(data); + } + + throw new Error(`Unable to find a resolver cache file. Tried: ${filePaths.join(', ')}`); +}