diff --git a/common/changes/@microsoft/rush/fix-warm-build-cache-writes_2026-09-24.json b/common/changes/@microsoft/rush/fix-warm-build-cache-writes_2026-09-24.json new file mode 100644 index 0000000000..45dfef1643 --- /dev/null +++ b/common/changes/@microsoft/rush/fix-warm-build-cache-writes_2026-09-24.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Allow operations to write build cache entries when their dependencies were not re-run because a previous iteration of a long-lived graph (such as the Rush daemon) produced a trusted result at the same state hash. Dependencies skipped by the user (e.g. `--only`) still block cache writes.", + "type": "patch" + } + ], + "packageName": "@microsoft/rush" +} diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 52f577b98c..c915abf8ca 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -15,7 +15,7 @@ import { } from '@rushstack/terminal'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import { OperationStatus } from './OperationStatus'; +import { OperationStatus, SUCCESS_STATUSES } from './OperationStatus'; import { CobuildLock, type ICobuildCompletedState } from '../cobuild/CobuildLock'; import { OperationBuildCache } from '../buildCache/OperationBuildCache'; import { RushConstants } from '../RushConstants'; @@ -116,7 +116,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } = this.#options; hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph: IOperationGraph, context: IOperationGraphContext) => { - graph.hooks.beforeDeleteResults.tap(PLUGIN_NAME, () => { + // The state hash at which each operation last completed successfully in an iteration of this graph + // in which cache writes were allowed for it (i.e. no dependency had an unknown state). + const trustedStateHashByOperation: Map = new Map(); + + graph.hooks.beforeDeleteResults.tap(PLUGIN_NAME, (operations: ReadonlySet) => { + for (const operation of operations) { + trustedStateHashByOperation.delete(operation); + } // Terminals and cobuild callbacks can retain the entire completed iteration, including other // projects' records. All of this scratch state is rebuilt by beforeExecuteIterationAsync. for (const cacheContext of this.#buildCacheContextByOperation.values()) { @@ -564,8 +571,31 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { switch (record.status) { case OperationStatus.Skipped: { - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; + // Skipping generally means we cannot guarantee integrity, so prevent cache writes in dependents. + // The exception is an operation that was not re-run because a previous iteration of this graph + // produced a trusted result at exactly the same state hash (e.g. a result retained by a + // long-lived graph such as the Rush daemon). Since the state hash of an operation covers the + // state hashes of all of its dependencies, a consumer's cache key fully describes this input. + if ( + blockCacheWrite || + record.operation.enabled === false || + trustedStateHashByOperation.get(operation) !== record.getStateHash() + ) { + blockCacheWrite = true; + trustedStateHashByOperation.delete(operation); + } + break; + } + + default: { + if (!blockCacheWrite && buildCacheContext && SUCCESS_STATUSES.has(record.status)) { + // The outputs of this operation were produced (or restored) in an iteration where cache + // writes were allowed, so they can be trusted by consumers in later iterations as long as + // the state hash is unchanged. + trustedStateHashByOperation.set(operation, record.getStateHash()); + } else { + trustedStateHashByOperation.delete(operation); + } break; } } diff --git a/libraries/rush-lib/src/logic/operations/test/CacheableOperationPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/CacheableOperationPlugin.test.ts new file mode 100644 index 0000000000..a9b3ad395a --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/test/CacheableOperationPlugin.test.ts @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +jest.mock('../../../utilities/Utilities'); +jest.mock('../OperationStateFile'); +jest.mock('../ProjectLogWritable', () => { + const actual = jest.requireActual('../ProjectLogWritable'); + const { TerminalWritable } = jest.requireActual('@rushstack/terminal'); + class MockTerminalWritable extends TerminalWritable { + protected onWriteChunk(): void { + /* noop */ + } + protected onClose(): void { + /* noop */ + } + } + return { + ...actual, + initializeProjectLogFilesAsync: jest.fn(async () => new MockTerminalWritable()) + }; +}); +jest.mock('../OperationMetadataManager', () => { + class MockOperationMetadataManager { + public readonly logFilenameIdentifier: string; + public readonly metadataFolderPath: string = '.rush/temp/operation/mock'; + public readonly stateFile: { state: undefined } = { state: undefined }; + public constructor({ operation }: { operation: { logFilenameIdentifier: string } }) { + this.logFilenameIdentifier = operation.logFilenameIdentifier; + } + public async saveAsync(): Promise { + /* noop */ + } + public async tryRestoreAsync(): Promise { + /* noop */ + } + public tryRestoreStopwatch(originalStopwatch: T): T { + return originalStopwatch; + } + } + return { OperationMetadataManager: MockOperationMetadataManager }; +}); +jest.mock('../../buildCache/OperationBuildCache', () => ({ + OperationBuildCache: { forOperation: jest.fn() } +})); + +import { MockWritable, StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; + +import type { IPhase } from '../../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; +import type { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; +import type { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; +import { PhasedCommandHooks, type IOperationGraphContext } from '../../../pluginFramework/PhasedCommandHooks'; +import type { IInputsSnapshot } from '../../incremental/InputsSnapshot'; +import { OperationBuildCache } from '../../buildCache/OperationBuildCache'; +import { CacheableOperationPlugin } from '../CacheableOperationPlugin'; +import { PhasedOperationPlugin } from '../PhasedOperationPlugin'; +import { OperationGraph } from '../OperationGraph'; +import { Operation } from '../Operation'; +import { OperationStatus } from '../OperationStatus'; +import type { IOperationRunner, IOperationRunnerContext } from '../IOperationRunner'; +import type { IExecutionResult } from '../IOperationExecutionResult'; +import type { OperationExecutionRecord } from '../OperationExecutionRecord'; + +const mockPhase: IPhase = { + name: 'phase', + allowWarningsOnSuccess: false, + associatedParameters: new Set(), + dependencies: { self: new Set(), upstream: new Set() }, + isSynthetic: false, + logFilenameIdentifier: 'phase', + missingScriptBehavior: 'silent' +}; + +class CacheableMockRunner implements IOperationRunner { + public readonly reportTiming: boolean = true; + public readonly silent: boolean = false; + public readonly cacheable: boolean = true; + public readonly warningsAreAllowed: boolean = false; + public readonly isNoOp: boolean = false; + public readonly name: string; + readonly #executions: string[]; + + public constructor(name: string, executions: string[]) { + this.name = name; + this.#executions = executions; + } + + public async executeAsync(context: IOperationRunnerContext): Promise { + this.#executions.push(this.name); + return OperationStatus.Success; + } + + public getConfigHash(): string { + return 'config'; + } +} + +interface ITestGraph { + graph: OperationGraph; + operations: Map; + localHashes: Map; + executions: string[]; + cacheWrites: string[]; + executeAsync(): Promise; +} + +/** + * Creates a linear chain of cacheable operations: names[0] <- names[1] <- ... (each depends on the previous). + */ +async function createTestGraphAsync(names: string[]): Promise { + const executions: string[] = []; + const cacheWrites: string[] = []; + const localHashes: Map = new Map(); + const operations: Map = new Map(); + const projectConfigurations: Map = new Map(); + + let previous: Operation | undefined; + for (const name of names) { + const project: RushConfigurationProject = { + packageName: name, + projectFolder: `/repo/${name}` + } as unknown as RushConfigurationProject; + projectConfigurations.set(project, { + getCacheDisabledReason: () => undefined + } as unknown as RushProjectConfiguration); + const operation: Operation = new Operation({ + runner: new CacheableMockRunner(name, executions), + logFilenameIdentifier: name, + phase: mockPhase, + project + }); + if (previous) { + operation.addDependency(previous); + } + previous = operation; + operations.set(name, operation); + localHashes.set(name, `${name}-v1`); + } + + jest.mocked(OperationBuildCache.forOperation).mockImplementation( + (record) => + ({ + tryRestoreFromCacheAsync: async () => false, + trySetCacheEntryAsync: async () => { + cacheWrites.push(record.operation.associatedProject.packageName); + return true; + } + }) as unknown as OperationBuildCache + ); + + const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); + const hooks: PhasedCommandHooks = new PhasedCommandHooks(); + new PhasedOperationPlugin().apply(hooks); + new CacheableOperationPlugin({ + allowWarningsInSuccessfulBuild: false, + buildCacheConfiguration: { + buildCacheEnabled: true, + cacheWriteEnabled: true + } as unknown as BuildCacheConfiguration, + cobuildConfiguration: undefined, + terminal, + excludeAppleDoubleFiles: false, + useDirectFileTransfersForBuildCache: false + }).apply(hooks); + + const graph: OperationGraph = new OperationGraph(new Set(operations.values()), { + quietMode: true, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [new MockWritable()], + abortController: new AbortController() + }); + await hooks.onGraphCreatedAsync.promise(graph, { + isIncrementalBuildAllowed: true, + projectConfigurations + } as unknown as IOperationGraphContext); + + const inputsSnapshot: IInputsSnapshot = { + hashes: new Map(), + rootDirectory: '/repo', + hasUncommittedChanges: false, + getTrackedFileHashesForOperation: () => new Map(), + getOperationOwnStateHash: (project: RushConfigurationProject) => localHashes.get(project.packageName)! + }; + + return { + graph, + operations, + localHashes, + executions, + cacheWrites, + executeAsync: async () => { + executions.length = 0; + cacheWrites.length = 0; + return await graph.executeAsync({ inputsSnapshot }); + } + }; +} + +function getStatus(testGraph: ITestGraph, result: IExecutionResult, name: string): OperationStatus { + return (result.operationResults.get(testGraph.operations.get(name)!) as OperationExecutionRecord).status; +} + +describe(CacheableOperationPlugin.name, () => { + it('writes cache entries for all operations in a cold iteration', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b']); + + const result: IExecutionResult = await testGraph.executeAsync(); + + expect(result.status).toBe(OperationStatus.Success); + expect(testGraph.executions).toEqual(['a', 'b']); + expect(testGraph.cacheWrites).toEqual(['a', 'b']); + }); + + it('writes a cache entry for an operation whose dependencies were retained from a previous iteration', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b', 'c']); + await testGraph.executeAsync(); + + testGraph.localHashes.set('c', 'c-v2'); + const result: IExecutionResult = await testGraph.executeAsync(); + + expect(result.status).toBe(OperationStatus.Success); + expect(getStatus(testGraph, result, 'a')).toBe(OperationStatus.Skipped); + expect(getStatus(testGraph, result, 'b')).toBe(OperationStatus.Skipped); + expect(testGraph.executions).toEqual(['c']); + expect(testGraph.cacheWrites).toEqual(['c']); + }); + + it('keeps allowing cache writes across several warm iterations', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b', 'c']); + await testGraph.executeAsync(); + + testGraph.localHashes.set('b', 'b-v2'); + await testGraph.executeAsync(); + expect(testGraph.executions).toEqual(['b', 'c']); + expect(testGraph.cacheWrites).toEqual(['b', 'c']); + + testGraph.localHashes.set('c', 'c-v2'); + await testGraph.executeAsync(); + expect(testGraph.executions).toEqual(['c']); + expect(testGraph.cacheWrites).toEqual(['c']); + }); + + it('does not write a cache entry when a dependency was skipped by the user (e.g. --only)', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b']); + await testGraph.executeAsync(); + + testGraph.operations.get('a')!.enabled = false; + testGraph.localHashes.set('b', 'b-v2'); + const result: IExecutionResult = await testGraph.executeAsync(); + + expect(getStatus(testGraph, result, 'a')).toBe(OperationStatus.Skipped); + expect(testGraph.executions).toEqual(['b']); + expect(testGraph.cacheWrites).toEqual([]); + }); + + it('does not write a cache entry when a dependency was never executed (cold --only)', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b']); + testGraph.operations.get('a')!.enabled = false; + + await testGraph.executeAsync(); + + expect(testGraph.executions).toEqual(['b']); + expect(testGraph.cacheWrites).toEqual([]); + }); + + it('does not trust a retained result that was produced while one of its dependencies was skipped', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b', 'c']); + const a: Operation = testGraph.operations.get('a')!; + + // Iteration 1: "a" is skipped by the user, so "b" and "c" execute without writing cache entries. + a.enabled = false; + await testGraph.executeAsync(); + expect(testGraph.executions).toEqual(['b', 'c']); + expect(testGraph.cacheWrites).toEqual([]); + + // Iteration 2: "a" executes for the first time, "b" is retained (same state hash), "c" changed. + a.enabled = true; + testGraph.localHashes.set('c', 'c-v2'); + const result: IExecutionResult = await testGraph.executeAsync(); + + expect(getStatus(testGraph, result, 'b')).toBe(OperationStatus.Skipped); + expect(testGraph.executions).toEqual(['a', 'c']); + // "b" was built against an unknown "a", so "c" must not write to the cache. + expect(testGraph.cacheWrites).toEqual(['a']); + }); + + it('does not trust results whose state hash changed without re-execution', async () => { + const testGraph: ITestGraph = await createTestGraphAsync(['a', 'b', 'c']); + await testGraph.executeAsync(); + + // "b" ignores dependency changes, so it is not re-run when "a" changes, but its state hash changes. + testGraph.operations.get('b')!.enabled = 'ignore-dependency-changes'; + testGraph.localHashes.set('a', 'a-v2'); + testGraph.localHashes.set('c', 'c-v2'); + const result: IExecutionResult = await testGraph.executeAsync(); + + expect(getStatus(testGraph, result, 'b')).toBe(OperationStatus.Skipped); + expect(testGraph.executions).toEqual(['a', 'c']); + expect(testGraph.cacheWrites).toEqual(['a']); + }); +});