From bf3c330031d207cdebeffc3f9e25a31acdefc343 Mon Sep 17 00:00:00 2001 From: Sean Larkin <3408176+TheLarkInn@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:50:56 +0000 Subject: [PATCH] [heft] Faster startup, same API (v2) Heft loads less at startup. Config, CLI, plugin framework and shared libraries load lazily. Common paths skip ajv, heft-config-file, argparse and fast-glob. Plugins load task code when the task runs. Node's compile cache is enabled at startup. heft --help: 407 ms -> 86 ms. First task: 406 ms -> 90 ms. Peak RSS: -31%. API reports unchanged. CLI output byte-identical. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/heft/bin/heft | 9 + apps/heft/config/heft.json | 13 + apps/heft/src/bootstrap/CompileCache.ts | 49 + .../heft/src/bootstrap/enableStartupCaches.ts | 9 + apps/heft/src/cli/CliConstants.ts | 50 + apps/heft/src/cli/HeftActionRunner.ts | 269 ++-- apps/heft/src/cli/HeftCommandLineParser.ts | 197 +-- .../heft/src/cli/HeftFullCommandLineParser.ts | 143 ++ apps/heft/src/cli/HelpFormatter.ts | 561 ++++++++ apps/heft/src/cli/HelpModel.ts | 61 + apps/heft/src/cli/LeanHeftCommandLine.ts | 634 ++++++++ apps/heft/src/cli/LeanParameterProvider.ts | 673 +++++++++ apps/heft/src/cli/actions/CleanAction.ts | 94 +- .../src/cli/actions/CleanActionExecution.ts | 68 + apps/heft/src/cli/actions/PhaseAction.ts | 17 +- apps/heft/src/cli/actions/PhaseScoping.ts | 137 ++ apps/heft/src/cli/actions/RunAction.ts | 115 +- .../heft/src/cli/test/LeanCommandLine.test.ts | 513 +++++++ .../src/configuration/HeftConfiguration.ts | 187 ++- .../configuration/HeftPluginConfiguration.ts | 47 +- .../src/configuration/HeftPluginDefinition.ts | 41 +- .../LeanConfigurationFileSpecification.ts | 163 +++ apps/heft/src/configuration/lean/LeanJson.ts | 81 ++ .../src/configuration/lean/LeanJsonSchema.ts | 997 +++++++++++++ .../lean/LeanProjectConfigurationFile.ts | 615 ++++++++ .../src/configuration/lean/LeanResolution.ts | 361 +++++ .../src/configuration/lean/LeanRigConfig.ts | 139 ++ .../src/configuration/lean/SchemaFastPath.ts | 92 ++ .../configuration/test/ConfigTestUtilities.ts | 78 + .../test/HeftConfiguration.test.ts | 190 +++ .../test/LeanHeftConfiguration.test.ts | 347 +++++ .../src/configuration/test/LeanJson.test.ts | 244 ++++ .../configuration/test/LeanJsonSchema.test.ts | 409 ++++++ .../test/LeanPluginConfigurationFile.test.ts | 305 ++++ .../configuration/test/LeanRigConfig.test.ts | 162 +++ .../test/ProjectPackageJson.test.ts | 115 ++ apps/heft/src/metrics/MetricsCollector.ts | 28 +- .../operations/OperationExecutionManager.ts | 402 ++++++ .../heft/src/operations/generateOperations.ts | 173 +++ .../runners/PhaseOperationRunner.ts | 10 +- .../operations/runners/TaskOperationRunner.ts | 95 +- .../test/OperationExecutionManager.test.ts | 265 ++++ .../heft/src/pluginFramework/HeftLifecycle.ts | 72 +- .../pluginFramework/HeftParameterManager.ts | 24 +- .../src/pluginFramework/HeftPluginHost.ts | 5 +- .../src/pluginFramework/HeftTaskSession.ts | 11 +- .../pluginFramework/InternalHeftSession.ts | 13 +- apps/heft/src/pluginFramework/TapableHooks.ts | 89 ++ .../logging/HeftChildReporter.ts | 11 +- .../pluginFramework/logging/LoggingManager.ts | 9 +- .../logging/test/MockScopedLogger.test.ts | 56 + apps/heft/src/plugins/CopyFilesPlugin.ts | 51 +- apps/heft/src/plugins/DeleteFilesPlugin.ts | 24 +- apps/heft/src/plugins/FileGlobSpecifier.ts | 56 +- apps/heft/src/plugins/NodeServicePlugin.ts | 6 +- apps/heft/src/plugins/SimpleGlob.ts | 288 ++++ apps/heft/src/plugins/test/SimpleGlob.test.ts | 156 ++ .../src/schemas/README-ModifyingSchemas.md | 11 + apps/heft/src/start.ts | 6 +- apps/heft/src/startWithVersionSelector.ts | 95 +- apps/heft/src/test/PublicApiContract.test.ts | 26 + apps/heft/src/utilities/CoreConfigFiles.ts | 262 +++- .../src/utilities/test/CliUtilities.test.ts | 37 + .../src/utilities/test/GitUtilities.test.ts | 3 + .../src/ApiExtractorPlugin.ts | 5 +- .../src/ApiExtractorRunner.ts | 6 +- .../heft-jest-plugin/src/JestPlugin.ts | 7 +- .../heft-jest-plugin/src/JestUtils.ts | 5 +- heft-plugins/heft-lint-plugin/src/Eslint.ts | 6 +- .../heft-lint-plugin/src/LintPlugin.ts | 21 +- .../src/TypeScriptPlugin.ts | 41 +- .../src/loadTypeScriptTool.ts | 7 +- .../src/ConfigurationFileAnnotation.ts | 14 + .../src/ConfigurationFileBase.ts | 22 +- libraries/node-core-library/config/heft.json | 23 + libraries/node-core-library/src/Executable.ts | 13 +- libraries/node-core-library/src/FileSystem.ts | 170 ++- libraries/node-core-library/src/Import.ts | 24 +- libraries/node-core-library/src/JsonSchema.ts | 88 +- .../src/JsonSchemaFastPath.ts | 1280 +++++++++++++++++ libraries/node-core-library/src/LockFile.ts | 11 +- .../src/test/JsonSchemaFastPath.test.ts | 133 ++ libraries/operation-graph/config/heft.json | 23 + libraries/rig-package/src/Helpers.ts | 16 +- libraries/rig-package/src/RigConfig.ts | 4 +- libraries/terminal/config/heft.json | 23 + libraries/ts-command-line/config/heft.json | 23 + .../includes/lazy-barrel/lazyBarrel.js | 92 ++ 88 files changed, 11677 insertions(+), 849 deletions(-) create mode 100644 apps/heft/src/bootstrap/CompileCache.ts create mode 100644 apps/heft/src/bootstrap/enableStartupCaches.ts create mode 100644 apps/heft/src/cli/CliConstants.ts create mode 100644 apps/heft/src/cli/HeftFullCommandLineParser.ts create mode 100644 apps/heft/src/cli/HelpFormatter.ts create mode 100644 apps/heft/src/cli/HelpModel.ts create mode 100644 apps/heft/src/cli/LeanHeftCommandLine.ts create mode 100644 apps/heft/src/cli/LeanParameterProvider.ts create mode 100644 apps/heft/src/cli/actions/CleanActionExecution.ts create mode 100644 apps/heft/src/cli/actions/PhaseScoping.ts create mode 100644 apps/heft/src/cli/test/LeanCommandLine.test.ts create mode 100644 apps/heft/src/configuration/lean/LeanConfigurationFileSpecification.ts create mode 100644 apps/heft/src/configuration/lean/LeanJson.ts create mode 100644 apps/heft/src/configuration/lean/LeanJsonSchema.ts create mode 100644 apps/heft/src/configuration/lean/LeanProjectConfigurationFile.ts create mode 100644 apps/heft/src/configuration/lean/LeanResolution.ts create mode 100644 apps/heft/src/configuration/lean/LeanRigConfig.ts create mode 100644 apps/heft/src/configuration/lean/SchemaFastPath.ts create mode 100644 apps/heft/src/configuration/test/ConfigTestUtilities.ts create mode 100644 apps/heft/src/configuration/test/HeftConfiguration.test.ts create mode 100644 apps/heft/src/configuration/test/LeanHeftConfiguration.test.ts create mode 100644 apps/heft/src/configuration/test/LeanJson.test.ts create mode 100644 apps/heft/src/configuration/test/LeanJsonSchema.test.ts create mode 100644 apps/heft/src/configuration/test/LeanPluginConfigurationFile.test.ts create mode 100644 apps/heft/src/configuration/test/LeanRigConfig.test.ts create mode 100644 apps/heft/src/configuration/test/ProjectPackageJson.test.ts create mode 100644 apps/heft/src/operations/OperationExecutionManager.ts create mode 100644 apps/heft/src/operations/generateOperations.ts create mode 100644 apps/heft/src/operations/test/OperationExecutionManager.test.ts create mode 100644 apps/heft/src/pluginFramework/TapableHooks.ts create mode 100644 apps/heft/src/pluginFramework/logging/test/MockScopedLogger.test.ts create mode 100644 apps/heft/src/plugins/SimpleGlob.ts create mode 100644 apps/heft/src/plugins/test/SimpleGlob.test.ts create mode 100644 apps/heft/src/test/PublicApiContract.test.ts create mode 100644 apps/heft/src/utilities/test/CliUtilities.test.ts create mode 100644 libraries/heft-config-file/src/ConfigurationFileAnnotation.ts create mode 100644 libraries/node-core-library/config/heft.json create mode 100644 libraries/node-core-library/src/JsonSchemaFastPath.ts create mode 100644 libraries/node-core-library/src/test/JsonSchemaFastPath.test.ts create mode 100644 libraries/operation-graph/config/heft.json create mode 100644 libraries/terminal/config/heft.json create mode 100644 libraries/ts-command-line/config/heft.json create mode 100644 rigs/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js diff --git a/apps/heft/bin/heft b/apps/heft/bin/heft index eb51ae89e3e..8ae7035ba28 100755 --- a/apps/heft/bin/heft +++ b/apps/heft/bin/heft @@ -1,2 +1,11 @@ #!/usr/bin/env node +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// Enable the V8 compile cache before anything else is loaded (Node.js >= 22.8). It honors the +// NODE_COMPILE_CACHE and NODE_DISABLE_COMPILE_CACHE environment variables and fails silently. +try { + require('../lib-commonjs/bootstrap/CompileCache.js').tryEnableCompileCache(); +} catch {} + require('../lib-commonjs/startWithVersionSelector.js'); diff --git a/apps/heft/config/heft.json b/apps/heft/config/heft.json index 76eac0e3d31..6e93614033e 100644 --- a/apps/heft/config/heft.json +++ b/apps/heft/config/heft.json @@ -22,6 +22,19 @@ } }, + // Make lib-commonjs/index.js load each re-exported module on first access, so that + // require("@rushstack/heft") stays cheap for plugins and tools that only need a few exports. + "lazy-barrel": { + "taskDependencies": ["typescript"], + "taskPlugin": { + "pluginPackage": "@rushstack/heft", + "pluginName": "run-script-plugin", + "options": { + "scriptPath": "./node_modules/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js" + } + } + }, + "copy-legacy-compatibility-start-js": { "taskDependencies": ["typescript"], "taskPlugin": { diff --git a/apps/heft/src/bootstrap/CompileCache.ts b/apps/heft/src/bootstrap/CompileCache.ts new file mode 100644 index 00000000000..9ee1dc6e4e8 --- /dev/null +++ b/apps/heft/src/bootstrap/CompileCache.ts @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// This module is loaded by bin/heft before anything else, so it intentionally only uses Node.js built-in modules +// (named imports avoid the interop helpers). +import { mkdirSync } from 'node:fs'; +import nodeModule from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +interface IModuleCompileCacheApi { + enableCompileCache?: () => unknown; + getCompileCacheDir?: () => string | undefined; +} + +/** + * Enables the V8 compile cache for the current process (Node.js >= 22.8), like `module.enableCompileCache()`: + * the cache folder is NODE_COMPILE_CACHE if set, otherwise `/node-compile-cache`, and + * NODE_DISABLE_COMPILE_CACHE disables it. Never throws; failures just leave the compile cache disabled. + * + * @returns the compile cache folder, or undefined if the compile cache is not enabled + */ +export function tryEnableCompileCache(): string | undefined { + try { + const compileCacheApi: IModuleCompileCacheApi = nodeModule as IModuleCompileCacheApi; + if (typeof compileCacheApi.enableCompileCache !== 'function') { + return undefined; + } + const env: NodeJS.ProcessEnv = process.env; + if (env.NODE_DISABLE_COMPILE_CACHE !== undefined) { + return undefined; + } + if (!env.NODE_COMPILE_CACHE) { + // Make sure that the default cache folder can be created with a single mkdir, because Node.js' recursive + // folder creation never returns for some unusable locations (e.g. TMPDIR pointing into /proc). + try { + mkdirSync(join(tmpdir(), 'node-compile-cache')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { + return undefined; + } + } + } + compileCacheApi.enableCompileCache(); + return compileCacheApi.getCompileCacheDir?.(); + } catch { + return undefined; + } +} diff --git a/apps/heft/src/bootstrap/enableStartupCaches.ts b/apps/heft/src/bootstrap/enableStartupCaches.ts new file mode 100644 index 00000000000..9412fc692ca --- /dev/null +++ b/apps/heft/src/bootstrap/enableStartupCaches.ts @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// This module is imported for its side effects by start.ts, before the rest of Heft is loaded. +// It enables the V8 compile cache (a no-op if bin/heft already enabled it). + +import { tryEnableCompileCache } from './CompileCache'; + +tryEnableCompileCache(); diff --git a/apps/heft/src/cli/CliConstants.ts b/apps/heft/src/cli/CliConstants.ts new file mode 100644 index 00000000000..f50b1ef25f5 --- /dev/null +++ b/apps/heft/src/cli/CliConstants.ts @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * Strings that define Heft's top-level command line. These are shared by the lean command-line implementation + * and the full ts-command-line based implementation, which must produce identical output. + */ +export const HEFT_TOOL_FILENAME: 'heft' = 'heft'; + +export const HEFT_TOOL_DESCRIPTION: string = 'Heft is a pluggable build system designed for web projects.'; + +export const DEBUG_PARAMETER_DESCRIPTION: string = + 'Show the full call stack if an error occurs while executing the tool'; + +export const UNMANAGED_PARAMETER_DESCRIPTION: string = + 'Disables the Heft version selector: When Heft is invoked via the shell path, normally it' + + " will examine the project's package.json dependencies and try to use the locally installed version" + + ' of Heft. Specify "--unmanaged" to force the invoked version of Heft to be used. This is useful for' + + ' example if you want to test a different version of Heft.'; + +export const VERBOSE_PARAMETER_DESCRIPTION: string = 'If specified, log information useful for debugging.'; + +export const CLEAN_ACTION_DOCUMENTATION: string = + 'Clean the project, removing temporary task folders and specified clean paths.'; + +/** + * The description of the remainder parameter that ts-command-line's `ScopedCommandLineAction` defines. + */ +export const SCOPED_ACTION_REMAINDER_DESCRIPTION: string = + 'Scoped parameters. Must be prefixed with "--", ex. "-- --scopedParameter ' + + 'foo --scopedFlag". For more information on available scoped parameters, use "-- --help".'; + +export function getRunActionDocumentation(watch: boolean): string { + return `Run a provided selection of Heft phases${watch ? ' in watch mode.' : ''}.`; +} + +export function getPhaseActionSummary(phaseName: string, watch: boolean): string { + return ( + `Runs to the ${phaseName} phase, including all transitive dependencies` + + (watch ? ', in watch mode.' : '.') + ); +} + +export function getPhaseActionDocumentation( + phaseName: string, + phaseDescription: string | undefined, + watch: boolean +): string { + return getPhaseActionSummary(phaseName, watch) + (phaseDescription ? ` ${phaseDescription}` : ''); +} diff --git a/apps/heft/src/cli/HeftActionRunner.ts b/apps/heft/src/cli/HeftActionRunner.ts index 1d61e1db0ac..8d14eb83c0d 100644 --- a/apps/heft/src/cli/HeftActionRunner.ts +++ b/apps/heft/src/cli/HeftActionRunner.ts @@ -1,19 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { fstatSync, statSync, type Stats } from 'node:fs'; import { performance } from 'node:perf_hooks'; -import { createInterface, type Interface as ReadlineInterface } from 'node:readline'; +import type * as ReadlineModule from 'node:readline'; import os from 'node:os'; import { AlreadyReportedError, InternalError, type IPackageJson } from '@rushstack/node-core-library'; import { Colorize, ConsoleTerminalProvider, type ITerminal } from '@rushstack/terminal'; -import { - type IOperationExecutionOptions, - type IWatchLoopState, +import type { + IOperationExecutionOptions, + IWatchLoopState, Operation, - OperationExecutionManager, OperationGroupRecord, - type OperationRequestRunCallback, + OperationRequestRunCallback, OperationStatus, WatchLoop } from '@rushstack/operation-graph'; @@ -22,16 +22,15 @@ import type { CommandLineParameterProvider, CommandLineStringListParameter } from '@rushstack/ts-command-line'; +import type { IRigConfig } from '@rushstack/rig-package'; import type { InternalHeftSession } from '../pluginFramework/InternalHeftSession'; -import type { HeftConfiguration } from '../configuration/HeftConfiguration'; +import { type HeftConfiguration, getRigConfigForConfigLoading } from '../configuration/HeftConfiguration'; import type { LoggingManager } from '../pluginFramework/logging/LoggingManager'; import type { HeftChildReporter } from '../pluginFramework/logging/HeftChildReporter'; import type { MetricsCollector } from '../metrics/MetricsCollector'; import { HeftParameterManager } from '../pluginFramework/HeftParameterManager'; -import { TaskOperationRunner } from '../operations/runners/TaskOperationRunner'; -import { PhaseOperationRunner } from '../operations/runners/PhaseOperationRunner'; -import type { IHeftPhase, HeftPhase } from '../pluginFramework/HeftPhase'; +import type { IHeftPhase } from '../pluginFramework/HeftPhase'; import type { IHeftAction, IHeftActionOptions } from './actions/IHeftAction'; import type { IHeftLifecycleCleanHookOptions, @@ -40,14 +39,23 @@ import type { IHeftLifecycleToolStartHookOptions } from '../pluginFramework/HeftLifecycleSession'; import type { HeftLifecycle } from '../pluginFramework/HeftLifecycle'; -import type { IHeftTask, HeftTask } from '../pluginFramework/HeftTask'; -import { deleteFilesAsync, type IDeleteOperation } from '../plugins/DeleteFilesPlugin'; +import type { IHeftTask } from '../pluginFramework/HeftTask'; +import type { IDeleteOperation } from '../plugins/DeleteFilesPlugin'; import { Constants } from '../utilities/Constants'; export interface IHeftActionRunnerOptions extends IHeftActionOptions { action: IHeftAction; } +/** + * The part of the `OperationExecutionManager` API that is used to run the operation graph. + */ +interface IOperationExecutionManager { + executeAsync( + executionOptions: IOperationExecutionOptions + ): Promise; +} + /** * Metadata for an operation that represents a task. * @public @@ -85,15 +93,46 @@ export function initializeHeft( const projectPackageJson: IPackageJson = heftConfiguration.projectPackageJson; terminal.writeVerboseLine(`Project: ${projectPackageJson.name}@${projectPackageJson.version}`); terminal.writeVerboseLine(`Project build folder: ${heftConfiguration.buildFolderPath}`); - if (heftConfiguration.rigConfig.rigFound) { - terminal.writeVerboseLine(`Rig package: ${heftConfiguration.rigConfig.rigPackageName}`); - terminal.writeVerboseLine(`Rig profile: ${heftConfiguration.rigConfig.rigProfile}`); + // Same data as heftConfiguration.rigConfig, without loading @rushstack/rig-package + const rigConfig: IRigConfig = getRigConfigForConfigLoading(heftConfiguration); + if (rigConfig.rigFound) { + terminal.writeVerboseLine(`Rig package: ${rigConfig.rigPackageName}`); + terminal.writeVerboseLine(`Rig profile: ${rigConfig.rigProfile}`); + } + // Heft's own package.json is only needed for this line. A ConsoleTerminalProvider discards verbose + // messages when verbose logging is disabled, so don't bother loading it in that case. + const { terminalProvider } = heftConfiguration; + if (!(terminalProvider instanceof ConsoleTerminalProvider) || terminalProvider.verboseEnabled) { + terminal.writeVerboseLine(`Heft version: ${heftConfiguration.heftPackageJson.version}`); } - terminal.writeVerboseLine(`Heft version: ${heftConfiguration.heftPackageJson.version}`); terminal.writeVerboseLine(`Node version: ${process.version}`); terminal.writeVerboseLine(''); } +function getReadlineModule(): typeof ReadlineModule { + // This module is loaded by every heft command, but node:readline is only needed once an action runs. + // process.getBuiltinModule() is not available before Node.js 20.16. + return typeof process.getBuiltinModule === 'function' + ? process.getBuiltinModule('node:readline') + : require('node:readline'); +} + +/** + * Returns true if the process's standard input is the null device (for example when the parent process spawned + * Heft with stdin set to "ignore", as Rush does for project commands). + */ +function isStdinNullDevice(): boolean { + if (process.platform === 'win32') { + return false; + } + try { + const stdinStats: Stats = fstatSync(0); + return stdinStats.isCharacterDevice() && stdinStats.rdev === statSync('/dev/null').rdev; + } catch { + return false; + } +} + let _cliAbortSignal: AbortSignal | undefined; export function ensureCliAbortSignal(terminal: ITerminal): AbortSignal { if (!_cliAbortSignal) { @@ -101,7 +140,20 @@ export function ensureCliAbortSignal(terminal: ITerminal): AbortSignal { // less gracefully if pressed a second time. const cliAbortController: AbortController = new AbortController(); _cliAbortSignal = cliAbortController.signal; - const cli: ReadlineInterface = createInterface(process.stdin, undefined, undefined, true); + + // Reading the null device immediately yields EOF, upon which the readline interface closes itself, so it could + // never receive a Ctrl+C keypress: Ctrl+C then reaches the process as a plain SIGINT either way. Skip creating + // the stdin stream and the interface in that case. + if (isStdinNullDevice()) { + return _cliAbortSignal; + } + + const cli: ReadlineModule.Interface = getReadlineModule().createInterface( + process.stdin, + undefined, + undefined, + true + ); let forceTerminate: boolean = false; cli.on('SIGINT', () => { cli.close(); @@ -132,16 +184,21 @@ export async function runWithLoggingAsync( abortSignal: AbortSignal, throwOnFailure?: boolean ): Promise { + // This module is loaded by every heft command, so only load operation-graph once an action runs. + const { OperationStatus: OperationStatusEnum } = await import( + '@rushstack/operation-graph/lib/OperationStatus' + ); + const startTime: number = performance.now(); loggingManager.resetScopedLoggerErrorsAndWarnings(); - let result: OperationStatus = OperationStatus.Failure; + let result: OperationStatus = OperationStatusEnum.Failure; // Execute the action operations let encounteredError: boolean = false; try { result = await fn(); - if (result === OperationStatus.Failure) { + if (result === OperationStatusEnum.Failure) { encounteredError = true; } } catch (e) { @@ -315,13 +372,24 @@ export class HeftActionRunner { initializeHeft(this.#heftConfiguration, terminal, this.parameterManager.defaultParameters.verbose); - const operations: ReadonlySet> = - this.#generateOperations(); + // The operation graph machinery is only needed once an action actually executes + const { generateOperations } = await import('../operations/generateOperations'); - const executionManager: OperationExecutionManager< - IHeftTaskOperationMetadata, - IHeftPhaseOperationMetadata - > = new OperationExecutionManager(operations); + const operations: ReadonlySet> = + generateOperations({ + internalHeftSession: this.#internalHeftSession, + selectedPhases: this.#action.selectedPhases, + terminal + }); + + // Watch mode uses the execution manager from @rushstack/operation-graph as-is: the time it takes to + // start rebuilding after a change is part of how watch mode's change detection behaves (e.g. watchers + // are restarted relative to when a rebuild began reading its inputs), so it is kept unchanged. Single + // runs use an equivalent execution manager that doesn't wait on a timer for every wave of ready + // operations. + const executionManager: IOperationExecutionManager = this.#action.watch + ? new (await import('@rushstack/operation-graph')).OperationExecutionManager(operations) + : new (await import('../operations/OperationExecutionManager')).OperationExecutionManager(operations); const cliAbortSignal: AbortSignal = ensureCliAbortSignal(this.#terminal); @@ -329,7 +397,7 @@ export class HeftActionRunner { await _startLifecycleAsync(this.#internalHeftSession); if (this.#action.watch) { - const watchLoop: WatchLoop = this.#createWatchLoop(executionManager); + const watchLoop: WatchLoop = await this.#createWatchLoopAsync(executionManager); if (process.send) { await watchLoop.runIPCAsync(); @@ -351,9 +419,10 @@ export class HeftActionRunner { } } - #createWatchLoop(executionManager: OperationExecutionManager): WatchLoop { + async #createWatchLoopAsync(executionManager: IOperationExecutionManager): Promise { + const { WatchLoop: WatchLoopClass } = await import('@rushstack/operation-graph'); const terminal: ITerminal = this.#terminal; - const watchLoop: WatchLoop = new WatchLoop({ + const watchLoop: WatchLoop = new WatchLoopClass({ onBeforeExecute: () => { // Write an empty line to the terminal for separation between iterations. We've already iterated // at this point, so log out that we're about to start a new run. @@ -374,7 +443,7 @@ export class HeftActionRunner { } async #executeOnceAsync( - executionManager: OperationExecutionManager, + executionManager: IOperationExecutionManager, abortSignal: AbortSignal, requestRun?: OperationRequestRunCallback ): Promise { @@ -432,147 +501,6 @@ export class HeftActionRunner { !requestRun ); } - - #generateOperations(): Set> { - const { selectedPhases } = this.#action; - - const operations: Map< - string, - Operation - > = new Map(); - const operationGroups: Map> = new Map(); - const internalHeftSession: InternalHeftSession = this.#internalHeftSession; - - let hasWarnedAboutSkippedPhases: boolean = false; - for (const phase of selectedPhases) { - // Warn if any dependencies are excluded from the list of selected phases - if (!hasWarnedAboutSkippedPhases) { - for (const dependencyPhase of phase.dependencyPhases) { - if (!selectedPhases.has(dependencyPhase)) { - // Only write once, and write with yellow to make it stand out without writing a warning to stderr - hasWarnedAboutSkippedPhases = true; - this.#terminal.writeLine( - Colorize.bold( - 'The provided list of phases does not contain all phase dependencies. You may need to run the ' + - 'excluded phases manually.' - ) - ); - break; - } - } - } - - // Create operation for the phase start node - const phaseOperation: Operation = _getOrCreatePhaseOperation( - internalHeftSession, - phase, - operations, - operationGroups - ); - - // Create operations for each task - for (const task of phase.tasks) { - const taskOperation: Operation = _getOrCreateTaskOperation( - internalHeftSession, - task, - operations, - operationGroups - ); - // Set the phase operation as a dependency of the task operation to ensure the phase operation runs first - taskOperation.addDependency(phaseOperation); - - // Set all dependency tasks as dependencies of the task operation - for (const dependencyTask of task.dependencyTasks) { - taskOperation.addDependency( - _getOrCreateTaskOperation(internalHeftSession, dependencyTask, operations, operationGroups) - ); - } - - // Set all tasks in a in a phase as dependencies of the consuming phase - for (const consumingPhase of phase.consumingPhases) { - if (this.#action.selectedPhases.has(consumingPhase)) { - // Set all tasks in a dependency phase as dependencies of the consuming phase to ensure the dependency - // tasks run first - const consumingPhaseOperation: Operation = _getOrCreatePhaseOperation( - internalHeftSession, - consumingPhase, - operations, - operationGroups - ); - consumingPhaseOperation.addDependency(taskOperation); - // This is purely to simplify the reported graph for phase circularities - consumingPhaseOperation.addDependency(phaseOperation); - } - } - } - } - - return new Set(operations.values()); - } -} - -function _getOrCreatePhaseOperation( - this: void, - internalHeftSession: InternalHeftSession, - phase: HeftPhase, - operations: Map, - operationGroups: Map> -): Operation { - const key: string = phase.phaseName; - - let operation: Operation | undefined = operations.get(key); - if (!operation) { - let group: OperationGroupRecord | undefined = operationGroups.get( - phase.phaseName - ); - if (!group) { - group = new OperationGroupRecord(phase.phaseName, { phase }); - operationGroups.set(phase.phaseName, group); - } - // Only create the operation. Dependencies are hooked up separately - operation = new Operation({ - group, - name: phase.phaseName, - runner: new PhaseOperationRunner({ phase, internalHeftSession }) - }); - operations.set(key, operation); - } - return operation; -} - -function _getOrCreateTaskOperation( - this: void, - internalHeftSession: InternalHeftSession, - task: HeftTask, - operations: Map, - operationGroups: Map> -): Operation { - const key: string = `${task.parentPhase.phaseName}.${task.taskName}`; - - let operation: Operation | undefined = operations.get( - key - ) as Operation; - if (!operation) { - const group: OperationGroupRecord | undefined = operationGroups.get( - task.parentPhase.phaseName - ); - if (!group) { - throw new InternalError( - `Task ${task.taskName} in phase ${task.parentPhase.phaseName} has no group. This should not happen.` - ); - } - operation = new Operation({ - group, - runner: new TaskOperationRunner({ - internalHeftSession, - task - }), - name: task.taskName, - metadata: { task, phase: task.parentPhase } - }); - operations.set(key, operation); - } - return operation; } async function _startLifecycleAsync(this: void, internalHeftSession: InternalHeftSession): Promise { @@ -623,6 +551,7 @@ async function _startLifecycleAsync(this: void, internalHeftSession: InternalHef // Delete the files if any were specified if (deleteOperations.length) { const rootFolderPath: string = internalHeftSession.heftConfiguration.buildFolderPath; + const { deleteFilesAsync } = await import('../plugins/DeleteFilesPlugin'); await deleteFilesAsync(rootFolderPath, deleteOperations, lifecycleLogger.terminal); } diff --git a/apps/heft/src/cli/HeftCommandLineParser.ts b/apps/heft/src/cli/HeftCommandLineParser.ts index 6b7ada39707..31d2b6fddf5 100644 --- a/apps/heft/src/cli/HeftCommandLineParser.ts +++ b/apps/heft/src/cli/HeftCommandLineParser.ts @@ -3,12 +3,6 @@ import os from 'node:os'; -import { - CommandLineParser, - type AliasCommandLineAction, - type CommandLineFlagParameter, - type CommandLineAction -} from '@rushstack/ts-command-line'; import { InternalError, AlreadyReportedError } from '@rushstack/node-core-library'; import { Terminal, @@ -21,70 +15,60 @@ import { MetricsCollector } from '../metrics/MetricsCollector'; import { HeftConfiguration } from '../configuration/HeftConfiguration'; import { InternalHeftSession } from '../pluginFramework/InternalHeftSession'; import { LoggingManager } from '../pluginFramework/logging/LoggingManager'; -import { CleanAction } from './actions/CleanAction'; -import { PhaseAction } from './actions/PhaseAction'; -import { RunAction } from './actions/RunAction'; import type { IHeftActionOptions } from './actions/IHeftAction'; -import { AliasAction } from './actions/AliasAction'; import { getToolParameterNamesFromArgs } from '../utilities/CliUtilities'; import { Constants } from '../utilities/Constants'; -import { HeftChildReporter } from '../pluginFramework/logging/HeftChildReporter'; +import type { HeftChildReporter } from '../pluginFramework/logging/HeftChildReporter'; +import { tryExecuteLeanCommandLineAsync } from './LeanHeftCommandLine'; /** - * This interfaces specifies values for parameters that must be parsed before the CLI - * is fully initialized. + * State shared by the lean and the full command-line implementations. */ -interface IPreInitializationArgumentValues { - debug?: boolean; - unmanaged?: boolean; +export interface IHeftCommandLineParserState { + readonly internalHeftSession: InternalHeftSession; + readonly childReporter: HeftChildReporter | undefined; + reportErrorAndSetExitCodeAsync(error: Error): Promise; } -const HEFT_TOOL_FILENAME: 'heft' = 'heft'; - -export class HeftCommandLineParser extends CommandLineParser { +/** + * Heft's command line. + * + * @remarks + * Most invocations are handled by a lean implementation that only defines the parameters of the invoked action and + * does not load ts-command-line's argparse-based parser. Anything that the lean implementation cannot handle with + * byte-identical results (help, errors, unusual syntax, etc.) is handled by the full ts-command-line based + * implementation in `HeftFullCommandLineParser`. + */ +export class HeftCommandLineParser { public readonly globalTerminal: ITerminal; - readonly #debugFlag: CommandLineFlagParameter; - readonly #unmanagedFlag: CommandLineFlagParameter; readonly #debug: boolean; readonly #terminalProvider: ITerminalProvider; readonly #childReporter: HeftChildReporter | undefined; readonly #loggingManager: LoggingManager; readonly #metricsCollector: MetricsCollector; readonly #heftConfiguration: HeftConfiguration; - #internalHeftSession: InternalHeftSession | undefined; public constructor() { - super({ - toolFilename: HEFT_TOOL_FILENAME, - toolDescription: 'Heft is a pluggable build system designed for web projects.' - }); - - // Initialize the debug flag as a parameter on the tool itself - this.#debugFlag = this.defineFlagParameter({ - parameterLongName: Constants.debugParameterLongName, - description: 'Show the full call stack if an error occurs while executing the tool' - }); - - // Initialize the unmanaged flag as a parameter on the tool itself. While this parameter - // is only used during version selection, we need to support parsing it here so that we - // don't throw due to an unrecognized parameter. - this.#unmanagedFlag = this.defineFlagParameter({ - parameterLongName: Constants.unmanagedParameterLongName, - description: - 'Disables the Heft version selector: When Heft is invoked via the shell path, normally it' + - " will examine the project's package.json dependencies and try to use the locally installed version" + - ' of Heft. Specify "--unmanaged" to force the invoked version of Heft to be used. This is useful for' + - ' example if you want to test a different version of Heft.' - }); - // Pre-initialize with known argument values to determine state of "--debug" - const preInitializationArgumentValues: IPreInitializationArgumentValues = - this.#getPreInitializationArgumentValues(); - this.#debug = !!preInitializationArgumentValues.debug; - - // Enable debug and verbose logging if the "--debug" flag is set - this.#childReporter = HeftChildReporter.tryInitialize(); + const toolParameters: Set = getToolParameterNamesFromArgs(process.argv); + this.#debug = toolParameters.has(Constants.debugParameterLongName); + + // Enable debug and verbose logging if the "--debug" flag is set. HeftChildReporter.tryInitialize() has no + // effect and returns undefined unless one of the Rush child reporter environment variables is set, so the + // module is only loaded in that case. + const { + _RUSH_REPORTER_CHILD_FD: childReporterFd, + _RUSH_REPORTER_CHILD_ACK_FD: childReporterAckFd + }: Record = process.env; + this.#childReporter = + childReporterFd === undefined && childReporterAckFd === undefined + ? undefined + : ( + require('../pluginFramework/logging/HeftChildReporter') as { + HeftChildReporter: typeof HeftChildReporter; + } + ).HeftChildReporter.tryInitialize(); this.#terminalProvider = this.#childReporter ?? new ConsoleTerminalProvider({ @@ -130,7 +114,6 @@ export class HeftCommandLineParser extends CommandLineParser { loggingManager: this.#loggingManager, metricsCollector: this.#metricsCollector }); - this.#internalHeftSession = internalHeftSession; const actionOptions: IHeftActionOptions = { internalHeftSession: internalHeftSession, @@ -140,90 +123,31 @@ export class HeftCommandLineParser extends CommandLineParser { heftConfiguration: this.#heftConfiguration }; - // Add the clean action, the run action, and the individual phase actions - this.addAction(new CleanAction(actionOptions)); - this.addAction(new RunAction(actionOptions)); - for (const phase of internalHeftSession.phases) { - this.addAction(new PhaseAction({ ...actionOptions, phase })); - } - - // Add the watch variant of the run action and the individual phase actions - this.addAction(new RunAction({ ...actionOptions, watch: true })); - for (const phase of internalHeftSession.phases) { - this.addAction(new PhaseAction({ ...actionOptions, phase, watch: true })); - } + const state: IHeftCommandLineParserState = { + internalHeftSession, + childReporter: this.#childReporter, + reportErrorAndSetExitCodeAsync: (error: Error) => this.#reportErrorAndSetExitCodeAsync(error) + }; - // Add the action aliases last, since we need the targets to be defined before we can add the aliases - const aliasActions: AliasCommandLineAction[] = []; - for (const [ - aliasName, - { actionName, defaultParameters } - ] of internalHeftSession.actionReferencesByAlias) { - const existingAction: CommandLineAction | undefined = this.tryGetAction(aliasName); - if (existingAction) { - throw new Error( - `The alias "${aliasName}" specified in heft.json cannot be used because an action ` + - 'with that name already exists.' - ); - } - const targetAction: CommandLineAction | undefined = this.tryGetAction(actionName); - if (!targetAction) { - throw new Error( - `The action "${actionName}" referred to by alias "${aliasName}" in heft.json could not be found.` - ); - } - aliasActions.push( - new AliasAction({ - terminal: this.globalTerminal, - toolFilename: HEFT_TOOL_FILENAME, - aliasName, - targetAction, - defaultParameters - }) - ); - } - // Add the alias actions. Do this in a second pass to disallow aliases that refer to other aliases. - for (const aliasAction of aliasActions) { - this.addAction(aliasAction); + // 0=node.exe, 1=script name + const leanResult: boolean | undefined = await tryExecuteLeanCommandLineAsync( + args ?? process.argv.slice(2), + actionOptions, + state + ); + if (leanResult !== undefined) { + return leanResult; } - return await super.executeAsync(args); + const { HeftFullCommandLineParser } = await import('./HeftFullCommandLineParser'); + const fullParser: InstanceType = new HeftFullCommandLineParser(state); + return await fullParser.defineActionsAndExecuteAsync(actionOptions, args); } catch (e) { await this.#reportErrorAndSetExitCodeAsync(e as Error); return false; } } - protected override async onExecuteAsync(): Promise { - try { - const selectedAction: CommandLineAction | undefined = this.selectedAction; - - let commandName: string = ''; - let unaliasedCommandName: string = ''; - - if (selectedAction) { - commandName = selectedAction.actionName; - if (selectedAction instanceof AliasAction) { - unaliasedCommandName = selectedAction.targetAction.actionName; - } else { - unaliasedCommandName = selectedAction.actionName; - } - } - - this.#internalHeftSession!.parsedCommandLine = { - commandName, - unaliasedCommandName - }; - this.#childReporter?.setCommandName(commandName); - await super.onExecuteAsync(); - } catch (e) { - await this.#reportErrorAndSetExitCodeAsync(e as Error); - } - - // If we make it here, things are fine and reset the exit code back to 0 - process.exitCode = 0; - } - #normalizeCwd(): void { const buildFolder: string = this.#heftConfiguration.buildFolderPath; const currentCwd: string = process.cwd(); @@ -238,25 +162,6 @@ export class HeftCommandLineParser extends CommandLineParser { } } - #getPreInitializationArgumentValues( - args: string[] = process.argv - ): IPreInitializationArgumentValues { - if (!this.#debugFlag) { - // The `this.#debugFlag` parameter (the parameter itself, not its value) - // has not yet been defined. Parameters need to be defined before we - // try to evaluate any parameters. This is to ensure that the - // `--debug` flag is defined correctly before we do this not-so-rigorous - // parameter parsing. - throw new InternalError('parameters have not yet been defined.'); - } - - const toolParameters: Set = getToolParameterNamesFromArgs(args); - return { - debug: toolParameters.has(this.#debugFlag.longName), - unmanaged: toolParameters.has(this.#unmanagedFlag.longName) - }; - } - async #reportErrorAndSetExitCodeAsync(error: Error): Promise { if (!(error instanceof AlreadyReportedError)) { if (this.#childReporter) { diff --git a/apps/heft/src/cli/HeftFullCommandLineParser.ts b/apps/heft/src/cli/HeftFullCommandLineParser.ts new file mode 100644 index 00000000000..bb2bb328422 --- /dev/null +++ b/apps/heft/src/cli/HeftFullCommandLineParser.ts @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + CommandLineParser, + type AliasCommandLineAction, + type CommandLineAction +} from '@rushstack/ts-command-line'; + +import { CleanAction } from './actions/CleanAction'; +import { PhaseAction } from './actions/PhaseAction'; +import { RunAction } from './actions/RunAction'; +import type { IHeftActionOptions } from './actions/IHeftAction'; +import { AliasAction } from './actions/AliasAction'; +import { Constants } from '../utilities/Constants'; +import { + DEBUG_PARAMETER_DESCRIPTION, + HEFT_TOOL_DESCRIPTION, + HEFT_TOOL_FILENAME, + UNMANAGED_PARAMETER_DESCRIPTION +} from './CliConstants'; +import type { IHeftCommandLineParserState } from './HeftCommandLineParser'; + +/** + * The complete ts-command-line (argparse) based command-line parser. This is the reference implementation of + * Heft's command line: it defines every action with every parameter before parsing anything. The lean + * implementation in `HeftCommandLineParser` falls back to this parser for anything that it cannot handle + * with byte-identical results (such as errors and unusual syntax). + */ +export class HeftFullCommandLineParser extends CommandLineParser { + readonly #state: IHeftCommandLineParserState; + + public constructor(state: IHeftCommandLineParserState) { + super({ + toolFilename: HEFT_TOOL_FILENAME, + toolDescription: HEFT_TOOL_DESCRIPTION + }); + + this.#state = state; + + // Initialize the debug flag as a parameter on the tool itself + this.defineFlagParameter({ + parameterLongName: Constants.debugParameterLongName, + description: DEBUG_PARAMETER_DESCRIPTION + }); + + // Initialize the unmanaged flag as a parameter on the tool itself. While this parameter + // is only used during version selection, we need to support parsing it here so that we + // don't throw due to an unrecognized parameter. + this.defineFlagParameter({ + parameterLongName: Constants.unmanagedParameterLongName, + description: UNMANAGED_PARAMETER_DESCRIPTION + }); + } + + /** + * Defines all actions and executes the command line. Errors thrown while defining the actions propagate + * to the caller, which reports them. + */ + public async defineActionsAndExecuteAsync(actionOptions: IHeftActionOptions, args?: string[]): Promise { + const { internalHeftSession } = actionOptions; + const { terminal } = actionOptions; + + // Add the clean action, the run action, and the individual phase actions + this.addAction(new CleanAction(actionOptions)); + this.addAction(new RunAction(actionOptions)); + for (const phase of internalHeftSession.phases) { + this.addAction(new PhaseAction({ ...actionOptions, phase })); + } + + // Add the watch variant of the run action and the individual phase actions + this.addAction(new RunAction({ ...actionOptions, watch: true })); + for (const phase of internalHeftSession.phases) { + this.addAction(new PhaseAction({ ...actionOptions, phase, watch: true })); + } + + // Add the action aliases last, since we need the targets to be defined before we can add the aliases + const aliasActions: AliasCommandLineAction[] = []; + for (const [ + aliasName, + { actionName, defaultParameters } + ] of internalHeftSession.actionReferencesByAlias) { + const existingAction: CommandLineAction | undefined = this.tryGetAction(aliasName); + if (existingAction) { + throw new Error( + `The alias "${aliasName}" specified in heft.json cannot be used because an action ` + + 'with that name already exists.' + ); + } + const targetAction: CommandLineAction | undefined = this.tryGetAction(actionName); + if (!targetAction) { + throw new Error( + `The action "${actionName}" referred to by alias "${aliasName}" in heft.json could not be found.` + ); + } + aliasActions.push( + new AliasAction({ + terminal, + toolFilename: HEFT_TOOL_FILENAME, + aliasName, + targetAction, + defaultParameters + }) + ); + } + // Add the alias actions. Do this in a second pass to disallow aliases that refer to other aliases. + for (const aliasAction of aliasActions) { + this.addAction(aliasAction); + } + + return await super.executeAsync(args); + } + + protected override async onExecuteAsync(): Promise { + try { + const selectedAction: CommandLineAction | undefined = this.selectedAction; + + let commandName: string = ''; + let unaliasedCommandName: string = ''; + + if (selectedAction) { + commandName = selectedAction.actionName; + if (selectedAction instanceof AliasAction) { + unaliasedCommandName = selectedAction.targetAction.actionName; + } else { + unaliasedCommandName = selectedAction.actionName; + } + } + + this.#state.internalHeftSession.parsedCommandLine = { + commandName, + unaliasedCommandName + }; + this.#state.childReporter?.setCommandName(commandName); + await super.onExecuteAsync(); + } catch (e) { + await this.#state.reportErrorAndSetExitCodeAsync(e as Error); + } + + // If we make it here, things are fine and reset the exit code back to 0 + process.exitCode = 0; + } +} diff --git a/apps/heft/src/cli/HelpFormatter.ts b/apps/heft/src/cli/HelpFormatter.ts new file mode 100644 index 00000000000..93bdefcc9e2 --- /dev/null +++ b/apps/heft/src/cli/HelpFormatter.ts @@ -0,0 +1,561 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// This is a faithful TypeScript port of the HelpFormatter from the "argparse" package, version 1.0.10 +// (lib/help/formatter.js; Copyright (C) 2012 by Vitaly Puzrin; MIT license), which is the version that +// @rushstack/ts-command-line uses. It exists so that Heft can render byte-identical help text without loading +// argparse. The behavior, including its quirks, must match the original exactly; it is covered by tests that +// compare its output with argparse's. +// +// Simplifications that do not affect the output for the parsers that ts-command-line builds: +// - mutually exclusive groups are not supported (ts-command-line does not create them) +// - `usage` is always generated (ts-command-line never provides one) +// - help strings are used as-is instead of being expanded with sprintf(); ts-command-line escapes every "%" +// before passing text to argparse, so sprintf() is the identity function for that text + +import { + ONE_OR_MORE, + OPTIONAL, + PARSER, + REMAINDER, + SUPPRESS, + ZERO_OR_MORE, + type IHelpAction, + type IHelpParser +} from './HelpModel'; + +const EOL: string = '\n'; + +function isOptional(action: IHelpAction): boolean { + return action.optionStrings.length !== 0; +} + +// All callers pass integers; like the original, a non-positive count produces an empty string. +function repeat(str: string, num: number): string { + return num > 0 ? str.repeat(num) : ''; +} + +const WHITESPACE_REGEXP: RegExp = /\s+/g; +const LONG_BREAK_REGEXP: RegExp = new RegExp(EOL + EOL + EOL + '+', 'g'); +const SPLIT_LINES_DELIMITERS: string[] = [' ', '.', ',', '!', '?']; +const SPLIT_LINES_DELIMITER_REGEXP: RegExp = new RegExp( + '[' + SPLIT_LINES_DELIMITERS.join('') + '][^' + SPLIT_LINES_DELIMITERS.join('') + ']*$' +); +const USAGE_PART_REGEXP: RegExp = new RegExp('\\(.*?\\)+|\\[.*?\\]+|\\S+', 'g'); + +function trimChars(str: string, chars: string): string { + let start: number = 0; + let end: number = str.length - 1; + while (chars.indexOf(str.charAt(start)) >= 0) { + start++; + } + while (chars.indexOf(str.charAt(end)) >= 0) { + end--; + } + return str.slice(start, end + 1); +} + +type HelpItem = () => string; + +class Section { + public readonly parent: Section | undefined; + readonly #heading: string | undefined; + readonly #items: HelpItem[] = []; + + public constructor(parent: Section | undefined, heading?: string) { + this.parent = parent; + this.#heading = heading; + } + + public addItem(item: HelpItem): void { + this.#items.push(item); + } + + public formatHelp(formatter: HelpFormatter): string { + // format the indented section + if (this.parent) { + formatter.indent(); + } + + const itemHelp: string = formatter.joinParts(this.#items.map((item: HelpItem) => item())); + + if (this.parent) { + formatter.dedent(); + } + + // return nothing if the section was empty + if (!itemHelp) { + return ''; + } + + // add the heading if the section was non-empty + let heading: string = ''; + if (this.#heading && this.#heading !== SUPPRESS) { + // The original indents the heading by `formatter.currentIndent`, which is not a property of its formatter + // (the property is named `_currentIndent`), so headings are never indented. + heading = this.#heading + ':' + EOL; + } + + // join the section-initialize newline, the heading and the help + return formatter.joinParts([EOL, heading, itemHelp, EOL]); + } +} + +export class HelpFormatter { + public currentIndent: number = 0; + + readonly #prog: string; + readonly #maxHelpPosition: number = 24; + readonly #width: number; + readonly #indentIncrement: number = 2; + #actionMaxLength: number = 0; + readonly #rootSection: Section; + #currentSection: Section; + + public constructor(prog: string) { + this.#prog = prog; + // Mirrors: (options.width || ((process.env.COLUMNS || 80) - 2)) + this.#width = ((process.env.COLUMNS || 80) as number) - 2; + this.#rootSection = new Section(undefined); + this.#currentSection = this.#rootSection; + } + + public indent(): void { + this.currentIndent += this.#indentIncrement; + } + + public dedent(): void { + this.currentIndent -= this.#indentIncrement; + if (this.currentIndent < 0) { + throw new Error('Indent decreased below 0.'); + } + } + + public startSection(heading: string): void { + this.indent(); + const section: Section = new Section(this.#currentSection, heading); + this.#addItem(() => section.formatHelp(this)); + this.#currentSection = section; + } + + public endSection(): void { + this.#currentSection = this.#currentSection.parent!; + this.dedent(); + } + + public addText(text: string | undefined): void { + if (text && text !== SUPPRESS) { + this.#addItem(() => this.#formatText(text)); + } + } + + public addUsage(actions: ReadonlyArray, prefix?: string): void { + this.#addItem(() => this.#formatUsage(actions, prefix)); + } + + public addArgument(action: IHelpAction): void { + if (action.help !== SUPPRESS) { + let invocationLength: number = this.#formatActionInvocation(action).length; + + if (action.subactions) { + this.indent(); + for (const subaction of action.subactions) { + const invocationNew: string = this.#formatActionInvocation(subaction); + invocationLength = Math.max(invocationLength, invocationNew.length); + } + this.dedent(); + } + + const actionLength: number = invocationLength + this.currentIndent; + this.#actionMaxLength = Math.max(this.#actionMaxLength, actionLength); + + this.#addItem(() => this.#formatAction(action)); + } + } + + public addArguments(actions: ReadonlyArray): void { + for (const action of actions) { + this.addArgument(action); + } + } + + public formatHelp(): string { + let help: string = this.#rootSection.formatHelp(this); + if (help) { + help = help.replace(LONG_BREAK_REGEXP, EOL + EOL); + help = trimChars(help, EOL) + EOL; + } + return help; + } + + public joinParts(partStrings: ReadonlyArray): string { + return partStrings.filter((part: string | undefined) => part && part !== SUPPRESS).join(''); + } + + #addItem(item: HelpItem): void { + this.#currentSection.addItem(item); + } + + #formatUsage(actions: ReadonlyArray, prefix: string | undefined): string { + if (!prefix && typeof prefix !== 'string') { + prefix = 'usage: '; + } + + let usage: string; + if (actions.length === 0) { + usage = this.#prog; + } else { + const prog: string = this.#prog; + const optionals: IHelpAction[] = []; + const positionals: IHelpAction[] = []; + + for (const action of actions) { + if (isOptional(action)) { + optionals.push(action); + } else { + positionals.push(action); + } + } + + const actionUsage: string = this.#formatActionsUsage([...optionals, ...positionals]); + usage = [prog, actionUsage].join(' '); + + const textWidth: number = this.#width - this.currentIndent; + if (prefix.length + usage.length > textWidth) { + // break usage into wrappable parts + const optionalUsage: string = this.#formatActionsUsage(optionals); + const positionalUsage: string = this.#formatActionsUsage(positionals); + + // `match()` returns null if there are no matches + const optionalParts: string[] | undefined = optionalUsage.match(USAGE_PART_REGEXP) ?? undefined; + const positionalParts: string[] = positionalUsage.match(USAGE_PART_REGEXP) || []; + + if (optionalParts!.join(' ') !== optionalUsage) { + throw new Error('assert "optionalParts.join(\' \') === optionalUsage"'); + } + if (positionalParts.join(' ') !== positionalUsage) { + throw new Error('assert "positionalParts.join(\' \') === positionalUsage"'); + } + + // helper for wrapping lines + const getLines: (parts: string[], indent: string, linePrefix?: string) => string[] = ( + parts: string[], + indent: string, + linePrefix?: string + ): string[] => { + const lines: string[] = []; + let line: string[] = []; + + let lineLength: number = linePrefix ? linePrefix.length - 1 : indent.length - 1; + + for (const part of parts) { + if (lineLength + 1 + part.length > textWidth) { + lines.push(indent + line.join(' ')); + line = []; + lineLength = indent.length - 1; + } + line.push(part); + lineLength += part.length + 1; + } + + // NOTE: an empty array is truthy, so this always adds a line (possibly containing only the indent) + if (line) { + lines.push(indent + line.join(' ')); + } + if (linePrefix) { + lines[0] = lines[0].substr(indent.length); + } + return lines; + }; + + let lines: string[]; + // if prog is short, follow it with optionals or positionals + if (prefix.length + prog.length <= 0.75 * textWidth) { + const indent: string = repeat(' ', prefix.length + prog.length + 1); + if (optionalParts) { + lines = [ + ...getLines([prog, ...optionalParts], indent, prefix), + ...getLines(positionalParts, indent) + ]; + } else if (positionalParts) { + lines = getLines([prog, ...positionalParts], indent, prefix); + } else { + lines = [prog]; + } + } else { + // if prog is long, put it on its own line + const indent: string = repeat(' ', prefix.length); + const parts: string[] = [...optionalParts!, ...positionalParts]; + lines = getLines(parts, indent); + if (lines.length > 1) { + lines = [...getLines(optionalParts!, indent), ...getLines(positionalParts, indent)]; + } + lines = [prog, ...lines]; + } + // join lines into usage + usage = lines.join(EOL); + } + } + + // prefix with 'usage:' + return prefix + usage + EOL + EOL; + } + + #formatActionsUsage(actions: ReadonlyArray): string { + const parts: (string | undefined)[] = []; + + // collect all actions format strings + for (const action of actions) { + if (action.help === SUPPRESS) { + // suppressed arguments are marked with None + parts.push(undefined); + } else if (!isOptional(action)) { + // produce all arg strings + parts.push(this.#formatArgs(action, action.dest)); + } else { + // produce the first way to invoke the option in brackets + const optionString: string = action.optionStrings[0]; + + let part: string; + // if the Optional doesn't take a value, format is: -s or --long + if (action.nargs === 0) { + part = '' + optionString; + } else { + // if the Optional takes a value, format is: -s ARGS or --long ARGS + const argsDefault: string = action.dest.toUpperCase(); + const argsString: string = this.#formatArgs(action, argsDefault); + part = optionString + ' ' + argsString; + } + // make it look optional if it's not required or in a group + if (!action.required) { + part = '[' + part + ']'; + } + parts.push(part); + } + } + + // join all the action items with spaces + let text: string = parts.filter((part: string | undefined) => !!part).join(' '); + + // clean up separators for mutually exclusive groups; remove empty groups + text = text.replace(/([\[(]) /g, '$1'); + text = text.replace(/ ([\])])/g, '$1'); + text = text.replace(/\[ *\]/g, ''); + text = text.replace(/\( *\)/g, ''); + text = text.replace(/\(([^|]*)\)/g, '$1'); + + text = text.trim(); + + // return the text + return text; + } + + #formatText(text: string): string { + const textWidth: number = this.#width - this.currentIndent; + const indentIncrement: string = repeat(' ', this.currentIndent); + return this.#fillText(text, textWidth, indentIncrement) + EOL + EOL; + } + + #formatAction(action: IHelpAction): string { + // determine the required width and the entry label + const helpPosition: number = Math.min(this.#actionMaxLength + 2, this.#maxHelpPosition); + const helpWidth: number = this.#width - helpPosition; + const actionWidth: number = helpPosition - this.currentIndent - 2; + let actionHeader: string = this.#formatActionInvocation(action); + let indentFirst: number = 0; + + // no help; start on same line and add a final newline + if (!action.help) { + actionHeader = repeat(' ', this.currentIndent) + actionHeader + EOL; + } else if (actionHeader.length <= actionWidth) { + // short action name; start on the same line and pad two spaces + actionHeader = + repeat(' ', this.currentIndent) + actionHeader + ' ' + repeat(' ', actionWidth - actionHeader.length); + indentFirst = 0; + } else { + // long action name; start on the next line + actionHeader = repeat(' ', this.currentIndent) + actionHeader + EOL; + indentFirst = helpPosition; + } + + // collect the pieces of the action help + const parts: string[] = [actionHeader]; + + // if there was help for the action, add lines of help text + if (action.help) { + const helpLines: string[] = this.#splitLines(action.help, helpWidth); + parts.push(repeat(' ', indentFirst) + helpLines[0] + EOL); + for (const line of helpLines.slice(1)) { + parts.push(repeat(' ', helpPosition) + line + EOL); + } + } else if (actionHeader.charAt(actionHeader.length - 1) !== EOL) { + // or add a newline if the description doesn't end with one + parts.push(EOL); + } + + // if there are any sub-actions, add their help as well + if (action.subactions) { + this.indent(); + for (const subaction of action.subactions) { + parts.push(this.#formatAction(subaction)); + } + this.dedent(); + } + + // return a single string + return this.joinParts(parts); + } + + #formatActionInvocation(action: IHelpAction): string { + if (!isOptional(action)) { + return this.#metavarFormatter(action, action.dest)(1)[0]; + } + + const parts: string[] = []; + + // if the Optional doesn't take a value, format is: -s, --long + if (action.nargs === 0) { + parts.push(...action.optionStrings); + } else { + // if the Optional takes a value, format is: -s ARGS, --long ARGS + const argsDefault: string = action.dest.toUpperCase(); + const argsString: string = this.#formatArgs(action, argsDefault); + for (const optionString of action.optionStrings) { + parts.push(optionString + ' ' + argsString); + } + } + return parts.join(', '); + } + + #metavarFormatter(action: IHelpAction, metavarDefault: string): (size: number) => string[] { + let result: string; + + if (action.metavar || action.metavar === '') { + result = action.metavar; + } else if (action.choices) { + const choices: ReadonlyArray | Record = action.choices; + let choicesString: string; + if (Array.isArray(choices)) { + choicesString = choices.join(','); + } else { + choicesString = Object.keys(choices).join(','); + } + result = '{' + choicesString + '}'; + } else { + result = metavarDefault; + } + + return (size: number): string[] => { + const metavars: string[] = []; + for (let i: number = 0; i < size; i += 1) { + metavars.push(result); + } + return metavars; + }; + } + + #formatArgs(action: IHelpAction, metavarDefault: string): string { + const buildMetavar: (size: number) => string[] = this.#metavarFormatter(action, metavarDefault); + + let metavars: string[]; + switch (action.nargs) { + case undefined: + metavars = buildMetavar(1); + return '' + metavars[0]; + case OPTIONAL: + metavars = buildMetavar(1); + return '[' + metavars[0] + ']'; + case ZERO_OR_MORE: + metavars = buildMetavar(2); + return '[' + metavars[0] + ' [' + metavars[1] + ' ...]]'; + case ONE_OR_MORE: + metavars = buildMetavar(2); + return '' + metavars[0] + ' [' + metavars[1] + ' ...]'; + case REMAINDER: + return '...'; + case PARSER: + metavars = buildMetavar(1); + return metavars[0] + ' ...'; + default: + metavars = buildMetavar(action.nargs as number); + return metavars.join(' '); + } + } + + #splitLines(text: string, width: number): string[] { + const lines: string[] = []; + + text = text.replace(/[\n\|\t]/g, ' '); + + text = text.trim(); + text = text.replace(WHITESPACE_REGEXP, ' '); + + // Wraps the text into lines of at most "width" characters, preferably after a delimiter. Note that the + // original implementation treats the index of a missing delimiter as NaN; this port preserves that behavior. + for (const line of text.split(EOL)) { + if (width >= line.length) { + lines.push(line); + continue; + } + + let wrapStart: number = 0; + let wrapEnd: number = width; + let delimiterIndex: number = 0; + while (wrapEnd <= line.length) { + if (wrapEnd !== line.length) { + delimiterIndex = (SPLIT_LINES_DELIMITER_REGEXP.exec(line.substring(wrapStart, wrapEnd)) || { index: NaN }) + .index; + wrapEnd = wrapStart + delimiterIndex + 1; + } + lines.push(line.substring(wrapStart, wrapEnd)); + wrapStart = wrapEnd; + wrapEnd += width; + } + if (wrapStart < line.length) { + lines.push(line.substring(wrapStart, wrapEnd)); + } + } + + return lines; + } + + #fillText(text: string, width: number, indent: string): string { + const lines: string[] = this.#splitLines(text, width).map((line: string) => indent + line); + return lines.join(EOL); + } +} + +/** + * Equivalent to argparse's `ArgumentParser.formatHelp()`. + */ +export function formatHelp(parser: IHelpParser): string { + const formatter: HelpFormatter = new HelpFormatter(parser.prog); + + // usage + formatter.addUsage(parser.actions); + + // description + formatter.addText(parser.description); + + // positionals, optionals and user-defined groups + for (const group of parser.groups) { + formatter.startSection(group.title); + formatter.addArguments(group.actions); + formatter.endSection(); + } + + // epilog + formatter.addText(parser.epilog); + + // determine help from format above + return formatter.formatHelp(); +} + +/** + * Equivalent to argparse's `ArgumentParser.formatUsage()`. + */ +export function formatUsage(parser: IHelpParser): string { + const formatter: HelpFormatter = new HelpFormatter(parser.prog); + formatter.addUsage(parser.actions); + return formatter.formatHelp(); +} diff --git a/apps/heft/src/cli/HelpModel.ts b/apps/heft/src/cli/HelpModel.ts new file mode 100644 index 00000000000..98814443223 --- /dev/null +++ b/apps/heft/src/cli/HelpModel.ts @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// Models of the argparse objects that its HelpFormatter reads. See HelpFormatter.ts, which is loaded only when +// help text is actually rendered. + +export const SUPPRESS: '==SUPPRESS==' = '==SUPPRESS=='; +export const OPTIONAL: '?' = '?'; +export const ZERO_OR_MORE: '*' = '*'; +export const ONE_OR_MORE: '+' = '+'; +export const PARSER: 'A...' = 'A...'; +export const REMAINDER: '...' = '...'; + +/** + * The properties of an argparse `Action` that the help formatter reads. + */ +export interface IHelpAction { + readonly optionStrings: ReadonlyArray; + readonly dest: string; + readonly nargs?: number | string; + readonly metavar?: string; + readonly help?: string; + readonly choices?: ReadonlyArray | Record; + readonly required?: boolean; + readonly subactions?: ReadonlyArray; +} + +/** + * The properties of an argparse `ArgumentGroup` that the help formatter reads. + */ +export interface IHelpActionGroup { + readonly title: string; + readonly actions: ReadonlyArray; +} + +/** + * The properties of an argparse `ArgumentParser` that are needed to render its help. + */ +export interface IHelpParser { + readonly prog: string; + readonly description: string | undefined; + readonly epilog: string | undefined; + /** + * All actions, in the order in which they were added to the parser. + */ + readonly actions: ReadonlyArray; + /** + * The action groups, in the order in which they were created. + */ + readonly groups: ReadonlyArray; +} + +/** + * The help action that argparse adds to every parser. + */ +export const HELP_ACTION: IHelpAction = { + optionStrings: ['-h', '--help'], + dest: SUPPRESS, + nargs: 0, + help: 'Show this help message and exit.' +}; diff --git a/apps/heft/src/cli/LeanHeftCommandLine.ts b/apps/heft/src/cli/LeanHeftCommandLine.ts new file mode 100644 index 00000000000..2241f8eae8c --- /dev/null +++ b/apps/heft/src/cli/LeanHeftCommandLine.ts @@ -0,0 +1,634 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { CommandLineFlagParameter, CommandLineParameter } from '@rushstack/ts-command-line'; +import { Colorize } from '@rushstack/terminal'; + +import type { HeftPhase } from '../pluginFramework/HeftPhase'; +import type { HeftActionRunner } from './HeftActionRunner'; +import type { IHeftAction, IHeftActionOptions } from './actions/IHeftAction'; +import type { IHeftCommandLineParserState } from './HeftCommandLineParser'; +import { LeanParameterProvider, type ILeanRegistration, type LeanParseResult } from './LeanParameterProvider'; +import { + definePhaseScopingParameters, + expandPhases, + getCleanSelectedPhases, + getPhaseActionSelectedPhases, + type IScopingParameters +} from './actions/PhaseScoping'; +import { + CLEAN_ACTION_DOCUMENTATION, + DEBUG_PARAMETER_DESCRIPTION, + HEFT_TOOL_DESCRIPTION, + HEFT_TOOL_FILENAME, + SCOPED_ACTION_REMAINDER_DESCRIPTION, + UNMANAGED_PARAMETER_DESCRIPTION, + VERBOSE_PARAMETER_DESCRIPTION, + getPhaseActionDocumentation, + getPhaseActionSummary, + getRunActionDocumentation +} from './CliConstants'; +import { HELP_ACTION, PARSER, type IHelpAction, type IHelpParser } from './HelpModel'; +import { Constants } from '../utilities/Constants'; + +// Matches ts-command-line's validation of action names +const ACTION_NAME_REGEXP: RegExp = /^[a-z][a-z0-9]*([-:][a-z0-9]+)*$/; + +// Matches long option strings that argparse treats as unrecognized options (no "=", spaces, etc.) +const PLAIN_LONG_OPTION_REGEXP: RegExp = /^--[a-z0-9]+(-[a-z0-9]+)*$/; + +// The names that ts-command-line registers on Heft's root parser; they are registered as ambiguous in each action. +const ROOT_PARAMETER_NAMES: readonly string[] = [ + Constants.debugParameterLongName, + Constants.unmanagedParameterLongName +]; + +type LeanActionKind = 'clean' | 'run' | 'phase'; + +interface ILeanActionDescriptor { + readonly kind: LeanActionKind; + readonly watch: boolean; + readonly phase?: HeftPhase; + readonly summary: string; +} + +interface ILeanAliasDescriptor { + readonly targetActionName: string; + readonly defaultParameters: readonly string[]; + readonly documentation: string; + readonly expansionMessage: string; +} + +/** + * A lean equivalent of a ts-command-line `CommandLineAction` that implements the members of {@link IHeftAction} + * that Heft uses. + */ +class LeanHeftAction extends LeanParameterProvider { + public readonly actionName: string; + public readonly watch: boolean; + #getSelectedPhases: (() => ReadonlySet) | undefined; + #selectedPhases: ReadonlySet | undefined; + #scopedParameters: ReadonlyArray | undefined; + + public constructor(actionName: string, watch: boolean) { + super(); + this.actionName = actionName; + this.watch = watch; + } + + /** + * Like the `selectedPhases` getter of Heft's actions, the selection is evaluated lazily, on first access. + */ + public get selectedPhases(): ReadonlySet { + if (!this.#selectedPhases) { + this.#selectedPhases = this.#getSelectedPhases!(); + } + return this.#selectedPhases; + } + + public setSelectedPhasesFactory(getSelectedPhases: () => ReadonlySet): void { + this.#getSelectedPhases = getSelectedPhases; + } + + /** + * Like ts-command-line's `ScopedCommandLineAction.parameters`, which includes the scoped parameters. + */ + public override get parameters(): ReadonlyArray { + if (this.#scopedParameters) { + return [...super.parameters, ...this.#scopedParameters]; + } else { + return super.parameters; + } + } + + public setScopedParameters(scopedParameters: ReadonlyArray): void { + this.#scopedParameters = scopedParameters; + } + + public asHeftAction(): IHeftAction { + return this as unknown as IHeftAction; + } +} + +/** + * Wraps an error that the full implementation would also throw, at the same point and in the same way. Such errors + * must be propagated rather than handled by falling back to the full implementation, because evaluating some + * definitions has side effects (for example, `HeftPhase.dependencyPhases` only throws on first access). + */ +class LeanDefinitionError { + public readonly error: unknown; + + public constructor(error: unknown) { + this.error = error; + } +} + +/** + * The outcome of a command line that was fully validated and parsed by the lean implementation. + * - 'execute': the action executes; this happens in ts-command-line's `onExecuteAsync()` stage + * - 'print': help (or a usage error) that ts-command-line prints while parsing, before `onExecuteAsync()` + */ +type LeanInvocation = + | { + readonly kind: 'execute'; + readonly actionName: string; + readonly unaliasedActionName: string; + executeAsync(): Promise; + } + | { + readonly kind: 'print'; + readonly helpParser: IHelpParser; + /** + * If specified, the usage is printed (instead of the help), followed by this error message. + */ + readonly usageError?: string; + }; + +/** + * Attempts to handle the command line without ts-command-line/argparse. Returns `undefined` if the lean + * implementation cannot guarantee results identical to the full implementation, in which case nothing has been + * modified and the caller must use the full implementation. + */ +export async function tryExecuteLeanCommandLineAsync( + args: readonly string[], + actionOptions: IHeftActionOptions, + state: IHeftCommandLineParserState +): Promise { + let invocation: LeanInvocation | undefined; + try { + invocation = await tryPrepareLeanInvocationAsync(args, actionOptions); + } catch (e) { + if (e instanceof LeanDefinitionError) { + throw e.error; + } + // Anything else is handled by the full implementation, which reports errors the canonical way. + invocation = undefined; + } + + if (!invocation) { + return undefined; + } + + if (invocation.kind === 'print') { + // This mirrors how ts-command-line and argparse print help and usage errors + const { formatHelp, formatUsage } = await import('./HelpFormatter'); + const { helpParser, usageError } = invocation; + if (usageError === undefined) { + process.stdout.write(formatHelp(helpParser)); + return true; + } else { + process.stdout.write(formatUsage(helpParser)); + // eslint-disable-next-line no-console + console.error(usageError); + return false; + } + } + + // This mirrors HeftFullCommandLineParser.onExecuteAsync() + try { + const { actionName, unaliasedActionName } = invocation; + state.internalHeftSession.parsedCommandLine = { + commandName: actionName, + unaliasedCommandName: unaliasedActionName + }; + state.childReporter?.setCommandName(actionName); + await invocation.executeAsync(); + } catch (e) { + await state.reportErrorAndSetExitCodeAsync(e as Error); + } + + // If we make it here, things are fine and reset the exit code back to 0 + process.exitCode = 0; + return true; +} + +async function tryPrepareLeanInvocationAsync( + args: readonly string[], + actionOptions: IHeftActionOptions +): Promise { + const { internalHeftSession, terminal } = actionOptions; + + // Enumerate the actions in the same order as the full implementation, and bail out on anything that would + // cause the full implementation to throw while defining them. + const actionsByName: Map = new Map(); + function tryAddAction(actionName: string, descriptor: ILeanActionDescriptor): boolean { + if (actionsByName.has(actionName) || !ACTION_NAME_REGEXP.test(actionName)) { + return false; + } + actionsByName.set(actionName, descriptor); + return true; + } + + tryAddAction('clean', { kind: 'clean', watch: false, summary: CLEAN_ACTION_DOCUMENTATION }); + tryAddAction('run', { kind: 'run', watch: false, summary: getRunActionDocumentation(false) }); + for (const phase of internalHeftSession.phases) { + const { phaseName } = phase; + const summary: string = getPhaseActionSummary(phaseName, false); + if (!tryAddAction(phaseName, { kind: 'phase', watch: false, phase, summary })) { + return undefined; + } + } + tryAddAction('run-watch', { kind: 'run', watch: true, summary: getRunActionDocumentation(true) }); + for (const phase of internalHeftSession.phases) { + const { phaseName } = phase; + const summary: string = getPhaseActionSummary(phaseName, true); + if (!tryAddAction(`${phaseName}-watch`, { kind: 'phase', watch: true, phase, summary })) { + return undefined; + } + } + + const aliasSummariesByName: Map = new Map(); + const aliasesByName: Map = new Map(); + for (const [ + aliasName, + { actionName, defaultParameters = [] } + ] of internalHeftSession.actionReferencesByAlias) { + if (actionsByName.has(aliasName) || !actionsByName.has(actionName) || !ACTION_NAME_REGEXP.test(aliasName)) { + return undefined; + } + // Matches ts-command-line's AliasCommandLineAction and Heft's AliasAction + const defaultParametersString: string = defaultParameters.join(' '); + const expandedCommand: string = `${HEFT_TOOL_FILENAME} ${actionName}${ + defaultParametersString ? ` ${defaultParametersString}` : '' + }`; + const summary: string = `An alias for "${expandedCommand}".`; + aliasSummariesByName.set(aliasName, summary); + aliasesByName.set(aliasName, { + targetActionName: actionName, + defaultParameters, + documentation: + `${summary} For more information on the aliased command, use ` + + `"${HEFT_TOOL_FILENAME} ${actionName} --help".`, + expansionMessage: `The "${HEFT_TOOL_FILENAME} ${aliasName}" alias was expanded to "${expandedCommand}".` + }); + } + + const { HeftActionRunner: HeftActionRunnerClass } = await import('./HeftActionRunner'); + function createPhaseAction( + phase: HeftPhase, + watch: boolean + ): { action: LeanHeftAction; actionRunner: HeftActionRunner } { + const action: LeanHeftAction = new LeanHeftAction(`${phase.phaseName}${watch ? '-watch' : ''}`, watch); + action.setSelectedPhasesFactory(() => getPhaseActionSelectedPhases(phase)); + const actionRunner: HeftActionRunner = new HeftActionRunnerClass({ + action: action.asHeftAction(), + ...actionOptions + }); + actionRunner.defineParameters(); + return { action, actionRunner }; + } + + // Define the parameters of every phase action in the same order as the full implementation does. Errors are + // propagated, since the full implementation would throw the same error at the same point. (The watch variant + // of a phase action defines the same parameters with the same names, so it doesn't need to be checked + // separately. The "clean" and "run" actions only define built-in parameters.) + const phaseActionsByPhase: Map = + new Map(); + for (const phase of internalHeftSession.phases) { + try { + phaseActionsByPhase.set(phase, createPhaseAction(phase, false)); + } catch (e) { + throw new LeanDefinitionError(e); + } + } + + // Ensure that registering the parameters of every phase action would succeed + const phaseActionRegistrations: Map = new Map(); + for (const { action } of phaseActionsByPhase.values()) { + const registration: ILeanRegistration | undefined = action.tryGetRegistration(ROOT_PARAMETER_NAMES); + if (!registration) { + return undefined; + } + phaseActionRegistrations.set(action, registration); + } + + // Process the tool-level arguments, i.e. the ones before the action name. This finds the action the same way + // as ts-command-line does when it expands aliases. + const actionNameIndex: number = args.findIndex((x) => !x.startsWith('-')); + const toolArgs: readonly string[] = actionNameIndex < 0 ? args : args.slice(0, actionNameIndex); + for (const arg of toolArgs) { + if (arg === '-h' || arg === '--help') { + return printRootHelp(actionsByName, aliasSummariesByName); + } else if (arg !== Constants.debugParameterLongName && arg !== Constants.unmanagedParameterLongName) { + if (actionNameIndex >= 0 || !isUnrecognizedToolOption(arg)) { + return undefined; + } + // argparse only reports unrecognized arguments after parsing succeeded, which it can't without an action + } + } + + if (actionNameIndex < 0) { + if (args.length === 0) { + // ts-command-line prints the help if no arguments are provided + return printRootHelp(actionsByName, aliasSummariesByName); + } + // argparse: the required positional is missing + return { + kind: 'print', + helpParser: getRootHelpParser(getActionSummaries(actionsByName, aliasSummariesByName)), + usageError: `${HEFT_TOOL_FILENAME}: error: too few arguments\n` + }; + } + + // The action name as specified on the command line, which may be an alias + const actionName: string = args[actionNameIndex]; + const alias: ILeanAliasDescriptor | undefined = aliasesByName.get(actionName); + const unaliasedActionName: string = alias ? alias.targetActionName : actionName; + const descriptor: ILeanActionDescriptor | undefined = actionsByName.get(unaliasedActionName); + if (!descriptor) { + return undefined; + } + + // Like ts-command-line, insert the alias's default parameters after the alias name. The parser of an alias + // registers the same parameters (and thus option strings) as the parser of its target action. + const actionArgs: readonly string[] = alias + ? [...alias.defaultParameters, ...args.slice(actionNameIndex + 1)] + : args.slice(actionNameIndex + 1); + const actionProg: string = `${HEFT_TOOL_FILENAME} ${actionName}`; + const unaliasedActionProg: string = `${HEFT_TOOL_FILENAME} ${unaliasedActionName}`; + function createExecuteInvocation(executeAsync: () => Promise): LeanInvocation { + return { + kind: 'execute', + actionName, + unaliasedActionName, + executeAsync: alias + ? async () => { + // This mirrors Heft's AliasAction + terminal.writeLine(alias.expansionMessage); + await executeAsync(); + } + : executeAsync + }; + } + + switch (descriptor.kind) { + case 'phase': { + const phase: HeftPhase = descriptor.phase!; + // The non-watch variant of the phase action was already created above + const { action, actionRunner } = descriptor.watch + ? createPhaseAction(phase, true) + : phaseActionsByPhase.get(phase)!; + const registration: ILeanRegistration | undefined = + phaseActionRegistrations.get(action) ?? action.tryGetRegistration(ROOT_PARAMETER_NAMES); + const parseResult: LeanParseResult | undefined = tryParseAndApply(action, registration, actionArgs, false); + if (parseResult?.kind === 'help') { + const documentation: string = + alias?.documentation ?? + getPhaseActionDocumentation(phase.phaseName, phase.phaseDescription, descriptor.watch); + return printHelp(action.getHelpParser(registration!, actionProg, documentation, undefined)); + } else if (parseResult?.kind !== 'ok') { + return undefined; + } + + return createExecuteInvocation(() => actionRunner.executeAsync()); + } + + case 'run': { + const documentation: string = getRunActionDocumentation(descriptor.watch); + const action: LeanHeftAction = new LeanHeftAction(unaliasedActionName, descriptor.watch); + action.defineCommandLineRemainder({ description: SCOPED_ACTION_REMAINDER_DESCRIPTION }); + const scopingParameters: IScopingParameters = definePhaseScopingParameters(action); + action.setSelectedPhasesFactory(() => + expandPhases( + scopingParameters.onlyParameter, + scopingParameters.toParameter, + scopingParameters.toExceptParameter, + internalHeftSession, + terminal + ) + ); + const actionRunner: HeftActionRunner = new HeftActionRunnerClass({ + action: action.asHeftAction(), + ...actionOptions + }); + + const registration: ILeanRegistration | undefined = action.tryGetRegistration(ROOT_PARAMETER_NAMES); + const parseResult: LeanParseResult | undefined = tryParseAndApply(action, registration, actionArgs, true); + if (parseResult?.kind === 'help') { + return printHelp( + action.getHelpParser(registration!, actionProg, alias?.documentation ?? documentation, undefined) + ); + } else if (parseResult?.kind !== 'ok') { + return undefined; + } + + // Evaluate the phase selection now, but only if doing so cannot report an error + for (const scopingParameter of [ + scopingParameters.onlyParameter, + scopingParameters.toParameter, + scopingParameters.toExceptParameter + ]) { + for (const phaseName of scopingParameter.values) { + if (!internalHeftSession.phasesByName.has(phaseName)) { + return undefined; + } + } + } + try { + // Throws if the selection is empty + if (action.selectedPhases.size === 0) { + return undefined; + } + } catch (e) { + return undefined; + } + + // Define the scoped parameters, like ScopedCommandLineAction does + const scopedParameterProvider: LeanParameterProvider = new LeanParameterProvider(); + actionRunner.defineParameters(scopedParameterProvider.asCommandLineParameterProvider()); + const scopedRegistration: ILeanRegistration | undefined = scopedParameterProvider.tryGetRegistration([ + ...ROOT_PARAMETER_NAMES, + ...registration!.registeredNames + ]); + + // ScopedCommandLineAction requires the remainder to start with "--", which is then discarded + const remainder: readonly string[] = parseResult.remainder ?? []; + if (remainder.length && remainder[0] !== '--') { + return undefined; + } + const scopedParseResult: LeanParseResult | undefined = tryParseAndApply( + scopedParameterProvider, + scopedRegistration, + remainder.slice(1), + false + ); + if (scopedParseResult?.kind === 'help') { + // ScopedCommandLineAction parses the scoped parameters (and prints their help) during execution. The + // scoping arguments are omitted from the help of an alias, since they don't apply to the alias itself. + const scopingArgs: string[] = []; + for (const parameter of action.parameters) { + parameter.appendToArgList(scopingArgs); + } + const scope: string = scopingArgs.join(' '); + const scopedHelpParser: IHelpParser = scopedParameterProvider.getHelpParser( + scopedRegistration!, + `${actionProg}${scope && !alias ? ` ${scope} --` : ''}`, + alias?.documentation ?? documentation, + Colorize.bold( + `For more information on available unscoped parameters, use "${unaliasedActionProg} --help"` + ) + ); + return createExecuteInvocation(async () => { + const { formatHelp } = await import('./HelpFormatter'); + process.stdout.write(formatHelp(scopedHelpParser)); + }); + } else if (scopedParseResult?.kind !== 'ok') { + return undefined; + } + action.setScopedParameters(scopedParameterProvider.parameters); + + return createExecuteInvocation(() => actionRunner.executeAsync()); + } + + case 'clean': { + const action: LeanHeftAction = new LeanHeftAction(unaliasedActionName, false); + const scopingParameters: IScopingParameters = definePhaseScopingParameters(action); + action.setSelectedPhasesFactory(() => + getCleanSelectedPhases(scopingParameters, internalHeftSession, terminal) + ); + const verboseFlag: CommandLineFlagParameter = action.defineFlagParameter({ + parameterLongName: Constants.verboseParameterLongName, + parameterShortName: Constants.verboseParameterShortName, + description: VERBOSE_PARAMETER_DESCRIPTION + }); + + const registration: ILeanRegistration | undefined = action.tryGetRegistration(ROOT_PARAMETER_NAMES); + const parseResult: LeanParseResult | undefined = tryParseAndApply(action, registration, actionArgs, false); + if (parseResult?.kind === 'help') { + return printHelp( + action.getHelpParser( + registration!, + actionProg, + alias?.documentation ?? CLEAN_ACTION_DOCUMENTATION, + undefined + ) + ); + } else if (parseResult?.kind !== 'ok') { + return undefined; + } + + return createExecuteInvocation(async () => { + const { executeCleanActionAsync } = await import('./actions/CleanActionExecution'); + await executeCleanActionAsync({ + action: action.asHeftAction(), + internalHeftSession, + terminal, + metricsCollector: actionOptions.metricsCollector, + isVerbose: verboseFlag.value + }); + }); + } + + default: + return undefined; + } +} + +/** + * Parses the arguments and, if successful, assigns the parameter values. Returns `undefined` if the full + * implementation must be used instead. + */ +function tryParseAndApply( + provider: LeanParameterProvider, + registration: ILeanRegistration | undefined, + args: readonly string[], + allowRemainder: boolean +): LeanParseResult | undefined { + if (!registration) { + return undefined; + } + + const result: LeanParseResult = provider.parseArguments(registration, args, allowRemainder); + if (result.kind === 'ok') { + if (!provider.tryApplyValues(result.data, allowRemainder ? result.remainder ?? [] : undefined)) { + return undefined; + } + } + + return result; +} + +/** + * Returns true if argparse would treat the argument as an unrecognized option of Heft's root parser, i.e. it is + * not a prefix of any of the root parser's option strings. + */ +function isUnrecognizedToolOption(arg: string): boolean { + if (!PLAIN_LONG_OPTION_REGEXP.test(arg)) { + return false; + } + for (const optionString of [...HELP_ACTION.optionStrings, ...ROOT_PARAMETER_NAMES]) { + if (optionString.startsWith(arg)) { + return false; + } + } + return true; +} + +function printHelp(helpParser: IHelpParser): LeanInvocation { + return { kind: 'print', helpParser }; +} + +function printRootHelp( + actionsByName: ReadonlyMap, + aliasSummariesByName: ReadonlyMap +): LeanInvocation { + return printHelp(getRootHelpParser(getActionSummaries(actionsByName, aliasSummariesByName))); +} + +function* getActionSummaries( + actionsByName: ReadonlyMap, + aliasSummariesByName: ReadonlyMap +): IterableIterator<[string, string]> { + for (const [actionName, { summary }] of actionsByName) { + yield [actionName, summary]; + } + yield* aliasSummariesByName; +} + +/** + * Models the argparse parser that ts-command-line builds for Heft's root command line. + * + * @param actionSummaries - the name and summary of each action, in the order in which they were added + */ +export function getRootHelpParser(actionSummaries: Iterable<[string, string]>): IHelpParser { + const subactions: IHelpAction[] = []; + for (const [actionName, summary] of actionSummaries) { + subactions.push({ + optionStrings: [], + dest: actionName, + help: summary + }); + } + + const subparsersAction: IHelpAction = { + optionStrings: [], + dest: 'action', + nargs: PARSER, + metavar: '', + subactions + }; + const debugAction: IHelpAction = { + optionStrings: [Constants.debugParameterLongName], + dest: 'debug', + nargs: 0, + help: DEBUG_PARAMETER_DESCRIPTION + }; + const unmanagedAction: IHelpAction = { + optionStrings: [Constants.unmanagedParameterLongName], + dest: 'unmanaged', + nargs: 0, + help: UNMANAGED_PARAMETER_DESCRIPTION + }; + + return { + prog: HEFT_TOOL_FILENAME, + description: HEFT_TOOL_DESCRIPTION, + epilog: Colorize.bold(`For detailed help about a specific command, use: ${HEFT_TOOL_FILENAME} -h`), + // ts-command-line adds the actions (and thus the subparsers) before it registers the tool's own parameters + actions: [HELP_ACTION, subparsersAction, debugAction, unmanagedAction], + groups: [ + { title: 'Positional arguments', actions: [subparsersAction] }, + { title: 'Optional arguments', actions: [HELP_ACTION, debugAction, unmanagedAction] } + ] + }; +} diff --git a/apps/heft/src/cli/LeanParameterProvider.ts b/apps/heft/src/cli/LeanParameterProvider.ts new file mode 100644 index 00000000000..923bbd622be --- /dev/null +++ b/apps/heft/src/cli/LeanParameterProvider.ts @@ -0,0 +1,673 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// IMPORTANT: This module must not load the "@rushstack/ts-command-line" package entry point, because doing so +// loads "argparse". Only the parameter classes are loaded (via their own modules, which are the same module +// instances that the package entry point re-exports), so the parameter objects handed to plugins are genuine +// ts-command-line instances. +import type { + CommandLineChoiceListParameter, + CommandLineChoiceParameter, + CommandLineFlagParameter, + CommandLineIntegerListParameter, + CommandLineIntegerParameter, + CommandLineParameter, + CommandLineParameterProvider, + CommandLineRemainder, + CommandLineStringListParameter, + CommandLineStringParameter, + ICommandLineChoiceDefinition, + ICommandLineChoiceListDefinition, + ICommandLineFlagDefinition, + ICommandLineIntegerDefinition, + ICommandLineIntegerListDefinition, + ICommandLineRemainderDefinition, + ICommandLineStringDefinition, + ICommandLineStringListDefinition +} from '@rushstack/ts-command-line'; +// The declarations of the package's own modules include the @internal members (such as _setValue()), which the +// package's public API rollup omits. +import { + CommandLineParameterKind, + type CommandLineParameter as InternalCommandLineParameter +} from '@rushstack/ts-command-line/lib/parameters/BaseClasses'; +import type { CommandLineChoiceParameter as ChoiceParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineChoiceParameter'; +import type { CommandLineChoiceListParameter as ChoiceListParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineChoiceListParameter'; +import { CommandLineFlagParameter as FlagParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineFlagParameter'; +import type { CommandLineIntegerParameter as IntegerParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineIntegerParameter'; +import type { CommandLineIntegerListParameter as IntegerListParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineIntegerListParameter'; +import type { CommandLineStringParameter as StringParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineStringParameter'; +import { CommandLineStringListParameter as StringListParameterClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineStringListParameter'; +import type { CommandLineRemainder as RemainderClass } from '@rushstack/ts-command-line/lib/parameters/CommandLineRemainder'; +import { SCOPING_PARAMETER_GROUP } from '@rushstack/ts-command-line/lib/Constants'; + +import { + HELP_ACTION, + REMAINDER, + SUPPRESS, + ZERO_OR_MORE, + type IHelpAction, + type IHelpActionGroup, + type IHelpParser +} from './HelpModel'; + +// Heft's built-in parameters are flags and string lists; the classes for the other kinds of parameters (and for +// the remainder) are only loaded if a parameter of that kind is defined. +let _choiceParameterClass: typeof ChoiceParameterClass | undefined; +let _choiceListParameterClass: typeof ChoiceListParameterClass | undefined; +let _integerParameterClass: typeof IntegerParameterClass | undefined; +let _integerListParameterClass: typeof IntegerListParameterClass | undefined; +let _stringParameterClass: typeof StringParameterClass | undefined; +let _remainderClass: typeof RemainderClass | undefined; + +const INTEGER_REGEXP: RegExp = /^[0-9]+$/; + +let _keyCounter: number = 0; + +/** + * Describes what a single command-line token means to a lean parser. + * - a parameter: the token is a registered option string of that parameter + * - 'help': the token is the argparse built-in help option + * - 'ambiguous': the token is registered only to report an ambiguity error + */ +export type LeanOptionTarget = InternalCommandLineParameter | 'help' | 'ambiguous'; + +/** + * A model of the argparse registrations that ts-command-line would perform for a parameter provider. + */ +export interface ILeanRegistration { + /** + * Maps every argparse option string of the provider to its meaning. + */ + readonly optionMap: ReadonlyMap; + /** + * The names that ts-command-line records as registered (used as "parent" names for nested parsers). + */ + readonly registeredNames: ReadonlySet; + /** + * Parameters that ts-command-line would report as ambiguous if they receive a value. + */ + readonly poisonedParameters: ReadonlySet; + /** + * The argparse registrations, in the order in which ts-command-line would perform them. + */ + readonly steps: ReadonlyArray; +} + +/** + * A single argparse registration performed by ts-command-line: either a parameter (with the option strings it is + * registered under, and its undocumented synonyms), or an ambiguous name that is registered only to report errors. + */ +export type LeanRegistrationStep = + | { + readonly parameter: InternalCommandLineParameter; + readonly optionStrings: readonly string[]; + readonly undocumentedSynonyms: readonly string[] | undefined; + } + | { readonly ambiguousName: string }; + + +/** + * The result of a lean parse. + * - 'ok': the arguments were fully understood; values are ready to be applied + * - 'help': the arguments request the help text of this provider + * - 'fallback': the lean parser cannot guarantee identical results; use the full ts-command-line parser + */ +export type LeanParseResult = + | { + readonly kind: 'ok'; + readonly data: Map; + readonly remainder?: string[]; + } + | { readonly kind: 'help' } + | { readonly kind: 'fallback' }; + +const FALLBACK: LeanParseResult = { kind: 'fallback' }; +const HELP: LeanParseResult = { kind: 'help' }; + +/** + * A lightweight stand-in for ts-command-line's `CommandLineParameterProvider` that does not depend on argparse. + * + * @remarks + * The parameter objects it creates are genuine ts-command-line parameter instances, and their values are + * assigned through the same `_setValue()` entry point, using the same data argparse would produce. This makes + * environment variable handling, defaults and validation identical to the full parser. Anything that the lean + * model cannot guarantee to be identical is reported as a "fallback", so the caller can use the full + * ts-command-line parser instead. + */ +export class LeanParameterProvider { + readonly #parameters: InternalCommandLineParameter[] = []; + readonly #parametersByLongName: Map = new Map(); + readonly #parametersByShortName: Map = new Map(); + #remainder: RemainderClass | undefined; + + public get parameters(): ReadonlyArray { + return this.#parameters as unknown as ReadonlyArray; + } + + public get remainder(): CommandLineRemainder | undefined { + return this.#remainder as unknown as CommandLineRemainder | undefined; + } + + /** + * Allows this object to be passed to APIs that expect a ts-command-line `CommandLineParameterProvider` and that + * only call the `define*Parameter()` methods. + * + * @remarks + * The package's public API declarations (dist/ts-command-line.d.ts) and its per-module declarations (lib-dts) + * describe the same runtime classes, but TypeScript treats them as unrelated types. The casts in this class + * bridge the two. + */ + public asCommandLineParameterProvider(): CommandLineParameterProvider { + return this as unknown as CommandLineParameterProvider; + } + + public defineChoiceParameter( + definition: ICommandLineChoiceDefinition + ): CommandLineChoiceParameter { + if (!_choiceParameterClass) { + _choiceParameterClass = ( + require('@rushstack/ts-command-line/lib/parameters/CommandLineChoiceParameter') as { + CommandLineChoiceParameter: typeof ChoiceParameterClass; + } + ).CommandLineChoiceParameter; + } + const parameter: InternalCommandLineParameter = new _choiceParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineChoiceParameter; + } + + public defineChoiceListParameter( + definition: ICommandLineChoiceListDefinition + ): CommandLineChoiceListParameter { + if (!_choiceListParameterClass) { + _choiceListParameterClass = ( + require('@rushstack/ts-command-line/lib/parameters/CommandLineChoiceListParameter') as { + CommandLineChoiceListParameter: typeof ChoiceListParameterClass; + } + ).CommandLineChoiceListParameter; + } + const parameter: InternalCommandLineParameter = new _choiceListParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineChoiceListParameter; + } + + public defineFlagParameter(definition: ICommandLineFlagDefinition): CommandLineFlagParameter { + const parameter: InternalCommandLineParameter = new FlagParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineFlagParameter; + } + + public defineIntegerParameter(definition: ICommandLineIntegerDefinition): CommandLineIntegerParameter { + if (!_integerParameterClass) { + _integerParameterClass = ( + require('@rushstack/ts-command-line/lib/parameters/CommandLineIntegerParameter') as { + CommandLineIntegerParameter: typeof IntegerParameterClass; + } + ).CommandLineIntegerParameter; + } + const parameter: InternalCommandLineParameter = new _integerParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineIntegerParameter; + } + + public defineIntegerListParameter( + definition: ICommandLineIntegerListDefinition + ): CommandLineIntegerListParameter { + if (!_integerListParameterClass) { + _integerListParameterClass = ( + require('@rushstack/ts-command-line/lib/parameters/CommandLineIntegerListParameter') as { + CommandLineIntegerListParameter: typeof IntegerListParameterClass; + } + ).CommandLineIntegerListParameter; + } + const parameter: InternalCommandLineParameter = new _integerListParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineIntegerListParameter; + } + + public defineStringParameter(definition: ICommandLineStringDefinition): CommandLineStringParameter { + if (!_stringParameterClass) { + _stringParameterClass = ( + require('@rushstack/ts-command-line/lib/parameters/CommandLineStringParameter') as { + CommandLineStringParameter: typeof StringParameterClass; + } + ).CommandLineStringParameter; + } + const parameter: InternalCommandLineParameter = new _stringParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineStringParameter; + } + + public defineStringListParameter( + definition: ICommandLineStringListDefinition + ): CommandLineStringListParameter { + const parameter: InternalCommandLineParameter = new StringListParameterClass(definition as never); + return this.#defineParameter(parameter) as {} as CommandLineStringListParameter; + } + + public defineCommandLineRemainder(definition: ICommandLineRemainderDefinition): CommandLineRemainder { + if (this.#remainder) { + throw new Error('defineRemainingArguments() has already been called for this provider'); + } + if (!_remainderClass) { + _remainderClass = ( + require('@rushstack/ts-command-line/lib/parameters/CommandLineRemainder') as { + CommandLineRemainder: typeof RemainderClass; + } + ).CommandLineRemainder; + } + this.#remainder = new _remainderClass(definition as never); + return this.#remainder as unknown as CommandLineRemainder; + } + + /** + * Identical to ts-command-line's `CommandLineParameterProvider.getParameterStringMap()`. + */ + public getParameterStringMap(): Record { + return getParameterStringMap(this.parameters); + } + + /** + * Models ts-command-line's `_registerDefinedParameters()` followed by the resulting argparse `addArgument()` + * calls. Returns `undefined` if ts-command-line or argparse would throw, or if the definitions use a feature that + * the lean parser does not model. + * + * @param parentParameterNames - the names registered by the parent parser(s), which ts-command-line registers + * as ambiguous in this provider. + */ + public tryGetRegistration(parentParameterNames: Iterable): ILeanRegistration | undefined { + const optionMap: Map = new Map(); + const registeredParametersByName: Map = new Map(); + const ambiguousNames: Set = new Set(); + const steps: LeanRegistrationStep[] = []; + + for (const helpOptionString of HELP_ACTION.optionStrings) { + optionMap.set(helpOptionString, 'help'); + } + + // argparse throws if an option string is registered more than once ("conflictHandler: error") + function tryAddOptionStrings(optionStrings: readonly string[], target: LeanOptionTarget): boolean { + for (const optionString of optionStrings) { + if (optionMap.has(optionString)) { + return false; + } + } + for (const optionString of optionStrings) { + optionMap.set(optionString, target); + } + return true; + } + + const parametersWithDuplicateShortNames: Set = new Set(); + for (const [shortName, shortNameParameters] of this.#parametersByShortName) { + if (shortNameParameters.length > 1) { + ambiguousNames.add(shortName); + for (const parameter of shortNameParameters) { + parametersWithDuplicateShortNames.add(parameter); + } + } + } + + for (const longNameParameters of this.#parametersByLongName.values()) { + const useScopedLongName: boolean = longNameParameters.length > 1; + for (const parameter of longNameParameters) { + if (useScopedLongName) { + if (!parameter.parameterScope) { + // ts-command-line throws "The parameter ... is defined multiple times with the same long name." + return undefined; + } + ambiguousNames.add(parameter.longName); + } + + if (parameter.required && parameter.environmentVariable) { + // ts-command-line patches the parameter's parse hooks for this case; not modeled. + return undefined; + } + + const { parameterGroup } = parameter; + if (parameterGroup !== undefined && typeof parameterGroup !== 'string') { + if ((parameterGroup as unknown) !== SCOPING_PARAMETER_GROUP) { + // ts-command-line throws "Unexpected parameter group" + return undefined; + } + } + + const { shortName, longName, scopedLongName, undocumentedSynonyms } = parameter; + const names: string[] = []; + if (shortName && !parametersWithDuplicateShortNames.has(parameter)) { + names.push(shortName); + } + if (!useScopedLongName) { + names.push(longName); + } + if (scopedLongName) { + names.push(scopedLongName); + } + + if (!tryAddOptionStrings(names, parameter)) { + return undefined; + } + + if (undocumentedSynonyms?.length) { + if (!tryAddOptionStrings(undocumentedSynonyms, parameter)) { + return undefined; + } + } + + steps.push({ parameter, optionStrings: names, undocumentedSynonyms }); + + for (const name of names) { + registeredParametersByName.set(name, parameter); + } + if (undocumentedSynonyms) { + for (const name of undocumentedSynonyms) { + registeredParametersByName.set(name, parameter); + } + } + } + } + + for (const parentParameterName of parentParameterNames) { + ambiguousNames.add(parentParameterName); + } + + const poisonedParameters: Set = new Set(); + for (const ambiguousName of ambiguousNames) { + const registeredParameter: InternalCommandLineParameter | undefined = + registeredParametersByName.get(ambiguousName); + if (registeredParameter) { + // ts-command-line reports "Ambiguous option" whenever this parameter receives a truthy value + poisonedParameters.add(registeredParameter); + } else if (!tryAddOptionStrings([ambiguousName], 'ambiguous')) { + return undefined; + } else { + steps.push({ ambiguousName }); + } + } + + return { + optionMap, + registeredNames: new Set(registeredParametersByName.keys()), + poisonedParameters, + steps + }; + } + + /** + * Models the argparse parser that ts-command-line would build for this provider, for rendering its help. + */ + public getHelpParser( + registration: ILeanRegistration, + prog: string, + description: string | undefined, + epilog: string | undefined + ): IHelpParser { + const actions: IHelpAction[] = [HELP_ACTION]; + const positionals: IHelpAction[] = []; + const optionals: IHelpAction[] = [HELP_ACTION]; + const groups: IHelpActionGroup[] = [ + { title: 'Positional arguments', actions: positionals }, + { title: 'Optional arguments', actions: optionals } + ]; + const customGroupActionsByName: Map = new Map(); + + for (const step of registration.steps) { + if ('ambiguousName' in step) { + const ambiguousAction: IHelpAction = { + optionStrings: [step.ambiguousName], + dest: SUPPRESS, + nargs: ZERO_OR_MORE, + help: SUPPRESS + }; + actions.push(ambiguousAction); + optionals.push(ambiguousAction); + continue; + } + + const { parameter, optionStrings, undocumentedSynonyms } = step; + let groupActions: IHelpAction[] = optionals; + const { parameterGroup } = parameter; + if (parameterGroup !== undefined) { + let customGroupActions: IHelpAction[] | undefined = customGroupActionsByName.get(parameterGroup); + if (!customGroupActions) { + customGroupActions = []; + customGroupActionsByName.set(parameterGroup, customGroupActions); + const parameterGroupName: string = typeof parameterGroup === 'string' ? parameterGroup : 'scoping'; + groups.push({ title: `Optional ${parameterGroupName} arguments`, actions: customGroupActions }); + } + groupActions = customGroupActions; + } + + const parameterAction: IHelpAction = { + optionStrings, + dest: parameter._parserKey!, + nargs: parameter.kind === CommandLineParameterKind.Flag ? 0 : undefined, + metavar: (parameter as { argumentName?: string }).argumentName, + help: getParameterHelp(parameter), + choices: + parameter.kind === CommandLineParameterKind.Choice || + parameter.kind === CommandLineParameterKind.ChoiceList + ? Array.from(parameter.alternatives) + : undefined, + required: parameter.required + }; + actions.push(parameterAction); + groupActions.push(parameterAction); + + if (undocumentedSynonyms?.length) { + const synonymsAction: IHelpAction = { + ...parameterAction, + optionStrings: undocumentedSynonyms, + help: SUPPRESS + }; + actions.push(synonymsAction); + groupActions.push(synonymsAction); + } + } + + if (this.#remainder) { + const remainderAction: IHelpAction = { + optionStrings: [], + dest: REMAINDER, + nargs: REMAINDER, + metavar: '"..."', + help: this.#remainder.description, + required: true + }; + actions.push(remainderAction); + positionals.push(remainderAction); + } + + return { prog, description, epilog, actions, groups }; + } + + /** + * Parses the arguments for this provider, accepting only a conservative subset of the syntax that argparse + * accepts: exact option strings, and option values that do not start with "-". + * + * @param allowRemainder - if true, a "--" token starts the remainder, which is returned including the "--" + */ + public parseArguments( + registration: ILeanRegistration, + args: readonly string[], + allowRemainder: boolean + ): LeanParseResult { + const { optionMap, poisonedParameters } = registration; + const data: Map = new Map(); + let remainder: string[] | undefined; + + for (let i: number = 0; i < args.length; i++) { + const arg: string = args[i]; + if (allowRemainder && arg === '--') { + remainder = args.slice(i); + break; + } + + const target: LeanOptionTarget | undefined = optionMap.get(arg); + if (target === undefined || target === 'ambiguous') { + return FALLBACK; + } else if (target === 'help') { + return HELP; + } else if (poisonedParameters.has(target)) { + return FALLBACK; + } + + if (target.kind === CommandLineParameterKind.Flag) { + data.set(target, true); + continue; + } + + const rawValue: string | undefined = args[++i]; + if (rawValue === undefined || rawValue.startsWith('-')) { + return FALLBACK; + } + + let value: string | number; + switch (target.kind) { + case CommandLineParameterKind.Choice: + case CommandLineParameterKind.ChoiceList: + if (!target.alternatives.has(rawValue)) { + return FALLBACK; + } + value = rawValue; + break; + case CommandLineParameterKind.Integer: + case CommandLineParameterKind.IntegerList: + if (!INTEGER_REGEXP.test(rawValue)) { + return FALLBACK; + } + value = parseInt(rawValue, 10); + break; + default: + value = rawValue; + break; + } + + switch (target.kind) { + case CommandLineParameterKind.ChoiceList: + case CommandLineParameterKind.IntegerList: + case CommandLineParameterKind.StringList: { + const values: (string | number)[] | undefined = data.get(target) as (string | number)[] | undefined; + if (values) { + values.push(value); + } else { + data.set(target, [value]); + } + break; + } + default: + if (data.has(target)) { + return FALLBACK; + } + data.set(target, value); + break; + } + } + + for (const parameter of this.#parameters) { + if (parameter.required && !data.has(parameter)) { + return FALLBACK; + } + } + + return { kind: 'ok', data, remainder }; + } + + /** + * Assigns the parsed values the same way ts-command-line's `_processParsedData()` does. Returns false if + * ts-command-line would throw. + */ + public tryApplyValues( + data: ReadonlyMap, + remainder?: string[] + ): boolean { + try { + for (const parameter of this.#parameters) { + // argparse provides `false` for omitted flags and `null` for other omitted options + const value: unknown = data.has(parameter) + ? data.get(parameter) + : parameter.kind === CommandLineParameterKind.Flag + ? false + : null; + parameter._setValue(value); + parameter._validateValue?.(); + } + + if (this.#remainder) { + this.#remainder._setValue(remainder ?? []); + } + } catch (e) { + return false; + } + + return true; + } + + #defineParameter(parameter: InternalCommandLineParameter): InternalCommandLineParameter { + parameter._parserKey = 'key_' + (_keyCounter++).toString(); + + this.#parameters.push(parameter); + + let longNameParameters: InternalCommandLineParameter[] | undefined = this.#parametersByLongName.get( + parameter.longName + ); + if (!longNameParameters) { + longNameParameters = []; + this.#parametersByLongName.set(parameter.longName, longNameParameters); + } + longNameParameters.push(parameter); + + if (parameter.shortName) { + let shortNameParameters: InternalCommandLineParameter[] | undefined = this.#parametersByShortName.get( + parameter.shortName + ); + if (!shortNameParameters) { + shortNameParameters = []; + this.#parametersByShortName.set(parameter.shortName, shortNameParameters); + } + shortNameParameters.push(parameter); + } + + return parameter; + } +} + +/** + * Identical to ts-command-line's `CommandLineParameterProvider.getParameterStringMap()`. + */ +export function getParameterStringMap(parameters: Iterable): Record { + const parameterMap: Record = {}; + for (const parameter of parameters as Iterable) { + const parameterName: string = parameter.scopedLongName || parameter.longName; + switch (parameter.kind) { + case CommandLineParameterKind.Flag: + case CommandLineParameterKind.Choice: + case CommandLineParameterKind.String: + case CommandLineParameterKind.Integer: + parameterMap[parameterName] = JSON.stringify(parameter.value); + break; + case CommandLineParameterKind.StringList: + case CommandLineParameterKind.IntegerList: + case CommandLineParameterKind.ChoiceList: + const arrayValue: ReadonlyArray | undefined = parameter.values; + parameterMap[parameterName] = arrayValue ? arrayValue.join(',') : ''; + break; + } + } + return parameterMap; +} + +/** + * The help text of a parameter, as computed by ts-command-line's `_registerParameter()`. + */ +function getParameterHelp(parameter: InternalCommandLineParameter): string { + let finalDescription: string = parameter.description; + + const supplementaryNotes: string[] = []; + parameter._getSupplementaryNotes(supplementaryNotes); + if (supplementaryNotes.length > 0) { + // If they left the period off the end of their sentence, then add one. + if (finalDescription.match(/[a-z0-9]"?\s*$/i)) { + finalDescription = finalDescription.trimEnd() + '.'; + } + // Append the supplementary text + finalDescription += ' ' + supplementaryNotes.join(' '); + } + + return finalDescription; +} diff --git a/apps/heft/src/cli/actions/CleanAction.ts b/apps/heft/src/cli/actions/CleanAction.ts index d9527c93bc6..89ad77d77bf 100644 --- a/apps/heft/src/cli/actions/CleanAction.ts +++ b/apps/heft/src/cli/actions/CleanAction.ts @@ -1,24 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { - CommandLineAction, - type CommandLineFlagParameter, - type CommandLineStringListParameter -} from '@rushstack/ts-command-line'; +import { CommandLineAction, type CommandLineFlagParameter } from '@rushstack/ts-command-line'; import type { ITerminal } from '@rushstack/terminal'; -import { OperationStatus } from '@rushstack/operation-graph'; import type { IHeftAction, IHeftActionOptions } from './IHeftAction'; import type { HeftPhase } from '../../pluginFramework/HeftPhase'; import type { InternalHeftSession } from '../../pluginFramework/InternalHeftSession'; import type { MetricsCollector } from '../../metrics/MetricsCollector'; -import type { HeftPhaseSession } from '../../pluginFramework/HeftPhaseSession'; -import type { HeftTaskSession } from '../../pluginFramework/HeftTaskSession'; import { Constants } from '../../utilities/Constants'; -import { definePhaseScopingParameters, expandPhases } from './RunAction'; -import { deleteFilesAsync, type IDeleteOperation } from '../../plugins/DeleteFilesPlugin'; -import { ensureCliAbortSignal, initializeHeft, runWithLoggingAsync } from '../HeftActionRunner'; +import { definePhaseScopingParameters, getCleanSelectedPhases, type IScopingParameters } from './PhaseScoping'; +import { CLEAN_ACTION_DOCUMENTATION, VERBOSE_PARAMETER_DESCRIPTION } from '../CliConstants'; export class CleanAction extends CommandLineAction implements IHeftAction { public readonly watch: boolean = false; @@ -26,92 +18,48 @@ export class CleanAction extends CommandLineAction implements IHeftAction { readonly #terminal: ITerminal; readonly #metricsCollector: MetricsCollector; readonly #verboseFlag: CommandLineFlagParameter; - readonly #toParameter: CommandLineStringListParameter; - readonly #toExceptParameter: CommandLineStringListParameter; - readonly #onlyParameter: CommandLineStringListParameter; + readonly #scopingParameters: IScopingParameters; #selectedPhases: ReadonlySet | undefined; public constructor(options: IHeftActionOptions) { super({ actionName: 'clean', - documentation: 'Clean the project, removing temporary task folders and specified clean paths.', - summary: 'Clean the project, removing temporary task folders and specified clean paths.' + documentation: CLEAN_ACTION_DOCUMENTATION, + summary: CLEAN_ACTION_DOCUMENTATION }); this.#terminal = options.terminal; this.#metricsCollector = options.metricsCollector; this.#internalHeftSession = options.internalHeftSession; - const { toParameter, toExceptParameter, onlyParameter } = definePhaseScopingParameters(this); - this.#toParameter = toParameter; - this.#toExceptParameter = toExceptParameter; - this.#onlyParameter = onlyParameter; + this.#scopingParameters = definePhaseScopingParameters(this); this.#verboseFlag = this.defineFlagParameter({ parameterLongName: Constants.verboseParameterLongName, parameterShortName: Constants.verboseParameterShortName, - description: 'If specified, log information useful for debugging.' + description: VERBOSE_PARAMETER_DESCRIPTION }); } public get selectedPhases(): ReadonlySet { if (!this.#selectedPhases) { - if ( - this.#onlyParameter.values.length || - this.#toParameter.values.length || - this.#toExceptParameter.values.length - ) { - this.#selectedPhases = expandPhases( - this.#onlyParameter, - this.#toParameter, - this.#toExceptParameter, - this.#internalHeftSession, - this.#terminal - ); - } else { - // No selected phases, clean everything - this.#selectedPhases = this.#internalHeftSession.phases; - } + this.#selectedPhases = getCleanSelectedPhases( + this.#scopingParameters, + this.#internalHeftSession, + this.#terminal + ); } return this.#selectedPhases; } protected override async onExecuteAsync(): Promise { - const { heftConfiguration } = this.#internalHeftSession; - const abortSignal: AbortSignal = ensureCliAbortSignal(this.#terminal); - - // Record this as the start of task execution. - this.#metricsCollector.setStartTime(); - initializeHeft(heftConfiguration, this.#terminal, this.#verboseFlag.value); - await runWithLoggingAsync( - this.#cleanFilesAsync.bind(this), - this, - this.#internalHeftSession.loggingManager, - this.#terminal, - this.#metricsCollector, - abortSignal - ); - } - - async #cleanFilesAsync(): Promise { - const deleteOperations: IDeleteOperation[] = []; - for (const phase of this.selectedPhases) { - // Add the temp folder and cache folder (if requested) for each task - const phaseSession: HeftPhaseSession = this.#internalHeftSession.getSessionForPhase(phase); - for (const task of phase.tasks) { - const taskSession: HeftTaskSession = phaseSession.getSessionForTask(task); - deleteOperations.push({ sourcePath: taskSession.tempFolderPath }); - } - // Add the manually specified clean operations - deleteOperations.push(...phase.cleanFiles); - } - - // Delete the files - if (deleteOperations.length) { - const rootFolderPath: string = this.#internalHeftSession.heftConfiguration.buildFolderPath; - await deleteFilesAsync(rootFolderPath, deleteOperations, this.#terminal); - } - - return deleteOperations.length === 0 ? OperationStatus.NoOp : OperationStatus.Success; + const { executeCleanActionAsync } = await import('./CleanActionExecution'); + await executeCleanActionAsync({ + action: this, + internalHeftSession: this.#internalHeftSession, + terminal: this.#terminal, + metricsCollector: this.#metricsCollector, + isVerbose: this.#verboseFlag.value + }); } } diff --git a/apps/heft/src/cli/actions/CleanActionExecution.ts b/apps/heft/src/cli/actions/CleanActionExecution.ts new file mode 100644 index 00000000000..c2752a33042 --- /dev/null +++ b/apps/heft/src/cli/actions/CleanActionExecution.ts @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { ITerminal } from '@rushstack/terminal'; +import { OperationStatus } from '@rushstack/operation-graph/lib/OperationStatus'; + +import type { IHeftAction } from './IHeftAction'; +import type { InternalHeftSession } from '../../pluginFramework/InternalHeftSession'; +import type { MetricsCollector } from '../../metrics/MetricsCollector'; +import type { HeftPhaseSession } from '../../pluginFramework/HeftPhaseSession'; +import type { HeftTaskSession } from '../../pluginFramework/HeftTaskSession'; +import { deleteFilesAsync, type IDeleteOperation } from '../../plugins/DeleteFilesPlugin'; +import { ensureCliAbortSignal, initializeHeft, runWithLoggingAsync } from '../HeftActionRunner'; + +export interface ICleanActionExecutionOptions { + readonly action: IHeftAction; + readonly internalHeftSession: InternalHeftSession; + readonly terminal: ITerminal; + readonly metricsCollector: MetricsCollector; + readonly isVerbose: boolean; +} + +/** + * Implements the "clean" action. Shared by the full and the lean command-line implementations. + */ +export async function executeCleanActionAsync(options: ICleanActionExecutionOptions): Promise { + const { action, internalHeftSession, terminal, metricsCollector, isVerbose } = options; + const { heftConfiguration } = internalHeftSession; + const abortSignal: AbortSignal = ensureCliAbortSignal(terminal); + + // Record this as the start of task execution. + metricsCollector.setStartTime(); + initializeHeft(heftConfiguration, terminal, isVerbose); + await runWithLoggingAsync( + () => cleanFilesAsync(action, internalHeftSession, terminal), + action, + internalHeftSession.loggingManager, + terminal, + metricsCollector, + abortSignal + ); +} + +async function cleanFilesAsync( + action: IHeftAction, + internalHeftSession: InternalHeftSession, + terminal: ITerminal +): Promise { + const deleteOperations: IDeleteOperation[] = []; + for (const phase of action.selectedPhases) { + // Add the temp folder and cache folder (if requested) for each task + const phaseSession: HeftPhaseSession = internalHeftSession.getSessionForPhase(phase); + for (const task of phase.tasks) { + const taskSession: HeftTaskSession = phaseSession.getSessionForTask(task); + deleteOperations.push({ sourcePath: taskSession.tempFolderPath }); + } + // Add the manually specified clean operations + deleteOperations.push(...phase.cleanFiles); + } + + // Delete the files + if (deleteOperations.length) { + const rootFolderPath: string = internalHeftSession.heftConfiguration.buildFolderPath; + await deleteFilesAsync(rootFolderPath, deleteOperations, terminal); + } + + return deleteOperations.length === 0 ? OperationStatus.NoOp : OperationStatus.Success; +} diff --git a/apps/heft/src/cli/actions/PhaseAction.ts b/apps/heft/src/cli/actions/PhaseAction.ts index 5a0abc3b696..313f6cd8141 100644 --- a/apps/heft/src/cli/actions/PhaseAction.ts +++ b/apps/heft/src/cli/actions/PhaseAction.ts @@ -4,9 +4,10 @@ import { CommandLineAction } from '@rushstack/ts-command-line'; import { HeftActionRunner } from '../HeftActionRunner'; -import { Selection } from '../../utilities/Selection'; import type { IHeftAction, IHeftActionOptions } from './IHeftAction'; import type { HeftPhase } from '../../pluginFramework/HeftPhase'; +import { getPhaseActionSelectedPhases } from './PhaseScoping'; +import { getPhaseActionDocumentation, getPhaseActionSummary } from '../CliConstants'; export interface IPhaseActionOptions extends IHeftActionOptions { phase: HeftPhase; @@ -24,13 +25,8 @@ export class PhaseAction extends CommandLineAction implements IHeftAction { const { phaseName, phaseDescription } = phase; super({ actionName: `${phaseName}${watch ? '-watch' : ''}`, - documentation: - `Runs to the ${phaseName} phase, including all transitive dependencies` + - (watch ? ', in watch mode.' : '.') + - (phaseDescription ? ` ${phaseDescription}` : ''), - summary: - `Runs to the ${phaseName} phase, including all transitive dependencies` + - (watch ? ', in watch mode.' : '.') + documentation: getPhaseActionDocumentation(phaseName, phaseDescription, watch), + summary: getPhaseActionSummary(phaseName, watch) }); this.watch = watch; @@ -41,10 +37,7 @@ export class PhaseAction extends CommandLineAction implements IHeftAction { public get selectedPhases(): ReadonlySet { if (!this.#selectedPhases) { - this.#selectedPhases = Selection.recursiveExpand( - [this.#phase], - (phase: HeftPhase) => phase.dependencyPhases - ); + this.#selectedPhases = getPhaseActionSelectedPhases(this.#phase); } return this.#selectedPhases; } diff --git a/apps/heft/src/cli/actions/PhaseScoping.ts b/apps/heft/src/cli/actions/PhaseScoping.ts new file mode 100644 index 00000000000..baf29b1720c --- /dev/null +++ b/apps/heft/src/cli/actions/PhaseScoping.ts @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { + CommandLineStringListParameter, + ICommandLineStringListDefinition, + ScopedCommandLineAction +} from '@rushstack/ts-command-line'; +// Same module (and therefore the same symbol) as `ScopedCommandLineAction.ScopingParameterGroup`, without loading +// the ts-command-line entry point (which loads argparse). +import { SCOPING_PARAMETER_GROUP as SCOPING_PARAMETER_GROUP_SYMBOL } from '@rushstack/ts-command-line/lib/Constants'; +import { AlreadyReportedError } from '@rushstack/node-core-library'; +import type { ITerminal } from '@rushstack/terminal'; + +import { Selection } from '../../utilities/Selection'; +import type { InternalHeftSession } from '../../pluginFramework/InternalHeftSession'; +import type { HeftPhase } from '../../pluginFramework/HeftPhase'; +import { Constants } from '../../utilities/Constants'; + +// The package's public API declarations and its per-module declarations declare distinct types for this symbol. +const SCOPING_PARAMETER_GROUP: typeof ScopedCommandLineAction.ScopingParameterGroup = + SCOPING_PARAMETER_GROUP_SYMBOL as unknown as typeof ScopedCommandLineAction.ScopingParameterGroup; + +/** + * The subset of a parameter provider that is needed to define the phase scoping parameters. + */ +export interface IPhaseScopingParameterProvider { + readonly actionName: string; + defineStringListParameter(definition: ICommandLineStringListDefinition): CommandLineStringListParameter; +} + +export interface IScopingParameters { + toParameter: CommandLineStringListParameter; + toExceptParameter: CommandLineStringListParameter; + onlyParameter: CommandLineStringListParameter; +} + +export function expandPhases( + onlyParameter: CommandLineStringListParameter, + toParameter: CommandLineStringListParameter, + toExceptParameter: CommandLineStringListParameter, + internalHeftSession: InternalHeftSession, + terminal: ITerminal +): Set { + const onlyPhases: Set = evaluatePhaseParameter(onlyParameter, internalHeftSession, terminal); + const toPhases: Set = evaluatePhaseParameter(toParameter, internalHeftSession, terminal); + const toExceptPhases: Set = evaluatePhaseParameter( + toExceptParameter, + internalHeftSession, + terminal + ); + + const expandFn: (phase: HeftPhase) => ReadonlySet = (phase: HeftPhase) => phase.dependencyPhases; + const selectedPhases: Set = Selection.union( + Selection.recursiveExpand(toPhases, expandFn), + Selection.recursiveExpand(Selection.directDependenciesOf(toExceptPhases, expandFn), expandFn), + onlyPhases + ); + if (selectedPhases.size === 0) { + throw new Error( + 'No phases were selected. Provide at least one phase to the ' + + `${JSON.stringify(Constants.toParameterLongName)}, ` + + `${JSON.stringify(Constants.toExceptParameterLongName)}, or ` + + `${JSON.stringify(Constants.onlyParameterLongName)} parameters.` + ); + } + return selectedPhases; +} + +function evaluatePhaseParameter( + phaseParameter: CommandLineStringListParameter, + internalHeftSession: InternalHeftSession, + terminal: ITerminal +): Set { + const parameterName: string = phaseParameter.longName; + const selection: Set = new Set(); + for (const rawSelector of phaseParameter.values) { + const phase: HeftPhase | undefined = internalHeftSession.phasesByName.get(rawSelector); + if (!phase) { + terminal.writeErrorLine( + `The phase name ${JSON.stringify(rawSelector)} passed to ${JSON.stringify(parameterName)} does ` + + 'not exist in heft.json.' + ); + throw new AlreadyReportedError(); + } + selection.add(phase); + } + return selection; +} + +export function definePhaseScopingParameters(action: IPhaseScopingParameterProvider): IScopingParameters { + return { + toParameter: action.defineStringListParameter({ + parameterLongName: Constants.toParameterLongName, + description: `The phase to ${action.actionName} to, including all transitive dependencies.`, + argumentName: 'PHASE', + parameterGroup: SCOPING_PARAMETER_GROUP + }), + toExceptParameter: action.defineStringListParameter({ + parameterLongName: Constants.toExceptParameterLongName, + description: `The phase to ${action.actionName} to (but not include), including all transitive dependencies.`, + argumentName: 'PHASE', + parameterGroup: SCOPING_PARAMETER_GROUP + }), + onlyParameter: action.defineStringListParameter({ + parameterLongName: Constants.onlyParameterLongName, + description: `The phase to ${action.actionName}.`, + argumentName: 'PHASE', + parameterGroup: SCOPING_PARAMETER_GROUP + }) + }; +} + +/** + * The phases selected by the "clean" action: the scoped selection if any scoping parameter was provided, + * otherwise all phases. + */ +export function getCleanSelectedPhases( + scopingParameters: IScopingParameters, + internalHeftSession: InternalHeftSession, + terminal: ITerminal +): ReadonlySet { + const { onlyParameter, toParameter, toExceptParameter } = scopingParameters; + if (onlyParameter.values.length || toParameter.values.length || toExceptParameter.values.length) { + return expandPhases(onlyParameter, toParameter, toExceptParameter, internalHeftSession, terminal); + } else { + // No selected phases, clean everything + return internalHeftSession.phases; + } +} + +/** + * The phases selected by a phase action: the phase and all of its transitive dependencies. + */ +export function getPhaseActionSelectedPhases(phase: HeftPhase): Set { + return Selection.recursiveExpand([phase], (p: HeftPhase) => p.dependencyPhases); +} diff --git a/apps/heft/src/cli/actions/RunAction.ts b/apps/heft/src/cli/actions/RunAction.ts index d69390dbbf1..b53c1e3dabe 100644 --- a/apps/heft/src/cli/actions/RunAction.ts +++ b/apps/heft/src/cli/actions/RunAction.ts @@ -1,102 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { - ScopedCommandLineAction, - type CommandLineParameterProvider, - type CommandLineStringListParameter -} from '@rushstack/ts-command-line'; -import { AlreadyReportedError } from '@rushstack/node-core-library'; +import { ScopedCommandLineAction, type CommandLineParameterProvider } from '@rushstack/ts-command-line'; import type { ITerminal } from '@rushstack/terminal'; -import { Selection } from '../../utilities/Selection'; import { HeftActionRunner } from '../HeftActionRunner'; import type { InternalHeftSession } from '../../pluginFramework/InternalHeftSession'; import type { IHeftAction, IHeftActionOptions } from './IHeftAction'; import type { HeftPhase } from '../../pluginFramework/HeftPhase'; -import { Constants } from '../../utilities/Constants'; +import { definePhaseScopingParameters, expandPhases, type IScopingParameters } from './PhaseScoping'; +import { getRunActionDocumentation } from '../CliConstants'; -export function expandPhases( - onlyParameter: CommandLineStringListParameter, - toParameter: CommandLineStringListParameter, - toExceptParameter: CommandLineStringListParameter, - internalHeftSession: InternalHeftSession, - terminal: ITerminal -): Set { - const onlyPhases: Set = evaluatePhaseParameter(onlyParameter, internalHeftSession, terminal); - const toPhases: Set = evaluatePhaseParameter(toParameter, internalHeftSession, terminal); - const toExceptPhases: Set = evaluatePhaseParameter( - toExceptParameter, - internalHeftSession, - terminal - ); - - const expandFn: (phase: HeftPhase) => ReadonlySet = (phase: HeftPhase) => phase.dependencyPhases; - const selectedPhases: Set = Selection.union( - Selection.recursiveExpand(toPhases, expandFn), - Selection.recursiveExpand(Selection.directDependenciesOf(toExceptPhases, expandFn), expandFn), - onlyPhases - ); - if (selectedPhases.size === 0) { - throw new Error( - 'No phases were selected. Provide at least one phase to the ' + - `${JSON.stringify(Constants.toParameterLongName)}, ` + - `${JSON.stringify(Constants.toExceptParameterLongName)}, or ` + - `${JSON.stringify(Constants.onlyParameterLongName)} parameters.` - ); - } - return selectedPhases; -} - -function evaluatePhaseParameter( - phaseParameter: CommandLineStringListParameter, - internalHeftSession: InternalHeftSession, - terminal: ITerminal -): Set { - const parameterName: string = phaseParameter.longName; - const selection: Set = new Set(); - for (const rawSelector of phaseParameter.values) { - const phase: HeftPhase | undefined = internalHeftSession.phasesByName.get(rawSelector); - if (!phase) { - terminal.writeErrorLine( - `The phase name ${JSON.stringify(rawSelector)} passed to ${JSON.stringify(parameterName)} does ` + - 'not exist in heft.json.' - ); - throw new AlreadyReportedError(); - } - selection.add(phase); - } - return selection; -} - -export interface IScopingParameters { - toParameter: CommandLineStringListParameter; - toExceptParameter: CommandLineStringListParameter; - onlyParameter: CommandLineStringListParameter; -} - -export function definePhaseScopingParameters(action: IHeftAction): IScopingParameters { - return { - toParameter: action.defineStringListParameter({ - parameterLongName: Constants.toParameterLongName, - description: `The phase to ${action.actionName} to, including all transitive dependencies.`, - argumentName: 'PHASE', - parameterGroup: ScopedCommandLineAction.ScopingParameterGroup - }), - toExceptParameter: action.defineStringListParameter({ - parameterLongName: Constants.toExceptParameterLongName, - description: `The phase to ${action.actionName} to (but not include), including all transitive dependencies.`, - argumentName: 'PHASE', - parameterGroup: ScopedCommandLineAction.ScopingParameterGroup - }), - onlyParameter: action.defineStringListParameter({ - parameterLongName: Constants.onlyParameterLongName, - description: `The phase to ${action.actionName}.`, - argumentName: 'PHASE', - parameterGroup: ScopedCommandLineAction.ScopingParameterGroup - }) - }; -} +// These are re-exported for compatibility with consumers of this module's path. +export { definePhaseScopingParameters, expandPhases, type IScopingParameters } from './PhaseScoping'; export class RunAction extends ScopedCommandLineAction implements IHeftAction { public readonly watch: boolean; @@ -104,36 +20,33 @@ export class RunAction extends ScopedCommandLineAction implements IHeftAction { readonly #internalHeftSession: InternalHeftSession; readonly #terminal: ITerminal; readonly #actionRunner: HeftActionRunner; - readonly #toParameter: CommandLineStringListParameter; - readonly #toExceptParameter: CommandLineStringListParameter; - readonly #onlyParameter: CommandLineStringListParameter; + readonly #scopingParameters: IScopingParameters; #selectedPhases: Set | undefined; public constructor(options: IHeftActionOptions) { + const documentation: string = getRunActionDocumentation(!!options.watch); super({ actionName: `run${options.watch ? '-watch' : ''}`, - documentation: `Run a provided selection of Heft phases${options.watch ? ' in watch mode.' : ''}.`, - summary: `Run a provided selection of Heft phases${options.watch ? ' in watch mode.' : ''}.` + documentation, + summary: documentation }); this.watch = options.watch ?? false; this.#terminal = options.terminal; this.#internalHeftSession = options.internalHeftSession; - const { toParameter, toExceptParameter, onlyParameter } = definePhaseScopingParameters(this); - this.#toParameter = toParameter; - this.#toExceptParameter = toExceptParameter; - this.#onlyParameter = onlyParameter; + this.#scopingParameters = definePhaseScopingParameters(this); this.#actionRunner = new HeftActionRunner({ action: this, ...options }); } public get selectedPhases(): ReadonlySet { if (!this.#selectedPhases) { + const { onlyParameter, toParameter, toExceptParameter } = this.#scopingParameters; this.#selectedPhases = expandPhases( - this.#onlyParameter, - this.#toParameter, - this.#toExceptParameter, + onlyParameter, + toParameter, + toExceptParameter, this.#internalHeftSession, this.#terminal ); diff --git a/apps/heft/src/cli/test/LeanCommandLine.test.ts b/apps/heft/src/cli/test/LeanCommandLine.test.ts new file mode 100644 index 00000000000..1fb1daa3817 --- /dev/null +++ b/apps/heft/src/cli/test/LeanCommandLine.test.ts @@ -0,0 +1,513 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// Differential tests: the lean command line (LeanParameterProvider + the argparse HelpFormatter port) must behave +// exactly like ts-command-line (which uses argparse) whenever it doesn't fall back to it. + +import { + AliasCommandLineAction, + CommandLineAction, + CommandLineParser, + type CommandLineParameter, + type CommandLineParameterProvider, + ScopedCommandLineAction +} from '@rushstack/ts-command-line'; +import { Colorize } from '@rushstack/terminal'; + +import { LeanParameterProvider, type ILeanRegistration, type LeanParseResult } from '../LeanParameterProvider'; +import { formatHelp, formatUsage } from '../HelpFormatter'; +import { getRootHelpParser } from '../LeanHeftCommandLine'; +import { + DEBUG_PARAMETER_DESCRIPTION, + HEFT_TOOL_DESCRIPTION, + HEFT_TOOL_FILENAME, + SCOPED_ACTION_REMAINDER_DESCRIPTION, + UNMANAGED_PARAMETER_DESCRIPTION +} from '../CliConstants'; + +const ROOT_PARAMETER_NAMES: string[] = ['--debug', '--unmanaged']; +const RUN_DOCUMENTATION: string = 'Run a provided selection of Heft phases.'; + +interface IDefinition { + kind: 'flag' | 'string' | 'stringList' | 'integer' | 'integerList' | 'choice' | 'choiceList'; + parameterLongName: string; + parameterShortName?: string; + parameterScope?: string; + description: string; + argumentName?: string; + alternatives?: string[]; + defaultValue?: string | number; + required?: boolean; + environmentVariable?: string; + undocumentedSynonyms?: string[]; +} + +// A small deterministic PRNG (Park-Miller), so that failures are reproducible +function createRandom(seed: number): (n: number) => number { + let state: number = seed; + return (n: number) => { + state = (state * 16807) % 2147483647; + return state % n; + }; +} + +function defineParameter(provider: CommandLineParameterProvider, definition: IDefinition): CommandLineParameter { + const { kind, ...rest } = definition; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const options: any = rest; + switch (kind) { + case 'flag': + return provider.defineFlagParameter(options); + case 'string': + return provider.defineStringParameter(options); + case 'stringList': + return provider.defineStringListParameter(options); + case 'integer': + return provider.defineIntegerParameter(options); + case 'integerList': + return provider.defineIntegerListParameter(options); + case 'choice': + return provider.defineChoiceParameter(options); + case 'choiceList': + return provider.defineChoiceListParameter(options); + } +} + +const BUILT_IN_DEFINITIONS: IDefinition[] = [ + { + kind: 'flag', + parameterLongName: '--verbose', + parameterShortName: '-v', + description: 'If specified, log information useful for debugging.' + }, + { kind: 'flag', parameterLongName: '--production', description: 'If specified, run Heft in production mode.' }, + { + kind: 'stringList', + parameterLongName: '--locales', + argumentName: 'LOCALE', + description: 'Use the specified locale for this run, if applicable.' + }, + { kind: 'flag', parameterLongName: '--clean', description: 'If specified, clean the outputs.' } +]; + +function getPluginDefinitions(scope: string): IDefinition[] { + return [ + { kind: 'flag', parameterLongName: '--fix', parameterScope: scope, description: 'Fix.' }, + { kind: 'integer', parameterLongName: '--count', argumentName: 'N', parameterScope: scope, description: 'n' }, + { + kind: 'integerList', + parameterLongName: '--nums', + argumentName: 'N', + parameterScope: scope, + description: 'n' + }, + { kind: 'string', parameterLongName: '--name', argumentName: 'TEXT', parameterScope: scope, description: 's' }, + { + kind: 'choice', + parameterLongName: '--color', + alternatives: ['red', 'blue'], + parameterScope: scope, + description: 'c' + }, + { + kind: 'choiceList', + parameterLongName: '--mons', + alternatives: ['a', 'b', 'c'], + parameterScope: scope, + description: 'm' + } + ]; +} + +const DEFINITION_SETS: IDefinition[][] = [ + BUILT_IN_DEFINITIONS, + [...BUILT_IN_DEFINITIONS, ...getPluginDefinitions('lint')], + [ + ...BUILT_IN_DEFINITIONS, + { + kind: 'string', + parameterLongName: '--req', + argumentName: 'R', + parameterScope: 'p', + description: 'r', + required: true + } + ], + [ + ...BUILT_IN_DEFINITIONS, + { + kind: 'string', + parameterLongName: '--name', + argumentName: 'T', + parameterScope: 'x', + description: 's', + defaultValue: 'd' + }, + { + kind: 'integer', + parameterLongName: '--count', + argumentName: 'N', + parameterScope: 'x', + description: 'n', + defaultValue: 42 + }, + { + kind: 'flag', + parameterLongName: '--eflag', + parameterScope: 'x', + description: 'e', + environmentVariable: 'HEFT_TEST_EFLAG' + }, + { + kind: 'string', + parameterLongName: '--estr', + argumentName: 'S', + parameterScope: 'x', + description: 'e', + environmentVariable: 'HEFT_TEST_ESTR' + } + ], + [...BUILT_IN_DEFINITIONS, ...getPluginDefinitions('one'), ...getPluginDefinitions('two')], + [ + ...BUILT_IN_DEFINITIONS, + { kind: 'flag', parameterLongName: '--vvv', parameterShortName: '-v', parameterScope: 'x', description: 'v' } + ], + [...BUILT_IN_DEFINITIONS, { kind: 'flag', parameterLongName: '--clean', parameterScope: 'x', description: 'c' }], + [...BUILT_IN_DEFINITIONS, { kind: 'flag', parameterLongName: '--debug', parameterScope: 'x', description: 'd' }], + [ + ...BUILT_IN_DEFINITIONS, + { kind: 'flag', parameterLongName: '--hhh', parameterShortName: '-h', parameterScope: 'x', description: 'h' } + ], + [ + ...BUILT_IN_DEFINITIONS, + { + kind: 'string', + parameterLongName: '--syn', + argumentName: 'S', + parameterScope: 'x', + description: 's', + undocumentedSynonyms: ['--old-syn'] + } + ] +]; + +class TestAction extends CommandLineAction { + public executed: boolean = false; + public constructor(actionName: string, documentation: string, definitions: IDefinition[]) { + super({ actionName, summary: documentation, documentation }); + for (const definition of definitions) { + defineParameter(this, definition); + } + } + protected override async onExecuteAsync(): Promise { + this.executed = true; + } +} + +class TestScopedAction extends ScopedCommandLineAction { + readonly #definitions: IDefinition[]; + public constructor(definitions: IDefinition[]) { + super({ actionName: 'run', summary: RUN_DOCUMENTATION, documentation: RUN_DOCUMENTATION }); + this.#definitions = definitions; + defineScopingParameters(this); + } + protected override onDefineScopedParameters(provider: CommandLineParameterProvider): void { + for (const definition of this.#definitions) { + defineParameter(provider, definition); + } + } + protected override async onExecuteAsync(): Promise { + // Nothing to do + } +} + +function defineScopingParameters(provider: CommandLineParameterProvider): void { + for (const parameterLongName of ['--to', '--to-except', '--only']) { + provider.defineStringListParameter({ + parameterLongName, + argumentName: 'PHASE', + description: `The phase ${parameterLongName}.`, + parameterGroup: ScopedCommandLineAction.ScopingParameterGroup + }); + } +} + +class TestParser extends CommandLineParser { + public constructor() { + super({ toolFilename: HEFT_TOOL_FILENAME, toolDescription: HEFT_TOOL_DESCRIPTION }); + this.defineFlagParameter({ parameterLongName: '--debug', description: DEBUG_PARAMETER_DESCRIPTION }); + this.defineFlagParameter({ parameterLongName: '--unmanaged', description: UNMANAGED_PARAMETER_DESCRIPTION }); + } +} + +interface IReferenceResult { + output: string; + error?: string; +} + +async function runReferenceAsync(parser: CommandLineParser, args: string[]): Promise { + let output: string = ''; + const stdoutWriteSpy: jest.SpyInstance = jest + .spyOn(process.stdout, 'write') + .mockImplementation((chunk: string | Uint8Array) => { + output += chunk; + return true; + }); + const consoleLogSpy: jest.SpyInstance = jest.spyOn(console, 'log').mockImplementation((...parts: unknown[]) => { + output += parts.join(' ') + '\n'; + }); + try { + await parser.executeWithoutErrorHandlingAsync(args); + return { output }; + } catch (e) { + return { output, error: (e as Error).message }; + } finally { + stdoutWriteSpy.mockRestore(); + consoleLogSpy.mockRestore(); + } +} + +function getValues(parameters: ReadonlyArray): unknown[] { + return parameters.map((parameter: CommandLineParameter) => [ + parameter.scopedLongName || parameter.longName, + 'values' in parameter ? parameter.values : parameter.value + ]); +} + +const EXTRA_ARGS: string[] = [ + 'x', + 'red', + 'blue', + 'a', + 'c', + '5', + '007', + '-5', + '1.5', + '', + 'hello world', + '--', + '-h', + '--help', + '--debug', + '--bogus', + '--verb', + '-vh', + '--locales=en' +]; + +function generateArgs(random: (n: number) => number, definitions: IDefinition[]): string[] { + const optionStrings: string[] = []; + for (const definition of definitions) { + optionStrings.push(definition.parameterLongName); + if (definition.parameterShortName) { + optionStrings.push(definition.parameterShortName); + } + if (definition.parameterScope) { + optionStrings.push(`--${definition.parameterScope}:${definition.parameterLongName.slice(2)}`); + } + optionStrings.push(...(definition.undocumentedSynonyms || [])); + } + const args: string[] = []; + const count: number = random(7); + for (let i: number = 0; i < count; i++) { + const pool: string[] = random(3) === 0 ? EXTRA_ARGS : optionStrings; + args.push(pool[random(pool.length)]); + } + return args; +} + +function createLeanProvider(definitions: IDefinition[]): [LeanParameterProvider, CommandLineParameter[]] { + const provider: LeanParameterProvider = new LeanParameterProvider(); + const parameters: CommandLineParameter[] = definitions.map((d: IDefinition) => + defineParameter(provider.asCommandLineParameterProvider(), d) + ); + return [provider, parameters]; +} + +function setColumns(columns: string | undefined): void { + if (columns === undefined) { + delete process.env.COLUMNS; + } else { + process.env.COLUMNS = columns; + } +} + +describe('LeanParameterProvider', () => { + const originalEnvironment: NodeJS.ProcessEnv = { ...process.env }; + afterEach(() => { + process.env = { ...originalEnvironment }; + }); + + it('parses exactly like ts-command-line whenever it does not fall back', async () => { + const random: (n: number) => number = createRandom(1); + let accepted: number = 0; + for (let i: number = 0; i < 600; i++) { + const definitions: IDefinition[] = DEFINITION_SETS[random(DEFINITION_SETS.length)]; + const args: string[] = generateArgs(random, definitions); + process.env.HEFT_TEST_EFLAG = ['1', '0', 'x', ''][random(4)]; + process.env.HEFT_TEST_ESTR = ['env', ''][random(2)]; + + const [provider, leanParameters] = createLeanProvider(definitions); + const registration: ILeanRegistration | undefined = provider.tryGetRegistration(ROOT_PARAMETER_NAMES); + + const parser: TestParser = new TestParser(); + const action: TestAction = new TestAction('build', 'Build.', definitions); + parser.addAction(action); + const referenceResult: IReferenceResult = await runReferenceAsync(parser, ['build', ...args]); + + if (!registration) { + // Falling back to ts-command-line is always correct + continue; + } + + const leanResult: LeanParseResult = provider.parseArguments(registration, args, false); + if (leanResult.kind === 'help') { + expect(referenceResult.error).toBeUndefined(); + expect(action.executed).toBe(false); + expect(referenceResult.output).toContain('usage: heft build'); + } else if (leanResult.kind === 'ok' && provider.tryApplyValues(leanResult.data)) { + accepted++; + expect(referenceResult.error).toBeUndefined(); + expect(action.executed).toBe(true); + expect(getValues(leanParameters)).toEqual(getValues(action.parameters)); + expect(provider.getParameterStringMap()).toEqual(action.getParameterStringMap()); + } + } + expect(accepted).toBeGreaterThan(50); + }); + + it('falls back when ts-command-line would fail to register the parameters', async () => { + for (const definitions of DEFINITION_SETS) { + const [provider] = createLeanProvider(definitions); + const registration: ILeanRegistration | undefined = provider.tryGetRegistration(ROOT_PARAMETER_NAMES); + + const parser: TestParser = new TestParser(); + parser.addAction(new TestAction('build', 'Build.', definitions)); + const referenceResult: IReferenceResult = await runReferenceAsync(parser, ['build', '--help']); + if (referenceResult.error !== undefined) { + expect(registration).toBeUndefined(); + } + } + }); +}); + +describe('HelpFormatter', () => { + const originalColumns: string | undefined = process.env.COLUMNS; + afterEach(() => { + setColumns(originalColumns); + }); + + const DOCUMENTATION: string[] = [ + 'Runs to the build phase, including all transitive dependencies.', + 'A supercalifragilisticexpialidociouswordwithoutanydelimiterswhatsoever that is long.', + 'Examine the package.json dependencies; 100% of "quoted" text, tabs\tand\nnewlines | pipes!' + ]; + + for (const columns of [undefined, '20', '40', '80', '120']) { + it(`renders action help like argparse (COLUMNS=${columns})`, async () => { + setColumns(columns); + for (const definitions of DEFINITION_SETS) { + for (const documentation of DOCUMENTATION) { + const [provider] = createLeanProvider(definitions); + const registration: ILeanRegistration | undefined = + provider.tryGetRegistration(ROOT_PARAMETER_NAMES); + if (!registration) { + continue; + } + const parser: TestParser = new TestParser(); + parser.addAction(new TestAction('build', documentation, definitions)); + const referenceResult: IReferenceResult = await runReferenceAsync(parser, ['build', '--help']); + expect(formatHelp(provider.getHelpParser(registration, 'heft build', documentation, undefined))).toEqual( + referenceResult.output + ); + } + } + }); + + it(`renders scoped and unscoped run help like argparse (COLUMNS=${columns})`, async () => { + setColumns(columns); + for (const definitions of DEFINITION_SETS) { + const unscopedProvider: LeanParameterProvider = new LeanParameterProvider(); + unscopedProvider.defineCommandLineRemainder({ description: SCOPED_ACTION_REMAINDER_DESCRIPTION }); + defineScopingParameters(unscopedProvider.asCommandLineParameterProvider()); + const unscopedRegistration: ILeanRegistration = unscopedProvider.tryGetRegistration(ROOT_PARAMETER_NAMES)!; + + const unscopedReference: IReferenceResult = await runReferenceAsync( + createRunParser(definitions), + ['run', '--help'] + ); + expect( + formatHelp(unscopedProvider.getHelpParser(unscopedRegistration, 'heft run', RUN_DOCUMENTATION, undefined)) + ).toEqual(unscopedReference.output); + + const [scopedProvider] = createLeanProvider(definitions); + const scopedRegistration: ILeanRegistration | undefined = scopedProvider.tryGetRegistration([ + ...ROOT_PARAMETER_NAMES, + ...unscopedRegistration.registeredNames + ]); + if (!scopedRegistration) { + continue; + } + const scopedReference: IReferenceResult = await runReferenceAsync(createRunParser(definitions), [ + 'run', + '--only', + 'build', + '--', + '--help' + ]); + expect( + formatHelp( + scopedProvider.getHelpParser( + scopedRegistration, + 'heft run --only build --', + RUN_DOCUMENTATION, + Colorize.bold('For more information on available unscoped parameters, use "heft run --help"') + ) + ) + ).toEqual(scopedReference.output); + } + }); + + it(`renders the root help and usage like argparse (COLUMNS=${columns})`, async () => { + setColumns(columns); + const parser: TestParser = new TestParser(); + const summaries: [string, string][] = []; + const actions: TestAction[] = []; + for (const [actionName, documentation] of [ + ['clean', 'Clean the project, removing temporary task folders and specified clean paths.'], + ['run', RUN_DOCUMENTATION], + ['build', DOCUMENTATION[0]], + ['trust-dev-cert-watch', DOCUMENTATION[2]] + ]) { + const action: TestAction = new TestAction(actionName, documentation, []); + actions.push(action); + parser.addAction(action); + summaries.push([actionName, action.summary]); + } + const alias: AliasCommandLineAction = new AliasCommandLineAction({ + toolFilename: HEFT_TOOL_FILENAME, + aliasName: 'start', + targetAction: actions[2], + defaultParameters: ['--serve'] + }); + parser.addAction(alias); + summaries.push(['start', alias.summary]); + + const helpReference: IReferenceResult = await runReferenceAsync(parser, ['--help']); + expect(formatHelp(getRootHelpParser(summaries))).toEqual(helpReference.output); + + const usageParser: TestParser = new TestParser(); + usageParser.addAction(new TestAction('build', 'Build.', [])); + const usageReference: IReferenceResult = await runReferenceAsync(usageParser, ['--version']); + expect(usageReference.error).toMatch(/too few arguments/); + expect(formatUsage(getRootHelpParser([['build', 'Build.']]))).toEqual(usageReference.output); + }); + } +}); + +function createRunParser(definitions: IDefinition[]): TestParser { + const parser: TestParser = new TestParser(); + parser.addAction(new TestScopedAction(definitions)); + return parser; +} diff --git a/apps/heft/src/configuration/HeftConfiguration.ts b/apps/heft/src/configuration/HeftConfiguration.ts index 85dc64200ba..fbbf62d0d83 100644 --- a/apps/heft/src/configuration/HeftConfiguration.ts +++ b/apps/heft/src/configuration/HeftConfiguration.ts @@ -3,16 +3,73 @@ import * as path from 'node:path'; -import { type IPackageJson, PackageJsonLookup, InternalError, Path } from '@rushstack/node-core-library'; +// Inline type specifiers keep the API report identical; TypeScript elides these imports at runtime +// eslint-disable-next-line @typescript-eslint/no-import-type-side-effects +import { type IPackageJson, type PackageJsonLookup, type InternalError } from '@rushstack/node-core-library'; import { Terminal, type ITerminalProvider, type ITerminal } from '@rushstack/terminal'; +// eslint-disable-next-line @typescript-eslint/no-import-type-side-effects import { type IProjectConfigurationFileSpecification, - ProjectConfigurationFile + type ProjectConfigurationFile } from '@rushstack/heft-config-file'; -import { type IRigConfig, RigConfig } from '@rushstack/rig-package'; +// eslint-disable-next-line @typescript-eslint/no-import-type-side-effects +import { type IRigConfig, type RigConfig } from '@rushstack/rig-package'; import { Constants } from '../utilities/Constants'; -import { RigPackageResolver, type IRigPackageResolver } from './RigPackageResolver'; +import type { RigPackageResolver, IRigPackageResolver } from './RigPackageResolver'; +import { getSharedLeanPackageJsonLookup, LeanBailError } from './lean/LeanResolution'; +import { tryLoadProjectConfigurationFileLean } from './lean/LeanConfigurationFileSpecification'; +import type { ILeanLoadResult } from './lean/LeanProjectConfigurationFile'; +import { LeanRigConfig, tryLoadRigConfigDataLean, type ILeanRigConfigData } from './lean/LeanRigConfig'; + +// These are loaded lazily, since they are not needed on Heft's startup path +function getPackageJsonLookupInstance(): PackageJsonLookup { + const { PackageJsonLookup: PackageJsonLookupClass } = require('@rushstack/node-core-library'); + return (PackageJsonLookupClass as typeof PackageJsonLookup).instance; +} + +function getInternalErrorClass(): typeof InternalError { + return require('@rushstack/node-core-library').InternalError; +} + +function getProjectConfigurationFileClass(): typeof ProjectConfigurationFile { + return require('@rushstack/heft-config-file').ProjectConfigurationFile; +} + +// Not a member of HeftConfiguration, to keep the public API unchanged +const _rigConfigsForConfigLoading: WeakMap IRigConfig> = new WeakMap(); + +/** + * Returns a rig config for Heft's own configuration loading. It has the same data as + * `HeftConfiguration.rigConfig`, but it only loads `@rushstack/rig-package` if one of its methods is called. + */ +export function getRigConfigForConfigLoading(heftConfiguration: HeftConfiguration): IRigConfig { + return _rigConfigsForConfigLoading.get(heftConfiguration)!(); +} + +function getRigConfigClass(): typeof RigConfig { + return require('@rushstack/rig-package').RigConfig; +} + +function getRigPackageResolverClass(): typeof RigPackageResolver { + return require('./RigPackageResolver').RigPackageResolver; +} + +/** + * Equivalent to `PackageJsonLookup.instance.tryGetPackageJsonFilePathFor(folderPath)`, without loading + * `@rushstack/node-core-library` in the common case. + */ +function tryGetPackageJsonFilePathFor(folderPath: string): { packageJsonPath: string | undefined; lean: boolean } { + let packageFolder: string | undefined; + try { + packageFolder = getSharedLeanPackageJsonLookup().tryGetPackageFolderFor(folderPath); + } catch { + // The lean lookup can't guarantee an identical result; use the original implementation + return { packageJsonPath: getPackageJsonLookupInstance().tryGetPackageJsonFilePathFor(folderPath), lean: false }; + } + + return { packageJsonPath: packageFolder ? path.join(packageFolder, 'package.json') : undefined, lean: true }; +} /** * @internal @@ -50,7 +107,12 @@ export class HeftConfiguration { #slashNormalizedBuildFolderPath: string | undefined; #projectConfigFolderPath: string | undefined; #tempFolderPath: string | undefined; + // Whether initialize() found the project's package.json with the shared lean lookup + #projectPackageJsonFromLeanLookup: boolean = false; + // The genuine RigConfig object. If #leanRigConfig is set, it is only created when it is needed. #rigConfig: IRigConfig | undefined; + // The rig data read without loading @rushstack/rig-package; its methods delegate to #rigConfig + #leanRigConfig: LeanRigConfig | undefined; #rigPackageResolver: RigPackageResolver | undefined; readonly #knownConfigurationFiles: Map> = new Map(); @@ -65,7 +127,8 @@ export class HeftConfiguration { */ public get slashNormalizedBuildFolderPath(): string { if (!this.#slashNormalizedBuildFolderPath) { - this.#slashNormalizedBuildFolderPath = Path.convertToSlashes(this.buildFolderPath); + // Equivalent to Path.convertToSlashes() from @rushstack/node-core-library + this.#slashNormalizedBuildFolderPath = this.buildFolderPath.split('\\').join('/'); } return this.#slashNormalizedBuildFolderPath; @@ -101,8 +164,15 @@ export class HeftConfiguration { * The rig.json configuration for this project, if present. */ public get rigConfig(): IRigConfig { + if (!this.#rigConfig && this.#leanRigConfig) { + // Returns the same object as RigConfig.loadForProjectFolder() calls by other code (like before) + this.#rigConfig = getRigConfigClass().loadForProjectFolder({ + projectFolderPath: this.buildFolderPath + }); + } + if (!this.#rigConfig) { - throw new InternalError( + throw new (getInternalErrorClass())( 'The rigConfig cannot be accessed until HeftConfiguration.checkForRigAsync() has been called' ); } @@ -114,7 +184,7 @@ export class HeftConfiguration { */ public get rigPackageResolver(): IRigPackageResolver { if (!this.#rigPackageResolver) { - this.#rigPackageResolver = new RigPackageResolver({ + this.#rigPackageResolver = new (getRigPackageResolverClass())({ buildFolder: this.buildFolderPath, projectPackageJson: this.projectPackageJson, rigConfig: this.rigConfig @@ -138,14 +208,26 @@ export class HeftConfiguration { * The Heft tool's package.json */ public get heftPackageJson(): IPackageJson { - return PackageJsonLookup.instance.tryLoadPackageJsonFor(__dirname)!; + return getPackageJsonLookupInstance().tryLoadPackageJsonFor(__dirname)!; } /** * The package.json of the project being built */ public get projectPackageJson(): IPackageJson { - return PackageJsonLookup.instance.tryLoadPackageJsonFor(this.buildFolderPath)!; + if (this.#projectPackageJsonFromLeanLookup) { + // Like the original implementation (where initialize() cached the package.json in PackageJsonLookup.instance), + // this returns the contents read at startup, without loading @rushstack/node-core-library. + try { + return getSharedLeanPackageJsonLookup().tryLoadPackageJsonFor(this.buildFolderPath)! as IPackageJson; + } catch (e) { + if (!(e instanceof LeanBailError)) { + throw e; + } + } + } + + return getPackageJsonLookupInstance().tryLoadPackageJsonFor(this.buildFolderPath)!; } /** @@ -159,6 +241,7 @@ export class HeftConfiguration { this.terminalProvider = terminalProvider; this.numberOfCores = numberOfCores; this.globalTerminal = new Terminal(terminalProvider); + _rigConfigsForConfigLoading.set(this, () => this.#leanRigConfig ?? this.rigConfig); } /** @@ -166,13 +249,27 @@ export class HeftConfiguration { * @internal */ public async _checkForRigAsync(): Promise { - if (!this.#rigConfig) { - this.#rigConfig = await RigConfig.loadForProjectFolderAsync({ + if (!this.#rigConfig && !this.#leanRigConfig) { + const leanRigConfigData: ILeanRigConfigData | undefined = tryLoadRigConfigDataLean(this.buildFolderPath); + if (leanRigConfigData) { + this.#leanRigConfig = new LeanRigConfig(leanRigConfigData, () => this.rigConfig); + return; + } + + // Use the original implementation, which reports errors in rig.json + this.#rigConfig = await getRigConfigClass().loadForProjectFolderAsync({ projectFolderPath: this.buildFolderPath }); } } + /** + * The value that the original implementation passed to the configuration file loaders. + */ + #getRigConfigForOriginalLoader(): IRigConfig | undefined { + return this.#leanRigConfig ? this.rigConfig : this.#rigConfig; + } + /** * Attempts to load a riggable project configuration file using blocking, synchronous I/O. * @param options - The options for the configuration file loader from `@rushstack/heft-config-file`. If invoking this function multiple times for the same file, reuse the same object. @@ -183,8 +280,18 @@ export class HeftConfiguration { options: IProjectConfigurationFileSpecification, terminal: ITerminal ): TConfigFile | undefined { + const leanResult: ILeanLoadResult | undefined = + this.#tryLoadProjectConfigurationFileLean(options, terminal); + if (leanResult) { + return leanResult.configurationFile; + } + const loader: ProjectConfigurationFile = this.#getConfigFileLoader(options); - return loader.tryLoadConfigurationFileForProject(terminal, this.buildFolderPath, this.#rigConfig); + return loader.tryLoadConfigurationFileForProject( + terminal, + this.buildFolderPath, + this.#getRigConfigForOriginalLoader() + ); } /** @@ -197,17 +304,25 @@ export class HeftConfiguration { options: IProjectConfigurationFileSpecification, terminal: ITerminal ): Promise { + const leanResult: ILeanLoadResult | undefined = + this.#tryLoadProjectConfigurationFileLean(options, terminal); + if (leanResult) { + return leanResult.configurationFile; + } + const loader: ProjectConfigurationFile = this.#getConfigFileLoader(options); - return loader.tryLoadConfigurationFileForProjectAsync(terminal, this.buildFolderPath, this.#rigConfig); + return loader.tryLoadConfigurationFileForProjectAsync( + terminal, + this.buildFolderPath, + this.#getRigConfigForOriginalLoader() + ); } /** * @internal */ public static initialize(options: IHeftConfigurationInitializationOptions): HeftConfiguration { - const packageJsonPath: string | undefined = PackageJsonLookup.instance.tryGetPackageJsonFilePathFor( - options.cwd - ); + const { packageJsonPath, lean } = tryGetPackageJsonFilePathFor(options.cwd); let buildFolderPath: string; if (packageJsonPath) { buildFolderPath = path.dirname(packageJsonPath); @@ -225,9 +340,47 @@ export class HeftConfiguration { ...options, buildFolderPath }); + configuration.#projectPackageJsonFromLeanLookup = lean; return configuration; } + /** + * Loads the configuration file without `@rushstack/heft-config-file` (and without compiling its schema), if the + * result is guaranteed to be identical. Returns `undefined` otherwise. + */ + #tryLoadProjectConfigurationFileLean( + options: IProjectConfigurationFileSpecification, + terminal: ITerminal + ): ILeanLoadResult | undefined { + // Same checks and side effects on the options object as #getConfigFileLoader() + const entry: IProjectConfigurationFileEntry | undefined = this.#knownConfigurationFiles.get( + options.projectRelativeFilePath + ) as IProjectConfigurationFileEntry | undefined; + if (entry) { + // Let #getConfigFileLoader() handle this + return undefined; + } + + Object.freeze(options); + + const leanRigConfig: LeanRigConfig | undefined = this.#leanRigConfig; + const rigConfig: IRigConfig | undefined = leanRigConfig ?? this.#rigConfig; + const leanResult: ILeanLoadResult | undefined = tryLoadProjectConfigurationFileLean( + options, + this.buildFolderPath, + rigConfig, + // The profile folder of a LeanRigConfig can be resolved without side effects + rigConfig === leanRigConfig + ); + if (leanResult) { + for (const message of leanResult.debugMessages) { + terminal.writeDebugLine(message); + } + } + + return leanResult; + } + #getConfigFileLoader( options: IProjectConfigurationFileSpecification ): ProjectConfigurationFile { @@ -238,7 +391,7 @@ export class HeftConfiguration { if (!entry) { entry = { options: Object.freeze(options), - loader: new ProjectConfigurationFile(options) + loader: new (getProjectConfigurationFileClass())(options) }; } else if (options !== entry.options) { throw new Error( diff --git a/apps/heft/src/configuration/HeftPluginConfiguration.ts b/apps/heft/src/configuration/HeftPluginConfiguration.ts index ac2ab9334c4..e5fa4e19795 100644 --- a/apps/heft/src/configuration/HeftPluginConfiguration.ts +++ b/apps/heft/src/configuration/HeftPluginConfiguration.ts @@ -1,7 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { JsonFile, JsonSchema } from '@rushstack/node-core-library'; +import * as fs from 'node:fs'; + +import type { JsonSchema } from '@rushstack/node-core-library'; import { HeftLifecyclePluginDefinition, @@ -12,6 +14,8 @@ import { } from './HeftPluginDefinition'; import type { IHeftConfigurationJsonPluginSpecifier } from '../utilities/CoreConfigFiles'; import heftPluginSchema from '../schemas/heft-plugin.schema.json'; +import { tryParseJsonLean } from './lean/LeanJson'; +import { tryValidateSchemaObject } from './lean/SchemaFastPath'; export interface IHeftPluginConfigurationJson { lifecyclePlugins?: IHeftLifecyclePluginDefinitionJson[]; @@ -20,9 +24,42 @@ export interface IHeftPluginConfigurationJson { const HEFT_PLUGIN_CONFIGURATION_FILENAME: 'heft-plugin.json' = 'heft-plugin.json'; -const _jsonSchema: JsonSchema = JsonSchema.fromLoadedObject(heftPluginSchema); +let _jsonSchema: JsonSchema | undefined; const _pluginConfigurationPromises: Map> = new Map(); +/** + * Loads and validates the heft-plugin.json file without loading ajv, if the result is guaranteed to be identical + * to `JsonFile.loadAndValidateAsync()`. Returns `undefined` otherwise (including for all error conditions). + */ +function _tryLoadHeftPluginConfigurationJsonLean(filePath: string): IHeftPluginConfigurationJson | undefined { + let fileText: string; + try { + fileText = fs.readFileSync(filePath, 'utf8'); + } catch { + return undefined; + } + + const parsed: { value: unknown } | undefined = tryParseJsonLean(fileText); + if (parsed && tryValidateSchemaObject(heftPluginSchema, parsed.value)) { + return parsed.value as IHeftPluginConfigurationJson; + } +} + +async function _loadHeftPluginConfigurationJsonAsync(filePath: string): Promise { + const leanResult: IHeftPluginConfigurationJson | undefined = _tryLoadHeftPluginConfigurationJsonLean(filePath); + if (leanResult) { + return leanResult; + } + + // Use the original implementation, which produces the canonical errors + const { JsonFile, JsonSchema: JsonSchemaClass } = await import('@rushstack/node-core-library'); + if (!_jsonSchema) { + _jsonSchema = JsonSchemaClass.fromLoadedObject(heftPluginSchema); + } + + return await JsonFile.loadAndValidateAsync(filePath, _jsonSchema); +} + /** * Loads and validates the heft-plugin.json file. */ @@ -66,10 +103,8 @@ export class HeftPluginConfiguration { _pluginConfigurationPromises.get(packageRoot); if (!heftPluginConfigurationPromise) { heftPluginConfigurationPromise = (async () => { - const heftPluginConfigurationJson: IHeftPluginConfigurationJson = await JsonFile.loadAndValidateAsync( - resolvedHeftPluginConfigurationJsonFilename, - _jsonSchema - ); + const heftPluginConfigurationJson: IHeftPluginConfigurationJson = + await _loadHeftPluginConfigurationJsonAsync(resolvedHeftPluginConfigurationJsonFilename); return new HeftPluginConfiguration(heftPluginConfigurationJson, packageRoot, packageName); })(); _pluginConfigurationPromises.set(packageRoot, heftPluginConfigurationPromise); diff --git a/apps/heft/src/configuration/HeftPluginDefinition.ts b/apps/heft/src/configuration/HeftPluginDefinition.ts index 822ef3453e3..2e220903a66 100644 --- a/apps/heft/src/configuration/HeftPluginDefinition.ts +++ b/apps/heft/src/configuration/HeftPluginDefinition.ts @@ -1,14 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as fs from 'node:fs'; import * as path from 'node:path'; -import { InternalError, JsonSchema } from '@rushstack/node-core-library'; +import type { JsonSchema, InternalError as InternalErrorType } from '@rushstack/node-core-library'; import type { IHeftPlugin } from '../pluginFramework/IHeftPlugin'; import type { IScopedLogger } from '../pluginFramework/logging/ScopedLogger'; import type { HeftLifecycleSession } from '../pluginFramework/HeftLifecycleSession'; import type { HeftTaskSession } from '../pluginFramework/HeftTaskSession'; +import { tryValidateSchemaFile } from './lean/SchemaFastPath'; /** * "baseParameter" from heft-plugin.schema.json @@ -186,6 +188,15 @@ export interface IHeftLifecyclePluginDefinitionJson extends IHeftPluginDefinitio export interface IHeftTaskPluginDefinitionJson extends IHeftPluginDefinitionJson {} +function getJsonSchemaClass(): typeof JsonSchema { + return require('@rushstack/node-core-library').JsonSchema; +} + +function getInternalErrorClass(): typeof InternalErrorType { + // Loaded lazily, since it is only needed for error reporting + return require('@rushstack/node-core-library').InternalError; +} + export interface IHeftPluginDefinitionOptions { heftPluginDefinitionJson: IHeftPluginDefinitionJson; packageName: string; @@ -196,6 +207,7 @@ export abstract class HeftPluginDefinitionBase { #heftPluginDefinitionJson: IHeftPluginDefinitionJson; #pluginPackageName: string; #resolvedEntryPoint: string; + #optionsSchemaPath: string | undefined; #optionsSchema: JsonSchema | undefined; protected constructor(options: IHeftPluginDefinitionOptions) { @@ -222,7 +234,14 @@ export abstract class HeftPluginDefinitionBase { options.packageRoot, options.heftPluginDefinitionJson.optionsSchema ); - this.#optionsSchema = JsonSchema.fromFile(resolvedSchemaPath); + // JsonSchema.fromFile() only checks that the file exists; the schema itself is loaded when the options are + // validated. Only construct the JsonSchema (which loads @rushstack/node-core-library) when it is needed. + if (!fs.existsSync(resolvedSchemaPath)) { + // Throws the canonical "Schema file not found" error + this.#optionsSchema = getJsonSchemaClass().fromFile(resolvedSchemaPath); + } + + this.#optionsSchemaPath = resolvedSchemaPath; } } @@ -288,12 +307,12 @@ export abstract class HeftPluginDefinitionBase { 'export a plugin class with a parameterless constructor.' ); } else { - throw new InternalError(`Could not load plugin from "${entryPointPath}": ${error}`); + throw new (getInternalErrorClass())(`Could not load plugin from "${entryPointPath}": ${error}`); } } if (!heftPlugin) { - throw new InternalError( + throw new (getInternalErrorClass())( `Plugin ${JSON.stringify(this.pluginName)} loaded from "${entryPointPath}" is null or undefined.` ); } @@ -301,7 +320,7 @@ export abstract class HeftPluginDefinitionBase { logger.terminal.writeVerboseLine(`Loaded plugin from "${entryPointPath}"`); if (typeof heftPlugin.apply !== 'function') { - throw new InternalError( + throw new (getInternalErrorClass())( `The plugin ${JSON.stringify(this.pluginName)} loaded from "${entryPointPath}" ` + 'doesn\'t define an "apply" function.' ); @@ -314,8 +333,18 @@ export abstract class HeftPluginDefinitionBase { * Validate the provided plugin options against the plugin's options schema, if one is provided. */ public validateOptions(options: unknown): void { - if (this.#optionsSchema) { + const optionsSchemaPath: string | undefined = this.#optionsSchemaPath; + if (optionsSchemaPath) { + if (tryValidateSchemaFile(optionsSchemaPath, options || {})) { + // Guaranteed to produce the same outcome as the ajv-based validation below + return; + } + try { + if (!this.#optionsSchema) { + this.#optionsSchema = getJsonSchemaClass().fromFile(optionsSchemaPath); + } + this.#optionsSchema.validateObject(options || {}, ''); } catch (error) { throw new Error( diff --git a/apps/heft/src/configuration/lean/LeanConfigurationFileSpecification.ts b/apps/heft/src/configuration/lean/LeanConfigurationFileSpecification.ts new file mode 100644 index 00000000000..eebc867c419 --- /dev/null +++ b/apps/heft/src/configuration/lean/LeanConfigurationFileSpecification.ts @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'node:path'; + +import type { IProjectConfigurationFileSpecification } from '@rushstack/heft-config-file'; +import type { IRigConfig } from '@rushstack/rig-package'; + +import { + type ILeanLoadResult, + type ILeanProjectConfigurationFileOptions, + type LeanInheritanceType, + LeanProjectConfigurationFile +} from './LeanProjectConfigurationFile'; +import { bail, getRigProfileFolder, getSharedLeanPackageJsonLookup, type LeanPackageJsonLookup } from './LeanResolution'; +import { tryLoadSchemaFile } from './SchemaFastPath'; + +// A JSONPath of the form "$.a.*.b", which only selects own properties and members of objects and arrays +const SIMPLE_JSON_PATH_REGEXP: RegExp = /^\$(\.(\*|[A-Za-z_][A-Za-z0-9_]*))+$/; + +const INHERITANCE_TYPES: ReadonlySet = new Set(['append', 'merge', 'replace']); + +type Resolver = ILeanProjectConfigurationFileOptions['customResolvers'][number]['resolve']; + +function getInheritanceType(propertyInheritance: unknown): LeanInheritanceType { + const inheritanceType: unknown = (propertyInheritance as { inheritanceType?: unknown } | undefined) + ?.inheritanceType; + if (typeof inheritanceType !== 'string' || !INHERITANCE_TYPES.has(inheritanceType)) { + // Includes custom inheritance functions, which must not be invoked twice + bail(); + } + + return inheritanceType as LeanInheritanceType; +} + +function createLeanOptions( + specification: IProjectConfigurationFileSpecification, + packageJsonLookup: LeanPackageJsonLookup, + rigConfig: IRigConfig | undefined +): ILeanProjectConfigurationFileOptions { + const { + projectRelativeFilePath, + jsonSchemaObject, + jsonSchemaPath, + jsonPathMetadata, + propertyInheritance, + propertyInheritanceDefaults, + customValidationFunction + } = specification; + if (typeof projectRelativeFilePath !== 'string' || customValidationFunction) { + // A custom validation function must not be invoked twice + bail(); + } + + let schemaObject: object | undefined; + if (jsonSchemaObject) { + schemaObject = jsonSchemaObject; + } else if (typeof jsonSchemaPath === 'string') { + schemaObject = tryLoadSchemaFile(jsonSchemaPath); + } + + if (typeof schemaObject !== 'object' || schemaObject === null) { + bail(); + } + + let configuredPropertyInheritance: Map | undefined; + if (propertyInheritance) { + configuredPropertyInheritance = new Map(); + for (const [propertyName, value] of Object.entries(propertyInheritance)) { + configuredPropertyInheritance.set(propertyName, getInheritanceType(value)); + } + } + + const customResolvers: { path: string[]; resolve: Resolver }[] = []; + if (jsonPathMetadata) { + for (const [jsonPath, metadata] of Object.entries(jsonPathMetadata)) { + const pathResolutionMethod: unknown = (metadata as { pathResolutionMethod?: unknown } | undefined) + ?.pathResolutionMethod; + if (pathResolutionMethod === undefined) { + // The original implementation leaves the values unchanged + continue; + } + + if (!SIMPLE_JSON_PATH_REGEXP.test(jsonPath)) { + bail(); + } + + let resolve: Resolver; + switch (pathResolutionMethod) { + case 'resolvePathRelativeToConfigurationFile': { + resolve = (propertyValue: string, configurationFilePath: string) => + path.resolve(path.dirname(configurationFilePath), propertyValue); + break; + } + + case 'resolvePathRelativeToProjectRoot': { + resolve = (propertyValue: string, configurationFilePath: string, projectFolderPath: string | undefined) => + projectFolderPath ? path.resolve(projectFolderPath, propertyValue) : bail(); + break; + } + + case 'NodeResolve': + case 'nodeResolve': { + resolve = (propertyValue: string, configurationFilePath: string) => + packageJsonLookup.resolveModule(propertyValue, path.dirname(configurationFilePath)); + break; + } + + default: { + // Custom resolvers must not be invoked twice + bail(); + } + } + + customResolvers.push({ path: jsonPath.split('.').slice(1), resolve }); + } + } + + const { array: arrayInheritance, object: objectInheritance } = propertyInheritanceDefaults ?? {}; + return { + projectRelativeFilePath, + jsonSchemaObject: schemaObject, + propertyInheritanceDefaults: { + array: arrayInheritance ? getInheritanceType(arrayInheritance) : 'append', + object: objectInheritance ? getInheritanceType(objectInheritance) : 'replace' + }, + propertyInheritance: configuredPropertyInheritance, + customResolvers, + packageJsonLookup, + getRigProfileFolder: (rigConfigToResolve: IRigConfig) => + rigConfigToResolve === rigConfig ? getRigProfileFolder(rigConfigToResolve) : bail() + }; +} + +/** + * The lean equivalent of `new ProjectConfigurationFile(specification).tryLoadConfigurationFileForProject()` from + * `@rushstack/heft-config-file` (using a fresh loader, like `HeftConfiguration.tryLoadProjectConfigurationFile()`). + * Returns `undefined` if the result might differ from the original implementation (including for all errors, + * unsupported options, and options that involve plugin-provided functions), in which case the original + * implementation must be used. + * + * @param isHeftRigConfig - Whether `rigConfig` is a RigConfig object created by Heft, whose profile folder can + * be resolved without side effects. + */ +export function tryLoadProjectConfigurationFileLean( + specification: IProjectConfigurationFileSpecification, + projectPath: string, + rigConfig: IRigConfig | undefined, + isHeftRigConfig: boolean +): ILeanLoadResult | undefined { + try { + const options: ILeanProjectConfigurationFileOptions = createLeanOptions( + specification as IProjectConfigurationFileSpecification, + getSharedLeanPackageJsonLookup(), + isHeftRigConfig ? rigConfig : undefined + ); + return new LeanProjectConfigurationFile( + options + ).tryLoadConfigurationFileForProjectAllowMissing(projectPath, rigConfig); + } catch { + return undefined; + } +} diff --git a/apps/heft/src/configuration/lean/LeanJson.ts b/apps/heft/src/configuration/lean/LeanJson.ts new file mode 100644 index 00000000000..945d712c78e --- /dev/null +++ b/apps/heft/src/configuration/lean/LeanJson.ts @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * A fast parser for the JSON dialect accepted by `JsonFile.parseString()` from `@rushstack/node-core-library` + * (which uses `jju.parse(text, { mode: 'json5', reserved_keys: 'replace' })`). + * + * The fast path supports standard JSON plus `//` and `/* *\/` comments and a single trailing comma after the + * last array element / object member. Comments are replaced with whitespace, trailing commas are removed, and + * the result is handed to the native `JSON.parse()`. + * + * This is exact for every input that it accepts: + * - `jju` (with `reserved_keys: 'replace'`) creates properties with `Object.defineProperty()` on a normal object, + * which matches `JSON.parse()` for duplicate keys (last value wins, first position kept) and for `__proto__` + * (an own data property). + * - Numbers use the same correctly-rounded conversion. + * + * Anything else (single-quoted strings, unquoted keys, hexadecimal numbers, `Infinity`, a BOM, exotic whitespace, + * U+2028/U+2029, unterminated comments, or any syntax error) returns `undefined`, and the caller must fall back + * to the real `JsonFile.parseString()`, which produces the canonical result or error message. + */ + +// A double-quoted string (group 1), or a comment. JSON5 comments are `//` to the end of the line and non-nested +// `/* */` blocks, outside of strings. Since regular expressions are executed natively, this is much faster than a +// character loop in cold (interpreted) JavaScript. +const STRING_OR_COMMENT_REGEXP: RegExp = /("(?:[^"\\]|\\.)*")|\/\/[^\n\r]*|\/\*[\s\S]*?\*\//g; + +// A double-quoted string (group 1), or a comma that follows the end of a value and precedes a closing bracket or +// brace (a JSON5 trailing comma). +const STRING_OR_TRAILING_COMMA_REGEXP: RegExp = + /("(?:[^"\\]|\\.)*")|(?<=[\]}"0-9a-zA-Z.+\-]\s*),(?=\s*[\]}])/g; + +const POSSIBLE_TRAILING_COMMA_REGEXP: RegExp = /,\s*[\]}]/; + +// Characters that jju treats differently than JSON.parse(): U+2028/U+2029 are line terminators, U+FEFF is +// whitespace, and a backslash followed by a line terminator is a JSON5 line continuation. +const UNSUPPORTED_SYNTAX_REGEXP: RegExp = /[\u2028\u2029\ufeff]|\\[\r\n]/; + +/** + * Returns the text with comments replaced by whitespace and JSON5-style trailing commas removed, or `undefined` if + * the text uses syntax that the fast path does not handle. + * + * @remarks + * Removing a comma that follows a value and precedes `]` or `}` can only produce valid JSON if the comma was a + * trailing comma, which JSON5 permits. Anything else that is not plain JSON (single quotes, unquoted keys, etc.) + * is left in place, so `JSON.parse()` rejects it. + */ +export function stripJsonCommentsAndTrailingCommas(text: string): string | undefined { + if (UNSUPPORTED_SYNTAX_REGEXP.test(text)) { + return undefined; + } + + let result: string = text; + if (result.indexOf('/') !== -1) { + // Keep strings, and replace comments with a space so that adjacent tokens stay separated + result = result.replace(STRING_OR_COMMENT_REGEXP, '$1 '); + } + + if (POSSIBLE_TRAILING_COMMA_REGEXP.test(result)) { + result = result.replace(STRING_OR_TRAILING_COMMA_REGEXP, '$1'); + } + + return result; +} + +/** + * Parses JSON text with the same result as `JsonFile.parseString()`, or returns `undefined` if the fast path + * cannot guarantee an identical result (including all syntax errors). + */ +export function tryParseJsonLean(text: string): { value: unknown } | undefined { + const stripped: string | undefined = stripJsonCommentsAndTrailingCommas(text); + if (stripped === undefined) { + return undefined; + } + + try { + return { value: JSON.parse(stripped) }; + } catch { + return undefined; + } +} diff --git a/apps/heft/src/configuration/lean/LeanJsonSchema.ts b/apps/heft/src/configuration/lean/LeanJsonSchema.ts new file mode 100644 index 00000000000..b29f9dfb134 --- /dev/null +++ b/apps/heft/src/configuration/lean/LeanJsonSchema.ts @@ -0,0 +1,997 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * A lean, *sound* fast path for JSON schema validation. + * + * Heft validates `heft.json`, every `heft-plugin.json`, and every plugin's options using `JsonSchema` from + * `@rushstack/node-core-library`, which compiles each schema with ajv (`strictSchema: true, allowUnionTypes: true`, + * using `ajv-draft-04` for draft-04 schemas). Compiling the schemas dominates Heft's startup time. + * + * This module answers a single question: "would ajv compile this schema without any error or warning, and accept + * this data?" It only returns `true` when it can prove that the answer is yes. In every other case (unsupported + * keyword, anything ajv's strict mode would log or reject, unusual data, or simply invalid data) it returns `false`, + * and the caller must fall back to the real `JsonSchema` validation, which produces the canonical behavior and + * error messages. + * + * The supported subset intentionally mirrors ajv's semantics rather than the JSON schema specification where the + * two differ (for example, keywords next to `$ref` are applied, and `required`/`properties` use `data[key] !== + * undefined`). + */ + +type JsonSchemaDraft = 'draft-04' | 'draft-07'; + +type JsonTypeName = 'array' | 'boolean' | 'integer' | 'null' | 'number' | 'object' | 'string'; + +interface ISchemaObject { + [key: string]: unknown; +} + +interface ISchemaPlan { + readonly root: ISchemaObject; + readonly regExpCache: Map; + readonly refTargets: Map; +} + +const JSON_TYPE_NAMES: ReadonlySet = new Set([ + 'array', + 'boolean', + 'integer', + 'null', + 'number', + 'object', + 'string' +]); + +// See JsonSchema.ts in @rushstack/node-core-library +const VENDOR_EXTENSION_KEY_PATTERN: RegExp = /^x-[a-z0-9]+-[a-z0-9]+(-[a-z0-9]+)*$/; + +// ajv-formats' "regex" format rejects patterns that use the unsupported "\Z" anchor +const Z_ANCHOR_REGEXP: RegExp = /[^\\]\\Z/; + +const SIMPLE_DEFINITION_REF_REGEXP: RegExp = /^#\/definitions\/([A-Za-z0-9_.-]+)$/; + +/** + * The data type(s) that each type-specific keyword applies to (ajv's keyword definitions). Used to replicate ajv's + * `strictTypes` checks. + */ +const KEYWORD_APPLICABLE_TYPE: ReadonlyMap = new Map([ + ['properties', 'object'], + ['patternProperties', 'object'], + ['additionalProperties', 'object'], + ['required', 'object'], + ['minProperties', 'object'], + ['maxProperties', 'object'], + ['items', 'array'], + ['minItems', 'array'], + ['maxItems', 'array'], + ['uniqueItems', 'array'], + ['minLength', 'string'], + ['maxLength', 'string'], + ['pattern', 'string'], + ['minimum', 'number'], + ['maximum', 'number'], + ['exclusiveMinimum', 'number'], + ['exclusiveMaximum', 'number'] +]); + +const planCache: WeakMap = new WeakMap(); + +class UnsupportedSchemaError extends Error {} + +function unsupported(): never { + throw new UnsupportedSchemaError(); +} + +function isPlainObject(value: unknown): value is ISchemaObject { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + + const prototype: unknown = Object.getPrototypeOf(value); + return prototype === Object.prototype; +} + +function isNonNegativeInteger(value: unknown): boolean { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} + +function isFiniteNumber(value: unknown): boolean { + return typeof value === 'number' && isFinite(value); +} + +function hasOwn(obj: object, key: string): boolean { + return Object.prototype.hasOwnProperty.call(obj, key); +} + +/** + * Checks that the value is JSON data that the validator can reason about exactly: `null`, booleans, finite numbers, + * strings, dense arrays, and objects with the standard prototype and without an own `__proto__` key. + * Symbol-keyed properties (such as configuration file annotations) are ignored, like they are by ajv. + */ +function isSimpleJsonData(value: unknown, depth: number): boolean { + if (depth > 256) { + return false; + } + + switch (typeof value) { + case 'string': + case 'boolean': + return true; + case 'number': + return isFinite(value); + case 'object': { + if (value === null) { + return true; + } + + if (Array.isArray(value)) { + if (Object.getPrototypeOf(value) !== Array.prototype) { + return false; + } + + for (let i: number = 0; i < value.length; i++) { + if (!(i in value) || !isSimpleJsonData(value[i], depth + 1)) { + return false; + } + } + + return true; + } + + if ( + Object.getPrototypeOf(value) !== Object.prototype || + // These own keys change the behavior of ajv's generated code or of fast-deep-equal + hasOwn(value, '__proto__') || + hasOwn(value, 'constructor') || + hasOwn(value, 'valueOf') || + hasOwn(value, 'toString') + ) { + return false; + } + + for (const key in value) { + if (hasOwn(value, key)) { + if (!isSimpleJsonData((value as Record)[key], depth + 1)) { + return false; + } + } else { + // Inherited enumerable property + return false; + } + } + + return true; + } + default: + return false; + } +} + +/** + * Deep equality with the semantics of `fast-deep-equal` (used by ajv), restricted to simple JSON data. + */ +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) { + return true; + } + + if (a && b && typeof a === 'object' && typeof b === 'object') { + if (Array.isArray(a) !== Array.isArray(b)) { + return false; + } + + if (Array.isArray(a)) { + const bArray: unknown[] = b as unknown[]; + if (a.length !== bArray.length) { + return false; + } + + for (let i: number = 0; i < a.length; i++) { + if (!deepEqual(a[i], bArray[i])) { + return false; + } + } + + return true; + } + + const aKeys: string[] = Object.keys(a); + if (aKeys.length !== Object.keys(b).length) { + return false; + } + + for (const key of aKeys) { + if (!hasOwn(b, key)) { + return false; + } + } + + for (const key of aKeys) { + if (!deepEqual((a as Record)[key], (b as Record)[key])) { + return false; + } + } + + return true; + } + + return false; +} + +function getSchemaTypes(schema: ISchemaObject): JsonTypeName[] { + const type: unknown = schema.type; + if (type === undefined) { + return []; + } + + return (Array.isArray(type) ? type : [type]) as JsonTypeName[]; +} + +// Replicates ajv's includesType() +function includesType(types: JsonTypeName[], type: JsonTypeName): boolean { + return types.includes(type) || (type === 'integer' && types.includes('number')); +} + +// Replicates ajv's hasApplicableType() +function hasApplicableType(schemaTypes: JsonTypeName[], keywordType: JsonTypeName): boolean { + return schemaTypes.includes(keywordType) || (keywordType === 'number' && schemaTypes.includes('integer')); +} + +class SchemaAnalyzer { + private readonly _root: ISchemaObject; + private readonly _draft: JsonSchemaDraft; + private readonly _vendorKeywords: ReadonlySet; + private readonly _regExpCache: Map = new Map(); + private readonly _refTargets: Map = new Map(); + // Schema objects analyzed with an empty type context (the root, $ref targets and definitions) + private readonly _analyzedWithEmptyContext: Set = new Set(); + + public constructor(root: ISchemaObject, draft: JsonSchemaDraft) { + this._root = root; + this._draft = draft; + const vendorKeywords: Set = new Set(); + for (const key of Object.keys(root)) { + if (VENDOR_EXTENSION_KEY_PATTERN.test(key)) { + vendorKeywords.add(key); + } + } + + this._vendorKeywords = vendorKeywords; + } + + public analyze(): ISchemaPlan { + this._analyzeSchema(this._root, [], true); + return { root: this._root, regExpCache: this._regExpCache, refTargets: this._refTargets }; + } + + private _getRegExp(pattern: unknown): RegExp { + if (typeof pattern !== 'string') { + unsupported(); + } + + let regExp: RegExp | undefined = this._regExpCache.get(pattern); + if (!regExp) { + if (Z_ANCHOR_REGEXP.test(pattern)) { + unsupported(); + } + + try { + // ajv uses the "u" flag (unicodeRegExp: true) + regExp = new RegExp(pattern, 'u'); + } catch { + unsupported(); + } + + this._regExpCache.set(pattern, regExp); + } + + return regExp; + } + + private _resolveRef(ref: unknown): ISchemaObject { + if (typeof ref !== 'string') { + unsupported(); + } + + let target: ISchemaObject | undefined = this._refTargets.get(ref); + if (target) { + return target; + } + + if (ref === '#') { + this._refTargets.set(ref, this._root); + return this._root; + } + + const match: RegExpExecArray | null = SIMPLE_DEFINITION_REF_REGEXP.exec(ref); + if (!match) { + unsupported(); + } + + const definitions: unknown = this._root.definitions; + if (!isPlainObject(definitions) || !hasOwn(definitions, match[1])) { + unsupported(); + } + + const definition: unknown = definitions[match[1]]; + if (!isPlainObject(definition)) { + unsupported(); + } + + target = definition; + this._refTargets.set(ref, target); + return target; + } + + private _analyzeSubschema(schema: unknown, contextTypes: JsonTypeName[]): void { + if (!isPlainObject(schema)) { + // Boolean schemas are not supported by draft-04, and are not used by Heft + unsupported(); + } + + this._analyzeSchema(schema, contextTypes, false); + } + + private _analyzeWithEmptyContext(schema: ISchemaObject): void { + if (!this._analyzedWithEmptyContext.has(schema)) { + this._analyzedWithEmptyContext.add(schema); + this._analyzeSchema(schema, [], schema === this._root); + } + } + + private _analyzeSchema(schema: ISchemaObject, contextTypes: JsonTypeName[], isRoot: boolean): void { + if (isRoot) { + this._analyzedWithEmptyContext.add(schema); + } + + const draft: JsonSchemaDraft = this._draft; + let hasRuleOtherThanRef: boolean = false; + + for (const key of Object.keys(schema)) { + const value: unknown = schema[key]; + if (key !== '$ref') { + hasRuleOtherThanRef = true; + } + + switch (key) { + case '$schema': { + if (!isRoot) { + unsupported(); + } + + break; + } + + case 'title': + case 'description': + case '$comment': { + if (typeof value !== 'string') { + unsupported(); + } + + break; + } + + case 'default': { + break; + } + + case 'examples': { + if (draft !== 'draft-07' || !Array.isArray(value)) { + unsupported(); + } + + break; + } + + case 'definitions': { + if (!isPlainObject(value)) { + unsupported(); + } + + for (const definitionName of Object.keys(value)) { + const definition: unknown = value[definitionName]; + if (!isPlainObject(definition)) { + unsupported(); + } + + this._analyzeWithEmptyContext(definition); + } + + break; + } + + case 'type': { + const types: unknown[] = Array.isArray(value) ? value : [value]; + if (types.length === 0 || new Set(types).size !== types.length) { + unsupported(); + } + + for (const type of types) { + if (typeof type !== 'string' || !JSON_TYPE_NAMES.has(type)) { + unsupported(); + } + } + + break; + } + + case 'enum': { + if (!Array.isArray(value) || value.length === 0 || !isSimpleJsonData(value, 0)) { + unsupported(); + } + + for (let i: number = 0; i < value.length; i++) { + for (let j: number = i + 1; j < value.length; j++) { + if (deepEqual(value[i], value[j])) { + unsupported(); + } + } + } + + break; + } + + case 'const': { + if (!isSimpleJsonData(value, 0)) { + unsupported(); + } + + break; + } + + case 'properties': { + if (!isPlainObject(value)) { + unsupported(); + } + + for (const propertyName of Object.keys(value)) { + if (propertyName in Object.prototype) { + // Includes "__proto__"; ajv's handling of these property names is unusual + unsupported(); + } + + this._analyzeSubschema(value[propertyName], []); + } + + break; + } + + case 'patternProperties': { + if (!isPlainObject(value)) { + unsupported(); + } + + for (const pattern of Object.keys(value)) { + if (pattern === '__proto__') { + unsupported(); + } + + this._getRegExp(pattern); + this._analyzeSubschema(value[pattern], []); + } + + break; + } + + case 'additionalProperties': { + if (typeof value !== 'boolean') { + this._analyzeSubschema(value, []); + } + + break; + } + + case 'required': { + if (!Array.isArray(value) || (draft === 'draft-04' && value.length === 0)) { + unsupported(); + } + + const seen: Set = new Set(); + for (const propertyName of value) { + if (typeof propertyName !== 'string' || seen.has(propertyName) || propertyName in Object.prototype) { + unsupported(); + } + + seen.add(propertyName); + } + + break; + } + + case 'items': { + // The array form (tuple validation) is not supported + this._analyzeSubschema(value, []); + break; + } + + case 'minItems': + case 'maxItems': + case 'minLength': + case 'maxLength': + case 'minProperties': + case 'maxProperties': { + if (!isNonNegativeInteger(value)) { + unsupported(); + } + + break; + } + + case 'uniqueItems': { + if (typeof value !== 'boolean') { + unsupported(); + } + + break; + } + + case 'pattern': { + this._getRegExp(value); + break; + } + + case 'minimum': + case 'maximum': { + if (!isFiniteNumber(value)) { + unsupported(); + } + + break; + } + + case 'exclusiveMinimum': + case 'exclusiveMaximum': { + // draft-04 uses a boolean modifier, which is not supported + if (draft !== 'draft-07' || !isFiniteNumber(value)) { + unsupported(); + } + + break; + } + + case 'allOf': + case 'anyOf': + case 'oneOf': { + // These are validated in place, so the subschemas inherit the type context; this is handled below + if (!Array.isArray(value) || value.length === 0) { + unsupported(); + } + + break; + } + + case 'not': { + // Validated in place; handled below + break; + } + + case '$ref': { + this._analyzeWithEmptyContext(this._resolveRef(value)); + break; + } + + default: { + if (!this._vendorKeywords.has(key)) { + // Unknown or unsupported keyword + unsupported(); + } + + break; + } + } + } + + // ajv's strict mode rejects a property that also matches a pattern property (allowMatchingProperties: false) + const properties: unknown = schema.properties; + const patternProperties: unknown = schema.patternProperties; + if (isPlainObject(properties) && isPlainObject(patternProperties)) { + for (const pattern of Object.keys(patternProperties)) { + const regExp: RegExp = this._getRegExp(pattern); + for (const propertyName of Object.keys(properties)) { + if (regExp.test(propertyName)) { + unsupported(); + } + } + } + } + + let dataTypes: JsonTypeName[] = contextTypes; + if (schema.$ref !== undefined && !hasRuleOtherThanRef) { + // ajv only evaluates the $ref, and skips the strictTypes checks for this schema object + return; + } + + // Replicate ajv's checkStrictTypes() (strictTypes: "log"). Any warning means the schema is not supported, + // so that the real ajv path logs it. + const types: JsonTypeName[] = getSchemaTypes(schema); + if (types.length) { + if (!contextTypes.length) { + dataTypes = types; + } else { + for (const type of types) { + if (!includesType(contextTypes, type)) { + unsupported(); + } + } + + // Replicates ajv's narrowSchemaTypes() + const narrowedTypes: JsonTypeName[] = []; + for (const contextType of contextTypes) { + if (includesType(types, contextType)) { + narrowedTypes.push(contextType); + } else if (types.includes('integer') && contextType === 'number') { + narrowedTypes.push('integer'); + } + } + + dataTypes = narrowedTypes; + } + } + + for (const key of Object.keys(schema)) { + const applicableType: JsonTypeName | undefined = KEYWORD_APPLICABLE_TYPE.get(key); + if (applicableType && !hasApplicableType(dataTypes, applicableType)) { + unsupported(); + } + } + + // In-place applicators inherit the (narrowed) type context + for (const key of ['allOf', 'anyOf', 'oneOf'] as const) { + const subschemas: unknown = schema[key]; + if (subschemas !== undefined) { + for (const subschema of subschemas as unknown[]) { + this._analyzeSubschema(subschema, dataTypes); + } + } + } + + if (schema.not !== undefined) { + this._analyzeSubschema(schema.not, dataTypes); + } + } +} + +function getSchemaDraft(schema: ISchemaObject): JsonSchemaDraft | undefined { + // Mirrors _inferJsonSchemaVersion() in JsonSchema.ts, restricted to the meta-schema URLs that ajv resolves. + const $schema: unknown = schema.$schema; + switch ($schema) { + case undefined: + case 'http://json-schema.org/draft-07/schema#': + case 'http://json-schema.org/draft-07/schema': + return 'draft-07'; + case 'http://json-schema.org/draft-04/schema#': + case 'http://json-schema.org/draft-04/schema': + return 'draft-04'; + default: + return undefined; + } +} + +function getSchemaPlan(schemaObject: object): ISchemaPlan | undefined { + let plan: ISchemaPlan | false | undefined = planCache.get(schemaObject); + if (plan === undefined) { + plan = false; + if (isPlainObject(schemaObject) && isSimpleJsonData(schemaObject, 0)) { + const draft: JsonSchemaDraft | undefined = getSchemaDraft(schemaObject); + if (draft) { + try { + plan = new SchemaAnalyzer(schemaObject, draft).analyze(); + } catch (e) { + if (!(e instanceof UnsupportedSchemaError)) { + throw e; + } + } + } + } + + planCache.set(schemaObject, plan); + } + + return plan || undefined; +} + +function matchesType(data: unknown, type: JsonTypeName): boolean { + switch (type) { + case 'string': + return typeof data === 'string'; + case 'number': + // Data is known to contain only finite numbers (strictNumbers: true) + return typeof data === 'number'; + case 'integer': + return typeof data === 'number' && Number.isInteger(data); + case 'boolean': + return typeof data === 'boolean'; + case 'null': + return data === null; + case 'array': + return Array.isArray(data); + case 'object': + return typeof data === 'object' && data !== null && !Array.isArray(data); + default: + return false; + } +} + +function countCodePoints(value: string): number { + let count: number = 0; + for (let i: number = 0; i < value.length; i++) { + const charCode: number = value.charCodeAt(i); + count++; + if (charCode >= 0xd800 && charCode <= 0xdbff && i + 1 < value.length) { + const nextCharCode: number = value.charCodeAt(i + 1); + if (nextCharCode >= 0xdc00 && nextCharCode <= 0xdfff) { + // Surrogate pair, counted as one character (ajv's ucs2length) + i++; + } + } + } + + return count; +} + +function validateNode(plan: ISchemaPlan, schema: ISchemaObject, data: unknown): boolean { + const type: unknown = schema.type; + if (type !== undefined) { + if (typeof type === 'string') { + if (!matchesType(data, type as JsonTypeName)) { + return false; + } + } else if (!(type as JsonTypeName[]).some((t: JsonTypeName) => matchesType(data, t))) { + return false; + } + } + + if (schema.$ref !== undefined && !validateNode(plan, plan.refTargets.get(schema.$ref as string)!, data)) { + return false; + } + + const enumValues: unknown = schema.enum; + if (enumValues !== undefined && !(enumValues as unknown[]).some((v: unknown) => deepEqual(data, v))) { + return false; + } + + if (schema.const !== undefined && !deepEqual(data, schema.const)) { + return false; + } + + const allOf: unknown = schema.allOf; + if (allOf !== undefined) { + // The order of evaluation doesn't affect the result, so evaluate subschemas without a $ref first: they are + // cheaper, and often reject the data. + for (const subschema of allOf as ISchemaObject[]) { + if (subschema.$ref === undefined && !validateNode(plan, subschema, data)) { + return false; + } + } + + for (const subschema of allOf as ISchemaObject[]) { + if (subschema.$ref !== undefined && !validateNode(plan, subschema, data)) { + return false; + } + } + } + + const anyOf: unknown = schema.anyOf; + if (anyOf !== undefined) { + if (!(anyOf as ISchemaObject[]).some((subschema: ISchemaObject) => validateNode(plan, subschema, data))) { + return false; + } + } + + const oneOf: unknown = schema.oneOf; + if (oneOf !== undefined) { + let passingCount: number = 0; + for (const subschema of oneOf as ISchemaObject[]) { + if (validateNode(plan, subschema, data) && ++passingCount > 1) { + return false; + } + } + + if (passingCount !== 1) { + return false; + } + } + + if (schema.not !== undefined && validateNode(plan, schema.not as ISchemaObject, data)) { + return false; + } + + switch (typeof data) { + case 'string': { + if (schema.minLength !== undefined || schema.maxLength !== undefined) { + const length: number = countCodePoints(data); + if (schema.minLength !== undefined && length < (schema.minLength as number)) { + return false; + } + + if (schema.maxLength !== undefined && length > (schema.maxLength as number)) { + return false; + } + } + + if (schema.pattern !== undefined && !plan.regExpCache.get(schema.pattern as string)!.test(data)) { + return false; + } + + break; + } + + case 'number': { + if (schema.minimum !== undefined && data < (schema.minimum as number)) { + return false; + } + + if (schema.maximum !== undefined && data > (schema.maximum as number)) { + return false; + } + + if (schema.exclusiveMinimum !== undefined && data <= (schema.exclusiveMinimum as number)) { + return false; + } + + if (schema.exclusiveMaximum !== undefined && data >= (schema.exclusiveMaximum as number)) { + return false; + } + + break; + } + + case 'object': { + if (data === null) { + break; + } + + if (Array.isArray(data)) { + if (schema.minItems !== undefined && data.length < (schema.minItems as number)) { + return false; + } + + if (schema.maxItems !== undefined && data.length > (schema.maxItems as number)) { + return false; + } + + const items: unknown = schema.items; + if (items !== undefined) { + for (const item of data) { + if (!validateNode(plan, items as ISchemaObject, item)) { + return false; + } + } + } + + if (schema.uniqueItems === true) { + for (let i: number = 1; i < data.length; i++) { + for (let j: number = 0; j < i; j++) { + if (deepEqual(data[i], data[j])) { + return false; + } + } + } + } + + break; + } + + const dataObject: Record = data as Record; + const required: unknown = schema.required; + if (required !== undefined) { + for (const propertyName of required as string[]) { + if (dataObject[propertyName] === undefined) { + return false; + } + } + } + + const dataKeys: string[] = Object.keys(dataObject); + if (schema.minProperties !== undefined && dataKeys.length < (schema.minProperties as number)) { + return false; + } + + if (schema.maxProperties !== undefined && dataKeys.length > (schema.maxProperties as number)) { + return false; + } + + const properties: ISchemaObject | undefined = schema.properties as ISchemaObject | undefined; + if (properties !== undefined) { + for (const propertyName of Object.keys(properties)) { + const propertyValue: unknown = dataObject[propertyName]; + if ( + propertyValue !== undefined && + !validateNode(plan, properties[propertyName] as ISchemaObject, propertyValue) + ) { + return false; + } + } + } + + const patternProperties: ISchemaObject | undefined = schema.patternProperties as + | ISchemaObject + | undefined; + const patterns: string[] | undefined = patternProperties ? Object.keys(patternProperties) : undefined; + if (patterns) { + for (const pattern of patterns) { + const regExp: RegExp = plan.regExpCache.get(pattern)!; + const patternSchema: ISchemaObject = patternProperties![pattern] as ISchemaObject; + for (const key of dataKeys) { + if (regExp.test(key) && !validateNode(plan, patternSchema, dataObject[key])) { + return false; + } + } + } + } + + const additionalProperties: unknown = schema.additionalProperties; + if (additionalProperties !== undefined && additionalProperties !== true) { + for (const key of dataKeys) { + if (properties !== undefined && hasOwn(properties, key)) { + continue; + } + + if (patterns && patterns.some((pattern: string) => plan.regExpCache.get(pattern)!.test(key))) { + continue; + } + + if ( + additionalProperties === false || + !validateNode(plan, additionalProperties as ISchemaObject, dataObject[key]) + ) { + return false; + } + } + } + + break; + } + + default: { + break; + } + } + + return true; +} + +/** + * Returns `true` if the schema is in the supported subset, i.e. `isDefinitelyValid()` can return `true` for it. + */ +export function isSchemaSupported(schemaObject: object): boolean { + if (!isPlainObject(schemaObject) || !isSimpleJsonData(schemaObject, 0)) { + return false; + } + + const draft: JsonSchemaDraft | undefined = getSchemaDraft(schemaObject); + if (!draft) { + return false; + } + + try { + new SchemaAnalyzer(schemaObject, draft).analyze(); + return true; + } catch (e) { + if (!(e instanceof UnsupportedSchemaError)) { + throw e; + } + + return false; + } +} + +/** + * Returns `true` only if `JsonSchema.fromLoadedObject(schemaObject).validateObject(data, ...)` from + * `@rushstack/node-core-library` is guaranteed to succeed without logging anything. A `false` result means + * "unknown": the caller must perform the real validation. + * + * @param schemaObject - The parsed schema. Results of the schema analysis are cached per object, so the object must + * not be mutated afterwards. + * @param data - The data to validate. + */ +export function isDefinitelyValid(schemaObject: object, data: unknown): boolean { + const plan: ISchemaPlan | undefined = getSchemaPlan(schemaObject); + if (!plan || !isSimpleJsonData(data, 0)) { + return false; + } + + return validateNode(plan, plan.root, data); +} diff --git a/apps/heft/src/configuration/lean/LeanProjectConfigurationFile.ts b/apps/heft/src/configuration/lean/LeanProjectConfigurationFile.ts new file mode 100644 index 00000000000..5772a20f5ad --- /dev/null +++ b/apps/heft/src/configuration/lean/LeanProjectConfigurationFile.ts @@ -0,0 +1,615 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type { IRigConfig } from '@rushstack/rig-package'; +import type { CONFIGURATION_FILE_FIELD_ANNOTATION } from '@rushstack/heft-config-file/lib/ConfigurationFileAnnotation'; + +import { stripJsonCommentsAndTrailingCommas } from './LeanJson'; +import { tryValidateSchemaObject } from './SchemaFastPath'; +import { bail, isNotExistError, type LeanPackageJsonLookup } from './LeanResolution'; + +/** + * The key of the annotation that `@rushstack/heft-config-file` attaches to every object in a loaded configuration + * file. It is the same symbol (from a dependency-free module of heft-config-file), so plugins can read the + * annotations with heft-config-file's APIs (e.g. `getObjectSourceFilePath()`), like before. + */ +export const LEAN_CONFIGURATION_FILE_FIELD_ANNOTATION: typeof CONFIGURATION_FILE_FIELD_ANNOTATION = + require('@rushstack/heft-config-file/lib/ConfigurationFileAnnotation').CONFIGURATION_FILE_FIELD_ANNOTATION; + +export interface ILeanConfigurationFileFieldAnnotation { + configurationFilePath: string | undefined; + originalValues: { [propertyName: string]: unknown }; + schemaPropertyOriginalValue?: string; +} + +interface IAnnotatedObject { + [LEAN_CONFIGURATION_FILE_FIELD_ANNOTATION]?: ILeanConfigurationFileFieldAnnotation; +} + +type JsonObject = { [key: string]: unknown } & IAnnotatedObject; + +export type LeanInheritanceType = 'append' | 'merge' | 'replace'; +type InheritanceType = LeanInheritanceType; + +export interface ILeanPropertyInheritanceDefaults { + array: InheritanceType; + object: InheritanceType; +} +type IPropertyInheritanceDefaults = ILeanPropertyInheritanceDefaults; + +interface IConfigurationFileEntry { + resolvedConfigurationFilePath: string; + parent: IConfigurationFileEntry | undefined; + configurationFile: JsonObject; + // The file's text, in plain JSON syntax + jsonText: string; +} + +/** + * A path in a configuration file, where `*` matches every member of an object or array. + */ +type JsonPathSegments = readonly string[]; + +export interface ILeanProjectConfigurationFileOptions { + projectRelativeFilePath: string; + jsonSchemaObject: object; + propertyInheritanceDefaults: IPropertyInheritanceDefaults; + /** + * The inheritance types configured for top-level properties (`propertyInheritance`). + */ + propertyInheritance?: ReadonlyMap; + /** + * The properties to resolve after loading, and the resolver to use for them (the equivalent of JSON path + * metadata with a path resolution method). + */ + customResolvers: ReadonlyArray<{ + path: JsonPathSegments; + resolve: (propertyValue: string, configurationFilePath: string, projectFolderPath: string | undefined) => string; + }>; + packageJsonLookup: LeanPackageJsonLookup; + /** + * Returns the same value as `rigConfig.getResolvedProfileFolder()`, without side effects on `rigConfig` + * (or bails). + */ + getRigProfileFolder: (rigConfig: IRigConfig) => string; +} + +export interface ILeanLoadResult { + /** + * The loaded configuration, or `undefined` if the file does not exist (only when `allowMissing` is set). + */ + configurationFile: T; + /** + * Debug messages that the original implementation would have written to the terminal. + */ + debugMessages: string[]; + /** + * All configuration files that were read. + */ + configurationFilePaths: string[]; +} + +const CONFIGURATION_FILE_MERGE_BEHAVIOR_FIELD_REGEX: RegExp = /^\$([^\.]+)\.inheritanceType$/; + +function hasOwn(obj: object, key: string): boolean { + return Object.prototype.hasOwnProperty.call(obj, key); +} + +function getAnnotation(obj: unknown): ILeanConfigurationFileFieldAnnotation | undefined { + return (obj as IAnnotatedObject)[LEAN_CONFIGURATION_FILE_FIELD_ANNOTATION]; +} + +/** + * Equivalent to `ConfigurationFileBase.getPropertyOriginalValue()`, for objects loaded by + * {@link LeanProjectConfigurationFile}. + */ +export function getLeanPropertyOriginalValue( + parentObject: object, + propertyName: string +): TValue | undefined { + const annotation: ILeanConfigurationFileFieldAnnotation | undefined = getAnnotation(parentObject); + if (annotation?.originalValues.hasOwnProperty(propertyName)) { + return annotation.originalValues[propertyName] as TValue; + } +} + +/** + * Equivalent to `ConfigurationFileBase.getObjectSourceFilePath()`, for objects loaded by + * {@link LeanProjectConfigurationFile}. + */ +export function getLeanObjectSourceFilePath(obj: object): string | undefined { + return getAnnotation(obj)?.configurationFilePath; +} + +/** + * A lean implementation of `ProjectConfigurationFile.loadConfigurationFileForProjectAsync()` from + * `@rushstack/heft-config-file`, for the options used by Heft for `config/heft.json`. It avoids loading + * heft-config-file, jsonpath-plus, ajv and node-core-library. + * + * `tryLoadConfigurationFileForProject()` returns `undefined` whenever the result might differ from the original + * implementation, including for all error conditions; the caller must then use the original implementation. + * The objects in a successful result are annotated exactly like the original (with a description-identical symbol). + */ +export class LeanProjectConfigurationFile { + private readonly _options: ILeanProjectConfigurationFileOptions; + private readonly _entryCache: Map = new Map(); + + public constructor(options: ILeanProjectConfigurationFileOptions) { + this._options = options; + } + + public tryLoadConfigurationFileForProject( + projectPath: string, + rigConfig: IRigConfig | undefined + ): ILeanLoadResult | undefined { + try { + return this._loadConfigurationFileForProject(projectPath, rigConfig, false) as + | ILeanLoadResult + | undefined; + } catch { + // Fall back to the original implementation, which produces the canonical result or error + return undefined; + } + } + + /** + * The equivalent of `ProjectConfigurationFile.tryLoadConfigurationFileForProject()`: the result's + * `configurationFile` is `undefined` if the file does not exist (in the project or in the rig). + */ + public tryLoadConfigurationFileForProjectAllowMissing( + projectPath: string, + rigConfig: IRigConfig | undefined + ): ILeanLoadResult | undefined { + try { + return this._loadConfigurationFileForProject(projectPath, rigConfig, true); + } catch { + return undefined; + } + } + + private _loadConfigurationFileForProject( + projectPath: string, + rigConfig: IRigConfig | undefined, + allowMissing: boolean + ): ILeanLoadResult { + const { projectRelativeFilePath, packageJsonLookup } = this._options; + const debugMessages: string[] = []; + const configurationFilePaths: string[] = []; + const projectConfigurationFilePath: string = path.resolve(projectPath, projectRelativeFilePath); + // The original implementation looks this up eagerly (and fails if a package.json can't be read) + const projectFolderPath: string | undefined = packageJsonLookup.tryGetPackageFolderFor(projectPath); + const visitedConfigurationFilePaths: Set = new Set(); + + const onFileNotFound: () => string | undefined = () => { + if (!rigConfig) { + return undefined; + } + + if (rigConfig.rigFound) { + const rigProfileFolder: string = this._options.getRigProfileFolder(rigConfig); + debugMessages.push( + `Configuration file "${projectConfigurationFilePath}" does not exist. Attempting to load via rig ` + + `("${rigProfileFolder}").` + ); + return path.resolve(rigProfileFolder, projectRelativeFilePath); + } else { + debugMessages.push(`No rig found for "${rigConfig.projectFolderPath}"`); + return undefined; + } + }; + + const entry: IConfigurationFileEntry | undefined = this._loadEntryWithCache( + projectConfigurationFilePath, + visitedConfigurationFilePaths, + configurationFilePaths, + allowMissing ? debugMessages : undefined, + onFileNotFound + ); + + if (!entry) { + return { configurationFile: undefined, debugMessages, configurationFilePaths }; + } + + const result: JsonObject = this._contextualizeAndFlatten(entry, projectFolderPath); + if (!tryValidateSchemaObject(this._options.jsonSchemaObject, result)) { + // Let the original implementation report the schema error + bail(); + } + + return { + configurationFile: result as unknown as TConfigurationFile, + debugMessages, + configurationFilePaths + }; + } + + /** + * Returns `undefined` only if the file is missing and `notFoundMessages` is provided, in which case the debug + * messages of the original implementation are appended to it. Otherwise, a missing file bails. + */ + private _loadEntryWithCache( + resolvedConfigurationFilePath: string, + visitedConfigurationFilePaths: Set, + configurationFilePaths: string[], + notFoundMessages?: string[], + onFileNotFound?: () => string | undefined + ): IConfigurationFileEntry | undefined { + if (visitedConfigurationFilePaths.has(resolvedConfigurationFilePath)) { + // A loop in the "extends" chain + bail(); + } + + visitedConfigurationFilePaths.add(resolvedConfigurationFilePath); + + let entry: IConfigurationFileEntry | undefined = this._entryCache.get(resolvedConfigurationFilePath); + if (!entry) { + let fileText: string; + try { + fileText = fs.readFileSync(resolvedConfigurationFilePath, 'utf8'); + } catch (e) { + if (!isNotExistError(e)) { + bail(); + } + + const fallbackPath: string | undefined = onFileNotFound?.(); + if (fallbackPath) { + const fallbackEntry: IConfigurationFileEntry | undefined = this._loadEntryWithCache( + fallbackPath, + visitedConfigurationFilePaths, + configurationFilePaths, + notFoundMessages + ); + if (fallbackEntry) { + return fallbackEntry; + } + } + + if (!notFoundMessages) { + bail(); + } + + notFoundMessages.push(`Configuration file "${resolvedConfigurationFilePath}" not found.`); + return undefined; + } + + const jsonText: string | undefined = stripJsonCommentsAndTrailingCommas(fileText); + if (jsonText === undefined) { + bail(); + } + + let parsedValue: unknown; + try { + parsedValue = JSON.parse(jsonText); + } catch { + bail(); + } + + if (typeof parsedValue !== 'object' || parsedValue === null) { + bail(); + } + + const configurationFile: JsonObject = parsedValue as JsonObject; + configurationFilePaths.push(resolvedConfigurationFilePath); + let parent: IConfigurationFileEntry | undefined; + const extendsValue: unknown = configurationFile.extends; + if (extendsValue) { + if (typeof extendsValue !== 'string') { + bail(); + } + + const resolvedParentConfigPath: string = this._options.packageJsonLookup.resolveModule( + extendsValue, + path.dirname(resolvedConfigurationFilePath) + ); + // A missing parent is an error ("cannot be resolved"), so it bails + parent = this._loadEntryWithCache( + resolvedParentConfigPath, + visitedConfigurationFilePaths, + configurationFilePaths + ); + } + + entry = { + resolvedConfigurationFilePath, + parent, + configurationFile, + jsonText + }; + this._entryCache.set(resolvedConfigurationFilePath, entry); + } else { + // The original implementation returns cached entries without re-checking their parents + for (let current: IConfigurationFileEntry | undefined = entry; current; current = current.parent) { + configurationFilePaths.push(current.resolvedConfigurationFilePath); + } + } + + return entry; + } + + private _contextualizeAndFlatten( + entry: IConfigurationFileEntry, + projectFolderPath: string | undefined + ): JsonObject { + const parentConfig: JsonObject = entry.parent + ? this._contextualizeAndFlatten(entry.parent, projectFolderPath) + : {}; + const currentConfig: JsonObject = this._contextualize(entry, projectFolderPath); + return this._mergeConfigurationFiles(parentConfig, currentConfig, entry.resolvedConfigurationFilePath); + } + + private _contextualize(entry: IConfigurationFileEntry, projectFolderPath: string | undefined): JsonObject { + // Deep copy, like the original implementation (which uses structuredClone()), since the entry is cached. + // Parsing the text again is faster than cloning in cold code, and the result is identical for JSON data. + const result: JsonObject = JSON.parse(entry.jsonText); + const { resolvedConfigurationFilePath } = entry; + annotateProperties(resolvedConfigurationFilePath, result); + + for (const { path: jsonPath, resolve } of this._options.customResolvers) { + forEachJsonPathMatch(result, jsonPath, 0, (parent: JsonObject, propertyName: string) => { + const propertyValue: unknown = parent[propertyName]; + if (typeof propertyValue !== 'string') { + bail(); + } + + parent[propertyName] = resolve(propertyValue, resolvedConfigurationFilePath, projectFolderPath); + }); + } + + return result; + } + + private _mergeConfigurationFiles( + parentConfiguration: JsonObject, + configurationJson: JsonObject, + resolvedConfigurationFilePath: string + ): JsonObject { + const ignoreProperties: Set = new Set(['extends', '$schema']); + const result: JsonObject = mergeObjects( + parentConfiguration, + configurationJson, + resolvedConfigurationFilePath, + this._options.propertyInheritanceDefaults, + this._options.propertyInheritance, + ignoreProperties + ); + getAnnotation(result)!.schemaPropertyOriginalValue = configurationJson.$schema as string | undefined; + return result; + } +} + +/** + * Invokes the callback for every property matched by the path, in the same order as jsonpath-plus. + * Only own properties match, and `*` matches the members of objects and arrays. + */ +function forEachJsonPathMatch( + value: unknown, + jsonPath: JsonPathSegments, + index: number, + callback: (parent: JsonObject, propertyName: string) => void +): void { + if (!value || typeof value !== 'object') { + return; + } + + const segment: string = jsonPath[index]; + const isLast: boolean = index === jsonPath.length - 1; + if (segment === '*') { + const keys: string[] = Array.isArray(value) ? Array.from(value.keys(), String) : Object.keys(value); + for (const key of keys) { + if (isLast) { + callback(value as JsonObject, key); + } else { + forEachJsonPathMatch((value as JsonObject)[key], jsonPath, index + 1, callback); + } + } + } else if (hasOwn(value, segment)) { + if (isLast) { + callback(value as JsonObject, segment); + } else { + forEachJsonPathMatch((value as JsonObject)[segment], jsonPath, index + 1, callback); + } + } +} + +function annotateProperties(resolvedConfigurationFilePath: string, root: unknown): void { + if (!root) { + return; + } + + const queue: Set = new Set([root]); + for (const obj of queue) { + if (obj && typeof obj === 'object') { + (obj as IAnnotatedObject)[LEAN_CONFIGURATION_FILE_FIELD_ANNOTATION] = { + configurationFilePath: resolvedConfigurationFilePath, + originalValues: { ...obj } + }; + + for (const objValue of Object.values(obj)) { + queue.add(objValue); + } + } + } +} + +function getPropertyOriginalValue(parentObject: object, propertyName: string): unknown { + return getLeanPropertyOriginalValue(parentObject, propertyName); +} + +/** + * A port of `ConfigurationFileBase.#mergeObjects()` from `@rushstack/heft-config-file`, for configurations without + * custom inheritance functions. Error conditions bail. + */ +function mergeObjects( + parentObject: JsonObject, + currentObject: JsonObject, + resolvedConfigurationFilePath: string, + defaultPropertyInheritance: IPropertyInheritanceDefaults, + configuredPropertyInheritance?: ReadonlyMap, + ignoreProperties?: Set +): JsonObject { + const resultAnnotation: ILeanConfigurationFileFieldAnnotation = { + configurationFilePath: resolvedConfigurationFilePath, + originalValues: {} + }; + const result: JsonObject = { + [LEAN_CONFIGURATION_FILE_FIELD_ANNOTATION]: resultAnnotation + }; + + const currentObjectPropertyNames: Set = new Set(Object.keys(currentObject)); + const inheritanceTypeMap: Map = new Map(); + const mergedPropertyNames: Set = new Set(Object.keys(parentObject)); + + for (const propertyName of currentObjectPropertyNames) { + if (ignoreProperties && ignoreProperties.has(propertyName)) { + continue; + } + + const inheritanceTypeMatches: RegExpMatchArray | null = propertyName.match( + CONFIGURATION_FILE_MERGE_BEHAVIOR_FIELD_REGEX + ); + if (inheritanceTypeMatches) { + const mergeTargetPropertyName: string = inheritanceTypeMatches[1]; + const inheritanceTypeRaw: unknown = currentObject[propertyName]; + if ( + !currentObjectPropertyNames.has(mergeTargetPropertyName) || + typeof inheritanceTypeRaw !== 'string' || + typeof currentObject[mergeTargetPropertyName] !== 'object' + ) { + bail(); + } + + switch (inheritanceTypeRaw.toLowerCase()) { + case 'append': + inheritanceTypeMap.set(mergeTargetPropertyName, 'append'); + break; + case 'merge': + inheritanceTypeMap.set(mergeTargetPropertyName, 'merge'); + break; + case 'replace': + inheritanceTypeMap.set(mergeTargetPropertyName, 'replace'); + break; + default: + bail(); + } + } else { + mergedPropertyNames.add(propertyName); + } + } + + for (const propertyName of mergedPropertyNames) { + const propertyValue: unknown = currentObject[propertyName]; + const parentPropertyValue: unknown = parentObject[propertyName]; + + let newValue: unknown; + const usePropertyValue: () => void = () => { + resultAnnotation.originalValues[propertyName] = getPropertyOriginalValue(currentObject, propertyName); + newValue = propertyValue; + }; + const useParentPropertyValue: () => void = () => { + resultAnnotation.originalValues[propertyName] = getPropertyOriginalValue(parentObject, propertyName); + newValue = parentPropertyValue; + }; + + if (propertyValue === null) { + if (parentPropertyValue !== undefined) { + resultAnnotation.originalValues[propertyName] = getPropertyOriginalValue(parentObject, propertyName); + } + + newValue = undefined; + } else if (propertyValue !== undefined && parentPropertyValue === undefined) { + usePropertyValue(); + } else if (parentPropertyValue !== undefined && propertyValue === undefined) { + useParentPropertyValue(); + } else if (propertyValue !== undefined && parentPropertyValue !== undefined) { + if (ignoreProperties && propertyName in Object.prototype) { + // At the top level, the original implementation looks up the property name in a plain object of + // configured inheritance types, which finds members of Object.prototype + bail(); + } + + let inheritanceType: InheritanceType | undefined = + inheritanceTypeMap.get(propertyName) ?? configuredPropertyInheritance?.get(propertyName); + if (!inheritanceType) { + if (Array.isArray(propertyValue) && Array.isArray(parentPropertyValue)) { + inheritanceType = defaultPropertyInheritance.array; + } else if ( + propertyValue && + parentPropertyValue && + typeof propertyValue === 'object' && + typeof parentPropertyValue === 'object' + ) { + inheritanceType = defaultPropertyInheritance.object; + } else { + inheritanceType = 'replace'; + } + } + + switch (inheritanceType) { + case 'replace': { + usePropertyValue(); + break; + } + + case 'append': { + if (!Array.isArray(propertyValue) || !Array.isArray(parentPropertyValue)) { + bail(); + } + + const parentAnnotation: ILeanConfigurationFileFieldAnnotation | undefined = + getAnnotation(parentPropertyValue); + const currentAnnotation: ILeanConfigurationFileFieldAnnotation | undefined = + getAnnotation(propertyValue); + if (!parentAnnotation || !currentAnnotation) { + // The original implementation would throw a TypeError + bail(); + } + + const newArray: unknown[] & IAnnotatedObject = [...parentPropertyValue, ...propertyValue]; + newArray[LEAN_CONFIGURATION_FILE_FIELD_ANNOTATION] = { + configurationFilePath: undefined, + originalValues: { + ...parentAnnotation.originalValues, + ...currentAnnotation.originalValues + } + }; + newValue = newArray; + break; + } + + case 'merge': { + if ( + parentPropertyValue === null || + propertyValue === null || + (propertyValue && typeof propertyValue !== 'object') || + (parentPropertyValue && typeof parentPropertyValue !== 'object') || + Array.isArray(propertyValue) || + Array.isArray(parentPropertyValue) + ) { + bail(); + } + + newValue = mergeObjects( + parentPropertyValue as JsonObject, + propertyValue as JsonObject, + resolvedConfigurationFilePath, + defaultPropertyInheritance + ); + break; + } + + default: { + bail(); + } + } + } + + if (newValue !== undefined) { + result[propertyName] = newValue; + } + } + + return result; +} diff --git a/apps/heft/src/configuration/lean/LeanResolution.ts b/apps/heft/src/configuration/lean/LeanResolution.ts new file mode 100644 index 00000000000..32983a8b73d --- /dev/null +++ b/apps/heft/src/configuration/lean/LeanResolution.ts @@ -0,0 +1,361 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { builtinModules } from 'node:module'; + +import { tryParseJsonLean } from './LeanJson'; + +/** + * Thrown when the lean implementation cannot guarantee a result identical to the original implementation. + * Callers catch it and fall back to the original (slower) code path, which produces the canonical result or error. + */ +export class LeanBailError extends Error {} + +export function bail(): never { + throw new LeanBailError(); +} + +/** + * Equivalent to `FileSystem.isNotExistError()` from `@rushstack/node-core-library`. + */ +export function isNotExistError(error: unknown): boolean { + const code: unknown = (error as NodeJS.ErrnoException | undefined)?.code; + return code === 'ENOENT' || code === 'ENOTDIR'; +} + +/** + * `stat()` with the error handling of the `resolve` package: missing entries return `undefined`, and any other + * error bails out. + */ +export function tryStat(filePath: string): fs.Stats | undefined { + try { + return fs.statSync(filePath, { throwIfNoEntry: false }); + } catch (e) { + if (isNotExistError(e)) { + return undefined; + } + + bail(); + } +} + +function isFileForResolve(filePath: string): boolean { + const stats: fs.Stats | undefined = tryStat(filePath); + return !!stats && (stats.isFile() || stats.isFIFO()); +} + +function isDirectoryForResolve(filePath: string): boolean { + const stats: fs.Stats | undefined = tryStat(filePath); + return !!stats && stats.isDirectory(); +} + +/** + * Equivalent to `FileSystem.getRealPath()` (which uses fs-extra's `realpathSync`, i.e. the JavaScript + * implementation of `fs.realpathSync`). + */ +export function getRealPath(linkPath: string): string { + return fs.realpathSync(linkPath); +} + +// The realpath function used by resolve@1.x (`fs.realpathSync.native` except on Windows). A missing file is +// returned unchanged. +const _resolveRealpathSync: (p: string) => string = + process.platform !== 'win32' && typeof fs.realpathSync.native === 'function' + ? fs.realpathSync.native + : fs.realpathSync; + +function realpathForResolve(filePath: string): string { + try { + return _resolveRealpathSync(filePath); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') { + bail(); + } + + return filePath; + } +} + +/** + * Replicates `node-modules-paths` from resolve@1.x (which, unlike Node.js, does not skip `node_modules` folders). + */ +function getNodeModulesPaths(start: string): string[] { + let prefix: string = '/'; + if (/^([A-Za-z]:)/.test(start)) { + prefix = ''; + } else if (/^\\\\/.test(start)) { + prefix = '\\\\'; + } + + const absoluteStart: string = path.resolve(start); + const paths: string[] = [absoluteStart]; + let parsed: path.ParsedPath = path.parse(absoluteStart); + while (parsed.dir !== paths[paths.length - 1]) { + paths.push(parsed.dir); + parsed = path.parse(parsed.dir); + } + + return paths.map((aPath: string) => path.resolve(prefix, aPath, 'node_modules')); +} + +/** + * Replicates `resolve.sync(request, { basedir, preserveSymlinks })` from resolve@1.x for a request of the + * form `/`. Bails for anything that would involve extension probing, directory + * resolution, `package.json` "main" fields, or a missing file. + */ +function resolveNodeModulesFile(request: string, baseFolder: string, preserveSymlinks: boolean = false): string { + for (const nodeModulesFolder of getNodeModulesPaths(baseFolder)) { + const candidate: string = path.join(nodeModulesFolder, request); + if (isDirectoryForResolve(path.dirname(candidate))) { + if (isFileForResolve(candidate)) { + return preserveSymlinks ? candidate : realpathForResolve(candidate); + } + + if (isFileForResolve(`${candidate}.js`) || isDirectoryForResolve(candidate)) { + // resolve would try harder here + bail(); + } + } + } + + // Not found; let the original implementation produce the error + bail(); +} + +const _definitelyValidPackageNameRegExp: RegExp = /^(@[a-z0-9\-_.]+\/)?[A-Za-z0-9\-][A-Za-z0-9\-_.]*$/; + +let _builtinModules: Set | undefined; +function isBuiltinModule(moduleName: string): boolean { + if (!_builtinModules) { + _builtinModules = new Set(builtinModules); + } + + return _builtinModules.has(moduleName); +} + +/** + * Equivalent to `RigConfig.getResolvedProfileFolder()` from `@rushstack/rig-package` (which uses + * `resolve.sync(\`\${rigPackageName}/package.json\`, { basedir: projectFolderPath })`, i.e. with the default + * `preserveSymlinks: true` of resolve@1.x), without loading the `resolve` package. Bails where the original would + * throw, or might differ. + */ +export function getRigProfileFolder(rigConfig: { + readonly rigFound: boolean; + readonly rigPackageName: string; + readonly projectFolderPath: string; + readonly relativeProfileFolderPath: string; +}): string { + const { rigFound, rigPackageName, projectFolderPath, relativeProfileFolderPath } = rigConfig; + if (!rigFound || typeof rigPackageName !== 'string' || !_definitelyValidPackageNameRegExp.test(rigPackageName)) { + bail(); + } + + const rigPackageJsonPath: string = resolveNodeModulesFile( + `${rigPackageName}/package.json`, + path.resolve(projectFolderPath), + true + ); + const profileFolder: string = path.join(path.dirname(rigPackageJsonPath), relativeProfileFolderPath); + if (!fs.existsSync(profileFolder)) { + bail(); + } + + return profileFolder; +} + +interface IPackageJsonLike { + name?: string; + version?: string; +} + +/** + * Lean equivalent of `PackageJsonLookup` from `@rushstack/node-core-library` with `loadExtraFields: true` + * (read-only; it does not share the cache of `PackageJsonLookup.instance`). Like the original, loaded package.json + * objects are frozen and cached by real path. + */ +export class LeanPackageJsonLookup { + private readonly _packageFolderCache: Map = new Map(); + private readonly _packageJsonCache: Map = new Map(); + + /** + * Returns the parsed package.json, `undefined` if the file does not exist, or bails if it can't be parsed + * exactly. + */ + private _tryLoadPackageJson(packageJsonPath: string): IPackageJsonLike | undefined { + let realPath: string; + try { + realPath = getRealPath(packageJsonPath); + } catch (e) { + if (isNotExistError(e)) { + return undefined; + } + + bail(); + } + + let packageJson: IPackageJsonLike | undefined = this._packageJsonCache.get(realPath); + if (!packageJson) { + let text: string; + try { + text = fs.readFileSync(realPath, 'utf8'); + } catch { + bail(); + } + + const parsed: { value: unknown } | undefined = tryParseJsonLean(text); + if (!parsed || typeof parsed.value !== 'object' || parsed.value === null) { + bail(); + } + + packageJson = Object.freeze(parsed.value) as IPackageJsonLike; + this._packageJsonCache.set(realPath, packageJson); + } + + return packageJson; + } + + /** + * Equivalent to `PackageJsonLookup.tryGetPackageFolderFor()`. + */ + public tryGetPackageFolderFor(fileOrFolderPath: string): string | undefined { + const resolvedPath: string = path.resolve(fileOrFolderPath); + if (this._packageFolderCache.has(resolvedPath)) { + return this._packageFolderCache.get(resolvedPath); + } + + const packageJson: IPackageJsonLike | undefined = this._tryLoadPackageJson(`${resolvedPath}/package.json`); + let result: string | undefined; + if (packageJson && packageJson.name) { + result = resolvedPath; + } else { + const parentFolder: string = path.dirname(resolvedPath); + result = !parentFolder || parentFolder === resolvedPath ? undefined : this.tryGetPackageFolderFor(parentFolder); + } + + this._packageFolderCache.set(resolvedPath, result); + return result; + } + + /** + * Equivalent to `PackageJsonLookup.tryLoadPackageJsonFor()`. Throws the original error if the "version" field is + * missing; bails for the other conditions in which the original implementation throws. + */ + public tryLoadPackageJsonFor(fileOrFolderPath: string): IPackageJsonLike | undefined { + const packageFolder: string | undefined = this.tryGetPackageFolderFor(fileOrFolderPath); + if (!packageFolder) { + return undefined; + } + + const jsonFilename: string = path.join(packageFolder, 'package.json'); + const packageJson: IPackageJsonLike | undefined = this._tryLoadPackageJson(jsonFilename); + if (!packageJson || !packageJson.name) { + bail(); + } + + if (!packageJson.version) { + // The same error as PackageJsonLookup.loadPackageJson() + throw new Error(`Error reading "${jsonFilename}":\n The required field "version" was not found`); + } + + return packageJson; + } + + /** + * Equivalent to `PackageJsonLookup.loadPackageJson(path.join(packageFolder, 'package.json'))`, bailing where + * that would throw. + */ + public loadPackageJsonForFolder(packageFolder: string): IPackageJsonLike { + const packageJson: IPackageJsonLike | undefined = this._tryLoadPackageJson( + path.join(packageFolder, 'package.json') + ); + if (!packageJson || !packageJson.name || !packageJson.version) { + bail(); + } + + return packageJson; + } + + /** + * Equivalent to `Import.resolvePackage({ packageName, baseFolderPath, allowSelfReference })`. + */ + public resolvePackage(packageName: string, baseFolderPath: string, allowSelfReference: boolean): string { + let normalizedRootPath: string; + try { + normalizedRootPath = getRealPath(baseFolderPath); + } catch { + bail(); + } + + if (allowSelfReference) { + const ownPackageFolder: string | undefined = this.tryGetPackageFolderFor(normalizedRootPath); + if (ownPackageFolder) { + const ownPackageJson: IPackageJsonLike = this.loadPackageJsonForFolder(ownPackageFolder); + if (ownPackageJson.name === packageName) { + return path.dirname(path.join(ownPackageFolder, 'package.json')); + } + } + } + + if ( + typeof packageName !== 'string' || + packageName.length > 214 || + !_definitelyValidPackageNameRegExp.test(packageName) || + packageName.split('/').some((segment: string) => segment === '.' || segment === '..') + ) { + // Let PackageName.parse() produce the error + bail(); + } + + return path.dirname(resolveNodeModulesFile(`${packageName}/package.json`, normalizedRootPath)); + } + + /** + * Equivalent to `Import.resolveModule({ modulePath, baseFolderPath })` / `Import.resolveModuleAsync()` for paths + * that refer to an existing file. + */ + public resolveModule(modulePath: string, baseFolderPath: string): string { + if (typeof modulePath !== 'string') { + bail(); + } + + if (path.isAbsolute(modulePath)) { + return modulePath; + } + + let normalizedRootPath: string; + try { + normalizedRootPath = getRealPath(baseFolderPath); + } catch { + bail(); + } + + if (modulePath.startsWith('.')) { + return path.resolve(normalizedRootPath, modulePath); + } + + const slashIndex: number = modulePath.indexOf('/'); + const moduleName: string = slashIndex === -1 ? modulePath : modulePath.slice(0, slashIndex); + if (isBuiltinModule(moduleName) || modulePath.indexOf('\\') !== -1 || modulePath.indexOf(':') !== -1) { + bail(); + } + + return resolveNodeModulesFile(modulePath, normalizedRootPath); + } +} + +let _sharedPackageJsonLookup: LeanPackageJsonLookup | undefined; + +/** + * The lean equivalent of `PackageJsonLookup.instance`: Heft's own package.json lookups share this cache for the + * lifetime of the process (like the original implementation), so that e.g. watch mode sees the same package.json + * contents as it did at startup. + */ +export function getSharedLeanPackageJsonLookup(): LeanPackageJsonLookup { + if (!_sharedPackageJsonLookup) { + _sharedPackageJsonLookup = new LeanPackageJsonLookup(); + } + + return _sharedPackageJsonLookup; +} diff --git a/apps/heft/src/configuration/lean/LeanRigConfig.ts b/apps/heft/src/configuration/lean/LeanRigConfig.ts new file mode 100644 index 00000000000..2fc738fdf76 --- /dev/null +++ b/apps/heft/src/configuration/lean/LeanRigConfig.ts @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type { IRigConfig } from '@rushstack/rig-package'; + +import { tryParseJsonLean } from './LeanJson'; + +// The validation rules of RigConfig in @rushstack/rig-package +const PACKAGE_NAME_REGEXP: RegExp = /^(@[A-Za-z0-9\-_\.]+\/)?[A-Za-z0-9\-_\.]+$/; +const RIG_NAME_REGEXP: RegExp = /-rig(-test)?$/; +const PROFILE_NAME_REGEXP: RegExp = /^[a-z0-9_\.]+(\-[a-z0-9_\.]+)*$/; + +/** + * The data members of `IRigConfig`. + */ +export interface ILeanRigConfigData { + readonly projectFolderOriginalPath: string; + readonly projectFolderPath: string; + readonly rigFound: boolean; + readonly filePath: string; + readonly rigPackageName: string; + readonly rigProfile: string; + readonly relativeProfileFolderPath: string; +} + +/** + * Reads `config/rig.json` with the same result as `RigConfig.loadForProjectFolderAsync()` from + * `@rushstack/rig-package`, without loading that package (and `jju`). Returns `undefined` if an identical result + * can't be guaranteed, including for every condition in which `RigConfig` throws. + */ +export function tryLoadRigConfigDataLean(projectFolderPath: string): ILeanRigConfigData | undefined { + const rigConfigFilePath: string = path.join(projectFolderPath, 'config/rig.json'); + let text: string; + try { + text = fs.readFileSync(rigConfigFilePath).toString(); + } catch (e) { + const code: unknown = (e as NodeJS.ErrnoException).code; + if (code === 'ENOENT' || code === 'ENOTDIR') { + // No rig config + return { + projectFolderOriginalPath: projectFolderPath, + projectFolderPath: path.resolve(projectFolderPath), + rigFound: false, + filePath: '', + rigPackageName: '', + rigProfile: '', + relativeProfileFolderPath: '' + }; + } + + return undefined; + } + + const parsed: { value: unknown } | undefined = tryParseJsonLean(text); + const json: unknown = parsed?.value; + if (typeof json !== 'object' || json === null || Array.isArray(json)) { + return undefined; + } + + // RigConfig parses with jju's defaults, which silently drop keys like "constructor" or "__proto__", so any + // unexpected key could make the results differ. + for (const key of Object.getOwnPropertyNames(json)) { + if (key !== '$schema' && key !== 'rigPackageName' && key !== 'rigProfile') { + return undefined; + } + } + + const { rigPackageName, rigProfile } = json as { rigPackageName?: unknown; rigProfile?: unknown }; + if ( + typeof rigPackageName !== 'string' || + !PACKAGE_NAME_REGEXP.test(rigPackageName) || + !RIG_NAME_REGEXP.test(rigPackageName) + ) { + return undefined; + } + + if (rigProfile !== undefined && (typeof rigProfile !== 'string' || !PROFILE_NAME_REGEXP.test(rigProfile))) { + return undefined; + } + + const effectiveRigProfile: string = rigProfile === undefined ? 'default' : rigProfile; + return { + projectFolderOriginalPath: projectFolderPath, + projectFolderPath: path.resolve(projectFolderPath), + rigFound: true, + filePath: rigConfigFilePath, + rigPackageName, + rigProfile: effectiveRigProfile, + relativeProfileFolderPath: 'profiles/' + effectiveRigProfile + }; +} + +/** + * An `IRigConfig` with the data that `RigConfig` would have for the project, whose methods delegate to the + * genuine `RigConfig` object (which is only created when it is needed). Heft uses it to load its own configuration + * files without loading `@rushstack/rig-package` on the startup path; `HeftConfiguration.rigConfig` still returns + * the genuine `RigConfig`. + */ +export class LeanRigConfig implements IRigConfig { + public readonly projectFolderOriginalPath: string; + public readonly projectFolderPath: string; + public readonly rigFound: boolean; + public readonly filePath: string; + public readonly rigPackageName: string; + public readonly rigProfile: string; + public readonly relativeProfileFolderPath: string; + + readonly #getRigConfig: () => IRigConfig; + + public constructor(data: ILeanRigConfigData, getRigConfig: () => IRigConfig) { + this.projectFolderOriginalPath = data.projectFolderOriginalPath; + this.projectFolderPath = data.projectFolderPath; + this.rigFound = data.rigFound; + this.filePath = data.filePath; + this.rigPackageName = data.rigPackageName; + this.rigProfile = data.rigProfile; + this.relativeProfileFolderPath = data.relativeProfileFolderPath; + this.#getRigConfig = getRigConfig; + } + + public getResolvedProfileFolder(): string { + return this.#getRigConfig().getResolvedProfileFolder(); + } + + public async getResolvedProfileFolderAsync(): Promise { + return await this.#getRigConfig().getResolvedProfileFolderAsync(); + } + + public tryResolveConfigFilePath(configFileRelativePath: string): string | undefined { + return this.#getRigConfig().tryResolveConfigFilePath(configFileRelativePath); + } + + public async tryResolveConfigFilePathAsync(configFileRelativePath: string): Promise { + return await this.#getRigConfig().tryResolveConfigFilePathAsync(configFileRelativePath); + } +} diff --git a/apps/heft/src/configuration/lean/SchemaFastPath.ts b/apps/heft/src/configuration/lean/SchemaFastPath.ts new file mode 100644 index 00000000000..99b29cda150 --- /dev/null +++ b/apps/heft/src/configuration/lean/SchemaFastPath.ts @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; + +import { tryParseJsonLean } from './LeanJson'; +import { isDefinitelyValid } from './LeanJsonSchema'; + +/** + * An optional native implementation of the schema fast path. Its contract is identical to + * {@link tryValidateSchemaObject}: `true` only if the original ajv-based validation is guaranteed to accept the + * schema and the data without logging anything. `undefined` means that it can't decide, and the JavaScript + * implementation is used instead. + */ +export interface INativeSchemaValidator { + validateObject(schemaObject: object, data: unknown): boolean | undefined; + /** + * Like `validateObject()`, for the schema that `JsonSchema.fromFile(schemaPath)` would load. + */ + validateFile?(schemaPath: string, data: unknown): boolean | undefined; +} + +let _nativeSchemaValidator: INativeSchemaValidator | undefined | false; + +/** + * The integration point for a native schema validator (see `src/native`). Returns `undefined` if no native + * implementation is available, in which case the JavaScript implementation is used. + */ +function tryGetNativeSchemaValidator(): INativeSchemaValidator | undefined { + if (_nativeSchemaValidator === undefined) { + _nativeSchemaValidator = false; + } + + return _nativeSchemaValidator || undefined; +} + +/** + * Returns `true` only if validating `data` against the schema with `JsonSchema` from + * `@rushstack/node-core-library` is guaranteed to succeed without logging anything. Otherwise (including when the + * data is invalid) returns `false`, and the caller must perform the original validation, which produces the + * canonical errors. + * + * @param schemaObject - The parsed schema. The analysis of the schema is cached per object, so the object must not be + * mutated. + */ +export function tryValidateSchemaObject(schemaObject: object, data: unknown): boolean { + const nativeResult: boolean | undefined = tryGetNativeSchemaValidator()?.validateObject(schemaObject, data); + if (nativeResult !== undefined) { + return nativeResult; + } + + return isDefinitelyValid(schemaObject, data); +} + +// Parsed schema files, by path. `false` means that the file can't be parsed by the lean parser. +const _schemaObjectsByPath: Map = new Map(); + +/** + * Loads a schema file with the same result as `JsonFile.load()`, or returns `undefined` if that can't be + * guaranteed. The result is cached per path. + */ +export function tryLoadSchemaFile(schemaPath: string): object | undefined { + let schemaObject: object | false | undefined = _schemaObjectsByPath.get(schemaPath); + if (schemaObject === undefined) { + schemaObject = false; + try { + const parsed: { value: unknown } | undefined = tryParseJsonLean(fs.readFileSync(schemaPath, 'utf8')); + if (parsed && typeof parsed.value === 'object' && parsed.value !== null) { + schemaObject = parsed.value; + } + } catch { + // Use the original implementation + } + + _schemaObjectsByPath.set(schemaPath, schemaObject); + } + + return schemaObject || undefined; +} + +/** + * Equivalent to {@link tryValidateSchemaObject} for the schema that `JsonSchema.fromFile(schemaPath)` would load. + */ +export function tryValidateSchemaFile(schemaPath: string, data: unknown): boolean { + const nativeResult: boolean | undefined = tryGetNativeSchemaValidator()?.validateFile?.(schemaPath, data); + if (nativeResult !== undefined) { + return nativeResult; + } + + const schemaObject: object | undefined = tryLoadSchemaFile(schemaPath); + return schemaObject !== undefined && tryValidateSchemaObject(schemaObject, data); +} diff --git a/apps/heft/src/configuration/test/ConfigTestUtilities.ts b/apps/heft/src/configuration/test/ConfigTestUtilities.ts new file mode 100644 index 00000000000..d591dc45163 --- /dev/null +++ b/apps/heft/src/configuration/test/ConfigTestUtilities.ts @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { CONFIGURATION_FILE_FIELD_ANNOTATION } from '@rushstack/heft-config-file/lib/ConfigurationFileAnnotation'; + +const ANNOTATION_DESCRIPTION: string = 'configuration-file-field-annotation'; + +/** + * Converts a loaded configuration into a plain structure that includes the annotations that + * `@rushstack/heft-config-file` attaches to every object (source file path and original values). + */ +export function describeValue(value: unknown, depth: number = 0): unknown { + if (depth > 50) { + throw new Error('Too deep'); + } + + if (typeof value !== 'object' || value === null) { + return Object.is(value, -0) ? '-0' : value; + } + + const annotationSymbol: symbol | undefined = Object.getOwnPropertySymbols(value).find( + (s: symbol) => s.description === ANNOTATION_DESCRIPTION + ); + const annotation: Record | undefined = annotationSymbol + ? (value as Record>)[annotationSymbol] + : undefined; + const result: Record = { + kind: Array.isArray(value) ? 'array' : 'object', + keys: Object.keys(value), + values: Object.values(value).map((v: unknown) => describeValue(v, depth + 1)), + otherSymbols: Object.getOwnPropertySymbols(value).filter((s: symbol) => s !== annotationSymbol).length + }; + if (annotation) { + // Plugins read the annotations with heft-config-file's APIs, which requires the identical symbol + result.annotationSymbolIsHeftConfigFiles = annotationSymbol === CONFIGURATION_FILE_FIELD_ANNOTATION; + result.annotation = { + configurationFilePath: annotation.configurationFilePath, + schemaPropertyOriginalValue: annotation.schemaPropertyOriginalValue, + hasSchemaPropertyOriginalValue: 'schemaPropertyOriginalValue' in annotation, + originalValues: describeValue({ ...(annotation.originalValues as object) }, depth + 1) + }; + } + + return result; +} + +export function writeFiles(rootFolder: string, files: Record): void { + for (const [relativePath, content] of Object.entries(files)) { + const filePath: string = path.join(rootFolder, relativePath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, typeof content === 'string' ? content : JSON.stringify(content, undefined, 2)); + } +} + +export function replaceAll(value: T, from: string, to: string): T { + if (typeof value === 'string') { + return value.split(from).join(to) as unknown as T; + } + + if (Array.isArray(value)) { + return value.map((item: unknown) => replaceAll(item, from, to)) as unknown as T; + } + + if (typeof value === 'object' && value !== null) { + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + result[replaceAll(key, from, to)] = replaceAll(item, from, to); + } + + return result as T; + } + + return value; +} + diff --git a/apps/heft/src/configuration/test/HeftConfiguration.test.ts b/apps/heft/src/configuration/test/HeftConfiguration.test.ts new file mode 100644 index 00000000000..1661815fc7a --- /dev/null +++ b/apps/heft/src/configuration/test/HeftConfiguration.test.ts @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; +import { ProjectConfigurationFile } from '@rushstack/heft-config-file'; + +import { HeftConfiguration } from '../HeftConfiguration'; + +interface ITestConfig { + name?: string; + list?: string[]; +} + +const TEST_CONFIG_SCHEMA: object = { + type: 'object', + additionalProperties: false, + properties: { + $schema: { type: 'string' }, + extends: { type: 'string' }, + name: { type: 'string' }, + list: { type: 'array', items: { type: 'string' } } + } +}; + +// Any loader instance can read the source-file annotation that heft-config-file attaches to loaded objects +const ANNOTATION_READER: ProjectConfigurationFile = new ProjectConfigurationFile({ + projectRelativeFilePath: 'config/unused.json', + jsonSchemaObject: {} +}); + +function plain(value: unknown): unknown { + return JSON.parse(JSON.stringify(value)); +} + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(value, undefined, 2)); +} + +// Exercises the public HeftConfiguration surface that plugins rely on, against a real project folder. +describe(HeftConfiguration.name, () => { + let rootFolder: string; + let projectFolder: string; + + beforeEach(() => { + rootFolder = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'heft-configuration-test-'))); + projectFolder = path.join(rootFolder, 'project'); + writeJson(path.join(projectFolder, 'package.json'), { name: 'test-project', version: '1.2.3' }); + const rigFolder: string = path.join(projectFolder, 'node_modules', 'test-rig'); + writeJson(path.join(rigFolder, 'package.json'), { name: 'test-rig', version: '1.0.0' }); + writeJson(path.join(rigFolder, 'profiles', 'default', 'config', 'test.json'), { + name: 'from-rig', + list: ['rig'] + }); + }); + + afterEach(() => { + fs.rmSync(rootFolder, { recursive: true, force: true }); + }); + + function initialize(cwd: string = projectFolder): HeftConfiguration { + return HeftConfiguration.initialize({ + cwd, + terminalProvider: new StringBufferTerminalProvider(), + numberOfCores: 3 + }); + } + + it('resolves the build folder from a nested working directory', () => { + const nested: string = path.join(projectFolder, 'src', 'deep'); + fs.mkdirSync(nested, { recursive: true }); + const configuration: HeftConfiguration = initialize(nested); + + expect(configuration.buildFolderPath).toBe(projectFolder); + expect(configuration.projectConfigFolderPath).toBe(path.join(projectFolder, 'config')); + expect(configuration.tempFolderPath).toBe(path.join(projectFolder, 'temp')); + expect(configuration.slashNormalizedBuildFolderPath).toBe(projectFolder.split(path.sep).join('/')); + expect(configuration.numberOfCores).toBe(3); + expect(configuration.projectPackageJson.name).toBe('test-project'); + expect(configuration.projectPackageJson.version).toBe('1.2.3'); + expect(configuration.heftPackageJson.name).toBe('@rushstack/heft'); + }); + + it('throws the documented error outside of a project folder', () => { + const outside: string = path.join(rootFolder, 'no-project'); + fs.mkdirSync(outside); + expect(() => initialize(outside)).toThrow('No package.json file found. Are you in a project folder?'); + }); + + it('requires _checkForRigAsync() before rigConfig is accessed', async () => { + const configuration: HeftConfiguration = initialize(); + expect(() => configuration.rigConfig).toThrow(/checkForRigAsync/); + await configuration._checkForRigAsync(); + expect(configuration.rigConfig.rigFound).toBe(false); + }); + + it('loads a riggable configuration file from the rig when the project does not provide one', async () => { + writeJson(path.join(projectFolder, 'config', 'rig.json'), { rigPackageName: 'test-rig' }); + const configuration: HeftConfiguration = initialize(); + await configuration._checkForRigAsync(); + expect(configuration.rigConfig.rigFound).toBe(true); + expect(configuration.rigConfig.rigPackageName).toBe('test-rig'); + expect(configuration.rigConfig.rigProfile).toBe('default'); + + const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); + const options: { + projectRelativeFilePath: string; + jsonSchemaObject: object; + propertyInheritance: { list: { inheritanceType: 'append' } }; + } = { + projectRelativeFilePath: 'config/test.json', + jsonSchemaObject: TEST_CONFIG_SCHEMA, + propertyInheritance: { list: { inheritanceType: 'append' } } + }; + + const fromRig: ITestConfig | undefined = await configuration.tryLoadProjectConfigurationFileAsync( + options, + terminal + ); + expect(plain(fromRig)).toEqual({ name: 'from-rig', list: ['rig'] }); + // Loaded objects are annotated with their source file (used by plugins via getObjectSourceFilePath) + expect(ANNOTATION_READER.getObjectSourceFilePath(fromRig!)).toBe( + path.join(projectFolder, 'node_modules', 'test-rig', 'profiles', 'default', 'config', 'test.json') + ); + expect(plain(configuration.tryLoadProjectConfigurationFile(options, terminal))).toEqual( + plain(fromRig) + ); + // The options object is frozen by the first load + expect(Object.isFrozen(options)).toBe(true); + }); + + it('applies "extends" with the requested property inheritance', async () => { + writeJson(path.join(projectFolder, 'config', 'base.json'), { name: 'base', list: ['a', 'b'] }); + writeJson(path.join(projectFolder, 'config', 'test.json'), { extends: './base.json', list: ['c'] }); + const configuration: HeftConfiguration = initialize(); + await configuration._checkForRigAsync(); + + const loaded: ITestConfig | undefined = await configuration.tryLoadProjectConfigurationFileAsync( + { + projectRelativeFilePath: 'config/test.json', + jsonSchemaObject: TEST_CONFIG_SCHEMA, + propertyInheritance: { list: { inheritanceType: 'append' } } + }, + new Terminal(new StringBufferTerminalProvider()) + ); + expect(plain(loaded)).toEqual({ name: 'base', list: ['a', 'b', 'c'] }); + expect(ANNOTATION_READER.getObjectSourceFilePath(loaded!)).toBe( + path.join(projectFolder, 'config', 'test.json') + ); + }); + + it('returns undefined when the file does not exist and there is no rig', async () => { + const configuration: HeftConfiguration = initialize(); + await configuration._checkForRigAsync(); + expect( + await configuration.tryLoadProjectConfigurationFileAsync( + { projectRelativeFilePath: 'config/missing.json', jsonSchemaObject: TEST_CONFIG_SCHEMA }, + new Terminal(new StringBufferTerminalProvider()) + ) + ).toBeUndefined(); + }); + + it('reports schema violations', async () => { + writeJson(path.join(projectFolder, 'config', 'test.json'), { name: 5 }); + const configuration: HeftConfiguration = initialize(); + await configuration._checkForRigAsync(); + await expect( + configuration.tryLoadProjectConfigurationFileAsync( + { projectRelativeFilePath: 'config/test.json', jsonSchemaObject: TEST_CONFIG_SCHEMA }, + new Terminal(new StringBufferTerminalProvider()) + ) + ).rejects.toThrow(/JSON validation failed/); + }); + + it('resolves riggable packages from the project first', async () => { + const configuration: HeftConfiguration = initialize(); + await configuration._checkForRigAsync(); + const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); + await expect(configuration.rigPackageResolver.resolvePackageAsync('test-rig', terminal)).resolves.toBe( + path.join(projectFolder, 'node_modules', 'test-rig') + ); + await expect( + configuration.rigPackageResolver.resolvePackageAsync('no-such-package', terminal) + ).rejects.toThrow(/Unable to resolve "no-such-package"/); + }); +}); diff --git a/apps/heft/src/configuration/test/LeanHeftConfiguration.test.ts b/apps/heft/src/configuration/test/LeanHeftConfiguration.test.ts new file mode 100644 index 00000000000..ff41eff4a63 --- /dev/null +++ b/apps/heft/src/configuration/test/LeanHeftConfiguration.test.ts @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import type { IRigConfig } from '@rushstack/rig-package'; +import { StringBufferTerminalProvider, Terminal, type ITerminal } from '@rushstack/terminal'; + +import { CoreConfigFiles, type IHeftConfigurationJson } from '../../utilities/CoreConfigFiles'; +import { HeftConfiguration, getRigConfigForConfigLoading } from '../HeftConfiguration'; +import { describeValue, replaceAll, writeFiles } from './ConfigTestUtilities'; + +const REPO_ROOT: string = path.resolve(__dirname, '../../../../..'); +interface ILoadOutcome { + value?: unknown; + error?: string; + debugOutput: string; +} + +async function createHeftConfigurationAsync(projectPath: string): Promise { + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectPath, + terminalProvider: new StringBufferTerminalProvider(true), + numberOfCores: 1 + }); + await heftConfiguration._checkForRigAsync(); + return heftConfiguration; +} + +function createTerminal(): { terminal: ITerminal; provider: StringBufferTerminalProvider } { + const provider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + return { terminal: new Terminal(provider), provider }; +} + +async function loadOriginalAsync(projectPath: string, rigConfig: IRigConfig): Promise { + const { terminal, provider } = createTerminal(); + try { + const value: IHeftConfigurationJson = await CoreConfigFiles._loadHeftConfigurationFileOriginalAsync( + terminal, + projectPath, + rigConfig + ); + return { value: describeValue(value), debugOutput: provider.getDebugOutput() }; + } catch (e) { + return { error: (e as Error).message, debugOutput: provider.getDebugOutput() }; + } +} + +function loadLean(projectPath: string, rigConfig: IRigConfig): ILoadOutcome | undefined { + const { terminal, provider } = createTerminal(); + const value: IHeftConfigurationJson | undefined = CoreConfigFiles._tryLoadHeftConfigurationFileLean( + terminal, + projectPath, + rigConfig + ); + return value === undefined ? undefined : { value: describeValue(value), debugOutput: provider.getDebugOutput() }; +} + +/** + * Loads the project's heft.json with both implementations, and checks the contract: the lean implementation either + * declines, or produces exactly the same result and debug output. Returns whether the lean path was taken. + * + * @param freshCopyPath - An identical copy of the project, used to check the public entry point. The original + * implementation caches results (including failures) per path, so a second load of the same path would not + * produce the same debug output. + */ +async function expectEquivalentAsync( + projectPath: string, + freshCopyPath?: string +): Promise<{ lean: boolean; original: ILoadOutcome }> { + // Like InternalHeftSession: the original implementation gets the genuine RigConfig, and the lean path gets + // getRigConfigForConfigLoading() + const heftConfiguration: HeftConfiguration = await createHeftConfigurationAsync(projectPath); + const rigConfig: IRigConfig = heftConfiguration.rigConfig; + const original: ILoadOutcome = await loadOriginalAsync(projectPath, rigConfig); + const lean: ILoadOutcome | undefined = loadLean( + projectPath, + getRigConfigForConfigLoading(await createHeftConfigurationAsync(projectPath)) + ); + if (lean) { + expect({ projectPath, ...lean }).toEqual({ projectPath, ...original }); + } + + // The public entry point must behave exactly like the original implementation + const combinedPath: string = freshCopyPath ?? projectPath; + const combinedRigConfig: IRigConfig = getRigConfigForConfigLoading( + await createHeftConfigurationAsync(combinedPath) + ); + const { terminal, provider } = createTerminal(); + let combined: ILoadOutcome; + try { + const value: IHeftConfigurationJson = await CoreConfigFiles.loadHeftConfigurationFileForProjectAsync( + terminal, + combinedPath, + combinedRigConfig + ); + combined = { value: describeValue(value), debugOutput: provider.getDebugOutput() }; + } catch (e) { + combined = { error: (e as Error).message, debugOutput: provider.getDebugOutput() }; + } + + if (freshCopyPath) { + combined = replaceAll(combined, freshCopyPath, projectPath); + } else if (!lean) { + // The original implementation returns its cached failure without logging again + combined.debugOutput = original.debugOutput; + } + + expect({ projectPath, ...combined }).toEqual({ projectPath, ...original }); + return { lean: !!lean, original }; +} + +// Loading every build-test project with the original implementation takes a while +jest.setTimeout(300000); + +describe('Lean heft.json loading', () => { + let tempFolder: string; + let fixtureCounter: number = 0; + + beforeAll(() => { + tempFolder = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'heft-lean-config-'))); + }); + + afterAll(() => { + fs.rmSync(tempFolder, { recursive: true, force: true }); + }); + + function createFixture(files: Record, suffix: string = ''): string { + const projectFolder: string = path.join(tempFolder, `fixture-${fixtureCounter}${suffix}`); + writeFiles(projectFolder, { + 'package.json': { name: 'fixture', version: '1.0.0' }, + 'node_modules/fake-plugin/package.json': { name: 'fake-plugin', version: '1.0.0' }, + 'node_modules/other-plugin/package.json': { name: 'other-plugin', version: '1.0.0' }, + ...files + }); + return projectFolder; + } + + async function expectFixtureEquivalentAsync( + files: Record + ): Promise<{ lean: boolean; original: ILoadOutcome }> { + fixtureCounter++; + const projectFolder: string = createFixture(files); + const freshCopyFolder: string = createFixture(files, '-copy'); + return await expectEquivalentAsync(projectFolder, freshCopyFolder); + } + + it('matches the original loader for every build-test project in the repo', async () => { + const buildTestsFolder: string = path.join(REPO_ROOT, 'build-tests'); + let projectCount: number = 0; + let leanCount: number = 0; + for (const projectName of fs.readdirSync(buildTestsFolder)) { + const projectPath: string = path.join(buildTestsFolder, projectName); + if ( + !fs.existsSync(path.join(projectPath, 'node_modules')) || + (!fs.existsSync(path.join(projectPath, 'config/heft.json')) && + !fs.existsSync(path.join(projectPath, 'config/rig.json'))) + ) { + continue; + } + + projectCount++; + const { lean, original } = await expectEquivalentAsync(projectPath); + if (lean) { + leanCount++; + } else { + // The lean path may only decline when the original fails + expect({ projectPath, error: original.error !== undefined }).toEqual({ projectPath, error: true }); + } + } + + expect(projectCount).toBeGreaterThan(10); + expect(leanCount).toBeGreaterThan(10); + }); + + it('matches the original loader for extends chains and inheritance annotations', async () => { + const { lean } = await expectFixtureEquivalentAsync({ + 'config/heft.json': `// A comment + { + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", + "extends": "./base/heft.json", + "heftPlugins": [{ "pluginPackage": "other-plugin", "options": { "a": [3] } }], + "aliasesByName": { + "$start.inheritanceType": "replace", + "start": { "actionName": "build-watch", "defaultParameters": ["--serve"] } + }, + "phasesByName": { + "build": { + "cleanFiles": [{ "includeGlobs": ["lib-esm"] }], + "tasksByName": { + "typescript": { + "taskPlugin": { + "pluginPackage": "fake-plugin", + "options": { "$list.inheritanceType": "replace", "list": [2], "nested": { "b": 2 } } + } + }, + "lint": null, + }, + }, + "$test.inheritanceType": "replace", + "test": { "phaseDependencies": ["build"] } + } + }`, + 'config/base/heft.json': { + extends: 'fake-plugin/heft-base.json', + heftPlugins: [{ pluginPackage: '@rushstack/heft', pluginName: 'x' }], + aliasesByName: { start: { actionName: 'start-old' }, other: { actionName: 'build' } }, + phasesByName: { + build: { + phaseDescription: 'Build', + cleanFiles: [{ includeGlobs: ['lib'] }], + tasksByName: { + typescript: { + taskPlugin: { + pluginPackage: 'fake-plugin', + options: { list: [1], nested: { a: 1 }, keep: true } + } + }, + lint: { taskDependencies: ['typescript'], taskPlugin: { pluginPackage: 'other-plugin' } } + } + }, + test: { tasksByName: { jest: { taskPlugin: { pluginPackage: 'fake-plugin' } } } } + } + }, + 'node_modules/fake-plugin/heft-base.json': { + phasesByName: { lint: { tasksByName: { eslint: { taskPlugin: { pluginPackage: 'fake-plugin' } } } } } + } + }); + expect(lean).toBe(true); + }); + + it('matches the original loader when heft.json comes from a rig', async () => { + const { lean, original } = await expectFixtureEquivalentAsync({ + 'config/rig.json': { rigPackageName: 'test-rig', rigProfile: 'library' }, + 'node_modules/test-rig/package.json': { name: 'test-rig', version: '1.0.0' }, + 'node_modules/test-rig/node_modules/rig-plugin/package.json': { name: 'rig-plugin', version: '1.0.0' }, + 'node_modules/test-rig/profiles/library/config/heft.json': { + extends: './heft-shared.json', + phasesByName: { build: { tasksByName: { rigged: { taskPlugin: { pluginPackage: 'rig-plugin' } } } } } + }, + 'node_modules/test-rig/profiles/library/config/heft-shared.json': { + heftPlugins: [{ pluginPackage: 'test-rig' }] + } + }); + expect(lean).toBe(true); + expect(original.debugOutput).toContain('Attempting to load via rig'); + }); + + it('matches the original loader for a symlinked (pnpm-style) rig package', async () => { + const rigFiles: Record = { + 'package.json': { name: 'linked-rig', version: '1.0.0' }, + 'node_modules/linked-rig-plugin/package.json': { name: 'linked-rig-plugin', version: '1.0.0' }, + 'profiles/default/config/heft.json': { + heftPlugins: [{ pluginPackage: 'linked-rig-plugin' }, { pluginPackage: 'linked-rig' }], + phasesByName: { build: { tasksByName: { t: { taskPlugin: { pluginPackage: 'linked-rig-plugin' } } } } } + } + }; + fixtureCounter++; + const projectFolders: string[] = []; + for (const suffix of ['', '-copy']) { + const projectFolder: string = createFixture({ 'config/rig.json': { rigPackageName: 'linked-rig' } }, suffix); + // Like pnpm, the package lives elsewhere and node_modules contains a symlink to it + const rigStoreFolder: string = path.join(projectFolder, '.store/linked-rig'); + writeFiles(rigStoreFolder, rigFiles); + fs.symlinkSync(rigStoreFolder, path.join(projectFolder, 'node_modules/linked-rig'), 'junction'); + projectFolders.push(projectFolder); + } + + const { lean, original } = await expectEquivalentAsync(projectFolders[0], projectFolders[1]); + expect(lean).toBe(true); + // The rig profile folder is reported via the symlink, while plugin packages resolve to real paths + expect(original.debugOutput).toContain(path.join(projectFolders[0], 'node_modules/linked-rig/profiles/default')); + }); + + it('declines for JSON5 syntax, and the combined loader matches the original', async () => { + for (const files of [ + { 'config/heft.json': "{ 'phasesByName': { build: { phaseDescription: 'x', }, }, }" }, + { 'config/heft.json': '\ufeff{ "phasesByName": {} }' } + ]) { + const { lean, original } = await expectFixtureEquivalentAsync(files); + expect({ files, lean, failed: original.error !== undefined }).toEqual({ files, lean: false, failed: false }); + } + }); + + it('declines for every error condition, and the combined loader reports the original error', async () => { + const errorFixtures: Record[] = [ + // Missing heft.json, without and with a rig + {}, + { 'config/rig.json': { rigPackageName: 'missing-rig' } }, + { + 'config/rig.json': { rigPackageName: 'empty-rig' }, + 'node_modules/empty-rig/package.json': { name: 'empty-rig', version: '1.0.0' }, + 'node_modules/empty-rig/profiles/default/.keep': '' + }, + // A missing rig profile (the original reports "The rig profile ... is not defined by the rig package") + { + 'config/rig.json': { rigPackageName: 'profile-rig', rigProfile: 'nope' }, + 'node_modules/profile-rig/package.json': { name: 'profile-rig', version: '1.0.0' }, + 'node_modules/profile-rig/profiles/default/config/heft.json': {} + }, + { + 'config/rig.json': '{ "rigPackageName": "profile-rig", "rigProfile": 5 }', + 'node_modules/profile-rig/package.json': { name: 'profile-rig', version: '1.0.0' }, + 'node_modules/profile-rig/profiles/default/config/heft.json': {} + }, + // Syntax errors (jju rejects a raw U+2028 in a string, but JSON.parse() would accept it) + { 'config/heft.json': '{ "phasesByName": { ' }, + { 'config/heft.json': '{ "phasesByName": { "build": { "phaseDescription": "a\u2028b" } } }' }, + // Schema violations, including the legacy schema + { 'config/heft.json': { phasesByName: { Build: {} } } }, + { 'config/heft.json': { unknownProperty: true } }, + { 'config/heft.json': { eventActions: [{ actionKind: 'copyFiles', heftEvent: 'pre-compile' }] } }, + { 'config/heft.json': { phasesByName: { build: { tasksByName: { t: {} } } } } }, + // Plugin resolution failures + { + 'config/heft.json': { + phasesByName: { build: { tasksByName: { t: { taskPlugin: { pluginPackage: 'missing' } } } } } + } + }, + { + 'config/heft.json': { + phasesByName: { build: { tasksByName: { t: { taskPlugin: { pluginPackage: 'Bad Name' } } } } } + } + }, + { 'config/heft.json': { heftPlugins: [{ pluginPackage: 5 }] } }, + // extends problems + { 'config/heft.json': { extends: './missing.json' } }, + { 'config/heft.json': { extends: 'missing-package/heft.json' } }, + { 'config/heft.json': { extends: './heft.json' } }, + { 'config/heft.json': { extends: './a.json' }, 'config/a.json': { extends: './heft.json' } }, + // Inheritance annotation problems + { 'config/heft.json': { '$phasesByName.inheritanceType': 'replace' } }, + { 'config/heft.json': { '$phasesByName.inheritanceType': 'bogus', phasesByName: {} } }, + { 'config/heft.json': { '$phasesByName.inheritanceType': 1, phasesByName: {} } }, + { + 'config/heft.json': { extends: './base.json', phasesByName: { build: { cleanFiles: { x: 1 } } } }, + 'config/base.json': { phasesByName: { build: { cleanFiles: [] } } } + } + ]; + for (const files of errorFixtures) { + const { lean, original } = await expectFixtureEquivalentAsync(files); + expect({ files, lean, failed: original.error !== undefined }).toEqual({ files, lean: false, failed: true }); + } + }); +}); diff --git a/apps/heft/src/configuration/test/LeanJson.test.ts b/apps/heft/src/configuration/test/LeanJson.test.ts new file mode 100644 index 00000000000..42a50931b97 --- /dev/null +++ b/apps/heft/src/configuration/test/LeanJson.test.ts @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { JsonFile } from '@rushstack/node-core-library'; + +import { tryParseJsonLean } from '../lean/LeanJson'; + +const REPO_ROOT: string = path.resolve(__dirname, '../../../../..'); + +// Compares two values exactly, including key order, property descriptors, prototypes and -0 +function isExactlyEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) { + return true; + } + + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) { + return false; + } + + if (Array.isArray(a) !== Array.isArray(b) || Object.getPrototypeOf(a) !== Object.getPrototypeOf(b)) { + return false; + } + + const aKeys: (string | symbol)[] = Reflect.ownKeys(a); + const bKeys: (string | symbol)[] = Reflect.ownKeys(b); + if (aKeys.length !== bKeys.length) { + return false; + } + + for (let i: number = 0; i < aKeys.length; i++) { + if (aKeys[i] !== bKeys[i]) { + return false; + } + + const aDescriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(a, aKeys[i])!; + const bDescriptor: PropertyDescriptor = Object.getOwnPropertyDescriptor(b, bKeys[i])!; + if ( + aDescriptor.enumerable !== bDescriptor.enumerable || + aDescriptor.writable !== bDescriptor.writable || + aDescriptor.configurable !== bDescriptor.configurable || + !isExactlyEqual(aDescriptor.value, bDescriptor.value) + ) { + return false; + } + } + + return true; +} + +type ReferenceResult = { value: unknown } | { error: string }; + +function parseWithReference(text: string): ReferenceResult { + try { + return { value: JsonFile.parseString(text) }; + } catch (e) { + return { error: (e as Error).message }; + } +} + +/** + * Asserts the soundness contract: whenever the lean parser returns a result, the reference parser must succeed + * with an identical result. + */ +function expectSound(text: string): { value: unknown } | undefined { + const leanResult: { value: unknown } | undefined = tryParseJsonLean(text); + if (leanResult) { + const referenceResult: ReferenceResult = parseWithReference(text); + if (!('value' in referenceResult) || !isExactlyEqual(leanResult.value, referenceResult.value)) { + throw new Error(`Lean JSON parser is unsound for ${JSON.stringify(text)}`); + } + } + + return leanResult; +} + +// A small deterministic PRNG, so that the fuzzing is reproducible +function createRandom(seed: number): () => number { + let state: number = seed; + return () => { + state = (state * 1103515245 + 12345) % 2147483648; + return state / 2147483648; + }; +} + +function findConfigFiles(): string[] { + const results: string[] = []; + for (const topFolder of ['apps', 'build-tests', 'heft-plugins', 'rigs']) { + const topFolderPath: string = path.join(REPO_ROOT, topFolder); + if (!fs.existsSync(topFolderPath)) { + continue; + } + + for (const projectName of fs.readdirSync(topFolderPath)) { + for (const relativePath of [ + 'package.json', + 'heft-plugin.json', + 'config/heft.json', + 'config/rig.json', + 'config/typescript.json' + ]) { + const filePath: string = path.join(topFolderPath, projectName, relativePath); + if (fs.existsSync(filePath)) { + results.push(filePath); + } + } + } + } + + return results; +} + +describe('LeanJson', () => { + it('matches JsonFile.parseString() for plain JSON and JSON with comments and trailing commas', () => { + const cases: [string, unknown][] = [ + ['{}', {}], + ['[]', []], + ['{"a": 1, "b": [true, false, null], "c": {"d": "e"}}', { a: 1, b: [true, false, null], c: { d: 'e' } }], + ['// comment\n{"a": 1}', { a: 1 }], + ['{"a": /* inline */ 1}', { a: 1 }], + ['{"a": 1, // trailing\n}', { a: 1 }], + ['{"a": [1, 2, /* x */ ], }', { a: [1, 2] }], + [ + '{"url": "https://example.com/a//b", "glob": "src/**/*.ts"}', + { url: 'https://example.com/a//b', glob: 'src/**/*.ts' } + ], + ['{"a": "/* not a comment */", "b": "// nor this"}', { a: '/* not a comment */', b: '// nor this' }], + ['{"escaped": "quote \\" // still a string"}', { escaped: 'quote " // still a string' }], + ['{"a": 1, "a": 2}', { a: 2 }], + ['{"n": -0}', { n: -0 }], + ['{"big": 1e400}', { big: Infinity }], + ['"just a string"', 'just a string'], + ['\r\n{\r\n "a": 1\r\n}\r\n', { a: 1 }] + ]; + for (const [text, expected] of cases) { + const result: { value: unknown } | undefined = expectSound(text); + expect(result).toBeDefined(); + expect(result!.value).toEqual(expected); + } + + // __proto__ is an own data property with both parsers + const protoResult: { value: unknown } | undefined = expectSound('{"__proto__": {"polluted": true}}'); + expect(protoResult).toBeDefined(); + expect(Object.getPrototypeOf(protoResult!.value)).toBe(Object.prototype); + expect(Object.keys(protoResult!.value as object)).toEqual(['__proto__']); + }); + + it('bails out for JSON5 features and syntax errors', () => { + const bailCases: string[] = [ + "{'a': 1}", + '{a: 1}', + '{"a": 0x10}', + '{"a": .5}', + '{"a": +1}', + '{"a": Infinity}', + '{"a": NaN}', + '\ufeff{"a": 1}', + '{"a": "line\u2028separator"}', + '{"a": 1} // comment\u2028 , "b": 2}', + '{"a": "continued \\\n line"}', + '[,]', + '{,}', + '[1,,]', + '{"a": 1,,}', + '{"a": 1 /* unterminated', + '{"a": "unterminated', + '{"a": 1}}', + '', + '// only a comment', + '{"a": 1 2}', + '1/*x*/2', + '{"a"\u00a0: 1}' + ]; + for (const text of bailCases) { + expect(expectSound(text)).toBeUndefined(); + } + }); + + it('matches JsonFile.parseString() for every config file in the repo', () => { + const files: string[] = findConfigFiles(); + let parsedCount: number = 0; + for (const filePath of files) { + if (expectSound(fs.readFileSync(filePath, 'utf8'))) { + parsedCount++; + } + } + + // The fast path should handle essentially all real config files + expect(parsedCount).toBeGreaterThan(files.length * 0.95); + }); + + it('is sound for randomly mutated config files', () => { + const files: string[] = findConfigFiles().slice(0, 200); + const texts: string[] = files.map((filePath: string) => fs.readFileSync(filePath, 'utf8').slice(0, 3000)); + const insertions: string[] = [ + '//x\n', + '/* c */', + '/*', + '*/', + '//', + ',', + ',,', + ' ,]', + ',}', + '"', + "'", + '\\', + '\\\n', + '\u2028', + '\ufeff', + '\t', + '\v', + 'NaN', + '0x1F', + '.5', + '-0', + '01', + '"__proto__":1,', + '"a":1,"a":2,', + '[,]', + '"//"', + '"/*"', + "'a'", + 'a:1,', + '\r', + '/**/', + '// */\n' + ]; + const random: () => number = createRandom(42); + for (let i: number = 0; i < 3000 && texts.length > 0; i++) { + let text: string = texts[Math.floor(random() * texts.length)]; + const insertionCount: number = 1 + Math.floor(random() * 3); + for (let j: number = 0; j < insertionCount; j++) { + const position: number = Math.floor(random() * (text.length + 1)); + text = + text.slice(0, position) + insertions[Math.floor(random() * insertions.length)] + text.slice(position); + } + + expectSound(text); + } + }); +}); diff --git a/apps/heft/src/configuration/test/LeanJsonSchema.test.ts b/apps/heft/src/configuration/test/LeanJsonSchema.test.ts new file mode 100644 index 00000000000..1bcb213627d --- /dev/null +++ b/apps/heft/src/configuration/test/LeanJsonSchema.test.ts @@ -0,0 +1,409 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { JsonFile, JsonSchema } from '@rushstack/node-core-library'; + +import { isDefinitelyValid, isSchemaSupported } from '../lean/LeanJsonSchema'; + +const SCHEMAS_FOLDER: string = path.resolve(__dirname, '../../schemas'); +const REPO_ROOT: string = path.resolve(__dirname, '../../../../..'); + +interface IReferenceResult { + compiledWithoutWarnings: boolean; + valid: boolean[]; +} + +/** + * Validates with the original implementation (ajv via node-core-library), capturing anything that ajv logs. + */ +function validateWithReference(schemaObject: object, dataList: unknown[]): IReferenceResult { + const logged: unknown[][] = []; + const spies: jest.SpyInstance[] = (['log', 'warn', 'error'] as const).map((method) => + jest.spyOn(console, method).mockImplementation((...args: unknown[]) => { + logged.push(args); + }) + ); + let jsonSchema: JsonSchema | undefined; + try { + jsonSchema = JsonSchema.fromLoadedObject(JSON.parse(JSON.stringify(schemaObject))); + jsonSchema.ensureCompiled(); + } catch { + jsonSchema = undefined; + } finally { + for (const spy of spies) { + spy.mockRestore(); + } + } + + return { + compiledWithoutWarnings: !!jsonSchema && logged.length === 0, + valid: dataList.map((data: unknown) => { + if (!jsonSchema) { + return false; + } + + try { + jsonSchema.validateObject(data as object, ''); + return true; + } catch { + return false; + } + }) + }; +} + +/** + * Asserts the soundness contract of the lean validator against the reference, and returns the lean verdicts. + */ +function expectSound(schemaObject: object, dataList: unknown[]): boolean[] { + const reference: IReferenceResult = validateWithReference(schemaObject, dataList); + if (isSchemaSupported(schemaObject) && !reference.compiledWithoutWarnings) { + throw new Error(`Schema is supported, but ajv throws or logs: ${JSON.stringify(schemaObject)}`); + } + + return dataList.map((data: unknown, i: number) => { + const leanValid: boolean = isDefinitelyValid(schemaObject, data); + if (leanValid && !reference.valid[i]) { + throw new Error( + `Unsound verdict for schema ${JSON.stringify(schemaObject)} and data ${JSON.stringify(data)}` + ); + } + + return leanValid; + }); +} + +function createRandom(seed: number): () => number { + let state: number = seed; + return () => { + state = (state * 1103515245 + 12345) % 2147483648; + return state / 2147483648; + }; +} + +const DRAFT_04: string = 'http://json-schema.org/draft-04/schema#'; +const DRAFT_07: string = 'http://json-schema.org/draft-07/schema#'; + +describe('LeanJsonSchema', () => { + it('supports all schemas that ship with Heft', () => { + for (const fileName of fs.readdirSync(SCHEMAS_FOLDER)) { + if (fileName.endsWith('.schema.json')) { + const schemaObject: object = JsonFile.load(path.join(SCHEMAS_FOLDER, fileName)); + expect([fileName, isSchemaSupported(schemaObject)]).toEqual([fileName, true]); + expect([fileName, validateWithReference(schemaObject, []).compiledWithoutWarnings]).toEqual([ + fileName, + true + ]); + } + } + }); + + it('never accepts a schema that ajv rejects or warns about', () => { + const cases: object[] = [ + // strictSchema: unknown keywords + { $schema: DRAFT_04, type: 'object', unknownKeyword: true }, + { $schema: DRAFT_04, type: 'object', examples: [] }, + { type: 'object', id: 'x' }, + // strictTypes: missing types / types not allowed by the context + { $schema: DRAFT_04, properties: { a: { type: 'string' } } }, + { $schema: DRAFT_04, type: 'string', minItems: 1 }, + { $schema: DRAFT_04, type: 'object', anyOf: [{ type: 'string' }] }, + { $schema: DRAFT_04, type: 'object', properties: { a: { pattern: '^a' } } }, + // strictTuples + { $schema: DRAFT_07, type: 'array', items: [{ type: 'string' }] }, + // meta-schema violations + { $schema: DRAFT_04, type: 'object', required: [] }, + { $schema: DRAFT_04, enum: ['a', 'a'] }, + { $schema: DRAFT_04, type: 'string', minLength: -1 }, + { $schema: DRAFT_04, type: 'nope' }, + { $schema: DRAFT_04, type: 'number', maximum: 1, exclusiveMaximum: true }, + { $schema: DRAFT_04, type: 'object', description: 1 }, + // regular expressions + { $schema: DRAFT_04, type: 'string', pattern: '(' }, + { $schema: DRAFT_04, type: 'string', pattern: 'a\\Z' }, + // properties that match patternProperties + { + $schema: DRAFT_04, + type: 'object', + properties: { abc: { type: 'string' } }, + patternProperties: { '^a': { type: 'string' } } + }, + // references + { $schema: DRAFT_04, $ref: '#/definitions/missing' }, + { $schema: DRAFT_04, $ref: 'http://example.com/schema.json' }, + // formats and unsupported meta-schemas + { $schema: DRAFT_04, type: 'string', format: 'uri' }, + { $schema: 'https://json-schema.org/draft/2020-12/schema', type: 'string' }, + // prototype property names + { $schema: DRAFT_04, type: 'object', required: ['constructor'] } + ]; + for (const schemaObject of cases) { + expect([schemaObject, isSchemaSupported(schemaObject)]).toEqual([schemaObject, false]); + expectSound(schemaObject, [{}, 'a', 1, []]); + } + }); + + it('matches ajv for supported keywords', () => { + const schemaObject: object = { + $schema: DRAFT_04, + type: 'object', + required: ['name'], + additionalProperties: false, + definitions: { + tag: { type: 'string', pattern: '^[a-z]+$', minLength: 2, maxLength: 5 } + }, + properties: { + name: { type: 'string', enum: ['a', 'b'] }, + count: { type: 'integer', minimum: 1, maximum: 3 }, + tags: { type: 'array', items: { $ref: '#/definitions/tag' }, uniqueItems: true, minItems: 1 }, + mode: { oneOf: [{ type: 'string' }, { type: 'number' }] }, + either: { anyOf: [{ type: 'string' }, { type: 'boolean' }], not: { enum: [false] } }, + both: { type: 'object', allOf: [{ required: ['x'] }, { required: ['y'] }] }, + fixed: { const: { a: [1, 'b'] } } + }, + patternProperties: { + '^x-': { type: 'number' } + } + }; + const dataList: unknown[] = [ + { name: 'a' }, + { name: 'c' }, + {}, + { name: 'a', extra: 1 }, + { name: 'a', 'x-1': 1 }, + { name: 'a', 'x-1': 'one' }, + { name: 'a', count: 2 }, + { name: 'a', count: 2.5 }, + { name: 'a', count: 4 }, + { name: 'a', tags: ['abc', 'de'] }, + { name: 'a', tags: ['abc', 'abc'] }, + { name: 'a', tags: [] }, + { name: 'a', tags: ['toolong'] }, + { name: 'a', tags: ['ab1'] }, + { name: 'a', tags: ['\ud83d\ude00\ud83d\ude00'] }, + { name: 'a', mode: 'x' }, + { name: 'a', mode: null }, + { name: 'a', either: true }, + { name: 'a', either: false }, + { name: 'a', either: 1 }, + { name: 'a', both: { x: 1, y: 2 } }, + { name: 'a', both: { x: 1 } }, + { name: 'a', fixed: { a: [1, 'b'] } }, + { name: 'a', fixed: { a: [1, 'c'] } } + ]; + const leanVerdicts: boolean[] = expectSound(schemaObject, dataList); + const referenceVerdicts: boolean[] = validateWithReference(schemaObject, dataList).valid; + // For supported schemas and simple JSON data, the verdicts are identical + expect(leanVerdicts).toEqual(referenceVerdicts); + expect(leanVerdicts.filter((x) => x).length).toBeGreaterThan(5); + }); + + it('defers for data that is not simple JSON', () => { + const schemaObject: object = { $schema: DRAFT_04, type: 'object' }; + expect(isDefinitelyValid(schemaObject, {})).toBe(true); + expect(isDefinitelyValid(schemaObject, Object.create(null))).toBe(false); + expect(isDefinitelyValid(schemaObject, { a: NaN })).toBe(false); + expect(isDefinitelyValid(schemaObject, { a: undefined })).toBe(false); + expect(isDefinitelyValid(schemaObject, { a: () => 1 })).toBe(false); + expect(isDefinitelyValid(schemaObject, JSON.parse('{"__proto__": {}}'))).toBe(false); + expect(isDefinitelyValid(schemaObject, { toString: 'x' })).toBe(false); + }); + + it('is sound for the config files in the repo and mutations of them', () => { + const random: () => number = createRandom(1234); + const pick: (values: T[]) => T = (values: T[]) => values[Math.floor(random() * values.length)]; + const primitives: unknown[] = [null, true, false, 0, 1, -1, 1.5, '', 'x', 'build', '--foo', '-x', '.js', [], {}]; + + function mutate(value: unknown, depth: number): unknown { + if (depth > 5) { + return value; + } + + if (Array.isArray(value)) { + const result: unknown[] = value.slice(); + if (result.length && random() < 0.3) { + result.splice(Math.floor(random() * result.length), 1); + } + + if (result.length && random() < 0.5) { + const i: number = Math.floor(random() * result.length); + result[i] = mutate(result[i], depth + 1); + } + + return random() < 0.1 ? pick(primitives) : result; + } + + if (value && typeof value === 'object') { + const result: Record = { ...(value as Record) }; + const keys: string[] = Object.keys(result); + if (keys.length && random() < 0.25) { + delete result[pick(keys)]; + } + + if (random() < 0.15) { + result[pick(['extra', 'pluginName', 'options', 'required', 'longName', 'build'])] = pick(primitives); + } + + if (keys.length && random() < 0.6) { + const key: string = pick(keys); + result[key] = mutate(result[key], depth + 1); + } + + return random() < 0.1 ? pick(primitives) : result; + } + + return random() < 0.5 ? pick(primitives) : value; + } + + const samplesBySchema: Map = new Map([ + ['heft.schema.json', []], + ['heft-plugin.schema.json', []] + ]); + for (const topFolder of ['apps', 'build-tests', 'heft-plugins', 'rigs']) { + const topFolderPath: string = path.join(REPO_ROOT, topFolder); + if (!fs.existsSync(topFolderPath)) { + continue; + } + + for (const projectName of fs.readdirSync(topFolderPath)) { + const heftJsonPath: string = path.join(topFolderPath, projectName, 'config/heft.json'); + if (fs.existsSync(heftJsonPath)) { + samplesBySchema.get('heft.schema.json')!.push(JsonFile.load(heftJsonPath)); + } + + const heftPluginJsonPath: string = path.join(topFolderPath, projectName, 'heft-plugin.json'); + if (fs.existsSync(heftPluginJsonPath)) { + samplesBySchema.get('heft-plugin.schema.json')!.push(JsonFile.load(heftPluginJsonPath)); + } + } + } + + for (const [schemaFileName, samples] of samplesBySchema) { + const schemaObject: object = JsonFile.load(path.join(SCHEMAS_FOLDER, schemaFileName)); + const dataList: unknown[] = [...samples]; + for (const sample of samples) { + for (let i: number = 0; i < 20; i++) { + dataList.push(mutate(JSON.parse(JSON.stringify(sample)), 0)); + } + } + + const leanVerdicts: boolean[] = expectSound(schemaObject, dataList); + // For the real config files (some of which are only valid after merging with the file that they extend), + // the verdicts are identical to ajv's, i.e. all valid files take the fast path + const referenceVerdicts: boolean[] = validateWithReference(schemaObject, samples).valid; + expect(leanVerdicts.slice(0, samples.length)).toEqual(referenceVerdicts); + expect(referenceVerdicts.filter((x) => x).length).toBeGreaterThan(samples.length / 2); + } + }); + + it('is sound for randomly generated schemas', () => { + const random: () => number = createRandom(99); + const pick: (values: T[]) => T = (values: T[]) => values[Math.floor(random() * values.length)]; + const types: string[] = ['string', 'number', 'integer', 'boolean', 'object', 'array', 'null']; + + function generateSchema(depth: number, withType: boolean): Record { + const schema: Record = {}; + const type: string = pick(types); + if (withType || random() < 0.7) { + schema.type = random() < 0.15 ? [type, pick(types)] : type; + } + + const r: number = random(); + if (type === 'object' && r < 0.8) { + schema.properties = { a: generateSchema(depth + 1, true), b: generateSchema(depth + 1, random() < 0.8) }; + if (random() < 0.5) { + schema.required = random() < 0.5 ? ['a'] : ['a', 'b']; + } + + if (random() < 0.5) { + schema.additionalProperties = random() < 0.6 ? false : generateSchema(depth + 1, true); + } + + if (random() < 0.3) { + schema.patternProperties = { [pick(['^x', '^[0-9]+$', 'b'])]: generateSchema(depth + 1, true) }; + } + + if (random() < 0.3) { + schema.anyOf = [{ required: ['a'] }, { required: ['c'] }]; + } + } else if (type === 'array' && r < 0.8) { + schema.items = generateSchema(depth + 1, true); + if (random() < 0.4) { + schema.minItems = Math.floor(random() * 3); + } + + if (random() < 0.4) { + schema.uniqueItems = true; + } + } else if (type === 'string' && r < 0.8) { + if (random() < 0.5) { + schema.pattern = pick(['^a', 'b$', '^[a-z]+$', '\\d', '^.{2,}$']); + } + + if (random() < 0.4) { + schema.enum = ['a', 'ab', pick(['x', 'b'])]; + } + + if (random() < 0.3) { + schema.maxLength = Math.floor(random() * 4); + } + } else if ((type === 'number' || type === 'integer') && r < 0.8) { + schema.minimum = pick([0, 1, -1]); + if (random() < 0.5) { + schema.maximum = pick([2, 10]); + } + } else if (depth < 3) { + schema[pick(['anyOf', 'oneOf', 'allOf'])] = [generateSchema(depth + 1, true), generateSchema(depth + 1, true)]; + } + + if (depth < 3 && random() < 0.1) { + schema.not = generateSchema(depth + 1, true); + } + + return schema; + } + + function generateData(depth: number): unknown { + const r: number = random(); + if (depth > 3 || r < 0.45) { + return pick([null, true, false, 0, 1, 2, -1, 1.5, 11, 'a', 'b', 'ab', 'abc', '', '1', 'x1', 'é']); + } + + if (r < 0.75) { + const result: Record = {}; + for (const key of ['a', 'b', 'c', 'x1', '12']) { + if (random() < 0.45) { + result[key] = generateData(depth + 1); + } + } + + return result; + } + + const result: unknown[] = []; + const length: number = Math.floor(random() * 4); + for (let i: number = 0; i < length; i++) { + result.push(i > 0 && random() < 0.3 ? result[0] : generateData(depth + 1)); + } + + return result; + } + + let fastPathCount: number = 0; + for (let i: number = 0; i < 150; i++) { + const schemaObject: Record = generateSchema(0, true); + schemaObject.$schema = random() < 0.5 ? DRAFT_04 : DRAFT_07; + const dataList: unknown[] = []; + for (let j: number = 0; j < 20; j++) { + dataList.push(generateData(0)); + } + + fastPathCount += expectSound(schemaObject, dataList).filter((x) => x).length; + } + + expect(fastPathCount).toBeGreaterThan(100); + }); +}); diff --git a/apps/heft/src/configuration/test/LeanPluginConfigurationFile.test.ts b/apps/heft/src/configuration/test/LeanPluginConfigurationFile.test.ts new file mode 100644 index 00000000000..0ebffb244d3 --- /dev/null +++ b/apps/heft/src/configuration/test/LeanPluginConfigurationFile.test.ts @@ -0,0 +1,305 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { JsonFile } from '@rushstack/node-core-library'; +import { + InheritanceType, + PathResolutionMethod, + ProjectConfigurationFile, + type IProjectConfigurationFileSpecification +} from '@rushstack/heft-config-file'; +import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; + +import { HeftConfiguration } from '../HeftConfiguration'; +import { describeValue, replaceAll, writeFiles } from './ConfigTestUtilities'; + +const REPO_ROOT: string = path.resolve(__dirname, '../../../../..'); + +interface ILoadOutcome { + value?: unknown; + error?: string; + debugOutput: string; + frozen: boolean; +} + +type SpecificationFactory = () => IProjectConfigurationFileSpecification; + +function loadSchema(relativePath: string): object | undefined { + const schemaPath: string = path.join(REPO_ROOT, relativePath); + return fs.existsSync(schemaPath) ? JsonFile.load(schemaPath) : undefined; +} + +// Specifications like the ones that Heft plugins pass to HeftConfiguration.tryLoadProjectConfigurationFileAsync() +const typeScriptSchema: object | undefined = loadSchema( + 'heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json' +); +const apiExtractorTaskSchema: object | undefined = loadSchema( + 'heft-plugins/heft-api-extractor-plugin/src/schemas/api-extractor-task.schema.json' +); +const specificationFactories: Record = { + typescript: () => ({ + projectRelativeFilePath: 'config/typescript.json', + jsonSchemaObject: typeScriptSchema!, + propertyInheritance: { + staticAssetsToCopy: { inheritanceType: InheritanceType.merge } + }, + jsonPathMetadata: { + '$.additionalModuleKindsToEmit.*.outFolderName': { + pathResolutionMethod: PathResolutionMethod.resolvePathRelativeToProjectRoot + } + } + }), + apiExtractorTask: () => ({ + projectRelativeFilePath: 'config/api-extractor-task.json', + jsonSchemaObject: apiExtractorTaskSchema! + }), + generic: () => ({ + projectRelativeFilePath: 'config/generic.json', + jsonSchemaObject: { + $schema: 'http://json-schema.org/draft-04/schema#', + type: 'object', + additionalProperties: false, + properties: { + $schema: { type: 'string' }, + extends: { type: 'string' }, + list: { type: 'array', items: { type: 'string' } }, + map: { type: 'object' }, + nested: { type: 'object', properties: { deep: { type: 'object' } } }, + relative: { type: 'string' }, + rooted: { type: 'string' }, + module: { type: 'string' } + } + }, + propertyInheritanceDefaults: { + array: { inheritanceType: InheritanceType.replace }, + object: { inheritanceType: InheritanceType.merge } + }, + propertyInheritance: { list: { inheritanceType: InheritanceType.append } }, + jsonPathMetadata: { + '$.relative': { pathResolutionMethod: PathResolutionMethod.resolvePathRelativeToConfigurationFile }, + '$.rooted': { pathResolutionMethod: PathResolutionMethod.resolvePathRelativeToProjectRoot }, + '$.module': { pathResolutionMethod: PathResolutionMethod.nodeResolve }, + '$.map.*': {} + } + }) +}; + +function createTerminal(): { terminal: Terminal; provider: StringBufferTerminalProvider } { + const provider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + return { terminal: new Terminal(provider), provider }; +} + +async function loadWithHeftConfigurationAsync( + projectFolder: string, + specification: IProjectConfigurationFileSpecification, + sync: boolean +): Promise { + const { terminal, provider } = createTerminal(); + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectFolder, + terminalProvider: provider, + numberOfCores: 1 + }); + await heftConfiguration._checkForRigAsync(); + try { + const value: unknown = sync + ? heftConfiguration.tryLoadProjectConfigurationFile(specification, terminal) + : await heftConfiguration.tryLoadProjectConfigurationFileAsync(specification, terminal); + return { value: describeValue(value), debugOutput: provider.getDebugOutput(), frozen: Object.isFrozen(specification) }; + } catch (e) { + return { error: (e as Error).message, debugOutput: provider.getDebugOutput(), frozen: Object.isFrozen(specification) }; + } +} + +async function loadOriginalAsync( + projectFolder: string, + specification: IProjectConfigurationFileSpecification +): Promise { + const { terminal, provider } = createTerminal(); + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectFolder, + terminalProvider: provider, + numberOfCores: 1 + }); + await heftConfiguration._checkForRigAsync(); + // What HeftConfiguration did before the lean path existed + Object.freeze(specification); + const loader: ProjectConfigurationFile = new ProjectConfigurationFile(specification); + try { + const value: unknown = await loader.tryLoadConfigurationFileForProjectAsync( + terminal, + heftConfiguration.buildFolderPath, + heftConfiguration.rigConfig + ); + return { value: describeValue(value), debugOutput: provider.getDebugOutput(), frozen: true }; + } catch (e) { + return { error: (e as Error).message, debugOutput: provider.getDebugOutput(), frozen: true }; + } +} + +jest.setTimeout(120000); + +describe('HeftConfiguration.tryLoadProjectConfigurationFile(Async) lean path', () => { + let tempFolder: string; + let fixtureCounter: number = 0; + + beforeAll(() => { + tempFolder = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'heft-lean-plugin-config-'))); + }); + + afterAll(() => { + fs.rmSync(tempFolder, { recursive: true, force: true }); + }); + + /** + * Loads the configuration file in (separate copies of) the fixture with the original loader and with + * HeftConfiguration (sync and async), and expects identical values, annotations, debug output, and errors. + */ + async function expectEquivalentAsync( + files: Record, + specificationName: string + ): Promise { + fixtureCounter++; + const folders: string[] = ['original', 'async', 'sync'].map((kind: string) => { + const projectFolder: string = path.join(tempFolder, `fixture-${fixtureCounter}-${kind}`); + writeFiles(projectFolder, { + 'package.json': { name: 'fixture', version: '1.0.0' }, + 'node_modules/some-package/data.json': '{}', + ...files + }); + return projectFolder; + }); + + const factory: SpecificationFactory = specificationFactories[specificationName]; + const original: ILoadOutcome = await loadOriginalAsync(folders[0], factory()); + const asyncOutcome: ILoadOutcome = replaceAll( + await loadWithHeftConfigurationAsync(folders[1], factory(), false), + folders[1], + folders[0] + ); + const syncOutcome: ILoadOutcome = replaceAll( + await loadWithHeftConfigurationAsync(folders[2], factory(), true), + folders[2], + folders[0] + ); + expect({ files, ...asyncOutcome }).toEqual({ files, ...original }); + expect({ files, ...syncOutcome }).toEqual({ files, ...original }); + return original; + } + + it('matches the original loader for typescript.json and api-extractor-task.json', async () => { + if (!typeScriptSchema || !apiExtractorTaskSchema) { + return; + } + + const typescriptCases: Record[] = [ + {}, + { 'config/typescript.json': '// comment\n{ "onlyResolveSymlinksInNodeModules": true, }' }, + { + 'config/typescript.json': { + extends: './base-typescript.json', + staticAssetsToCopy: { fileExtensions: ['.png'], includeGlobs: ['a/**'] }, + additionalModuleKindsToEmit: [{ moduleKind: 'esnext', outFolderName: 'lib-esm' }] + }, + 'config/base-typescript.json': { + staticAssetsToCopy: { fileExtensions: ['.css'], excludeGlobs: ['x'] }, + additionalModuleKindsToEmit: [{ moduleKind: 'commonjs', outFolderName: 'lib-commonjs' }] + } + }, + { + 'config/rig.json': { rigPackageName: 'plugin-rig' }, + 'node_modules/plugin-rig/package.json': { name: 'plugin-rig', version: '1.0.0' }, + 'node_modules/plugin-rig/profiles/default/config/typescript.json': { useTranspilerWorker: true } + }, + { + 'config/rig.json': { rigPackageName: 'plugin-rig' }, + 'node_modules/plugin-rig/package.json': { name: 'plugin-rig', version: '1.0.0' }, + 'node_modules/plugin-rig/profiles/default/.keep': '' + }, + // Errors + { 'config/typescript.json': { notAnOption: true } }, + { 'config/typescript.json': '{ "useTranspilerWorker": ' }, + { 'config/typescript.json': { extends: './missing.json' } }, + { 'config/rig.json': { rigPackageName: 'missing-rig' } }, + { + 'config/rig.json': { rigPackageName: 'plugin-rig', rigProfile: 'nope' }, + 'node_modules/plugin-rig/package.json': { name: 'plugin-rig', version: '1.0.0' }, + 'node_modules/plugin-rig/profiles/default/.keep': '' + } + ]; + for (const files of typescriptCases) { + await expectEquivalentAsync(files, 'typescript'); + } + + const apiExtractorTaskCases: Record[] = [ + {}, + { 'config/api-extractor-task.json': { runInWatchMode: true } }, + { 'config/api-extractor-task.json': { runInWatchMode: 'yes' } } + ]; + for (const files of apiExtractorTaskCases) { + await expectEquivalentAsync(files, 'apiExtractorTask'); + } + }); + + it('matches the original loader for inheritance options and path resolution methods', async () => { + const cases: Record[] = [ + { + 'config/generic.json': { + extends: '../base/generic.json', + list: ['b'], + map: { b: 2 }, + nested: { deep: { y: 1 } }, + relative: './file.txt', + rooted: 'src', + module: 'some-package/data.json' + }, + 'base/generic.json': { list: ['a'], map: { a: 1 }, nested: { deep: { x: 1 } }, rooted: 'lib' } + }, + { + 'config/generic.json': { + extends: 'some-package/data.json', + '$list.inheritanceType': 'replace', + list: ['c'] + } + }, + { 'config/generic.json': { module: './local.js' } }, + // Errors + { 'config/generic.json': { module: 'missing-package/x.json' } }, + { 'config/generic.json': { module: 'fs' } }, + { 'config/generic.json': { list: 'not-an-array' } } + ]; + for (const files of cases) { + await expectEquivalentAsync(files, 'generic'); + } + }); + + it('matches the original loader for the config files of the build-test projects', async () => { + if (!typeScriptSchema) { + return; + } + + const buildTestsFolder: string = path.join(REPO_ROOT, 'build-tests'); + let count: number = 0; + for (const projectName of fs.readdirSync(buildTestsFolder)) { + const projectFolder: string = path.join(buildTestsFolder, projectName); + if (!fs.existsSync(path.join(projectFolder, 'node_modules')) || !fs.existsSync(path.join(projectFolder, 'package.json'))) { + continue; + } + + count++; + const original: ILoadOutcome = await loadOriginalAsync(projectFolder, specificationFactories.typescript()); + const lean: ILoadOutcome = await loadWithHeftConfigurationAsync( + projectFolder, + specificationFactories.typescript(), + false + ); + expect({ projectFolder, ...lean }).toEqual({ projectFolder, ...original }); + } + + expect(count).toBeGreaterThan(10); + }); +}); diff --git a/apps/heft/src/configuration/test/LeanRigConfig.test.ts b/apps/heft/src/configuration/test/LeanRigConfig.test.ts new file mode 100644 index 00000000000..347f5824162 --- /dev/null +++ b/apps/heft/src/configuration/test/LeanRigConfig.test.ts @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { RigConfig } from '@rushstack/rig-package'; +import { StringBufferTerminalProvider } from '@rushstack/terminal'; + +import { HeftConfiguration, getRigConfigForConfigLoading } from '../HeftConfiguration'; +import { LeanRigConfig, tryLoadRigConfigDataLean, type ILeanRigConfigData } from '../lean/LeanRigConfig'; +import { writeFiles } from './ConfigTestUtilities'; + +const REPO_ROOT: string = path.resolve(__dirname, '../../../../..'); + +function getRigConfigData(rigConfig: RigConfig | LeanRigConfig): ILeanRigConfigData { + const { + projectFolderOriginalPath, + projectFolderPath, + rigFound, + filePath, + rigPackageName, + rigProfile, + relativeProfileFolderPath + } = rigConfig; + return { + projectFolderOriginalPath, + projectFolderPath, + rigFound, + filePath, + rigPackageName, + rigProfile, + relativeProfileFolderPath + }; +} + +function loadOriginal(projectFolder: string): ILeanRigConfigData | { error: string } { + try { + return getRigConfigData(RigConfig.loadForProjectFolder({ projectFolderPath: projectFolder, bypassCache: true })); + } catch (e) { + return { error: (e as Error).message }; + } +} + +describe('LeanRigConfig', () => { + let tempFolder: string; + let counter: number = 0; + + beforeAll(() => { + tempFolder = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'heft-lean-rig-'))); + }); + + afterAll(() => { + fs.rmSync(tempFolder, { recursive: true, force: true }); + }); + + function createProject(rigJson: string | object | undefined): string { + const projectFolder: string = path.join(tempFolder, `project-${counter++}`); + writeFiles(projectFolder, { + 'package.json': { name: 'project', version: '1.0.0' }, + ...(rigJson !== undefined ? { 'config/rig.json': rigJson } : {}) + }); + return projectFolder; + } + + it('reads the same data as RigConfig, or declines', () => { + const acceptedCases: (string | object | undefined)[] = [ + undefined, + { rigPackageName: 'example-rig' }, + { rigPackageName: '@scope/example-rig', rigProfile: 'library' }, + { $schema: 'x', rigPackageName: 'example-rig-test', rigProfile: 'a-b.c_d' }, + '// A comment\n{ "rigPackageName": "example-rig", }', + '{ "rigPackageName": "first-rig", "rigPackageName": "second-rig" }' + ]; + for (const rigJson of acceptedCases) { + const projectFolder: string = createProject(rigJson); + expect({ rigJson, data: tryLoadRigConfigDataLean(projectFolder) }).toEqual({ + rigJson, + data: loadOriginal(projectFolder) + }); + } + + // RigConfig either throws for these, or jju produces a different object than JSON.parse() + const declinedCases: (string | object)[] = [ + {}, + { rigPackageName: 'not-a-rig-package' }, + { rigPackageName: 'bad name-rig' }, + { rigPackageName: 'example-rig', rigProfile: 'Bad' }, + { rigPackageName: 'example-rig', rigProfile: 5 }, + { rigPackageName: 'example-rig', unknownField: true }, + { rigPackageName: 'example-rig', constructor: 1 }, + '{ "__proto__": {}, "rigPackageName": "example-rig" }', + "{ rigPackageName: 'example-rig' }", + '{ "rigPackageName": ', + '[]' + ]; + for (const rigJson of declinedCases) { + const projectFolder: string = createProject(rigJson); + expect({ rigJson, data: tryLoadRigConfigDataLean(projectFolder) }).toEqual({ rigJson, data: undefined }); + } + }); + + it('reads the same data as RigConfig for every project in the repo', () => { + let count: number = 0; + for (const topFolder of ['apps', 'build-tests', 'heft-plugins', 'libraries']) { + const topFolderPath: string = path.join(REPO_ROOT, topFolder); + for (const projectName of fs.readdirSync(topFolderPath)) { + const projectFolder: string = path.join(topFolderPath, projectName); + if (fs.existsSync(path.join(projectFolder, 'package.json'))) { + count++; + const data: ILeanRigConfigData | undefined = tryLoadRigConfigDataLean(projectFolder); + expect({ projectFolder, data }).toEqual({ projectFolder, data: loadOriginal(projectFolder) }); + } + } + } + + expect(count).toBeGreaterThan(50); + }); + + it('keeps HeftConfiguration.rigConfig a genuine RigConfig', async () => { + const projectFolder: string = createProject({ rigPackageName: 'example-rig', rigProfile: 'library' }); + writeFiles(projectFolder, { + 'node_modules/example-rig/package.json': { name: 'example-rig', version: '1.0.0' }, + 'node_modules/example-rig/profiles/library/config/x.json': '{}' + }); + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectFolder, + terminalProvider: new StringBufferTerminalProvider(), + numberOfCores: 1 + }); + expect(() => heftConfiguration.rigConfig).toThrow(/cannot be accessed until/); + await heftConfiguration._checkForRigAsync(); + + const leanRigConfig: unknown = getRigConfigForConfigLoading(heftConfiguration); + expect(leanRigConfig).toBeInstanceOf(LeanRigConfig); + + const rigConfig: RigConfig = heftConfiguration.rigConfig as RigConfig; + expect(rigConfig).toBeInstanceOf(RigConfig); + // The same object as other callers get from @rushstack/rig-package's cache + expect(rigConfig).toBe(RigConfig.loadForProjectFolder({ projectFolderPath: heftConfiguration.buildFolderPath })); + expect(heftConfiguration.rigConfig).toBe(rigConfig); + expect(getRigConfigData(leanRigConfig as LeanRigConfig)).toEqual(getRigConfigData(rigConfig)); + + // The methods of the lean object delegate to the genuine object + expect((leanRigConfig as LeanRigConfig).getResolvedProfileFolder()).toEqual(rigConfig.getResolvedProfileFolder()); + expect((leanRigConfig as LeanRigConfig).tryResolveConfigFilePath('config/x.json')).toEqual( + path.join(projectFolder, 'node_modules/example-rig/profiles/library/config/x.json') + ); + }); + + it('reports rig.json errors like the original implementation', async () => { + const projectFolder: string = createProject({ rigPackageName: 'not-a-rig-package' }); + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectFolder, + terminalProvider: new StringBufferTerminalProvider(), + numberOfCores: 1 + }); + const original: { error: string } = loadOriginal(projectFolder) as { error: string }; + await expect(heftConfiguration._checkForRigAsync()).rejects.toThrow(original.error); + }); +}); diff --git a/apps/heft/src/configuration/test/ProjectPackageJson.test.ts b/apps/heft/src/configuration/test/ProjectPackageJson.test.ts new file mode 100644 index 00000000000..798376acf9e --- /dev/null +++ b/apps/heft/src/configuration/test/ProjectPackageJson.test.ts @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { PackageJsonLookup, type IPackageJson } from '@rushstack/node-core-library'; +import { StringBufferTerminalProvider } from '@rushstack/terminal'; + +import { HeftConfiguration } from '../HeftConfiguration'; +import { replaceAll, writeFiles } from './ConfigTestUtilities'; + +type Outcome = { value: IPackageJson; frozen: boolean } | { error: string }; + +function getOutcome(getPackageJson: () => IPackageJson): Outcome { + try { + const value: IPackageJson = getPackageJson(); + return { value, frozen: Object.isFrozen(value) }; + } catch (e) { + return { error: (e as Error).message }; + } +} + +/** + * Heft 1.3.1: HeftConfiguration.initialize() looked up the project with PackageJsonLookup.instance (which parses and + * caches the package.json), and the projectPackageJson getter returned PackageJsonLookup.instance's cached object. + */ +function simulateOriginal(projectFolder: string, editAfterStartup: () => void): Outcome[] { + const lookup: PackageJsonLookup = new PackageJsonLookup({ loadExtraFields: true }); + const packageJsonPath: string = lookup.tryGetPackageJsonFilePathFor(projectFolder)!; + const buildFolderPath: string = path.dirname(packageJsonPath); + editAfterStartup(); + return [ + getOutcome(() => lookup.tryLoadPackageJsonFor(buildFolderPath)!), + getOutcome(() => lookup.tryLoadPackageJsonFor(buildFolderPath)!) + ]; +} + +function runHeft(projectFolder: string, editAfterStartup: () => void): Outcome[] { + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectFolder, + terminalProvider: new StringBufferTerminalProvider(), + numberOfCores: 1 + }); + editAfterStartup(); + return [ + getOutcome(() => heftConfiguration.projectPackageJson), + getOutcome(() => heftConfiguration.projectPackageJson) + ]; +} + +describe('HeftConfiguration.projectPackageJson', () => { + let tempFolder: string; + let counter: number = 0; + + beforeAll(() => { + tempFolder = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'heft-project-package-json-'))); + }); + + afterAll(() => { + fs.rmSync(tempFolder, { recursive: true, force: true }); + }); + + function expectSameAsOriginal(packageJson: object | string, edit?: object | string): Outcome[] { + const folders: string[] = ['original', 'heft'].map((kind: string) => { + const projectFolder: string = path.join(tempFolder, `project-${counter}-${kind}`); + writeFiles(projectFolder, { 'package.json': packageJson, 'src/.keep': '' }); + return projectFolder; + }); + counter++; + + const editFor: (folder: string) => () => void = (folder: string) => () => { + if (edit !== undefined) { + // e.g. an edit in watch mode, after Heft started + writeFiles(folder, { 'package.json': edit }); + } + }; + const original: Outcome[] = simulateOriginal(path.join(folders[0], 'src'), editFor(folders[0])); + const heft: Outcome[] = runHeft(path.join(folders[1], 'src'), editFor(folders[1])); + expect(replaceAll(heft, folders[1], folders[0])).toEqual(original); + return heft; + } + + it('returns the frozen contents read at startup, like Heft 1.3.1', () => { + const outcomes: Outcome[] = expectSameAsOriginal({ name: 'project', version: '1.0.0', extra: { a: 1 } }); + expect(outcomes[0]).toMatchObject({ frozen: true, value: { name: 'project', extra: { a: 1 } } }); + }); + + it('ignores edits made after startup (e.g. in watch mode), like Heft 1.3.1', () => { + const outcomes: Outcome[] = expectSameAsOriginal( + { name: 'project', version: '1.0.0' }, + { name: 'project', version: '2.0.0', added: true } + ); + expect(outcomes[0]).toMatchObject({ value: { version: '1.0.0' } }); + // An edit that makes the file invalid doesn't matter either + expectSameAsOriginal({ name: 'project', version: '1.0.0' }, '{ invalid'); + }); + + it('reports a missing "version" field with the original error, every time', () => { + const outcomes: Outcome[] = expectSameAsOriginal({ name: 'project' }); + expect(outcomes[0]).toMatchObject({ error: expect.stringContaining('The required field "version"') }); + }); + + it('returns the same object from every call', () => { + const projectFolder: string = path.join(tempFolder, `project-${counter++}-identity`); + writeFiles(projectFolder, { 'package.json': { name: 'project', version: '1.0.0' } }); + const heftConfiguration: HeftConfiguration = HeftConfiguration.initialize({ + cwd: projectFolder, + terminalProvider: new StringBufferTerminalProvider(), + numberOfCores: 1 + }); + expect(heftConfiguration.projectPackageJson).toBe(heftConfiguration.projectPackageJson); + }); +}); diff --git a/apps/heft/src/metrics/MetricsCollector.ts b/apps/heft/src/metrics/MetricsCollector.ts index d1e03b27352..0f30a86f32e 100644 --- a/apps/heft/src/metrics/MetricsCollector.ts +++ b/apps/heft/src/metrics/MetricsCollector.ts @@ -4,10 +4,16 @@ import * as os from 'node:os'; import { performance } from 'node:perf_hooks'; +// This is intentionally not an `import type`: it is only used as a type (so it is elided from the emitted +// JavaScript and tapable is loaded lazily), but the emitted declarations and the API report must keep the +// original `import { AsyncParallelHook } from 'tapable'` form. +// eslint-disable-next-line @typescript-eslint/consistent-type-imports import { AsyncParallelHook } from 'tapable'; import { InternalError } from '@rushstack/node-core-library'; +import { createAsyncParallelHook, defineLazyProperty } from '../pluginFramework/TapableHooks'; + /** * @public */ @@ -100,8 +106,14 @@ export interface IPerformanceData { * A simple performance metrics collector. A plugin is required to pipe data anywhere. */ export class MetricsCollector { - public readonly recordMetricsHook: AsyncParallelHook = - new AsyncParallelHook(['recordMetricsHookOptions']); + declare public readonly recordMetricsHook: AsyncParallelHook; + + // Creating the hook requires loading tapable, so the hook is created when the property is first accessed. + readonly #isRecordMetricsHookMaterialized: () => boolean = defineLazyProperty( + this, + 'recordMetricsHook', + () => createAsyncParallelHook(['recordMetricsHookOptions']) + ); #bootDurationMs: number | undefined; #startTimeMs: number | undefined; @@ -139,6 +151,16 @@ export class MetricsCollector { throw new InternalError('The command name must be specified.'); } + // If the hook was never accessed or has neither taps nor interceptors, nothing can observe the metrics, + // so skip gathering them. + if (!this.#isRecordMetricsHookMaterialized()) { + return; + } + const recordMetricsHook: AsyncParallelHook = this.recordMetricsHook; + if (typeof recordMetricsHook.isUsed === 'function' && !recordMetricsHook.isUsed()) { + return; + } + const filledPerformanceData: IPerformanceData = { taskTotalExecutionMs: performance.now() - startTimeMs, ...(performanceData || {}) @@ -164,7 +186,7 @@ export class MetricsCollector { commandParameters: parameters || {} }; - await this.recordMetricsHook.promise({ + await recordMetricsHook.promise({ metricName: 'inner_loop_heft', metricData }); diff --git a/apps/heft/src/operations/OperationExecutionManager.ts b/apps/heft/src/operations/OperationExecutionManager.ts new file mode 100644 index 00000000000..7573f829a97 --- /dev/null +++ b/apps/heft/src/operations/OperationExecutionManager.ts @@ -0,0 +1,402 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Async, MinimumHeap } from '@rushstack/node-core-library'; +import type { + IExecuteOperationContext, + IOperationExecutionOptions, + IOperationState, + Operation, + OperationGroupRecord +} from '@rushstack/operation-graph'; +import { OperationStatus } from '@rushstack/operation-graph/lib/OperationStatus'; + +// This module is self-contained on purpose (fewer modules to resolve and load before the first task runs). + +interface ISortableOperation> { + name: string | undefined; + criticalPathLength?: number | undefined; + weight: number; + consumers: Set; +} + +// The functions below are the same as the ones in @rushstack/operation-graph's calculateCriticalPath module. + +function calculateCriticalPathLengths>(operations: Iterable): T[] { + // Clone the set of operations as an array, so that we can sort it. + const queue: T[] = Array.from(operations); + + // Create a collection for detecting visited nodes + const cycleDetectorStack: Set = new Set(); + for (const operation of queue) { + calculateCriticalPathLength(operation, cycleDetectorStack); + } + + return queue; +} + +function calculateShortestPath>(startOperation: T, endOperation: T): T[] { + // Map of each operation to the most optimal parent + const parents: Map = new Map([[endOperation, undefined]]); + let finalParent: T | undefined; + + // Run a breadth-first search to find the shortest path between the start and end operations + outer: for (const [operation] of parents) { + for (const consumer of operation.consumers) { + // Since this is a breadth-first traversal, the first encountered path to a given node + // will be tied for shortest, so only the first encountered path needs to be tracked + if (!parents.has(consumer)) { + parents.set(consumer, operation); + } + + if (consumer === startOperation) { + finalParent = operation; + break outer; + } + } + } + + if (!finalParent) { + throw new Error(`Could not find a path from "${startOperation.name}" to "${endOperation.name}"`); + } + + // Walk back up the path from the end operation to the start operation + let currentOperation: T = finalParent; + const path: T[] = [startOperation]; + while (currentOperation !== undefined) { + path.push(currentOperation); + currentOperation = parents.get(currentOperation)!; + } + return path; +} + +function calculateCriticalPathLength>( + operation: T, + dependencyChain: Set +): number { + if (dependencyChain.has(operation)) { + // Ensure we have the shortest path to the cycle + const shortestPath: T[] = calculateShortestPath(operation, operation); + + throw new Error( + 'A cyclic dependency was encountered:\n ' + + shortestPath.map((visitedTask) => visitedTask.name).join('\n -> ') + ); + } + + let { criticalPathLength } = operation; + + if (criticalPathLength !== undefined) { + // This has been visited already + return criticalPathLength; + } + + criticalPathLength = 0; + if (operation.consumers.size) { + dependencyChain.add(operation); + for (const consumer of operation.consumers) { + criticalPathLength = Math.max( + criticalPathLength, + calculateCriticalPathLength(consumer, dependencyChain) + ); + } + dependencyChain.delete(operation); + } + // Include the contribution from the current operation + criticalPathLength += operation.weight ?? 1; + + // Record result + operation.criticalPathLength = criticalPathLength; + + return criticalPathLength; +} + +interface IQueueItem { + task: () => Promise; + priority: number; +} + +function getSignal(): [Promise, () => void] { + let resolver: () => void; + const promise: Promise = new Promise((resolve) => { + resolver = resolve; + }); + return [promise, resolver!]; +} + +/** + * A priority queue of work items, consumed via async iteration. + * + * @remarks + * This is equivalent to the `WorkQueue` in `@rushstack/operation-graph`, except that: + * - the consumer is woken up with `setImmediate()` instead of `setTimeout()`. Both batch every item that was + * pushed while the current macrotask (including its microtasks) runs, so that items are dequeued in priority + * order; however `setTimeout()` imposes a minimum delay of 1ms for every wave of newly-ready operations. + * - it observes the caller's abort signal directly and is stopped via {@link WorkQueue.stop} when execution + * completes, instead of the caller aborting a dedicated `AbortController` (dispatching an abort event). + */ +class WorkQueue { + readonly #queue: MinimumHeap; + readonly #abortSignal: AbortSignal; + readonly #onAbort: () => void; + readonly #stopPromise: Promise; + + #isStopped: boolean = false; + #resolveStop!: () => void; + #pushPromise: Promise; + #resolvePush: () => void; + #resolvePushImmediate: NodeJS.Immediate | undefined; + + public constructor(abortSignal: AbortSignal) { + // Sort by priority descending. Thus the comparator returns a negative number if a has higher priority than b. + this.#queue = new MinimumHeap((a: IQueueItem, b: IQueueItem) => b.priority - a.priority); + this.#abortSignal = abortSignal; + this.#stopPromise = new Promise((resolve) => { + this.#resolveStop = resolve; + }); + this.#onAbort = () => this.stop(); + if (abortSignal.aborted) { + this.stop(); + } else { + abortSignal.addEventListener('abort', this.#onAbort, { once: true }); + } + + [this.#pushPromise, this.#resolvePush] = getSignal(); + this.#resolvePushImmediate = undefined; + } + + public async *[Symbol.asyncIterator](): AsyncIterableIterator<() => Promise> { + while (!this.#isStopped) { + while (this.#queue.size > 0) { + const item: IQueueItem = this.#queue.poll()!; + yield item.task; + } + + await Promise.race([this.#pushPromise, this.#stopPromise]); + } + } + + /** + * Ends the iteration. Items that have not started yet resolve with `OperationStatus.Aborted`. + * This happens automatically when the abort signal is aborted. + */ + public stop(): void { + this.#isStopped = true; + this.#resolveStop(); + } + + /** + * Stops observing the abort signal. + */ + public detachAbortSignal(): void { + this.#abortSignal.removeEventListener('abort', this.#onAbort); + } + + public pushAsync(task: () => Promise, priority: number): Promise { + return new Promise((resolve, reject) => { + this.#queue.push({ + task: () => task().then(resolve, reject), + priority + }); + + // ESLINT: "Promises must be awaited, end with a call to .catch, end with a call to .then ..." + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.#stopPromise.finally(() => resolve(OperationStatus.Aborted)); + + this.#resolvePushDebounced(); + }); + } + + #resolvePushDebounced(): void { + if (!this.#resolvePushImmediate) { + this.#resolvePushImmediate = setImmediate(() => { + this.#resolvePushImmediate = undefined; + this.#resolvePush(); + + [this.#pushPromise, this.#resolvePush] = getSignal(); + }); + } + } +} + + +/** + * Executes a graph of operations, honoring their dependencies and the requested parallelism. + * + * @remarks + * This mirrors `OperationExecutionManager` from `@rushstack/operation-graph` (same scheduling order, logging, + * hooks and result), but uses a {@link WorkQueue} that does not add a timer delay for every wave of + * newly-ready operations. + */ +export class OperationExecutionManager { + /** + * The set of operations that will be executed + */ + readonly #operations: Operation[]; + /** + * The total number of non-silent operations in the graph. + * Silent operations are generally used to simplify the construction of the graph. + */ + readonly #trackedOperationCount: number; + + readonly #groupRecords: Set>; + + public constructor(operations: ReadonlySet>) { + let trackedOperationCount: number = 0; + for (const operation of operations) { + if (!operation.runner?.silent) { + // Only count non-silent operations + trackedOperationCount++; + } + } + + this.#trackedOperationCount = trackedOperationCount; + + this.#operations = calculateCriticalPathLengths(operations); + + this.#groupRecords = new Set(Array.from(this.#operations, (e) => e.group).filter((e) => e !== undefined)); + + for (const consumer of operations) { + for (const dependency of consumer.dependencies) { + if (!operations.has(dependency)) { + throw new Error( + `Operation ${JSON.stringify(consumer.name)} declares a dependency on operation ` + + `${JSON.stringify(dependency.name)} that is not in the set of operations to execute.` + ); + } + } + } + } + + /** + * Executes all operations which have been registered, returning a promise which is resolved when all the + * operations are completed successfully, or rejects when any operation fails. + */ + public async executeAsync( + executionOptions: IOperationExecutionOptions + ): Promise { + let hasReportedFailures: boolean = false; + + const { abortSignal, parallelism, terminal, requestRun } = executionOptions; + + if (abortSignal.aborted) { + return OperationStatus.Aborted; + } + + const startedGroups: Set = new Set(); + const finishedGroups: Set = new Set(); + + const maxParallelism: number = Math.min(this.#operations.length, parallelism); + + for (const groupRecord of this.#groupRecords) { + groupRecord.reset(); + } + + for (const operation of this.#operations) { + operation.reset(); + } + + terminal.writeVerboseLine(`Executing a maximum of ${maxParallelism} simultaneous tasks...`); + + // The work queue stops when the abort signal is aborted, or when it is stopped below. + const workQueue: WorkQueue = new WorkQueue(abortSignal); + try { + + const executionContext: IExecuteOperationContext = { + terminal, + abortSignal, + + requestRun, + + queueWork: (workFn: () => Promise, priority: number): Promise => { + return workQueue.pushAsync(workFn, priority); + }, + + beforeExecute: (operation: Operation): void => { + // Initialize group if uninitialized and log the group name + const { group, runner } = operation; + if (group) { + if (!startedGroups.has(group)) { + startedGroups.add(group); + group.startTimer(); + terminal.writeLine(` ---- ${group.name} started ---- `); + executionOptions.beforeExecuteOperationGroup?.(group); + } + } + if (!runner?.silent) { + executionOptions.beforeExecuteOperation?.(operation); + } + }, + + afterExecute: ( + operation: Operation, + state: IOperationState + ): void => { + const { group, runner } = operation; + if (group) { + group.setOperationAsComplete(operation, state); + } + + if (state.status === OperationStatus.Failure) { + // This operation failed. Mark it as such and all reachable dependents as blocked. + // Failed operations get reported, even if silent. + // Generally speaking, silent operations shouldn't be able to fail, so this is a safety measure. + const message: string | undefined = state.error?.message; + if (message) { + terminal.writeErrorLine(message); + } + hasReportedFailures = true; + } + + if (!runner?.silent) { + executionOptions.afterExecuteOperation?.(operation); + } + + if (group) { + // Log out the group name and duration if it is the last operation in the group + if (group?.finished && !finishedGroups.has(group)) { + finishedGroups.add(group); + const finishedLoggingWord: string = group.hasFailures + ? 'encountered an error' + : group.hasCancellations + ? 'cancelled' + : 'finished'; + terminal.writeLine( + ` ---- ${group.name} ${finishedLoggingWord} (${group.duration.toFixed(3)}s) ---- ` + ); + executionOptions.afterExecuteOperationGroup?.(group); + } + } + } + }; + + const workQueuePromise: Promise = Async.forEachAsync( + workQueue, + (workFn: () => Promise) => workFn(), + { + concurrency: maxParallelism + } + ); + + await Promise.all(this.#operations.map((record: Operation) => record._executeAsync(executionContext))); + + // Terminate queue execution. + workQueue.stop(); + await workQueuePromise; + } finally { + // Cleanup resources + workQueue.detachAbortSignal(); + } + + const finalStatus: OperationStatus = + this.#trackedOperationCount === 0 + ? OperationStatus.NoOp + : abortSignal.aborted + ? OperationStatus.Aborted + : hasReportedFailures + ? OperationStatus.Failure + : OperationStatus.Success; + + return finalStatus; + } +} diff --git a/apps/heft/src/operations/generateOperations.ts b/apps/heft/src/operations/generateOperations.ts new file mode 100644 index 00000000000..750fd28a33e --- /dev/null +++ b/apps/heft/src/operations/generateOperations.ts @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { InternalError } from '@rushstack/node-core-library'; +import { Colorize, type ITerminal } from '@rushstack/terminal'; +import type { Operation as IOperation } from '@rushstack/operation-graph'; +// Deep imports avoid loading the rest of the package (e.g. WatchLoop). These resolve to the same +// modules (and thus the same classes) that the package entry point re-exports. +import { Operation } from '@rushstack/operation-graph/lib/Operation'; +import { OperationGroupRecord } from '@rushstack/operation-graph/lib/OperationGroupRecord'; + +import type { IHeftPhaseOperationMetadata, IHeftTaskOperationMetadata } from '../cli/HeftActionRunner'; +import type { HeftPhase } from '../pluginFramework/HeftPhase'; +import type { HeftTask } from '../pluginFramework/HeftTask'; +import type { InternalHeftSession } from '../pluginFramework/InternalHeftSession'; +import { PhaseOperationRunner } from './runners/PhaseOperationRunner'; +import { TaskOperationRunner } from './runners/TaskOperationRunner'; + +export interface IGenerateOperationsOptions { + internalHeftSession: InternalHeftSession; + selectedPhases: ReadonlySet; + terminal: ITerminal; +} + +/** + * Creates the operation graph (one silent operation per phase, plus one operation per task) for the + * selected phases. + */ +export function generateOperations( + options: IGenerateOperationsOptions +): Set> { + const { internalHeftSession, selectedPhases, terminal } = options; + + const operations: Map< + string, + Operation + > = new Map(); + const operationGroups: Map> = new Map(); + + let hasWarnedAboutSkippedPhases: boolean = false; + for (const phase of selectedPhases) { + // Warn if any dependencies are excluded from the list of selected phases + if (!hasWarnedAboutSkippedPhases) { + for (const dependencyPhase of phase.dependencyPhases) { + if (!selectedPhases.has(dependencyPhase)) { + // Only write once, and write with yellow to make it stand out without writing a warning to stderr + hasWarnedAboutSkippedPhases = true; + terminal.writeLine( + Colorize.bold( + 'The provided list of phases does not contain all phase dependencies. You may need to run the ' + + 'excluded phases manually.' + ) + ); + break; + } + } + } + + // Create operation for the phase start node + const phaseOperation: Operation = _getOrCreatePhaseOperation( + internalHeftSession, + phase, + operations, + operationGroups + ); + + // Create operations for each task + for (const task of phase.tasks) { + const taskOperation: Operation = _getOrCreateTaskOperation( + internalHeftSession, + task, + operations, + operationGroups + ); + // Set the phase operation as a dependency of the task operation to ensure the phase operation runs first + taskOperation.addDependency(phaseOperation); + + // Set all dependency tasks as dependencies of the task operation + for (const dependencyTask of task.dependencyTasks) { + taskOperation.addDependency( + _getOrCreateTaskOperation(internalHeftSession, dependencyTask, operations, operationGroups) + ); + } + + // Set all tasks in a in a phase as dependencies of the consuming phase + for (const consumingPhase of phase.consumingPhases) { + if (selectedPhases.has(consumingPhase)) { + // Set all tasks in a dependency phase as dependencies of the consuming phase to ensure the dependency + // tasks run first + const consumingPhaseOperation: Operation = _getOrCreatePhaseOperation( + internalHeftSession, + consumingPhase, + operations, + operationGroups + ); + consumingPhaseOperation.addDependency(taskOperation); + // This is purely to simplify the reported graph for phase circularities + consumingPhaseOperation.addDependency(phaseOperation); + } + } + } + } + + // The declarations under "lib/" are distinct from the rolled-up declarations of the package entry point, + // but describe the same runtime classes. + return new Set(operations.values()) as unknown as Set< + IOperation + >; +} + +function _getOrCreatePhaseOperation( + this: void, + internalHeftSession: InternalHeftSession, + phase: HeftPhase, + operations: Map, + operationGroups: Map> +): Operation { + const key: string = phase.phaseName; + + let operation: Operation | undefined = operations.get(key); + if (!operation) { + let group: OperationGroupRecord | undefined = operationGroups.get( + phase.phaseName + ); + if (!group) { + group = new OperationGroupRecord(phase.phaseName, { phase }); + operationGroups.set(phase.phaseName, group); + } + // Only create the operation. Dependencies are hooked up separately + operation = new Operation({ + group, + name: phase.phaseName, + runner: new PhaseOperationRunner({ phase, internalHeftSession }) + }); + operations.set(key, operation); + } + return operation; +} + +function _getOrCreateTaskOperation( + this: void, + internalHeftSession: InternalHeftSession, + task: HeftTask, + operations: Map, + operationGroups: Map> +): Operation { + const key: string = `${task.parentPhase.phaseName}.${task.taskName}`; + + let operation: Operation | undefined = operations.get( + key + ) as Operation; + if (!operation) { + const group: OperationGroupRecord | undefined = operationGroups.get( + task.parentPhase.phaseName + ); + if (!group) { + throw new InternalError( + `Task ${task.taskName} in phase ${task.parentPhase.phaseName} has no group. This should not happen.` + ); + } + operation = new Operation({ + group, + runner: new TaskOperationRunner({ + internalHeftSession, + task + }), + name: task.taskName, + metadata: { task, phase: task.parentPhase } + }); + operations.set(key, operation); + } + return operation; +} diff --git a/apps/heft/src/operations/runners/PhaseOperationRunner.ts b/apps/heft/src/operations/runners/PhaseOperationRunner.ts index fd9f46488e8..04cf024a7b4 100644 --- a/apps/heft/src/operations/runners/PhaseOperationRunner.ts +++ b/apps/heft/src/operations/runners/PhaseOperationRunner.ts @@ -1,13 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { - type IOperationRunner, - type IOperationRunnerContext, - OperationStatus -} from '@rushstack/operation-graph'; +import type { IOperationRunner, IOperationRunnerContext } from '@rushstack/operation-graph'; +import { OperationStatus } from '@rushstack/operation-graph/lib/OperationStatus'; -import { deleteFilesAsync, type IDeleteOperation } from '../../plugins/DeleteFilesPlugin'; +import type { IDeleteOperation } from '../../plugins/DeleteFilesPlugin'; import type { HeftPhase } from '../../pluginFramework/HeftPhase'; import type { HeftPhaseSession } from '../../pluginFramework/HeftPhaseSession'; import type { InternalHeftSession } from '../../pluginFramework/InternalHeftSession'; @@ -64,6 +61,7 @@ export class PhaseOperationRunner implements IOperationRunner { // Delete the files if any were specified if (deleteOperations.length) { const rootFolderPath: string = internalHeftSession.heftConfiguration.buildFolderPath; + const { deleteFilesAsync } = await import('../../plugins/DeleteFilesPlugin'); await deleteFilesAsync(rootFolderPath, deleteOperations, cleanLogger.terminal); } diff --git a/apps/heft/src/operations/runners/TaskOperationRunner.ts b/apps/heft/src/operations/runners/TaskOperationRunner.ts index 86b642235f0..64967053929 100644 --- a/apps/heft/src/operations/runners/TaskOperationRunner.ts +++ b/apps/heft/src/operations/runners/TaskOperationRunner.ts @@ -1,25 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { createHash, type Hash } from 'node:crypto'; +import type { Hash } from 'node:crypto'; -import { glob } from 'fast-glob'; - -import { - type IOperationRunner, - type IOperationRunnerContext, - OperationStatus -} from '@rushstack/operation-graph'; +import type { IOperationRunner, IOperationRunnerContext } from '@rushstack/operation-graph'; +import { OperationStatus } from '@rushstack/operation-graph/lib/OperationStatus'; import { AlreadyReportedError, InternalError } from '@rushstack/node-core-library'; import type { HeftTask } from '../../pluginFramework/HeftTask'; -import { - copyFilesAsync, - type ICopyOperation, - asAbsoluteCopyOperation, - asRelativeCopyOperation -} from '../../plugins/CopyFilesPlugin'; -import { deleteFilesAsync } from '../../plugins/DeleteFilesPlugin'; +import type { ICopyOperation } from '../../plugins/CopyFilesPlugin'; import type { HeftTaskSession, IHeftTaskFileOperations, @@ -28,13 +17,51 @@ import type { } from '../../pluginFramework/HeftTaskSession'; import type { HeftPhaseSession } from '../../pluginFramework/HeftPhaseSession'; import type { InternalHeftSession } from '../../pluginFramework/InternalHeftSession'; -import { watchGlobAsync, type IGlobOptions } from '../../plugins/FileGlobSpecifier'; -import { - type IWatchedFileState, - type IWatchFileSystem, +import type { GlobFn, IGlobOptions } from '../../plugins/FileGlobSpecifier'; +import type { + IWatchedFileState, + IWatchFileSystem, WatchFileSystemAdapter } from '../../utilities/WatchFileSystemAdapter'; +// The modules below are only needed for specific features (file operations, watch mode, globbing), so they +// are loaded on first use to keep them out of the startup path. +type CopyFilesPluginModule = typeof import('../../plugins/CopyFilesPlugin'); +type DeleteFilesPluginModule = typeof import('../../plugins/DeleteFilesPlugin'); +type FileGlobSpecifierModule = typeof import('../../plugins/FileGlobSpecifier'); +type WatchFileSystemAdapterModule = typeof import('../../utilities/WatchFileSystemAdapter'); + +let _copyFilesPluginModulePromise: Promise | undefined; +let _deleteFilesPluginModulePromise: Promise | undefined; +let _watchModulesPromise: Promise<[WatchFileSystemAdapterModule, FileGlobSpecifierModule]> | undefined; +let _fastGlobPromise: Promise | undefined; + +function loadCopyFilesPluginModuleAsync(): Promise { + return (_copyFilesPluginModulePromise ??= import('../../plugins/CopyFilesPlugin')); +} + +function loadDeleteFilesPluginModuleAsync(): Promise { + return (_deleteFilesPluginModulePromise ??= import('../../plugins/DeleteFilesPlugin')); +} + +function loadWatchModulesAsync(): Promise<[WatchFileSystemAdapterModule, FileGlobSpecifierModule]> { + return (_watchModulesPromise ??= Promise.all([ + import('../../utilities/WatchFileSystemAdapter'), + import('../../plugins/FileGlobSpecifier') + ])); +} + +/** + * Loads "fast-glob" the first time a plugin globs, and then forwards to it. + */ +const globAsync: GlobFn = async ( + pattern: string | string[], + options?: IGlobOptions | undefined +): Promise => { + const glob: GlobFn = await (_fastGlobPromise ??= import('fast-glob').then((fastGlob) => fastGlob.glob)); + return await glob(pattern, options); +}; + export interface ITaskOperationRunnerOptions { internalHeftSession: InternalHeftSession; task: HeftTask; @@ -106,6 +133,11 @@ export class TaskOperationRunner implements IOperationRunner { return OperationStatus.Aborted; } + // These modules are only used in watch mode, where they are accessed synchronously below. + const watchModules: [WatchFileSystemAdapterModule, FileGlobSpecifierModule] | undefined = isWatchMode + ? await loadWatchModulesAsync() + : undefined; + if (!this.#fileOperations && hooks.registerFileOperations.isUsed()) { const fileOperations: IHeftTaskFileOperations = await hooks.registerFileOperations.promise({ copyOperations: new Set(), @@ -115,6 +147,10 @@ export class TaskOperationRunner implements IOperationRunner { let copyConfigHash: string | undefined; const { copyOperations } = fileOperations; if (copyOperations.size > 0) { + const [{ asAbsoluteCopyOperation, asRelativeCopyOperation }, { createHash }] = await Promise.all([ + loadCopyFilesPluginModuleAsync(), + import('node:crypto') + ]); // Do this here so that we only need to do it once for each Heft invocation const hasher: Hash | undefined = createHash('sha256'); const absolutePathCopyOperations: Set = new Set(); @@ -143,7 +179,10 @@ export class TaskOperationRunner implements IOperationRunner { let watchFileSystemAdapter: WatchFileSystemAdapter | undefined; const getWatchFileSystemAdapter = (): WatchFileSystemAdapter => { if (!watchFileSystemAdapter) { - watchFileSystemAdapter = this.#watchFileSystemAdapter ||= new WatchFileSystemAdapter(); + if (!watchModules) { + throw new InternalError(`The WatchFileSystemAdapter is only available in watch mode.`); + } + watchFileSystemAdapter = this.#watchFileSystemAdapter ||= new watchModules[0].WatchFileSystemAdapter(); watchFileSystemAdapter.setBaseline(); } return watchFileSystemAdapter; @@ -161,7 +200,7 @@ export class TaskOperationRunner implements IOperationRunner { // Create the options and provide a utility method to obtain paths to copy const runHookOptions: IHeftTaskRunHookOptions = { abortSignal, - globAsync: glob + globAsync }; // Run the plugin run hook @@ -173,7 +212,7 @@ export class TaskOperationRunner implements IOperationRunner { pattern: string | string[], options: IGlobOptions = {} ): Promise> => { - return watchGlobAsync(pattern, { + return watchModules![1].watchGlobAsync(pattern, { ...options, fs: getWatchFileSystemAdapter() }); @@ -215,9 +254,15 @@ export class TaskOperationRunner implements IOperationRunner { const { copyOperations, deleteOperations } = this.#fileOperations; const copyConfigHash: string | undefined = this.#copyConfigHash; + const shouldDelete: boolean = deleteOperations.size > 0; + const [copyFilesPluginModule, deleteFilesPluginModule] = await Promise.all([ + copyConfigHash ? loadCopyFilesPluginModuleAsync() : undefined, + shouldDelete ? loadDeleteFilesPluginModuleAsync() : undefined + ]); + await Promise.all([ copyConfigHash - ? copyFilesAsync( + ? copyFilesPluginModule!.copyFilesAsync( copyOperations, logger.terminal, `${taskSession.tempFolderPath}/file-copy.json`, @@ -225,8 +270,8 @@ export class TaskOperationRunner implements IOperationRunner { isWatchMode ? getWatchFileSystemAdapter() : undefined ) : Promise.resolve(), - deleteOperations.size > 0 - ? deleteFilesAsync(rootFolderPath, deleteOperations, logger.terminal) + shouldDelete + ? deleteFilesPluginModule!.deleteFilesAsync(rootFolderPath, deleteOperations, logger.terminal) : Promise.resolve() ]); } diff --git a/apps/heft/src/operations/test/OperationExecutionManager.test.ts b/apps/heft/src/operations/test/OperationExecutionManager.test.ts new file mode 100644 index 00000000000..cc603fb10ae --- /dev/null +++ b/apps/heft/src/operations/test/OperationExecutionManager.test.ts @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; +import { + type IOperationRunner, + type IOperationRunnerContext, + Operation, + OperationExecutionManager as LibraryOperationExecutionManager, + OperationGroupRecord, + OperationStatus +} from '@rushstack/operation-graph'; + +import { OperationExecutionManager as HeftOperationExecutionManager } from '../OperationExecutionManager'; + +// Heft's OperationExecutionManager is meant to behave exactly like the one in @rushstack/operation-graph, +// except for how quickly it starts newly-ready operations. These tests run both on the same graphs and +// compare everything that is observable: hook calls, log output, operation states and the overall result. + +interface IOperationSpec { + name: string; + group?: string; + dependencies?: string[]; + silent?: boolean; + weight?: number; + result?: OperationStatus | 'throw'; + // Number of microtask turns the runner takes to complete + ticks?: number; +} + +interface IRunOptions { + parallelism: number; + abortAfter?: string; + preAborted?: boolean; +} + +interface IExecutionRecord { + events: string[]; + output: string; + status: OperationStatus; + states: string[]; +} + +interface IExecutionManager { + executeAsync: LibraryOperationExecutionManager['executeAsync']; +} + +type ExecutionManagerFactory = (operations: ReadonlySet) => IExecutionManager; + +const createLibraryExecutionManager: ExecutionManagerFactory = (operations) => + new LibraryOperationExecutionManager(operations); +const createHeftExecutionManager: ExecutionManagerFactory = (operations) => + new HeftOperationExecutionManager(operations); + +class RecordingRunner implements IOperationRunner { + public readonly name: string; + public readonly silent: boolean; + readonly #spec: IOperationSpec; + readonly #events: string[]; + readonly #onDone: (name: string) => void; + + public constructor(spec: IOperationSpec, events: string[], onDone: (name: string) => void) { + this.name = spec.name; + this.silent = !!spec.silent; + this.#spec = spec; + this.#events = events; + this.#onDone = onDone; + } + + public async executeAsync(context: IOperationRunnerContext): Promise { + this.#events.push(`run:${this.name}:${context.isFirstRun}`); + for (let i: number = 0; i < (this.#spec.ticks ?? 0); i++) { + await Promise.resolve(); + } + this.#events.push(`done:${this.name}`); + this.#onDone(this.name); + if (this.#spec.result === 'throw') { + throw new Error(`${this.name} threw`); + } + return this.#spec.result ?? OperationStatus.Success; + } +} + +async function runAsync( + createExecutionManager: ExecutionManagerFactory, + specs: IOperationSpec[], + options: IRunOptions +): Promise { + const events: string[] = []; + const abortController: AbortController = new AbortController(); + const onDone = (name: string): void => { + if (name === options.abortAfter) { + abortController.abort(); + } + }; + + const groups: Map = new Map(); + const operations: Map = new Map(); + for (const spec of specs) { + let group: OperationGroupRecord | undefined; + if (spec.group) { + group = groups.get(spec.group); + if (!group) { + group = new OperationGroupRecord(spec.group); + groups.set(spec.group, group); + } + } + operations.set( + spec.name, + new Operation({ + name: spec.name, + group, + weight: spec.weight, + runner: new RecordingRunner(spec, events, onDone) + }) + ); + } + for (const spec of specs) { + for (const dependency of spec.dependencies ?? []) { + operations.get(spec.name)!.addDependency(operations.get(dependency)!); + } + } + + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(true); + const manager: IExecutionManager = createExecutionManager(new Set(operations.values())); + if (options.preAborted) { + abortController.abort(); + } + const status: OperationStatus = await manager.executeAsync({ + abortSignal: abortController.signal, + parallelism: options.parallelism, + terminal: new Terminal(terminalProvider), + beforeExecuteOperation: (operation: Operation) => events.push(`before:${operation.name}`), + afterExecuteOperation: (operation: Operation) => events.push(`after:${operation.name}`), + beforeExecuteOperationGroup: (group: OperationGroupRecord) => events.push(`group-start:${group.name}`), + afterExecuteOperationGroup: (group: OperationGroupRecord) => events.push(`group-end:${group.name}`) + }); + + const output: string = [ + terminalProvider.getOutput({ normalizeSpecialCharacters: true }), + terminalProvider.getVerboseOutput({ normalizeSpecialCharacters: true }), + terminalProvider.getErrorOutput({ normalizeSpecialCharacters: true }) + ] + .join('\n') + .replace(/\(\d+\.\d+s\)/g, '()'); + const states: string[] = Array.from(operations.values(), (operation: Operation) => { + return `${operation.name}:${operation.state?.status}:${operation.state?.error?.message ?? ''}`; + }); + return { events, output, status, states }; +} + +async function expectSameBehaviorAsync( + specs: IOperationSpec[], + options: IRunOptions +): Promise { + const expected: IExecutionRecord = await runAsync(createLibraryExecutionManager, specs, options); + const actual: IExecutionRecord = await runAsync(createHeftExecutionManager, specs, options); + expect(actual).toEqual(expected); + return actual; +} + +// A two-phase graph shaped like the one that heft generates: a silent operation per phase, which the +// phase's tasks depend on, and which depends on all tasks of the phases it consumes. +const heftLikeGraph: IOperationSpec[] = [ + { name: 'build', group: 'build', silent: true }, + { name: 'typescript', group: 'build', dependencies: ['build'], ticks: 3 }, + { name: 'lint', group: 'build', dependencies: ['build', 'typescript'], ticks: 1 }, + { name: 'api-extractor', group: 'build', dependencies: ['build', 'typescript'], ticks: 2 }, + { name: 'copy', group: 'build', dependencies: ['build'] }, + { name: 'test', group: 'test', silent: true, dependencies: ['build', 'typescript', 'lint', 'copy'] }, + { name: 'jest', group: 'test', dependencies: ['test'], ticks: 4 }, + { name: 'report', group: 'test', dependencies: ['test', 'jest'] } +]; + +describe('OperationExecutionManager (heft)', () => { + it('matches @rushstack/operation-graph for a heft-like graph', async () => { + for (const parallelism of [1, 2, 8]) { + const record: IExecutionRecord = await expectSameBehaviorAsync(heftLikeGraph, { parallelism }); + expect(record.status).toEqual(OperationStatus.Success); + } + }); + + it('matches @rushstack/operation-graph when many operations become ready at the same time', async () => { + const specs: IOperationSpec[] = [{ name: 'root', group: 'g', silent: true }]; + for (let i: number = 0; i < 20; i++) { + specs.push({ name: `op${i}`, group: 'g', dependencies: ['root'], weight: (i * 7) % 5, ticks: i % 3 }); + } + for (let i: number = 0; i < 10; i++) { + specs.push({ name: `tail${i}`, group: 'h', dependencies: [`op${i}`, `op${19 - i}`], ticks: i % 2 }); + } + for (const parallelism of [1, 3, 16]) { + await expectSameBehaviorAsync(specs, { parallelism }); + } + }); + + it('matches @rushstack/operation-graph when operations fail or throw', async () => { + const specs: IOperationSpec[] = heftLikeGraph.map((spec: IOperationSpec) => { + if (spec.name === 'typescript') { + return { ...spec, result: OperationStatus.Failure }; + } else if (spec.name === 'copy') { + return { ...spec, result: 'throw' }; + } else { + return spec; + } + }); + const record: IExecutionRecord = await expectSameBehaviorAsync(specs, { parallelism: 4 }); + expect(record.status).toEqual(OperationStatus.Failure); + }); + + it('matches @rushstack/operation-graph when execution is aborted', async () => { + const record: IExecutionRecord = await expectSameBehaviorAsync(heftLikeGraph, { + parallelism: 2, + abortAfter: 'lint' + }); + expect(record.status).toEqual(OperationStatus.Aborted); + + const preAbortedRecord: IExecutionRecord = await expectSameBehaviorAsync(heftLikeGraph, { + parallelism: 2, + preAborted: true + }); + expect(preAbortedRecord.status).toEqual(OperationStatus.Aborted); + }); + + it('matches @rushstack/operation-graph when there is nothing to run', async () => { + const record: IExecutionRecord = await expectSameBehaviorAsync( + [{ name: 'only-silent', group: 'g', silent: true }], + { parallelism: 4 } + ); + expect(record.status).toEqual(OperationStatus.NoOp); + }); + + it('reports dependency cycles like @rushstack/operation-graph', () => { + const createCycle = (): Set => { + const a: Operation = new Operation({ name: 'a' }); + const b: Operation = new Operation({ name: 'b' }); + const c: Operation = new Operation({ name: 'c' }); + a.addDependency(b); + b.addDependency(c); + c.addDependency(a); + return new Set([a, b, c]); + }; + + let expectedMessage: string | undefined; + try { + createLibraryExecutionManager(createCycle()); + } catch (e) { + expectedMessage = (e as Error).message; + } + expect(expectedMessage).toMatch(/^A cyclic dependency was encountered:/); + expect(() => createHeftExecutionManager(createCycle())).toThrow(expectedMessage); + }); + + it('rejects dependencies that are not in the set of operations like @rushstack/operation-graph', () => { + const createOperations = (): Set => { + const a: Operation = new Operation({ name: 'a' }); + const b: Operation = new Operation({ name: 'b' }); + a.addDependency(b); + return new Set([a]); + }; + const expectedMessage: string = + 'Operation "a" declares a dependency on operation "b" that is not in the set of operations to execute.'; + expect(() => createLibraryExecutionManager(createOperations())).toThrow(expectedMessage); + expect(() => createHeftExecutionManager(createOperations())).toThrow(expectedMessage); + }); +}); diff --git a/apps/heft/src/pluginFramework/HeftLifecycle.ts b/apps/heft/src/pluginFramework/HeftLifecycle.ts index b227208d0e1..efdafcdb4ca 100644 --- a/apps/heft/src/pluginFramework/HeftLifecycle.ts +++ b/apps/heft/src/pluginFramework/HeftLifecycle.ts @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { AsyncParallelHook, SyncHook } from 'tapable'; - import { InternalError } from '@rushstack/node-core-library'; import { HeftPluginConfiguration } from '../configuration/HeftPluginConfiguration'; @@ -14,19 +12,20 @@ import type { HeftPluginDefinitionBase } from '../configuration/HeftPluginDefinition'; import type { IHeftLifecyclePlugin, IHeftPlugin } from './IHeftPlugin'; -import { +import type { HeftLifecycleSession, - type IHeftLifecycleCleanHookOptions, - type IHeftLifecycleHooks, - type IHeftLifecycleToolStartHookOptions, - type IHeftLifecycleToolFinishHookOptions, - type IHeftLifecycleSession, - type IHeftTaskStartHookOptions, - type IHeftTaskFinishHookOptions, - type IHeftPhaseStartHookOptions, - type IHeftPhaseFinishHookOptions + IHeftLifecycleCleanHookOptions, + IHeftLifecycleHooks, + IHeftLifecycleToolStartHookOptions, + IHeftLifecycleToolFinishHookOptions, + IHeftLifecycleSession, + IHeftTaskStartHookOptions, + IHeftTaskFinishHookOptions, + IHeftPhaseStartHookOptions, + IHeftPhaseFinishHookOptions } from './HeftLifecycleSession'; import type { ScopedLogger } from './logging/ScopedLogger'; +import { createAsyncParallelHook, createSyncHook, defineLazyProperty } from './TapableHooks'; export interface IHeftLifecycleContext { lifecycleSession?: HeftLifecycleSession; @@ -68,16 +67,37 @@ export class HeftLifecycle extends HeftPluginHost { this.#internalHeftSession = internalHeftSession; this.#lifecyclePluginSpecifiers = lifecyclePluginSpecifiers; - this.#lifecycleHooks = { - clean: new AsyncParallelHook(), - toolStart: new AsyncParallelHook(), - toolFinish: new AsyncParallelHook(), - recordMetrics: internalHeftSession.metricsCollector.recordMetricsHook, - taskStart: new SyncHook(['task']), - taskFinish: new SyncHook(['task']), - phaseStart: new SyncHook(['phase']), - phaseFinish: new SyncHook(['phase']) - }; + // The hooks are created on first access, since creating them requires loading tapable, which is not + // needed if Heft exits without running the lifecycle (e.g. when printing help). The properties are + // defined in the same order, with the same constructor arguments, as a plain object literal would have. + const lifecycleHooks: IHeftLifecycleHooks = {} as IHeftLifecycleHooks; + defineLazyProperty(lifecycleHooks, 'clean', () => + createAsyncParallelHook() + ); + defineLazyProperty(lifecycleHooks, 'toolStart', () => + createAsyncParallelHook() + ); + defineLazyProperty(lifecycleHooks, 'toolFinish', () => + createAsyncParallelHook() + ); + defineLazyProperty( + lifecycleHooks, + 'recordMetrics', + () => internalHeftSession.metricsCollector.recordMetricsHook + ); + defineLazyProperty(lifecycleHooks, 'taskStart', () => + createSyncHook(['task']) + ); + defineLazyProperty(lifecycleHooks, 'taskFinish', () => + createSyncHook(['task']) + ); + defineLazyProperty(lifecycleHooks, 'phaseStart', () => + createSyncHook(['phase']) + ); + defineLazyProperty(lifecycleHooks, 'phaseFinish', () => + createSyncHook(['phase']) + ); + this.#lifecycleHooks = lifecycleHooks; } protected async applyPluginsInternalAsync(): Promise { @@ -87,8 +107,12 @@ export class HeftLifecycle extends HeftPluginHost { const loadPluginPromises: Promise>[] = []; for (const [pluginDefinition, lifecycleContext] of this.#lifecycleContextByDefinition) { if (!lifecycleContext.lifecycleSession) { - // Generate the plugin-specific session - lifecycleContext.lifecycleSession = new HeftLifecycleSession({ + // Generate the plugin-specific session. The session implementation is only loaded if there are + // lifecycle plugins to apply. + const { HeftLifecycleSession: HeftLifecycleSessionClass } = require('./HeftLifecycleSession') as { + HeftLifecycleSession: typeof HeftLifecycleSession; + }; + lifecycleContext.lifecycleSession = new HeftLifecycleSessionClass({ debug: this.#internalHeftSession.debug, heftConfiguration: this.#internalHeftSession.heftConfiguration, loggingManager: this.#internalHeftSession.loggingManager, diff --git a/apps/heft/src/pluginFramework/HeftParameterManager.ts b/apps/heft/src/pluginFramework/HeftParameterManager.ts index 17538c851f9..60b696526e0 100644 --- a/apps/heft/src/pluginFramework/HeftParameterManager.ts +++ b/apps/heft/src/pluginFramework/HeftParameterManager.ts @@ -2,18 +2,20 @@ // See LICENSE in the project root for license information. import { InternalError } from '@rushstack/node-core-library'; -import { - type CommandLineParameter, - type CommandLineParameterProvider, - CommandLineParameterKind, - type CommandLineChoiceParameter, - type CommandLineChoiceListParameter, - type CommandLineFlagParameter, - type CommandLineIntegerParameter, - type CommandLineIntegerListParameter, - type CommandLineStringParameter, - type CommandLineStringListParameter +import type { + CommandLineParameter, + CommandLineParameterProvider, + CommandLineChoiceParameter, + CommandLineChoiceListParameter, + CommandLineFlagParameter, + CommandLineIntegerParameter, + CommandLineIntegerListParameter, + CommandLineStringParameter, + CommandLineStringListParameter } from '@rushstack/ts-command-line'; +// Import the enum from its defining module (the package index re-exports this same object), since loading +// the package index also loads the argparse-based command line parser, which is not otherwise needed here. +import { CommandLineParameterKind } from '@rushstack/ts-command-line/lib/parameters/BaseClasses'; import type { HeftPluginDefinitionBase, diff --git a/apps/heft/src/pluginFramework/HeftPluginHost.ts b/apps/heft/src/pluginFramework/HeftPluginHost.ts index df957c3f613..9ff3bf543fa 100644 --- a/apps/heft/src/pluginFramework/HeftPluginHost.ts +++ b/apps/heft/src/pluginFramework/HeftPluginHost.ts @@ -1,13 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { SyncHook } from 'tapable'; +import type { SyncHook } from 'tapable'; import { InternalError } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import type { HeftPluginDefinitionBase } from '../configuration/HeftPluginDefinition'; import type { IHeftPlugin } from './IHeftPlugin'; +import { createSyncHook } from './TapableHooks'; export abstract class HeftPluginHost { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -46,7 +47,7 @@ export abstract class HeftPluginHost { const pluginHookName: string = this.getPluginHookName(pluginToAccessPackage, pluginToAccessName); let pluginAccessRequestHook: SyncHook | undefined = this.#pluginAccessRequestHooks.get(pluginHookName); if (!pluginAccessRequestHook) { - pluginAccessRequestHook = new SyncHook(['pluginAccessor']); + pluginAccessRequestHook = createSyncHook(['pluginAccessor']); this.#pluginAccessRequestHooks.set(pluginHookName, pluginAccessRequestHook); } if (pluginAccessRequestHook.taps.some((t) => t.name === requestorName)) { diff --git a/apps/heft/src/pluginFramework/HeftTaskSession.ts b/apps/heft/src/pluginFramework/HeftTaskSession.ts index aa06e38fcc1..f773b3d0c75 100644 --- a/apps/heft/src/pluginFramework/HeftTaskSession.ts +++ b/apps/heft/src/pluginFramework/HeftTaskSession.ts @@ -1,6 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +// This is intentionally not an `import type`: it is only used as types (so it is elided from the emitted +// JavaScript and tapable is loaded lazily), but the emitted declarations and the API report must keep the +// original `import { AsyncParallelHook, AsyncSeriesWaterfallHook } from 'tapable'` form. +// eslint-disable-next-line @typescript-eslint/consistent-type-imports import { AsyncParallelHook, AsyncSeriesWaterfallHook } from 'tapable'; import { InternalError } from '@rushstack/node-core-library'; @@ -15,6 +19,7 @@ import type { ICopyOperation } from '../plugins/CopyFilesPlugin'; import type { HeftPluginHost } from './HeftPluginHost'; import type { GlobFn, WatchGlobFn } from '../plugins/FileGlobSpecifier'; import type { IWatchFileSystem } from '../utilities/WatchFileSystemAdapter'; +import { createAsyncParallelHook, createAsyncSeriesWaterfallHook } from './TapableHooks'; /** * The type of {@link IHeftTaskSession.parsedCommandLine}, which exposes details about the @@ -282,9 +287,9 @@ export class HeftTaskSession implements IHeftTaskSession { this.metricsCollector = metricsCollector; this.taskName = task.taskName; this.hooks = { - run: new AsyncParallelHook(['runHookOptions']), - runIncremental: new AsyncParallelHook(['runIncrementalHookOptions']), - registerFileOperations: new AsyncSeriesWaterfallHook(['fileOperations']) + run: createAsyncParallelHook(['runHookOptions']), + runIncremental: createAsyncParallelHook(['runIncrementalHookOptions']), + registerFileOperations: createAsyncSeriesWaterfallHook(['fileOperations']) }; // Guaranteed to be unique since phases are uniquely named, tasks are uniquely named within diff --git a/apps/heft/src/pluginFramework/InternalHeftSession.ts b/apps/heft/src/pluginFramework/InternalHeftSession.ts index b4c3dfff514..0d6071b97a6 100644 --- a/apps/heft/src/pluginFramework/InternalHeftSession.ts +++ b/apps/heft/src/pluginFramework/InternalHeftSession.ts @@ -5,7 +5,7 @@ import { Async, InternalError } from '@rushstack/node-core-library'; import { Constants } from '../utilities/Constants'; import { HeftLifecycle } from './HeftLifecycle'; -import { HeftPhaseSession } from './HeftPhaseSession'; +import type { HeftPhaseSession } from './HeftPhaseSession'; import { HeftPhase } from './HeftPhase'; import { CoreConfigFiles, @@ -14,7 +14,7 @@ import { } from '../utilities/CoreConfigFiles'; import type { MetricsCollector } from '../metrics/MetricsCollector'; import type { LoggingManager } from './logging/LoggingManager'; -import type { HeftConfiguration } from '../configuration/HeftConfiguration'; +import { type HeftConfiguration, getRigConfigForConfigLoading } from '../configuration/HeftConfiguration'; import type { HeftPluginDefinitionBase } from '../configuration/HeftPluginDefinition'; import type { HeftTask } from './HeftTask'; import type { HeftParameterManager } from './HeftParameterManager'; @@ -69,7 +69,8 @@ export class InternalHeftSession { await CoreConfigFiles.loadHeftConfigurationFileForProjectAsync( options.heftConfiguration.globalTerminal, options.heftConfiguration.buildFolderPath, - options.heftConfiguration.rigConfig + // Same data as heftConfiguration.rigConfig, without loading @rushstack/rig-package unless needed + getRigConfigForConfigLoading(options.heftConfiguration) ); const internalHeftSession: InternalHeftSession = new InternalHeftSession(heftConfigurationJson, options); @@ -157,7 +158,11 @@ export class InternalHeftSession { public getSessionForPhase(phase: HeftPhase): HeftPhaseSession { let phaseSession: HeftPhaseSession | undefined = this.#phaseSessionsByPhase.get(phase); if (!phaseSession) { - phaseSession = new HeftPhaseSession({ internalHeftSession: this, phase }); + // The phase and task session implementations are only loaded once a phase actually runs + const { HeftPhaseSession: HeftPhaseSessionClass } = require('./HeftPhaseSession') as { + HeftPhaseSession: typeof HeftPhaseSession; + }; + phaseSession = new HeftPhaseSessionClass({ internalHeftSession: this, phase }); this.#phaseSessionsByPhase.set(phase, phaseSession); } return phaseSession; diff --git a/apps/heft/src/pluginFramework/TapableHooks.ts b/apps/heft/src/pluginFramework/TapableHooks.ts new file mode 100644 index 00000000000..2cee1e1bbb4 --- /dev/null +++ b/apps/heft/src/pluginFramework/TapableHooks.ts @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { AsyncParallelHook, AsyncSeriesWaterfallHook, SyncHook } from 'tapable'; + +// Loading the "tapable" package index requires every hook implementation, and Heft only needs these three. +// The individual modules are the same files that the package index re-exports, so the classes are +// identical to the ones obtained via `import { SyncHook } from 'tapable'`. +let _syncHookClass: typeof SyncHook | undefined; +let _asyncParallelHookClass: typeof AsyncParallelHook | undefined; +let _asyncSeriesWaterfallHookClass: typeof AsyncSeriesWaterfallHook | undefined; + +/** + * Creates a tapable `SyncHook` with the specified tap argument names. + */ +export function createSyncHook(tapArgumentNames?: string[]): SyncHook { + if (!_syncHookClass) { + _syncHookClass = require('tapable/lib/SyncHook') as typeof SyncHook; + } + return new _syncHookClass(tapArgumentNames); +} + +/** + * Creates a tapable `AsyncParallelHook` with the specified tap argument names. + */ +export function createAsyncParallelHook(tapArgumentNames?: string[]): AsyncParallelHook { + if (!_asyncParallelHookClass) { + _asyncParallelHookClass = require('tapable/lib/AsyncParallelHook') as typeof AsyncParallelHook; + } + return new _asyncParallelHookClass(tapArgumentNames); +} + +/** + * Creates a tapable `AsyncSeriesWaterfallHook` with the specified tap argument names. + */ +export function createAsyncSeriesWaterfallHook(tapArgumentNames?: string[]): AsyncSeriesWaterfallHook { + if (!_asyncSeriesWaterfallHookClass) { + _asyncSeriesWaterfallHookClass = + require('tapable/lib/AsyncSeriesWaterfallHook') as typeof AsyncSeriesWaterfallHook; + } + return new _asyncSeriesWaterfallHookClass(tapArgumentNames); +} + +/** + * Defines an enumerable property on `target` whose value is created by `factory` when the property is first + * read. Once read (or assigned), the property is replaced with an ordinary writable data property, so it then + * behaves exactly like a property that was assigned in a constructor or object literal. + * + * @returns A function that reports whether the property has been materialized, i.e. whether it has been + * read (invoking `factory`) or assigned. While it has not been materialized, nothing can have observed its value. + */ +export function defineLazyProperty( + target: TTarget, + key: TKey, + factory: () => TTarget[TKey] +): () => boolean { + let materialized: boolean = false; + let currentValue: TTarget[TKey]; + const materialize = (value: TTarget[TKey]): void => { + materialized = true; + currentValue = value; + const descriptor: PropertyDescriptor | undefined = Object.getOwnPropertyDescriptor(target, key); + if (descriptor?.configurable) { + Object.defineProperty(target, key, { + value, + writable: true, + enumerable: true, + configurable: true + }); + } + // Otherwise the target was frozen or sealed; keep serving the value from the accessor. + }; + + Object.defineProperty(target, key, { + enumerable: true, + configurable: true, + get(): TTarget[TKey] { + if (!materialized) { + materialize(factory()); + } + return currentValue; + }, + set(value: TTarget[TKey]): void { + materialize(value); + } + }); + + return () => materialized; +} diff --git a/apps/heft/src/pluginFramework/logging/HeftChildReporter.ts b/apps/heft/src/pluginFramework/logging/HeftChildReporter.ts index 471c053b88e..440598feefc 100644 --- a/apps/heft/src/pluginFramework/logging/HeftChildReporter.ts +++ b/apps/heft/src/pluginFramework/logging/HeftChildReporter.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as crypto from 'node:crypto'; +import type * as crypto from 'node:crypto'; import * as fs from 'node:fs'; import { EOL } from 'node:os'; @@ -34,6 +34,11 @@ interface IReporterEventScope { readonly commandName?: string; } +// node:crypto is comparatively expensive to load and is only needed when the reporter channel is active. +function randomUUID(): string { + return (require('node:crypto') as typeof crypto).randomUUID(); +} + function readDescriptorFd(env: Record, name: string): number | undefined { const raw: string | undefined = env[name]; if (raw === undefined || !/^\d+$/.test(raw)) { @@ -185,7 +190,7 @@ export class HeftChildReporter implements ITerminalProvider { private constructor(descriptorFd: number, sourceVersion: string, context: IReporterChildContext) { this._descriptorFd = descriptorFd; this._sourceVersion = sourceVersion; - this._sessionId = crypto.randomUUID(); + this._sessionId = randomUUID(); this.parentReporterName = context.reporter; this.terminalWidth = context.terminalWidth; this.supportsColor = context.color; @@ -301,7 +306,7 @@ export class HeftChildReporter implements ITerminalProvider { } : { kind: 'tool', toolName: loggerName }; const diagnostic: Record = { - diagnosticId: crypto.randomUUID(), + diagnosticId: randomUUID(), code: 'RUSH_EXTERNAL_TOOL_PROBLEM', category: 'operation', severity, diff --git a/apps/heft/src/pluginFramework/logging/LoggingManager.ts b/apps/heft/src/pluginFramework/logging/LoggingManager.ts index 6ad0609fcd6..47e5a3e983d 100644 --- a/apps/heft/src/pluginFramework/logging/LoggingManager.ts +++ b/apps/heft/src/pluginFramework/logging/LoggingManager.ts @@ -9,7 +9,8 @@ import { import type { ITerminalProvider } from '@rushstack/terminal'; import type { HeftChildReporter } from './HeftChildReporter'; -import { ScopedLogger } from './ScopedLogger'; +import type { ScopedLogger } from './ScopedLogger'; + export interface ILoggingManagerOptions { terminalProvider: ITerminalProvider; childReporter?: HeftChildReporter; @@ -51,7 +52,11 @@ export class LoggingManager { if (existingScopedLogger) { throw new Error(`A named logger with name ${JSON.stringify(loggerName)} has already been requested.`); } else { - const scopedLogger: ScopedLogger = new ScopedLogger({ + // The logger implementation is only loaded once the first logger is requested + const { ScopedLogger: ScopedLoggerClass } = require('./ScopedLogger') as { + ScopedLogger: typeof ScopedLogger; + }; + const scopedLogger: ScopedLogger = new ScopedLoggerClass({ loggerName, terminalProvider: this.#options.terminalProvider, getShouldPrintStacks: () => this.#shouldPrintStacks, diff --git a/apps/heft/src/pluginFramework/logging/test/MockScopedLogger.test.ts b/apps/heft/src/pluginFramework/logging/test/MockScopedLogger.test.ts new file mode 100644 index 00000000000..a9b4edbb214 --- /dev/null +++ b/apps/heft/src/pluginFramework/logging/test/MockScopedLogger.test.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; + +import { MockScopedLogger } from '../MockScopedLogger'; + +// MockScopedLogger is deep-imported by plugin test suites ("@rushstack/heft/lib/pluginFramework/logging/MockScopedLogger"), +// so its observable behavior is part of Heft's de-facto public surface. +describe(MockScopedLogger.name, () => { + it('records errors and warnings without writing to the terminal', () => { + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(); + const logger: MockScopedLogger = new MockScopedLogger(new Terminal(terminalProvider)); + + expect(logger.loggerName).toBe('mockLogger'); + expect(logger.hasErrors).toBe(false); + + const error: Error = new Error('an error'); + const warning: Error = new Error('a warning'); + logger.emitError(error); + logger.emitWarning(warning); + + expect(logger.hasErrors).toBe(true); + expect(logger.errors).toEqual([error]); + expect(logger.warnings).toEqual([warning]); + expect(terminalProvider.getOutput()).toBe(''); + expect(terminalProvider.getErrorOutput()).toBe(''); + }); + + it('resets errors and warnings in place', () => { + const logger: MockScopedLogger = new MockScopedLogger(new Terminal(new StringBufferTerminalProvider())); + const errors: Error[] = logger.errors; + const warnings: Error[] = logger.warnings; + logger.emitError(new Error('e')); + logger.emitWarning(new Error('w')); + + logger.resetErrorsAndWarnings(); + + expect(logger.hasErrors).toBe(false); + expect(logger.errors).toBe(errors); + expect(logger.warnings).toBe(warnings); + expect(errors).toHaveLength(0); + expect(warnings).toHaveLength(0); + }); + + it('exposes the provided terminal', () => { + const terminalProvider: StringBufferTerminalProvider = new StringBufferTerminalProvider(); + const terminal: Terminal = new Terminal(terminalProvider); + const logger: MockScopedLogger = new MockScopedLogger(terminal); + + logger.terminal.writeLine('hello'); + + expect(logger.terminal).toBe(terminal); + expect(terminalProvider.getOutput()).toBe('hello[n]'); + }); +}); diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index 2c9d47347a1..bc73580b3aa 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { createHash } from 'node:crypto'; +import type * as crypto from 'node:crypto'; import type * as fs from 'node:fs'; import * as path from 'node:path'; @@ -9,22 +9,39 @@ import { AlreadyExistsBehavior, FileSystem, Async } from '@rushstack/node-core-l import type { ITerminal } from '@rushstack/terminal'; import { Constants } from '../utilities/Constants'; -import { +import type { asAbsoluteFileSelectionSpecifier, getFileSelectionSpecifierPathsAsync, - type IFileSelectionSpecifier + IFileSelectionSpecifier } from './FileGlobSpecifier'; import type { HeftConfiguration } from '../configuration/HeftConfiguration'; import type { IHeftTaskPlugin } from '../pluginFramework/IHeftPlugin'; import type { IHeftTaskSession, IHeftTaskFileOperations } from '../pluginFramework/HeftTaskSession'; import type { WatchFileSystemAdapter } from '../utilities/WatchFileSystemAdapter'; -import { - type IIncrementalBuildInfo, +import type { + IIncrementalBuildInfo, makePathRelative, tryReadBuildInfoAsync, writeBuildInfoAsync } from '../pluginFramework/IncrementalBuildInfo'; +// The globbing and incremental build info helpers are only needed once files are actually copied, so they are +// not loaded when the plugin is loaded and applied. +function getFileGlobSpecifierModule(): { + asAbsoluteFileSelectionSpecifier: typeof asAbsoluteFileSelectionSpecifier; + getFileSelectionSpecifierPathsAsync: typeof getFileSelectionSpecifierPathsAsync; +} { + return require('./FileGlobSpecifier'); +} + +function getIncrementalBuildInfoModule(): { + makePathRelative: typeof makePathRelative; + tryReadBuildInfoAsync: typeof tryReadBuildInfoAsync; + writeBuildInfoAsync: typeof writeBuildInfoAsync; +} { + return require('../pluginFramework/IncrementalBuildInfo'); +} + /** * Used to specify a selection of files to copy from a specific source folder to one * or more destination folders. @@ -84,7 +101,7 @@ export function asAbsoluteCopyOperation( rootFolderPath: string, copyOperation: ICopyOperation ): ICopyOperation { - const absoluteCopyOperation: ICopyOperation = asAbsoluteFileSelectionSpecifier( + const absoluteCopyOperation: ICopyOperation = getFileGlobSpecifierModule().asAbsoluteFileSelectionSpecifier( rootFolderPath, copyOperation ); @@ -98,12 +115,13 @@ export function asRelativeCopyOperation( rootFolderPath: string, copyOperation: ICopyOperation ): ICopyOperation { + const { makePathRelative: makeRelative } = getIncrementalBuildInfoModule(); return { ...copyOperation, destinationFolders: copyOperation.destinationFolders.map((folder) => - makePathRelative(folder, rootFolderPath) + makeRelative(folder, rootFolderPath) ), - sourcePath: copyOperation.sourcePath && makePathRelative(copyOperation.sourcePath, rootFolderPath) + sourcePath: copyOperation.sourcePath && makeRelative(copyOperation.sourcePath, rootFolderPath) }; } @@ -136,10 +154,11 @@ async function _getCopyDescriptorsAsync( // "sourcePath" is required to be a folder. To copy a single file, put the parent folder in "sourcePath" // and the filename in "includeGlobs". const sourceFolder: string = copyConfiguration.sourcePath!; - const sourceFiles: Map = await getFileSelectionSpecifierPathsAsync({ - fileGlobSpecifier: copyConfiguration, - fileSystemAdapter - }); + const sourceFiles: Map = + await getFileGlobSpecifierModule().getFileSelectionSpecifierPathsAsync({ + fileGlobSpecifier: copyConfiguration, + fileSystemAdapter + }); // Dedupe and throw if a double-write is detected for (const destinationFolderPath of copyConfiguration.destinationFolders) { @@ -196,7 +215,9 @@ async function _copyFilesInnerAsync( return; } - let oldBuildInfo: IIncrementalBuildInfo | undefined = await tryReadBuildInfoAsync(buildInfoPath); + const { tryReadBuildInfoAsync: tryReadBuildInfo, writeBuildInfoAsync: writeBuildInfo } = + getIncrementalBuildInfoModule(); + let oldBuildInfo: IIncrementalBuildInfo | undefined = await tryReadBuildInfo(buildInfoPath); if (oldBuildInfo && oldBuildInfo.configHash !== configHash) { terminal.writeVerboseLine(`File copy configuration changed, discarding incremental state.`); oldBuildInfo = undefined; @@ -216,6 +237,8 @@ async function _copyFilesInnerAsync( allInputFiles.add(copyDescriptor.sourcePath); } + // node:crypto is comparatively expensive to load, so only load it once there is something to hash. + const { createHash }: typeof crypto = require('node:crypto'); await Async.forEachAsync( allInputFiles, async (inputFilePath: string) => { @@ -284,7 +307,7 @@ async function _copyFilesInnerAsync( `linked ${linkedFileCount} file${linkedFileCount === 1 ? '' : 's'}` ); - await writeBuildInfoAsync(buildInfo, buildInfoPath); + await writeBuildInfo(buildInfo, buildInfoPath); } const PLUGIN_NAME: 'copy-files-plugin' = 'copy-files-plugin'; diff --git a/apps/heft/src/plugins/DeleteFilesPlugin.ts b/apps/heft/src/plugins/DeleteFilesPlugin.ts index 73976dad8f4..249e1ec1452 100644 --- a/apps/heft/src/plugins/DeleteFilesPlugin.ts +++ b/apps/heft/src/plugins/DeleteFilesPlugin.ts @@ -7,15 +7,24 @@ import { FileSystem, Async } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import { Constants } from '../utilities/Constants'; -import { +import type { getFileSelectionSpecifierPathsAsync, asAbsoluteFileSelectionSpecifier, - type IFileSelectionSpecifier + IFileSelectionSpecifier } from './FileGlobSpecifier'; import type { HeftConfiguration } from '../configuration/HeftConfiguration'; import type { IHeftTaskPlugin } from '../pluginFramework/IHeftPlugin'; import type { IHeftTaskSession, IHeftTaskFileOperations } from '../pluginFramework/HeftTaskSession'; +// The globbing helpers are only needed once files are actually deleted, so they are not loaded when the +// plugin is loaded and applied. +function getFileGlobSpecifierModule(): { + asAbsoluteFileSelectionSpecifier: typeof asAbsoluteFileSelectionSpecifier; + getFileSelectionSpecifierPathsAsync: typeof getFileSelectionSpecifierPathsAsync; +} { + return require('./FileGlobSpecifier'); +} + /** * Used to specify a selection of source files to delete from the specified source folder. * @@ -40,17 +49,18 @@ async function _getPathsToDeleteAsync( filesToDelete: new Set(), foldersToDelete: new Set() }; + const { + asAbsoluteFileSelectionSpecifier: asAbsoluteSpecifier, + getFileSelectionSpecifierPathsAsync: getSpecifierPathsAsync + } = getFileGlobSpecifierModule(); await Async.forEachAsync( deleteOperations, async (deleteOperation: IDeleteOperation) => { - const absoluteSpecifier: IDeleteOperation = asAbsoluteFileSelectionSpecifier( - rootFolderPath, - deleteOperation - ); + const absoluteSpecifier: IDeleteOperation = asAbsoluteSpecifier(rootFolderPath, deleteOperation); // Glob the files under the source path and add them to the set of files to delete - const sourcePaths: Map = await getFileSelectionSpecifierPathsAsync({ + const sourcePaths: Map = await getSpecifierPathsAsync({ fileGlobSpecifier: absoluteSpecifier, includeFolders: true }); diff --git a/apps/heft/src/plugins/FileGlobSpecifier.ts b/apps/heft/src/plugins/FileGlobSpecifier.ts index 96d3223dd5a..facaff7a65f 100644 --- a/apps/heft/src/plugins/FileGlobSpecifier.ts +++ b/apps/heft/src/plugins/FileGlobSpecifier.ts @@ -4,11 +4,43 @@ import type * as fs from 'node:fs'; import * as path from 'node:path'; -import glob, { type FileSystemAdapter, type Entry } from 'fast-glob'; +import type { default as FastGlob, FileSystemAdapter, Entry } from 'fast-glob'; import { Async } from '@rushstack/node-core-library'; import type { IWatchFileSystemAdapter, IWatchedFileState } from '../utilities/WatchFileSystemAdapter'; +import { trySimpleGlobAsync } from './SimpleGlob'; + +let _fastGlob: typeof FastGlob | undefined; + +/** + * fast-glob is expensive to load (~70 modules), so it is only loaded when a glob is actually evaluated. + * This returns the same function object that `import glob from 'fast-glob'` would provide. + */ +export function getFastGlob(): typeof FastGlob { + if (!_fastGlob) { + _fastGlob = require('fast-glob') as typeof FastGlob; + } + return _fastGlob; +} + +/** + * Returns the `glob` function exported by fast-glob. + */ +export function getGlobFn(): GlobFn { + return getFastGlob().glob; +} + +// Characters that fast-glob's escapePath() may rewrite on any platform. A string that contains none of +// these characters is returned unchanged by escapePath(), so fast-glob does not need to be loaded. +const POSSIBLE_GLOB_SYMBOLS_REGEXP: RegExp = /[()*?[\]{|}!+@\\]/; + +function escapePath(pattern: string): string { + if (typeof pattern === 'string' && pattern.length > 0 && !POSSIBLE_GLOB_SYMBOLS_REGEXP.test(pattern)) { + return pattern; + } + return getFastGlob().escapePath(pattern); +} /** * Used to specify a selection of one or more files. @@ -118,7 +150,7 @@ export async function watchGlobAsync( throw new Error(`"cwd" must be set in the options passed to "watchGlobAsync"`); } - const rawFiles: string[] = await glob(pattern, options); + const rawFiles: string[] = await getFastGlob()(pattern, options); const results: Map = new Map(); await Async.forEachAsync( @@ -141,7 +173,23 @@ export async function getFileSelectionSpecifierPathsAsync( options: IGetFileSelectionSpecifierPathsOptions ): Promise> { const { fileGlobSpecifier, includeFolders, fileSystemAdapter } = options; - const rawEntries: Entry[] = await glob(fileGlobSpecifier.includeGlobs!, { + const { excludeGlobs } = fileGlobSpecifier; + if ( + !fileSystemAdapter && + (excludeGlobs === undefined || (Array.isArray(excludeGlobs) && !excludeGlobs.length)) + ) { + // Common patterns can be evaluated with identical results without loading fast-glob + const simpleGlobResults: Map | undefined = await trySimpleGlobAsync( + fileGlobSpecifier.includeGlobs, + fileGlobSpecifier.sourcePath, + !includeFolders + ); + if (simpleGlobResults) { + return simpleGlobResults; + } + } + + const rawEntries: Entry[] = await getFastGlob()(fileGlobSpecifier.includeGlobs!, { fs: fileSystemAdapter, cwd: fileGlobSpecifier.sourcePath, ignore: fileGlobSpecifier.excludeGlobs, @@ -205,7 +253,7 @@ function getIncludedGlobPatterns(fileGlobSpecifier: IFileSelectionSpecifier): st escapedFileExtension = fileExtension; } - escapedFileExtension = glob.escapePath(escapedFileExtension); + escapedFileExtension = escapePath(escapedFileExtension); escapedFileExtensions.add(escapedFileExtension); } diff --git a/apps/heft/src/plugins/NodeServicePlugin.ts b/apps/heft/src/plugins/NodeServicePlugin.ts index d66c97ca695..2ccdf5864a9 100644 --- a/apps/heft/src/plugins/NodeServicePlugin.ts +++ b/apps/heft/src/plugins/NodeServicePlugin.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as child_process from 'node:child_process'; +import type * as child_process from 'node:child_process'; import * as process from 'node:process'; import { InternalError, SubprocessTerminator } from '@rushstack/node-core-library'; @@ -289,7 +289,9 @@ export default class NodeServicePlugin implements IHeftTaskPlugin { this.#clearTimeout(); this.#logger.terminal.writeLine(`Invoking command: "${this.#shellCommand!}"`); - const childProcess: child_process.ChildProcess = child_process.spawn(this.#shellCommand!, { + // node:child_process is only loaded once the service actually needs to be launched. + const { spawn }: typeof child_process = require('node:child_process'); + const childProcess: child_process.ChildProcess = spawn(this.#shellCommand!, { shell: true, ...SubprocessTerminator.RECOMMENDED_OPTIONS }); diff --git a/apps/heft/src/plugins/SimpleGlob.ts b/apps/heft/src/plugins/SimpleGlob.ts new file mode 100644 index 00000000000..8d40fca4893 --- /dev/null +++ b/apps/heft/src/plugins/SimpleGlob.ts @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +/** + * This module evaluates a small subset of glob patterns without loading fast-glob (which costs ~70 modules + * at startup). It is used for the patterns that Heft generates most commonly (clean operations, file + * extension selections, etc.) and produces exactly the entries that fast-glob would produce when called by + * `getFileSelectionSpecifierPathsAsync()` with `{ dot: true, absolute: true, objectMode: true }` and no custom + * file system adapter or ignore patterns. + * + * Whenever a pattern, a file system entry, or an error falls outside of what is modeled here (for example + * symbolic links, unusual characters, or any error other than a missing path), `undefined` is returned and the + * caller falls back to fast-glob, which then produces its usual results or errors. Since evaluating globs has no + * side effects, falling back after a partial evaluation is safe. + */ + +type ISimplePattern = + // A pattern without glob syntax, e.g. "lib" or "temp/build". fast-glob stats these directly. + | { kind: 'literal'; relativePath: string } + // "**/*" (recursive) or "*" (top level only) + | { kind: 'any'; recursive: boolean } + // "**/*", e.g. "**/*.json": matches entries at any depth whose name ends with the suffix + | { kind: 'suffix'; suffix: string } + // "*", e.g. "build.*": matches top-level entries whose name starts with the prefix + | { kind: 'prefix'; prefix: string }; + +// A literal path segment that fast-glob treats as static and matches verbatim. +const LITERAL_SEGMENT_REGEXP: RegExp = /^[A-Za-z0-9_.-]+$/; +const DOTS_ONLY_REGEXP: RegExp = /^\.+$/; +// A file extension (without the leading dot) that has no special meaning in globs or brace expansions. +const EXTENSION_REGEXP: RegExp = /^[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*$/; +const RECURSIVE_EXTENSION_PATTERN_REGEXP: RegExp = /^\*\*\/\*\.([^/{}]+)$/; +const RECURSIVE_EXTENSION_LIST_PATTERN_REGEXP: RegExp = /^\*\*\/\*\.\{([^/{}]+)\}$/; +const PREFIX_PATTERN_REGEXP: RegExp = /^([A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.)\*$/; +// Entry names that fast-glob may treat differently: line terminators are not matched by "." in the regular +// expressions generated by micromatch, and backslashes are rewritten by fast-glob's path normalization. +const UNSUPPORTED_NAME_CHARACTERS_REGEXP: RegExp = /[\\\n\r\u2028\u2029]/; + +function parsePattern(pattern: unknown): ISimplePattern[] | undefined { + if (typeof pattern !== 'string') { + return undefined; + } + + if (pattern === '**/*') { + return [{ kind: 'any', recursive: true }]; + } + + if (pattern === '*') { + return [{ kind: 'any', recursive: false }]; + } + + let match: RegExpExecArray | null = RECURSIVE_EXTENSION_PATTERN_REGEXP.exec(pattern); + if (match) { + const extension: string = match[1]; + return EXTENSION_REGEXP.test(extension) ? [{ kind: 'suffix', suffix: `.${extension}` }] : undefined; + } + + match = RECURSIVE_EXTENSION_LIST_PATTERN_REGEXP.exec(pattern); + if (match) { + // fast-glob expands braces into one pattern per alternative + const extensions: string[] = match[1].split(','); + if (extensions.length < 2 || !extensions.every((extension) => EXTENSION_REGEXP.test(extension))) { + return undefined; + } + return extensions.map((extension) => ({ kind: 'suffix', suffix: `.${extension}` })); + } + + match = PREFIX_PATTERN_REGEXP.exec(pattern); + if (match) { + return [{ kind: 'prefix', prefix: match[1] }]; + } + + const segments: string[] = pattern.split('/'); + for (const segment of segments) { + if (!LITERAL_SEGMENT_REGEXP.test(segment) || DOTS_ONLY_REGEXP.test(segment)) { + return undefined; + } + } + + return [{ kind: 'literal', relativePath: pattern }]; +} + +/** + * Same shape as the objects that fast-glob creates for entries that it obtains via `fs.lstat()`. + */ +class DirentFromStats { + public readonly name: string; + public readonly isBlockDevice: () => boolean; + public readonly isCharacterDevice: () => boolean; + public readonly isDirectory: () => boolean; + public readonly isFIFO: () => boolean; + public readonly isFile: () => boolean; + public readonly isSocket: () => boolean; + public readonly isSymbolicLink: () => boolean; + + public constructor(name: string, stats: fs.Stats) { + this.name = name; + this.isBlockDevice = stats.isBlockDevice.bind(stats); + this.isCharacterDevice = stats.isCharacterDevice.bind(stats); + this.isDirectory = stats.isDirectory.bind(stats); + this.isFIFO = stats.isFIFO.bind(stats); + this.isFile = stats.isFile.bind(stats); + this.isSocket = stats.isSocket.bind(stats); + this.isSymbolicLink = stats.isSymbolicLink.bind(stats); + } +} + +function isNotExistError(error: NodeJS.ErrnoException): boolean { + return error.code === 'ENOENT'; +} + +function lstatAsync(filePath: string): Promise { + return new Promise((resolve) => { + fs.lstat(filePath, (error: NodeJS.ErrnoException | null, stats: fs.Stats) => { + if (error) { + // A missing path produces no entry; any other error is left to fast-glob. + resolve(isNotExistError(error) ? undefined : false); + } else { + resolve(stats); + } + }); + }); +} + +function readFolderAsync(folderPath: string): Promise { + return new Promise((resolve) => { + fs.readdir(folderPath, { withFileTypes: true }, (error: NodeJS.ErrnoException | null, dirents) => { + if (error) { + resolve(isNotExistError(error) ? undefined : false); + } else { + resolve(dirents); + } + }); + }); +} + +// Same as the path joining performed by fast-glob's directory walker, which always uses '/'. +function joinPathSegments(a: string, b: string): string { + if (a === '') { + return b; + } + return a.endsWith('/') ? a + b : `${a}/${b}`; +} + +// Same as the absolute path transformation performed by fast-glob. +function toAbsolutePath(cwd: string, relativePath: string): string { + return path.resolve(cwd, relativePath).replace(/\\/g, '/'); +} + +interface IFolderToRead { + folderPath: string; + relativeFolderPath: string; +} + +/** + * Returns the same entries that fast-glob would return (in objectMode, with `dot: true` and `absolute: true`) + * for the specified patterns, or `undefined` if the patterns or the file system contents are not supported. + */ +export async function trySimpleGlobAsync( + patterns: unknown, + cwd: unknown, + onlyFiles: boolean +): Promise | undefined> { + if (!Array.isArray(patterns) || patterns.length === 0 || typeof cwd !== 'string' || cwd === '') { + return undefined; + } + + const literalPaths: string[] = []; + let matchAnyRecursive: boolean = false; + let matchAnyTopLevel: boolean = false; + const suffixes: string[] = []; + const prefixes: string[] = []; + for (const pattern of patterns) { + const simplePatterns: ISimplePattern[] | undefined = parsePattern(pattern); + if (!simplePatterns) { + return undefined; + } + + for (const simplePattern of simplePatterns) { + switch (simplePattern.kind) { + case 'literal': + literalPaths.push(simplePattern.relativePath); + break; + case 'any': + if (simplePattern.recursive) { + matchAnyRecursive = true; + } else { + matchAnyTopLevel = true; + } + break; + case 'suffix': + suffixes.push(simplePattern.suffix); + break; + case 'prefix': + prefixes.push(simplePattern.prefix); + break; + } + } + } + + const isRecursive: boolean = matchAnyRecursive || suffixes.length > 0; + const hasDynamicPatterns: boolean = isRecursive || matchAnyTopLevel || prefixes.length > 0; + + // Relative path -> entry. fast-glob de-duplicates entries by their relative path. + const entriesByRelativePath: Map = new Map(); + + const literalStats: (fs.Stats | undefined | false)[] = await Promise.all( + literalPaths.map((literalPath: string) => lstatAsync(path.resolve(cwd, literalPath))) + ); + for (let i: number = 0; i < literalPaths.length; i++) { + const stats: fs.Stats | undefined | false = literalStats[i]; + if (stats === false || (stats && stats.isSymbolicLink())) { + return undefined; + } + if (stats && (!onlyFiles || stats.isFile())) { + const literalPath: string = literalPaths[i]; + if (!entriesByRelativePath.has(literalPath)) { + entriesByRelativePath.set( + literalPath, + new DirentFromStats(literalPath, stats) as unknown as fs.Dirent + ); + } + } + } + + if (hasDynamicPatterns) { + let foldersToRead: IFolderToRead[] = [{ folderPath: path.resolve(cwd), relativeFolderPath: '' }]; + let isTopLevel: boolean = true; + while (foldersToRead.length > 0) { + const folderContents: (fs.Dirent[] | undefined | false)[] = await Promise.all( + foldersToRead.map(({ folderPath }) => readFolderAsync(folderPath)) + ); + + const nextFoldersToRead: IFolderToRead[] = []; + for (let i: number = 0; i < foldersToRead.length; i++) { + const dirents: fs.Dirent[] | undefined | false = folderContents[i]; + if (dirents === false) { + return undefined; + } + if (dirents === undefined) { + if (isTopLevel) { + // fast-glob ignores a missing root folder + continue; + } + // A folder was removed while it was being read + return undefined; + } + + const { folderPath, relativeFolderPath } = foldersToRead[i]; + for (const dirent of dirents) { + const { name } = dirent; + if (dirent.isSymbolicLink() || UNSUPPORTED_NAME_CHARACTERS_REGEXP.test(name)) { + return undefined; + } + + const relativePath: string = joinPathSegments(relativeFolderPath, name); + const isMatch: boolean = + matchAnyRecursive || + (isTopLevel && matchAnyTopLevel) || + suffixes.some((suffix: string) => name.endsWith(suffix)) || + (isTopLevel && prefixes.some((prefix: string) => name.startsWith(prefix))); + if (isMatch && (!onlyFiles || dirent.isFile()) && !entriesByRelativePath.has(relativePath)) { + entriesByRelativePath.set(relativePath, dirent); + } + + if (isRecursive && dirent.isDirectory()) { + nextFoldersToRead.push({ + folderPath: joinPathSegments(folderPath, name), + relativeFolderPath: relativePath + }); + } + } + } + + foldersToRead = nextFoldersToRead; + isTopLevel = false; + } + } + + const results: Map = new Map(); + for (const [relativePath, dirent] of entriesByRelativePath) { + results.set(toAbsolutePath(cwd, relativePath), dirent); + } + return results; +} diff --git a/apps/heft/src/plugins/test/SimpleGlob.test.ts b/apps/heft/src/plugins/test/SimpleGlob.test.ts new file mode 100644 index 00000000000..3ff50a16258 --- /dev/null +++ b/apps/heft/src/plugins/test/SimpleGlob.test.ts @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import glob from 'fast-glob'; + +import { trySimpleGlobAsync } from '../SimpleGlob'; + +// These tests verify that trySimpleGlobAsync() returns exactly what fast-glob returns for the options used by +// getFileSelectionSpecifierPathsAsync(), and that it declines (returns undefined) when it cannot guarantee that. + +const TREE: string[] = [ + 'a.txt', + '.txt', + '..txt', + 'b.json', + 'x.d.ts', + 'd.ts', + 'build/a.txt', + 'build.x/b.txt', + 'build./c.json', + 'build.json', + 'lib/nested/deep/file.txt', + 'lib/nested/.hidden/file.json', + 'lib/nested/.hidden/.dotfile', + '.cache/entry.txt', + 'temp/build/out.d.ts', + 'temp/test/out.js', + 'with space.txt', + 'emptydir/' +]; + +function createTree(root: string): void { + for (const entry of TREE) { + const fullPath: string = path.join(root, entry); + if (entry.endsWith('/')) { + fs.mkdirSync(fullPath, { recursive: true }); + } else { + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.writeFileSync(fullPath, entry); + } + } +} + +async function fastGlobAsync(patterns: string[], cwd: string, onlyFiles: boolean): Promise { + const entries: glob.Entry[] = await glob(patterns, { + cwd, + onlyFiles, + dot: true, + absolute: true, + objectMode: true + }); + return describeResults(new Map(entries.map((entry) => [entry.path, entry.dirent as fs.Dirent]))); +} + +function describeResults(results: Map): string[] { + return Array.from(results, ([filePath, dirent]) => + [filePath, dirent.name, dirent.isFile(), dirent.isDirectory(), dirent.isSymbolicLink()].join('|') + ).sort(); +} + +describe('trySimpleGlobAsync', () => { + let root: string; + + beforeAll(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'heft-simple-glob-')); + createTree(root); + }); + + afterAll(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + const supportedPatternSets: string[][] = [ + ['**/*'], + ['*'], + ['**/*.txt'], + ['**/*.d.ts'], + ['**/*.{txt,json}'], + ['build.*'], + ['build', 'build.*'], + ['lib', 'temp/build', 'nonexistent', '.cache'], + ['build.json', '**/*.json'], + ['a.txt', 'a.txt'], + ['*', '**/*.txt', 'temp/test'] + ]; + + for (const patterns of supportedPatternSets) { + for (const onlyFiles of [true, false]) { + it(`matches fast-glob for ${JSON.stringify(patterns)} (onlyFiles: ${onlyFiles})`, async () => { + const expected: string[] = await fastGlobAsync(patterns, root, onlyFiles); + const actual: Map | undefined = await trySimpleGlobAsync( + patterns, + root, + onlyFiles + ); + expect(actual).toBeDefined(); + expect(describeResults(actual!)).toEqual(expected); + }); + } + } + + it('matches fast-glob when the root folder does not exist', async () => { + const cwd: string = path.join(root, 'does-not-exist'); + const actual: Map | undefined = await trySimpleGlobAsync(['**/*', 'lib'], cwd, false); + expect(describeResults(actual!)).toEqual(await fastGlobAsync(['**/*', 'lib'], cwd, false)); + }); + + const unsupportedPatternSets: string[][] = [ + ['*.txt'], + ['**'], + ['lib/**/*.txt'], + ['!lib'], + ['[ab].txt'], + ['**/*.{txt}'], + ['./lib'], + ['../lib'], + ['lib/'], + ['b*'], + ['**/*.t?t'], + [''] + ]; + + for (const patterns of unsupportedPatternSets) { + it(`declines ${JSON.stringify(patterns)}`, async () => { + expect(await trySimpleGlobAsync(patterns, root, false)).toBeUndefined(); + }); + } + + it('declines when a symbolic link is encountered', async () => { + const linkRoot: string = fs.mkdtempSync(path.join(os.tmpdir(), 'heft-simple-glob-link-')); + try { + fs.writeFileSync(path.join(linkRoot, 'file.txt'), ''); + try { + fs.symlinkSync(path.join(linkRoot, 'file.txt'), path.join(linkRoot, 'link.txt')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EPERM') { + // Creating symbolic links requires elevated privileges on some Windows configurations + return; + } + throw error; + } + expect(await trySimpleGlobAsync(['**/*'], linkRoot, false)).toBeUndefined(); + expect(await trySimpleGlobAsync(['link.txt'], linkRoot, false)).toBeUndefined(); + } finally { + fs.rmSync(linkRoot, { recursive: true, force: true }); + } + }); + + it('declines when the root is not a folder', async () => { + expect(await trySimpleGlobAsync(['**/*'], path.join(root, 'a.txt'), false)).toBeUndefined(); + }); +}); diff --git a/apps/heft/src/schemas/README-ModifyingSchemas.md b/apps/heft/src/schemas/README-ModifyingSchemas.md index 1819095e9d0..98ee01e64e0 100644 --- a/apps/heft/src/schemas/README-ModifyingSchemas.md +++ b/apps/heft/src/schemas/README-ModifyingSchemas.md @@ -2,3 +2,14 @@ If you change the Heft schemas, be sure to update the example files under **schemas/templates**. The templates are used as a reference when updating the website documentation. + +## Startup performance + +At startup, Heft validates **heft.json**, every **heft-plugin.json** and every plugin's options without +compiling the schemas with ajv, using the fast path in **src/configuration/lean/**. The fast path only +accepts data when it can prove that the original `JsonSchema` validation would accept it; otherwise the +original validation runs (and produces the usual error messages). + +The schemas in this folder must stay within the subset of JSON schema that the fast path supports +(see `LeanJsonSchema.ts`), otherwise every Heft invocation will load and compile ajv again. The unit test +**src/configuration/test/LeanJsonSchema.test.ts** fails if a schema in this folder is not supported. diff --git a/apps/heft/src/start.ts b/apps/heft/src/start.ts index 0fa5a60c557..0cbaeb5d7d8 100644 --- a/apps/heft/src/start.ts +++ b/apps/heft/src/start.ts @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { HeftCommandLineParser } from './cli/HeftCommandLineParser'; - // Launching via lib-commonjs/start.js bypasses the version selector. Use that for debugging Heft. +// Enable the compile cache and the module resolution cache before the rest of Heft is loaded +import './bootstrap/enableStartupCaches'; +import { HeftCommandLineParser } from './cli/HeftCommandLineParser'; + const parser: HeftCommandLineParser = new HeftCommandLineParser(); parser diff --git a/apps/heft/src/startWithVersionSelector.ts b/apps/heft/src/startWithVersionSelector.ts index e9cc3b950f6..697e84087be 100644 --- a/apps/heft/src/startWithVersionSelector.ts +++ b/apps/heft/src/startWithVersionSelector.ts @@ -5,43 +5,56 @@ // NOTE: Since startWithVersionSelector.ts is loaded in the same process as start.ts, any dependencies that // we import here may become side-by-side versions. We want to minimize any dependencies. -import * as path from 'node:path'; -import * as fs from 'node:fs'; +// This file is on the startup path of every Heft invocation, so it intentionally only uses Node.js +// built-in modules (named imports avoid the interop helpers) and inlines the few constants it needs. +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; import type { IPackageJson } from '@rushstack/node-core-library'; -import { getToolParameterNamesFromArgs } from './utilities/CliUtilities'; -import { Constants } from './utilities/Constants'; +// These must match the values in ./utilities/Constants.ts +const HEFT_PACKAGE_NAME: '@rushstack/heft' = '@rushstack/heft'; +const UNMANAGED_PARAMETER_LONG_NAME: '--unmanaged' = '--unmanaged'; +const DEBUG_PARAMETER_LONG_NAME: '--debug' = '--debug'; // Excerpted from PackageJsonLookup.tryGetPackageFolderFor() function tryGetPackageFolderFor(resolvedFileOrFolderPath: string): string | undefined { - // Two lookups are required, because get() cannot distinguish the undefined value - // versus a missing key. - // if (this._packageFolderCache.has(resolvedFileOrFolderPath)) { - // return this._packageFolderCache.get(resolvedFileOrFolderPath); - // } - - // Is resolvedFileOrFolderPath itself a folder with a package.json file? If so, return it. - if (fs.existsSync(path.join(resolvedFileOrFolderPath, 'package.json'))) { - // this._packageFolderCache.set(resolvedFileOrFolderPath, resolvedFileOrFolderPath); - return resolvedFileOrFolderPath; - } + // Walk upwards until a folder containing a package.json file is found + let currentFolder: string = resolvedFileOrFolderPath; + for (;;) { + // Is currentFolder itself a folder with a package.json file? If so, return it. + if (existsSync(join(currentFolder, 'package.json'))) { + return currentFolder; + } - // Otherwise go up one level - const parentFolder: string | undefined = path.dirname(resolvedFileOrFolderPath); - if (!parentFolder || parentFolder === resolvedFileOrFolderPath) { - // We reached the root directory without finding a package.json file, - // so cache the negative result - // this._packageFolderCache.set(resolvedFileOrFolderPath, undefined); - return undefined; // no match - } + // Otherwise go up one level + const parentFolder: string | undefined = dirname(currentFolder); + if (!parentFolder || parentFolder === currentFolder) { + // We reached the root directory without finding a package.json file + return undefined; // no match + } - // Recurse upwards, caching every step along the way - const parentResult: string | undefined = tryGetPackageFolderFor(parentFolder); - // Cache the parent's answer as well - // this._packageFolderCache.set(resolvedFileOrFolderPath, parentResult); + currentFolder = parentFolder; + } +} - return parentResult; +/** + * Returns the tool parameter names that precede the action name. This is a copy of + * `getToolParameterNamesFromArgs()` from ./utilities/CliUtilities.ts, inlined to avoid loading extra modules. + */ +function getToolParameterNamesFromArgs(argv: string[] = process.argv): Set { + const toolParameters: Set = new Set(); + // Skip the first two arguments, which are the path to the Node executable and the path to the Heft + // entrypoint. The remaining arguments are the tool arguments. Grab them until we reach a non-"-"-prefixed + // argument. We can do this simple parsing because the Heft tool only has simple optional flags. + for (let i: number = 2; i < argv.length; ++i) { + const arg: string = argv[i]; + if (!arg.startsWith('-')) { + break; + } + toolParameters.add(arg); + } + return toolParameters; } /** @@ -51,18 +64,18 @@ function tryGetPackageFolderFor(resolvedFileOrFolderPath: string): string | unde */ function tryStartLocalHeft(): boolean { const toolParameters: Set = getToolParameterNamesFromArgs(); - if (toolParameters.has(Constants.unmanagedParameterLongName)) { + if (toolParameters.has(UNMANAGED_PARAMETER_LONG_NAME)) { console.log( - `Bypassing the Heft version selector because ${JSON.stringify(Constants.unmanagedParameterLongName)} ` + + `Bypassing the Heft version selector because ${JSON.stringify(UNMANAGED_PARAMETER_LONG_NAME)} ` + 'was specified.' ); console.log(); return false; - } else if (toolParameters.has(Constants.debugParameterLongName)) { + } else if (toolParameters.has(DEBUG_PARAMETER_LONG_NAME)) { // The unmanaged flag could be undiscoverable if it's not in their locally installed version console.log( 'Searching for a locally installed version of Heft. Use the ' + - `${JSON.stringify(Constants.unmanagedParameterLongName)} flag if you want to avoid this.` + `${JSON.stringify(UNMANAGED_PARAMETER_LONG_NAME)} flag if you want to avoid this.` ); } @@ -71,8 +84,8 @@ function tryStartLocalHeft(): boolean { if (projectFolder) { let heftEntryPoint: string; try { - const packageJsonPath: string = path.join(projectFolder, 'package.json'); - const packageJsonContent: string = fs.readFileSync(packageJsonPath).toString(); + const packageJsonPath: string = join(projectFolder, 'package.json'); + const packageJsonContent: string = readFileSync(packageJsonPath).toString(); let packageJson: IPackageJson; try { packageJson = JSON.parse(packageJsonContent); @@ -82,8 +95,8 @@ function tryStartLocalHeft(): boolean { // Does package.json have a dependency on Heft? if ( - !(packageJson.dependencies && packageJson.dependencies[Constants.heftPackageName]) && - !(packageJson.devDependencies && packageJson.devDependencies[Constants.heftPackageName]) + !(packageJson.dependencies && packageJson.dependencies[HEFT_PACKAGE_NAME]) && + !(packageJson.devDependencies && packageJson.devDependencies[HEFT_PACKAGE_NAME]) ) { // No explicit dependency on Heft return false; @@ -91,13 +104,13 @@ function tryStartLocalHeft(): boolean { // To avoid a loading the "resolve" NPM package, let's assume that the Heft dependency must be // installed as "/node_modules/@rushstack/heft". - const heftFolder: string = path.join(projectFolder, 'node_modules', Constants.heftPackageName); + const heftFolder: string = join(projectFolder, 'node_modules', HEFT_PACKAGE_NAME); // Try the new output layout first, then fall back to the legacy layout - const commonJsHeftEntryPoint: string = path.join(heftFolder, 'lib-commonjs', 'start.js'); - if (!fs.existsSync(commonJsHeftEntryPoint)) { - const legacyHeftEntryPoint: string = path.join(heftFolder, 'lib', 'start.js'); - if (!fs.existsSync(legacyHeftEntryPoint)) { + const commonJsHeftEntryPoint: string = join(heftFolder, 'lib-commonjs', 'start.js'); + if (!existsSync(commonJsHeftEntryPoint)) { + const legacyHeftEntryPoint: string = join(heftFolder, 'lib', 'start.js'); + if (!existsSync(legacyHeftEntryPoint)) { throw new Error( `Unable to find Heft entry point: ${commonJsHeftEntryPoint} or ${legacyHeftEntryPoint}` ); diff --git a/apps/heft/src/test/PublicApiContract.test.ts b/apps/heft/src/test/PublicApiContract.test.ts new file mode 100644 index 00000000000..969a27db74b --- /dev/null +++ b/apps/heft/src/test/PublicApiContract.test.ts @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as heftIndex from '../index'; +import { HeftConfiguration } from '../configuration/HeftConfiguration'; +import { MetricsCollector } from '../metrics/MetricsCollector'; + +// These tests pin the *runtime* surface of the package entry point. The type surface is pinned by the +// API report (common/reviews/api/heft.api.md); this guards against bundling/lazy-loading changes that keep the +// types identical but change what `require('@rushstack/heft')` returns at runtime. +describe('@rushstack/heft runtime entry point', () => { + it('exports exactly the expected runtime values', () => { + expect(Object.keys(heftIndex).sort()).toEqual(['HeftConfiguration', '_MetricsCollector']); + expect(typeof heftIndex.HeftConfiguration).toBe('function'); + expect(typeof heftIndex._MetricsCollector).toBe('function'); + }); + + it('re-exports the same objects as the deep-importable modules (no duplicated module state)', () => { + expect(heftIndex.HeftConfiguration).toBe(HeftConfiguration); + expect(heftIndex._MetricsCollector).toBe(MetricsCollector); + }); + + it('keeps the static factory used by the CLI', () => { + expect(typeof HeftConfiguration.initialize).toBe('function'); + }); +}); diff --git a/apps/heft/src/utilities/CoreConfigFiles.ts b/apps/heft/src/utilities/CoreConfigFiles.ts index ff221359a7d..a0c07a2ca1d 100644 --- a/apps/heft/src/utilities/CoreConfigFiles.ts +++ b/apps/heft/src/utilities/CoreConfigFiles.ts @@ -3,19 +3,25 @@ import * as path from 'node:path'; -import { - ProjectConfigurationFile, - InheritanceType, - PathResolutionMethod, - type IJsonPathMetadataResolverOptions -} from '@rushstack/heft-config-file'; -import { Import, PackageJsonLookup, InternalError } from '@rushstack/node-core-library'; +import type { ProjectConfigurationFile, IJsonPathMetadataResolverOptions } from '@rushstack/heft-config-file'; import type { ITerminal } from '@rushstack/terminal'; import type { IRigConfig } from '@rushstack/rig-package'; import type { IDeleteOperation } from '../plugins/DeleteFilesPlugin'; import type { INodeServicePluginConfiguration } from '../plugins/NodeServicePlugin'; import { Constants } from './Constants'; +import { + LeanProjectConfigurationFile, + getLeanPropertyOriginalValue, + type ILeanLoadResult +} from '../configuration/lean/LeanProjectConfigurationFile'; +import { + bail, + getRigProfileFolder, + getSharedLeanPackageJsonLookup, + type LeanPackageJsonLookup +} from '../configuration/lean/LeanResolution'; +import { LeanRigConfig } from '../configuration/lean/LeanRigConfig'; export interface IHeftConfigurationJsonActionReference { actionName: string; @@ -61,6 +67,124 @@ export interface IHeftConfigurationJson { let _heftConfigFileLoader: ProjectConfigurationFile | undefined; let _nodeServiceConfigurationLoader: ProjectConfigurationFile | undefined; +let _leanHeftConfigFileLoader: LeanProjectConfigurationFile | undefined; + +function _isGenuineRigConfig(rigConfig: IRigConfig): boolean { + // A genuine RigConfig can only exist if @rushstack/rig-package has already been loaded + const { RigConfig } = require('@rushstack/rig-package'); + return rigConfig instanceof RigConfig; +} + +/** + * Loads heft.json using the lean loader. Returns `undefined` if the result might differ from the original + * `@rushstack/heft-config-file` loader (including every error condition), in which case the original loader + * must be used. + */ +function _tryLoadHeftConfigurationFileLean( + projectPath: string, + rigConfig: IRigConfig | undefined +): ILeanLoadResult | undefined { + if (!_leanHeftConfigFileLoader) { + const packageJsonLookup: LeanPackageJsonLookup = getSharedLeanPackageJsonLookup(); + let heftPluginPackageFolder: string | undefined; + // Keep in sync with the pluginPackageResolver in loadHeftConfigurationFileForProjectAsync() + const resolvePluginPackage: (propertyValue: string, configurationFilePath: string) => string = ( + propertyValue: string, + configurationFilePath: string + ) => { + if (propertyValue === Constants.heftPackageName) { + if (!heftPluginPackageFolder) { + heftPluginPackageFolder = packageJsonLookup.tryGetPackageFolderFor(__dirname); + } + + if (!heftPluginPackageFolder) { + bail(); + } + + return heftPluginPackageFolder; + } else { + return packageJsonLookup.resolvePackage(propertyValue, path.dirname(configurationFilePath), true); + } + }; + + _leanHeftConfigFileLoader = new LeanProjectConfigurationFile({ + projectRelativeFilePath: `${Constants.projectConfigFolderName}/${Constants.heftConfigurationFilename}`, + jsonSchemaObject: require('../schemas/heft.schema.json'), + propertyInheritanceDefaults: { + array: 'append', + object: 'merge' + }, + customResolvers: [ + { path: ['heftPlugins', '*', 'pluginPackage'], resolve: resolvePluginPackage }, + { + path: ['phasesByName', '*', 'tasksByName', '*', 'taskPlugin', 'pluginPackage'], + resolve: resolvePluginPackage + } + ], + packageJsonLookup, + // Never call rigConfig.getResolvedProfileFolder() here: RigConfig caches the profile folder before checking + // that it exists, so a failed call would change the behavior of the original implementation (which the + // lean path falls back to). For the rig configs created by Heft, this also avoids loading the "resolve" + // package on the startup path. + getRigProfileFolder: (rigConfigToResolve: IRigConfig) => + rigConfigToResolve instanceof LeanRigConfig || _isGenuineRigConfig(rigConfigToResolve) + ? getRigProfileFolder(rigConfigToResolve) + : bail() + }); + } + + return _leanHeftConfigFileLoader.tryLoadConfigurationFileForProject(projectPath, rigConfig); +} + +/** + * The pluginPackage field was resolved to the root of the package, but we also want to have + * the original plugin package name in the config file. + */ +function _normalizeHeftConfigurationFile( + configurationFile: IHeftConfigurationJson, + getOriginalPluginPackage: (rawSpecifier: IHeftConfigurationJsonPluginSpecifier) => string +): IHeftConfigurationJson { + function getUpdatedPluginSpecifier( + rawSpecifier: IHeftConfigurationJsonPluginSpecifier + ): IHeftConfigurationJsonPluginSpecifier { + const pluginPackageName: string = getOriginalPluginPackage(rawSpecifier); + const newSpecifier: IHeftConfigurationJsonPluginSpecifier = { + ...rawSpecifier, + pluginPackageRoot: rawSpecifier.pluginPackage, + pluginPackage: pluginPackageName + }; + return newSpecifier; + } + + const phasesByName: IHeftConfigurationJsonPhases = {}; + + const normalizedConfigurationFile: IHeftConfigurationJson = { + ...configurationFile, + heftPlugins: configurationFile.heftPlugins?.map(getUpdatedPluginSpecifier) ?? [], + phasesByName + }; + + for (const [phaseName, phase] of Object.entries(configurationFile.phasesByName || {})) { + const tasksByName: IHeftConfigurationJsonTasks = {}; + phasesByName[phaseName] = { + ...phase, + tasksByName + }; + + for (const [taskName, task] of Object.entries(phase.tasksByName || {})) { + if (task.taskPlugin) { + tasksByName[taskName] = { + ...task, + taskPlugin: getUpdatedPluginSpecifier(task.taskPlugin) + }; + } else { + tasksByName[taskName] = task; + } + } + } + + return normalizedConfigurationFile; +} export class CoreConfigFiles { public static heftConfigurationProjectRelativeFilePath: string = `${Constants.projectConfigFolderName}/${Constants.heftConfigurationFilename}`; @@ -75,6 +199,60 @@ export class CoreConfigFiles { projectPath: string, rigConfig?: IRigConfig | undefined ): Promise { + const leanResult: IHeftConfigurationJson | undefined = CoreConfigFiles._tryLoadHeftConfigurationFileLean( + terminal, + projectPath, + rigConfig + ); + if (leanResult) { + return leanResult; + } + + // Use the original implementation, which produces the canonical errors + return await CoreConfigFiles._loadHeftConfigurationFileOriginalAsync(terminal, projectPath, rigConfig); + } + + /** + * Loads heft.json without using `@rushstack/heft-config-file`. Returns `undefined` if the result could differ + * from {@link CoreConfigFiles._loadHeftConfigurationFileOriginalAsync}, including for every error condition. + * @internal + */ + public static _tryLoadHeftConfigurationFileLean( + terminal: ITerminal, + projectPath: string, + rigConfig: IRigConfig | undefined + ): IHeftConfigurationJson | undefined { + const leanResult: ILeanLoadResult | undefined = _tryLoadHeftConfigurationFileLean( + projectPath, + rigConfig + ); + if (leanResult) { + for (const message of leanResult.debugMessages) { + terminal.writeDebugLine(message); + } + + return _normalizeHeftConfigurationFile( + leanResult.configurationFile, + (rawSpecifier: IHeftConfigurationJsonPluginSpecifier) => + getLeanPropertyOriginalValue(rawSpecifier, 'pluginPackage')! + ); + } + } + + /** + * Loads heft.json using `@rushstack/heft-config-file` (the original implementation). + * @internal + */ + public static async _loadHeftConfigurationFileOriginalAsync( + terminal: ITerminal, + projectPath: string, + rigConfig: IRigConfig | undefined + ): Promise { + const HeftConfigFile: typeof import('@rushstack/heft-config-file') = await import( + '@rushstack/heft-config-file' + ); + const { Import, PackageJsonLookup, InternalError } = await import('@rushstack/node-core-library'); + if (!_heftConfigFileLoader) { let heftPluginPackageFolder: string | undefined; @@ -110,24 +288,24 @@ export class CoreConfigFiles { const schemaObject: object = await import('../schemas/heft.schema.json'); // eslint-disable-next-line require-atomic-updates - _heftConfigFileLoader = new ProjectConfigurationFile({ + _heftConfigFileLoader = new HeftConfigFile.ProjectConfigurationFile({ projectRelativeFilePath: CoreConfigFiles.heftConfigurationProjectRelativeFilePath, jsonSchemaObject: schemaObject, propertyInheritanceDefaults: { - array: { inheritanceType: InheritanceType.append }, - object: { inheritanceType: InheritanceType.merge } + array: { inheritanceType: HeftConfigFile.InheritanceType.append }, + object: { inheritanceType: HeftConfigFile.InheritanceType.merge } }, jsonPathMetadata: { // Use a custom resolver for the plugin packages, since the NodeResolve algorithm will resolve to the // package.json exports/module property, which may or may not exist. '$.heftPlugins.*.pluginPackage': { - pathResolutionMethod: PathResolutionMethod.custom, + pathResolutionMethod: HeftConfigFile.PathResolutionMethod.custom, customResolver: pluginPackageResolver }, // Use a custom resolver for the plugin packages, since the NodeResolve algorithm will resolve to the // package.json exports/module property, which may or may not exist. '$.phasesByName.*.tasksByName.*.taskPlugin.pluginPackage': { - pathResolutionMethod: PathResolutionMethod.custom, + pathResolutionMethod: HeftConfigFile.PathResolutionMethod.custom, customResolver: pluginPackageResolver } } @@ -158,7 +336,7 @@ export class CoreConfigFiles { // that we follow the "extends" chain for the entire config file. const legacySchemaObject: object = await import('../schemas/heft-legacy.schema.json'); const legacyConfigFileLoader: ProjectConfigurationFile = - new ProjectConfigurationFile({ + new HeftConfigFile.ProjectConfigurationFile({ projectRelativeFilePath: CoreConfigFiles.heftConfigurationProjectRelativeFilePath, jsonSchemaObject: legacySchemaObject }); @@ -177,51 +355,14 @@ export class CoreConfigFiles { ); } - // The pluginPackage field was resolved to the root of the package, but we also want to have - // the original plugin package name in the config file. - function getUpdatedPluginSpecifier( - rawSpecifier: IHeftConfigurationJsonPluginSpecifier - ): IHeftConfigurationJsonPluginSpecifier { - const pluginPackageName: string = heftConfigFileLoader.getPropertyOriginalValue({ - parentObject: rawSpecifier, - propertyName: 'pluginPackage' - })!; - const newSpecifier: IHeftConfigurationJsonPluginSpecifier = { - ...rawSpecifier, - pluginPackageRoot: rawSpecifier.pluginPackage, - pluginPackage: pluginPackageName - }; - return newSpecifier; - } - - const phasesByName: IHeftConfigurationJsonPhases = {}; - - const normalizedConfigurationFile: IHeftConfigurationJson = { - ...configurationFile, - heftPlugins: configurationFile.heftPlugins?.map(getUpdatedPluginSpecifier) ?? [], - phasesByName - }; - - for (const [phaseName, phase] of Object.entries(configurationFile.phasesByName || {})) { - const tasksByName: IHeftConfigurationJsonTasks = {}; - phasesByName[phaseName] = { - ...phase, - tasksByName - }; - - for (const [taskName, task] of Object.entries(phase.tasksByName || {})) { - if (task.taskPlugin) { - tasksByName[taskName] = { - ...task, - taskPlugin: getUpdatedPluginSpecifier(task.taskPlugin) - }; - } else { - tasksByName[taskName] = task; - } - } - } - - return normalizedConfigurationFile; + return _normalizeHeftConfigurationFile( + configurationFile, + (rawSpecifier: IHeftConfigurationJsonPluginSpecifier) => + heftConfigFileLoader.getPropertyOriginalValue({ + parentObject: rawSpecifier, + propertyName: 'pluginPackage' + })! + ); } public static async tryLoadNodeServiceConfigurationFileAsync( @@ -230,9 +371,12 @@ export class CoreConfigFiles { rigConfig?: IRigConfig | undefined ): Promise { if (!_nodeServiceConfigurationLoader) { + const HeftConfigFile: typeof import('@rushstack/heft-config-file') = await import( + '@rushstack/heft-config-file' + ); const schemaObject: object = await import('../schemas/node-service.schema.json'); // eslint-disable-next-line require-atomic-updates - _nodeServiceConfigurationLoader = new ProjectConfigurationFile({ + _nodeServiceConfigurationLoader = new HeftConfigFile.ProjectConfigurationFile({ projectRelativeFilePath: CoreConfigFiles.nodeServiceConfigurationProjectRelativeFilePath, jsonSchemaObject: schemaObject }); diff --git a/apps/heft/src/utilities/test/CliUtilities.test.ts b/apps/heft/src/utilities/test/CliUtilities.test.ts new file mode 100644 index 00000000000..2cb5712807e --- /dev/null +++ b/apps/heft/src/utilities/test/CliUtilities.test.ts @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { getToolParameterNamesFromArgs } from '../CliUtilities'; + +// The version selector and the command line parser both rely on this "tool parameters precede the action" +// scan to find --debug / --unmanaged before the full parser runs. +describe(getToolParameterNamesFromArgs.name, () => { + function scan(...args: string[]): string[] { + return Array.from(getToolParameterNamesFromArgs(['node', 'heft', ...args])); + } + + it('returns the dash-prefixed arguments that precede the action name', () => { + expect(scan('--debug', '--unmanaged', 'build', '--clean')).toEqual(['--debug', '--unmanaged']); + }); + + it('stops at the first argument that does not start with a dash', () => { + expect(scan('build', '--debug')).toEqual([]); + expect(scan('--debug', 'run', '--only', 'build', '--', '--unmanaged')).toEqual(['--debug']); + }); + + it('includes every dash-prefixed token, including unknown flags and "--"', () => { + expect(scan('-h', '--nosuch', '--', '--debug')).toEqual(['-h', '--nosuch', '--', '--debug']); + }); + + it('de-duplicates repeated flags', () => { + expect(scan('--debug', '--debug')).toEqual(['--debug']); + }); + + it('returns an empty set when there are no tool arguments', () => { + expect(scan()).toEqual([]); + }); + + it('skips the node executable and script path', () => { + expect(Array.from(getToolParameterNamesFromArgs(['--debug', '--unmanaged']))).toEqual([]); + }); +}); diff --git a/apps/heft/src/utilities/test/GitUtilities.test.ts b/apps/heft/src/utilities/test/GitUtilities.test.ts index 9b51b6d1f1d..4eca3460177 100644 --- a/apps/heft/src/utilities/test/GitUtilities.test.ts +++ b/apps/heft/src/utilities/test/GitUtilities.test.ts @@ -5,6 +5,9 @@ import * as path from 'node:path'; import { GitUtilities, type GitignoreFilterFn } from '../GitUtilities'; import { PackageJsonLookup } from '@rushstack/node-core-library'; +// These tests spawn git, which can exceed the default 5 second timeout on a heavily loaded machine +jest.setTimeout(60_000); + describe('GitUtilities', () => { describe('checkIgnoreAsync', () => { const projectRoot: string = PackageJsonLookup.instance.tryGetPackageFolderFor(__dirname)!; diff --git a/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorPlugin.ts b/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorPlugin.ts index fb15b8008f8..24347e4026c 100644 --- a/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorPlugin.ts +++ b/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorPlugin.ts @@ -11,7 +11,7 @@ import type { ConfigurationFile } from '@rushstack/heft'; -import { invokeApiExtractorAsync } from './ApiExtractorRunner'; +import type { invokeApiExtractorAsync as InvokeApiExtractorAsync } from './ApiExtractorRunner'; import apiExtractorConfigSchema from './schemas/api-extractor-task.schema.json'; // eslint-disable-next-line @rushstack/no-new-null @@ -204,6 +204,9 @@ export default class ApiExtractorPlugin implements IHeftTaskPlugin { printApiReportDiffOption === 'always' || (printApiReportDiffOption === 'production' && production); // Run API Extractor + // The runner is only loaded when the task runs. + const { invokeApiExtractorAsync }: { invokeApiExtractorAsync: typeof InvokeApiExtractorAsync } = + require('./ApiExtractorRunner'); await invokeApiExtractorAsync({ apiExtractor, apiExtractorConfiguration, diff --git a/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorRunner.ts b/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorRunner.ts index ae5fa860b2d..4dae14e9ca4 100644 --- a/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorRunner.ts +++ b/heft-plugins/heft-api-extractor-plugin/src/ApiExtractorRunner.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as semver from 'semver'; +import type * as semver from 'semver'; import type { IScopedLogger } from '@rushstack/heft'; import { FileError, InternalError } from '@rushstack/node-core-library'; @@ -65,7 +65,9 @@ export async function invokeApiExtractorAsync( terminal.writeLine(`Using API Extractor version ${apiExtractor.Extractor.version}`); - const apiExtractorVersion: semver.SemVer | null = semver.parse(apiExtractor.Extractor.version); + // Deep import: the 'semver' package index loads ~45 modules. The SemVer class is the same object. + const parseSemVer: typeof semver.parse = require('semver/functions/parse'); + const apiExtractorVersion: semver.SemVer | null = parseSemVer(apiExtractor.Extractor.version); if ( !apiExtractorVersion || apiExtractorVersion.major < MIN_SUPPORTED_MAJOR_VERSION || diff --git a/heft-plugins/heft-jest-plugin/src/JestPlugin.ts b/heft-plugins/heft-jest-plugin/src/JestPlugin.ts index 32eefb41729..103a49147dc 100644 --- a/heft-plugins/heft-jest-plugin/src/JestPlugin.ts +++ b/heft-plugins/heft-jest-plugin/src/JestPlugin.ts @@ -9,7 +9,7 @@ import * as path from 'node:path'; import type { AggregatedResult } from '@jest/reporters'; import type { Config } from '@jest/types'; -import { resolveRunner, resolveSequencer, resolveTestEnvironment, resolveWatchPlugin } from 'jest-resolve'; +import type * as JestResolveModule from 'jest-resolve'; import type { HeftConfiguration, @@ -970,7 +970,10 @@ function _getJsonPathMetadata( return path.join(PLUGIN_PACKAGE_FOLDER, restOfPath); } - // Use the Jest-provided resolvers to resolve the module paths + // Use the Jest-provided resolvers to resolve the module paths. "jest-resolve" (and its dependencies) is only + // loaded when a Jest configuration file is processed. + const { resolveRunner, resolveSequencer, resolveTestEnvironment, resolveWatchPlugin }: typeof JestResolveModule = + require('jest-resolve'); switch (parsedPropertyName) { case 'testRunner': return resolveRunner(/*resolver:*/ undefined, { diff --git a/heft-plugins/heft-jest-plugin/src/JestUtils.ts b/heft-plugins/heft-jest-plugin/src/JestUtils.ts index 89e6b5069c8..bdfcc33951c 100644 --- a/heft-plugins/heft-jest-plugin/src/JestUtils.ts +++ b/heft-plugins/heft-jest-plugin/src/JestUtils.ts @@ -4,7 +4,7 @@ import * as path from 'node:path'; import { createHash } from 'node:crypto'; -import { default as JestResolver } from 'jest-resolve'; +import type * as JestResolveModule from 'jest-resolve'; import type { TransformOptions } from '@jest/transform'; import { FileSystem } from '@rushstack/node-core-library'; @@ -46,6 +46,9 @@ export const jestResolve = ( // eslint-disable-next-line @rushstack/no-new-null ): string | null => { const { key, filePath, rootDir, optional } = options; + // Loaded on first use: "jest-resolve" and its dependencies are only needed when a Jest configuration is processed. + const JestResolver: typeof JestResolveModule.default = (require('jest-resolve') as typeof JestResolveModule) + .default; const module: string | null = JestResolver.findNodeModule(replaceRootDirInPath(rootDir, filePath), { basedir: rootDir, resolver: resolver || undefined diff --git a/heft-plugins/heft-lint-plugin/src/Eslint.ts b/heft-plugins/heft-lint-plugin/src/Eslint.ts index 23c6e6fc5c8..031c6ca772c 100644 --- a/heft-plugins/heft-lint-plugin/src/Eslint.ts +++ b/heft-plugins/heft-lint-plugin/src/Eslint.ts @@ -7,7 +7,7 @@ import { performance } from 'node:perf_hooks'; import type * as TEslint from 'eslint'; import type * as TEslintLegacy from 'eslint-8'; -import * as semver from 'semver'; +import type * as semver from 'semver'; import stableStringify from 'json-stable-stringify-without-jsonify'; import { Async, FileError, FileSystem, Path } from '@rushstack/node-core-library'; @@ -160,7 +160,9 @@ export class Eslint extends LinterBase { async #initInnerAsync(heftConfiguration: HeftConfiguration, logger: IScopedLogger): Promise { // Locate the tslint linter if enabled - this.#tslintConfigFilePath = await Tslint.resolveTslintConfigFilePathAsync(heftConfiguration); + this.#tslintConfigFilePath = await getTslintClass().resolveTslintConfigFilePathAsync(heftConfiguration); if (this.#tslintConfigFilePath) { this.#tslintToolPath = await heftConfiguration.rigPackageResolver.resolvePackageAsync( 'tslint', @@ -231,7 +240,7 @@ export default class LintPlugin implements IHeftTaskPlugin { } // Locate the eslint linter if enabled - this.#eslintConfigFilePath = await Eslint.resolveEslintConfigFilePathAsync(heftConfiguration); + this.#eslintConfigFilePath = await getEslintClass().resolveEslintConfigFilePathAsync(heftConfiguration); if (this.#eslintConfigFilePath) { logger.terminal.writeVerboseLine(`ESLint config file path: ${this.#eslintConfigFilePath}`); this.#eslintToolPath = await heftConfiguration.rigPackageResolver.resolvePackageAsync( @@ -261,7 +270,7 @@ export default class LintPlugin implements IHeftTaskPlugin { const lintOperations: (() => Promise)[] = []; if (this.#eslintConfigFilePath && this.#eslintToolPath) { - const eslintLinter: Eslint = await Eslint.initializeAsync({ + const eslintLinter: Eslint = await getEslintClass().initializeAsync({ tsProgram, fix, sarifLogPath, @@ -278,7 +287,7 @@ export default class LintPlugin implements IHeftTaskPlugin { } if (this.#tslintConfigFilePath && this.#tslintToolPath) { - const tslintLinter: Tslint = await Tslint.initializeAsync({ + const tslintLinter: Tslint = await getTslintClass().initializeAsync({ tsProgram, fix, scopedLogger: taskSession.logger, diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts index 32dff7658e2..9fca0ead0fa 100644 --- a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts +++ b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts @@ -4,11 +4,16 @@ import * as path from 'node:path'; import type * as TTypescript from 'typescript'; +// Only used as a type (the runtime class is loaded below), so this import is elided from the emitted JavaScript. +// It is not written as "import type" because that would change the published API report. +// eslint-disable-next-line @typescript-eslint/consistent-type-imports import { SyncHook } from 'tapable'; import { FileSystem } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; -import { ProjectConfigurationFile, InheritanceType, PathResolutionMethod } from '@rushstack/heft-config-file'; +// The '@rushstack/heft-config-file' package is only loaded when a tsconfig file is loaded (see below). The enum-like +// InheritanceType/PathResolutionMethod values used here are plain strings. +import type { ProjectConfigurationFile, InheritanceType, PathResolutionMethod } from '@rushstack/heft-config-file'; import type { HeftConfiguration, IHeftTaskSession, @@ -20,7 +25,7 @@ import type { ConfigurationFile } from '@rushstack/heft'; -import { TypeScriptBuilder, type ITypeScriptBuilderConfiguration } from './TypeScriptBuilder'; +import type { TypeScriptBuilder, ITypeScriptBuilderConfiguration } from './TypeScriptBuilder'; import anythingSchema from './schemas/anything.schema.json'; import typescriptConfigSchema from './schemas/typescript.schema.json'; import { getTsconfigFilePath } from './tsconfigLoader'; @@ -144,12 +149,13 @@ const TYPESCRIPT_LOADER_CONFIG: ConfigurationFile.IProjectConfigurationFileSpeci propertyInheritance: { staticAssetsToCopy: { // When merging objects, arrays will be automatically appended - inheritanceType: InheritanceType.merge + inheritanceType: 'merge' as InheritanceType.merge } }, jsonPathMetadata: { '$.additionalModuleKindsToEmit.*.outFolderName': { - pathResolutionMethod: PathResolutionMethod.resolvePathRelativeToProjectRoot + pathResolutionMethod: + 'resolvePathRelativeToProjectRoot' as PathResolutionMethod.resolvePathRelativeToProjectRoot } } }; @@ -200,17 +206,20 @@ export async function loadPartialTsconfigFileAsync( } else { // Ensure that the file loader has been initialized. if (!_partialTsconfigFileLoader) { - _partialTsconfigFileLoader = new ProjectConfigurationFile({ + const { + ProjectConfigurationFile: ProjectConfigurationFileClass + }: typeof import('@rushstack/heft-config-file') = require('@rushstack/heft-config-file'); + _partialTsconfigFileLoader = new ProjectConfigurationFileClass({ projectRelativeFilePath: typeScriptConfigurationJson?.project || 'tsconfig.json', jsonSchemaObject: anythingSchema, propertyInheritance: { compilerOptions: { - inheritanceType: InheritanceType.merge + inheritanceType: 'merge' as InheritanceType.merge } }, jsonPathMetadata: { '$.compilerOptions.outDir': { - pathResolutionMethod: PathResolutionMethod.custom, + pathResolutionMethod: 'custom' as PathResolutionMethod.custom, customResolver( resolverOptions: ConfigurationFile.IJsonPathMetadataResolverOptions ): string { @@ -246,9 +255,19 @@ interface ITypeScriptConfigurationJsonAndPartialTsconfigFile { partialTsconfigFile: IPartialTsconfig | undefined; } +// Loading the "tapable" package index requires every hook implementation; this plugin only needs SyncHook. +// The class object is identical to the one exported by the package index. +let _syncHookClass: typeof SyncHook | undefined; +function getSyncHookClass(): typeof SyncHook { + if (!_syncHookClass) { + _syncHookClass = require('tapable/lib/SyncHook') as typeof SyncHook; + } + return _syncHookClass; +} + export default class TypeScriptPlugin implements IHeftTaskPlugin { public accessor: ITypeScriptPluginAccessor = { - onChangedFilesHook: new SyncHook(['changedFilesHookOptions']) + onChangedFilesHook: new (getSyncHookClass())(['changedFilesHookOptions']) }; public apply(taskSession: IHeftTaskSession, heftConfiguration: HeftConfiguration): void { @@ -386,8 +405,10 @@ export default class TypeScriptPlugin implements IHeftTaskPlugin { } }; - // Run the builder - const typeScriptBuilder: TypeScriptBuilder = new TypeScriptBuilder(typeScriptBuilderConfiguration); + // Run the builder. The builder module (and the TypeScript tooling it loads) is only needed when the task runs. + const { TypeScriptBuilder: TypeScriptBuilderClass }: typeof import('./TypeScriptBuilder') = + require('./TypeScriptBuilder'); + const typeScriptBuilder: TypeScriptBuilder = new TypeScriptBuilderClass(typeScriptBuilderConfiguration); return typeScriptBuilder; } diff --git a/heft-plugins/heft-typescript-plugin/src/loadTypeScriptTool.ts b/heft-plugins/heft-typescript-plugin/src/loadTypeScriptTool.ts index 5fb9ad86fb8..5c993091a95 100644 --- a/heft-plugins/heft-typescript-plugin/src/loadTypeScriptTool.ts +++ b/heft-plugins/heft-typescript-plugin/src/loadTypeScriptTool.ts @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +// Only used as a type (see loadTypeScriptToolAsync), so this import is elided from the emitted JavaScript. +// It is not written as "import type" because that would change the published API report. +// eslint-disable-next-line @typescript-eslint/consistent-type-imports import semver from 'semver'; import type { HeftConfiguration } from '@rushstack/heft'; @@ -70,7 +73,9 @@ export async function loadTypeScriptToolAsync( const compilerPackageJsonFilename: string = `${typeScriptToolPath}/package.json`; const packageJson: IPackageJson = await JsonFile.loadAsync(compilerPackageJsonFilename); const typescriptVersion: string = packageJson.version; - const typescriptParsedVersion: semver.SemVer | null = semver.parse(typescriptVersion); + // Deep import: the 'semver' package index loads ~45 modules. The SemVer class is the same object. + const parseSemVer: typeof semver.parse = require('semver/functions/parse'); + const typescriptParsedVersion: semver.SemVer | null = parseSemVer(typescriptVersion); if (!typescriptParsedVersion) { throw new Error( `Unable to parse version "${typescriptVersion}" for TypeScript compiler package in: ` + diff --git a/libraries/heft-config-file/src/ConfigurationFileAnnotation.ts b/libraries/heft-config-file/src/ConfigurationFileAnnotation.ts new file mode 100644 index 00000000000..a610b3d5f1b --- /dev/null +++ b/libraries/heft-config-file/src/ConfigurationFileAnnotation.ts @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * The key of the annotation that is attached to every object loaded from a configuration file, which records the + * source file and the original property values. + * + * @remarks + * This module has no dependencies, so that tools can recognize (and produce) annotated objects without loading + * the rest of this package. + */ +export const CONFIGURATION_FILE_FIELD_ANNOTATION: unique symbol = Symbol( + 'configuration-file-field-annotation' +); diff --git a/libraries/heft-config-file/src/ConfigurationFileBase.ts b/libraries/heft-config-file/src/ConfigurationFileBase.ts index 0382fe2099d..9ccc9cf74c0 100644 --- a/libraries/heft-config-file/src/ConfigurationFileBase.ts +++ b/libraries/heft-config-file/src/ConfigurationFileBase.ts @@ -3,11 +3,23 @@ import * as nodeJsPath from 'node:path'; -import { JSONPath } from 'jsonpath-plus'; +import type { JSONPath as JSONPathFunction } from 'jsonpath-plus'; import { JsonSchema, JsonFile, Import, FileSystem } from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; +let _jsonPath: typeof JSONPathFunction | undefined; + +/** + * jsonpath-plus is only needed when jsonPathMetadata is specified, so it is loaded on first use. + */ +function _getJsonPath(): typeof JSONPathFunction { + if (!_jsonPath) { + _jsonPath = require('jsonpath-plus').JSONPath as typeof JSONPathFunction; + } + return _jsonPath; +} + interface IConfigurationJson { extends?: string; } @@ -155,9 +167,9 @@ export { PathResolutionMethod }; /* eslint-enable @typescript-eslint/typedef,@typescript-eslint/no-redeclare,@typescript-eslint/no-namespace,@typescript-eslint/naming-convention */ const CONFIGURATION_FILE_MERGE_BEHAVIOR_FIELD_REGEX: RegExp = /^\$([^\.]+)\.inheritanceType$/; -export const CONFIGURATION_FILE_FIELD_ANNOTATION: unique symbol = Symbol( - 'configuration-file-field-annotation' -); +import { CONFIGURATION_FILE_FIELD_ANNOTATION } from './ConfigurationFileAnnotation'; + +export { CONFIGURATION_FILE_FIELD_ANNOTATION }; export interface IAnnotatedField< TField, @@ -660,7 +672,7 @@ export abstract class ConfigurationFileBase { diff --git a/libraries/node-core-library/config/heft.json b/libraries/node-core-library/config/heft.json new file mode 100644 index 00000000000..860f73615d3 --- /dev/null +++ b/libraries/node-core-library/config/heft.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", + + "extends": "decoupled-local-node-rig/profiles/default/config/heft.json", + "phasesByName": { + "build": { + "tasksByName": { + // Make lib-commonjs/index.js load each re-exported module on first access, so that consumers + // only pay for the parts of this package that they actually use. + "lazy-barrel": { + "taskDependencies": ["typescript"], + "taskPlugin": { + "pluginPackage": "@rushstack/heft", + "pluginName": "run-script-plugin", + "options": { + "scriptPath": "./node_modules/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js" + } + } + } + } + } + } +} diff --git a/libraries/node-core-library/src/Executable.ts b/libraries/node-core-library/src/Executable.ts index 049cf741391..76c0bceabe5 100644 --- a/libraries/node-core-library/src/Executable.ts +++ b/libraries/node-core-library/src/Executable.ts @@ -2,6 +2,8 @@ // See LICENSE in the project root for license information. import * as os from 'node:os'; +// This import is only used for types (the module is loaded lazily); the value-style import keeps the API report stable. +// eslint-disable-next-line @typescript-eslint/consistent-type-imports import * as child_process from 'node:child_process'; import * as path from 'node:path'; @@ -11,6 +13,13 @@ import { PosixModeBits } from './PosixModeBits'; import { Text } from './Text'; import { InternalError } from './InternalError'; +/** + * node:child_process (which also loads the net, dgram and stream implementations) is only loaded when needed. + */ +function _getChildProcess(): typeof child_process { + return require('node:child_process'); +} + const OS_PLATFORM: NodeJS.Platform = os.platform(); /** @@ -481,7 +490,7 @@ export class Executable { const normalizedCommandLine: ICommandLineOptions = _buildCommandLineFixup(resolvedPath, args, context); - return child_process.spawnSync(normalizedCommandLine.path, normalizedCommandLine.args, spawnOptions); + return _getChildProcess().spawnSync(normalizedCommandLine.path, normalizedCommandLine.args, spawnOptions); } /** @@ -534,7 +543,7 @@ export class Executable { const normalizedCommandLine: ICommandLineOptions = _buildCommandLineFixup(resolvedPath, args, context); - return child_process.spawn(normalizedCommandLine.path, normalizedCommandLine.args, spawnOptions); + return _getChildProcess().spawn(normalizedCommandLine.path, normalizedCommandLine.args, spawnOptions); } /** {@inheritDoc Executable.(waitForExitAsync:3)} */ diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index 7eff1995b8b..66a49148aa0 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -3,13 +3,37 @@ import * as nodeJsPath from 'node:path'; import * as fs from 'node:fs'; -import * as fsPromises from 'node:fs/promises'; +import type * as fsPromises from 'node:fs/promises'; -import * as fsx from 'fs-extra'; +import type * as FsExtra from 'fs-extra'; import { Text, type NewlineKind, Encoding } from './Text'; import { PosixModeBits } from './PosixModeBits'; +let _fsx: typeof FsExtra | undefined; + +/** + * fs-extra (and graceful-fs, which it loads) is relatively expensive to load, so it is loaded on first use. + * + * @remarks + * Where fs-extra re-exports the exact same function object as `node:fs` (for example `existsSync`, + * `readFileSync`, `readdirSync`, `realpathSync` and `unlinkSync`), or where it just wraps a `node:fs` function + * that graceful-fs does not patch (`rm`, `unlink`), the `node:fs` function is called directly instead. + */ +function _getFsx(): typeof FsExtra { + if (!_fsx) { + _fsx = require('fs-extra') as typeof FsExtra; + } + return _fsx; +} + +/** + * node:fs/promises is only needed by some of the asynchronous APIs, so it is loaded on first use. + */ +function _getFsPromises(): typeof fsPromises { + return require('node:fs/promises'); +} + /** * An alias for the Node.js `fs.Stats` object. * @@ -411,7 +435,7 @@ export class FileSystem { */ public static exists(path: string): boolean { return _wrapException(() => { - return fsx.existsSync(path); + return fs.existsSync(path); }); } @@ -421,7 +445,7 @@ export class FileSystem { public static async existsAsync(path: string): Promise { return await _wrapExceptionAsync(() => { return new Promise((resolve: (result: boolean) => void) => { - fsx.exists(path, resolve); + _getFsx().exists(path, resolve); }); }); } @@ -434,7 +458,7 @@ export class FileSystem { */ public static getStatistics(path: string): FileSystemStats { return _wrapException(() => { - return fsx.statSync(path); + return _getFsx().statSync(path); }); } @@ -443,7 +467,7 @@ export class FileSystem { */ public static async getStatisticsAsync(path: string): Promise { return await _wrapExceptionAsync(() => { - return fsx.stat(path); + return _getFsx().stat(path); }); } @@ -456,7 +480,7 @@ export class FileSystem { */ public static updateTimes(path: string, times: IFileSystemUpdateTimeParameters): void { return _wrapException(() => { - fsx.utimesSync(path, times.accessedTime, times.modifiedTime); + _getFsx().utimesSync(path, times.accessedTime, times.modifiedTime); }); } @@ -467,7 +491,7 @@ export class FileSystem { await _wrapExceptionAsync(() => { // This cast is needed because the fs-extra typings require both parameters // to have the same type (number or Date), whereas Node.js does not require that. - return fsx.utimes(path, times.accessedTime as number, times.modifiedTime as number); + return _getFsx().utimes(path, times.accessedTime as number, times.modifiedTime as number); }); } @@ -488,7 +512,7 @@ export class FileSystem { */ public static async changePosixModeBitsAsync(path: string, mode: PosixModeBits): Promise { await _wrapExceptionAsync(() => { - return fsx.chmod(path, mode); + return _getFsx().chmod(path, mode); }); } @@ -554,7 +578,7 @@ export class FileSystem { }; try { - fsx.moveSync(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); + _getFsx().moveSync(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); } catch (error) { if (options.ensureFolderExists) { if (!FileSystem.isNotExistError(error as Error)) { @@ -563,7 +587,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(options.destinationPath); FileSystem.ensureFolder(folderPath); - fsx.moveSync(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); + _getFsx().moveSync(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); } else { throw error; } @@ -582,7 +606,7 @@ export class FileSystem { }; try { - await fsx.move(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); + await _getFsx().move(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); } catch (error) { if (options.ensureFolderExists) { if (!FileSystem.isNotExistError(error as Error)) { @@ -591,7 +615,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(options.destinationPath); await FileSystem.ensureFolderAsync(nodeJsPath.dirname(folderPath)); - await fsx.move(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); + await _getFsx().move(options.sourcePath, options.destinationPath, { overwrite: options.overwrite }); } else { throw error; } @@ -612,7 +636,7 @@ export class FileSystem { */ public static ensureFolder(folderPath: string): void { _wrapException(() => { - fsx.ensureDirSync(folderPath); + _getFsx().ensureDirSync(folderPath); }); } @@ -621,7 +645,7 @@ export class FileSystem { */ public static async ensureFolderAsync(folderPath: string): Promise { await _wrapExceptionAsync(() => { - return fsx.ensureDir(folderPath); + return _getFsx().ensureDir(folderPath); }); } @@ -638,7 +662,7 @@ export class FileSystem { ...options }; - const fileNames: string[] = fsx.readdirSync(folderPath); + const fileNames: string[] = fs.readdirSync(folderPath); if (options.absolutePaths) { return fileNames.map((fileName) => nodeJsPath.resolve(folderPath, fileName)); } else { @@ -660,7 +684,7 @@ export class FileSystem { ...options }; - const fileNames: string[] = await fsx.readdir(folderPath); + const fileNames: string[] = await _getFsx().readdir(folderPath); if (options.absolutePaths) { return fileNames.map((fileName) => nodeJsPath.resolve(folderPath, fileName)); } else { @@ -683,7 +707,7 @@ export class FileSystem { ...options }; - const folderEntries: FolderItem[] = fsx.readdirSync(folderPath, { withFileTypes: true }); + const folderEntries: FolderItem[] = fs.readdirSync(folderPath, { withFileTypes: true }); if (options.absolutePaths) { return folderEntries.map((folderEntry) => { folderEntry.name = nodeJsPath.resolve(folderPath, folderEntry.name); @@ -708,7 +732,7 @@ export class FileSystem { ...options }; - const folderEntries: FolderItem[] = await fsPromises.readdir(folderPath, { withFileTypes: true }); + const folderEntries: FolderItem[] = await _getFsPromises().readdir(folderPath, { withFileTypes: true }); if (options.absolutePaths) { return folderEntries.map((folderEntry) => { folderEntry.name = nodeJsPath.resolve(folderPath, folderEntry.name); @@ -729,7 +753,7 @@ export class FileSystem { */ public static deleteFolder(folderPath: string): void { _wrapException(() => { - fsx.removeSync(folderPath); + _getFsx().removeSync(folderPath); }); } @@ -738,7 +762,12 @@ export class FileSystem { */ public static async deleteFolderAsync(folderPath: string): Promise { await _wrapExceptionAsync(() => { - return fsx.remove(folderPath); + // This is exactly what fs-extra's remove() does (graceful-fs does not wrap fs.rm) + return new Promise((resolve: () => void, reject: (error: Error) => void) => { + fs.rm(folderPath, { recursive: true, force: true }, (error: Error | null) => + error != null ? reject(error) : resolve() + ); + }); }); } @@ -752,7 +781,7 @@ export class FileSystem { */ public static ensureEmptyFolder(folderPath: string): void { _wrapException(() => { - fsx.emptyDirSync(folderPath); + _getFsx().emptyDirSync(folderPath); }); } @@ -761,7 +790,7 @@ export class FileSystem { */ public static async ensureEmptyFolderAsync(folderPath: string): Promise { await _wrapExceptionAsync(() => { - return fsx.emptyDir(folderPath); + return _getFsx().emptyDir(folderPath); }); } @@ -795,7 +824,7 @@ export class FileSystem { } try { - fsx.writeFileSync(filePath, contents, { encoding: options.encoding }); + _getFsx().writeFileSync(filePath, contents, { encoding: options.encoding }); } catch (error) { if (options.ensureFolderExists) { if (!FileSystem.isNotExistError(error as Error)) { @@ -804,7 +833,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(filePath); FileSystem.ensureFolder(folderPath); - fsx.writeFileSync(filePath, contents, { encoding: options.encoding }); + _getFsx().writeFileSync(filePath, contents, { encoding: options.encoding }); } else { throw error; } @@ -838,7 +867,7 @@ export class FileSystem { let fd: number | undefined; try { - fd = fsx.openSync(filePath, 'w'); + fd = _getFsx().openSync(filePath, 'w'); } catch (error) { if (!options?.ensureFolderExists || !FileSystem.isNotExistError(error as Error)) { throw error; @@ -846,14 +875,14 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(filePath); FileSystem.ensureFolder(folderPath); - fd = fsx.openSync(filePath, 'w'); + fd = _getFsx().openSync(filePath, 'w'); } try { // In practice this loop will have exactly 1 iteration, but the spec allows // for a writev call to write fewer bytes than requested while (toCopy.length) { - let bytesWritten: number = fsx.writevSync(fd, toCopy); + let bytesWritten: number = _getFsx().writevSync(fd, toCopy); let buffersWritten: number = 0; while (buffersWritten < toCopy.length) { const bytesInCurrentBuffer: number = toCopy[buffersWritten].byteLength; @@ -877,7 +906,7 @@ export class FileSystem { } } } finally { - fsx.closeSync(fd); + _getFsx().closeSync(fd); } }); } @@ -901,7 +930,7 @@ export class FileSystem { } try { - await fsx.writeFile(filePath, contents, { encoding: options.encoding }); + await _getFsx().writeFile(filePath, contents, { encoding: options.encoding }); } catch (error) { if (options.ensureFolderExists) { if (!FileSystem.isNotExistError(error as Error)) { @@ -910,7 +939,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(filePath); await FileSystem.ensureFolderAsync(folderPath); - await fsx.writeFile(filePath, contents, { encoding: options.encoding }); + await _getFsx().writeFile(filePath, contents, { encoding: options.encoding }); } else { throw error; } @@ -933,7 +962,7 @@ export class FileSystem { let handle: fsPromises.FileHandle | undefined; try { - handle = await fsPromises.open(filePath, 'w'); + handle = await _getFsPromises().open(filePath, 'w'); } catch (error) { if (!options?.ensureFolderExists || !FileSystem.isNotExistError(error as Error)) { throw error; @@ -941,7 +970,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(filePath); await FileSystem.ensureFolderAsync(folderPath); - handle = await fsPromises.open(filePath, 'w'); + handle = await _getFsPromises().open(filePath, 'w'); } try { @@ -1003,7 +1032,7 @@ export class FileSystem { } try { - fsx.appendFileSync(filePath, contents, { encoding: options.encoding }); + _getFsx().appendFileSync(filePath, contents, { encoding: options.encoding }); } catch (error) { if (options.ensureFolderExists) { if (!FileSystem.isNotExistError(error as Error)) { @@ -1012,7 +1041,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(filePath); FileSystem.ensureFolder(folderPath); - fsx.appendFileSync(filePath, contents, { encoding: options.encoding }); + _getFsx().appendFileSync(filePath, contents, { encoding: options.encoding }); } else { throw error; } @@ -1039,7 +1068,7 @@ export class FileSystem { } try { - await fsx.appendFile(filePath, contents, { encoding: options.encoding }); + await _getFsx().appendFile(filePath, contents, { encoding: options.encoding }); } catch (error) { if (options.ensureFolderExists) { if (!FileSystem.isNotExistError(error as Error)) { @@ -1048,7 +1077,7 @@ export class FileSystem { const folderPath: string = nodeJsPath.dirname(filePath); await FileSystem.ensureFolderAsync(folderPath); - await fsx.appendFile(filePath, contents, { encoding: options.encoding }); + await _getFsx().appendFile(filePath, contents, { encoding: options.encoding }); } else { throw error; } @@ -1104,7 +1133,7 @@ export class FileSystem { */ public static readFileToBuffer(filePath: string): Buffer { return _wrapException(() => { - return fsx.readFileSync(filePath); + return fs.readFileSync(filePath); }); } @@ -1112,8 +1141,26 @@ export class FileSystem { * An async version of {@link FileSystem.readFileToBuffer}. */ public static async readFileToBufferAsync(filePath: string): Promise { - return await _wrapExceptionAsync(() => { - return fsx.readFile(filePath); + return await _wrapExceptionAsync(async () => { + // fs-extra's readFile() is graceful-fs's, which calls fs.readFile() and only handles EMFILE/ENFILE errors + // (by queueing and retrying); any other result is passed through unchanged. So call fs.readFile() directly, + // which avoids loading fs-extra, and only use fs-extra to retry in the EMFILE/ENFILE case. + const { data, error } = await new Promise( + (resolve: (result: { data?: Buffer; error?: NodeJS.ErrnoException }) => void) => { + fs.readFile(filePath, (readError: NodeJS.ErrnoException | null, readData: Buffer) => + resolve(readError ? { error: readError } : { data: readData }) + ); + } + ); + if (error) { + if (error.code !== 'EMFILE' && error.code !== 'ENFILE') { + throw error; + } + + return await _getFsx().readFile(filePath); + } + + return data!; }); } @@ -1140,7 +1187,7 @@ export class FileSystem { } _wrapException(() => { - fsx.copySync(options.sourcePath, options.destinationPath, { + _getFsx().copySync(options.sourcePath, options.destinationPath, { errorOnExist: options.alreadyExistsBehavior === AlreadyExistsBehavior.Error, overwrite: options.alreadyExistsBehavior === AlreadyExistsBehavior.Overwrite }); @@ -1163,7 +1210,7 @@ export class FileSystem { } await _wrapExceptionAsync(() => { - return fsx.copy(options.sourcePath, options.destinationPath, { + return _getFsx().copy(options.sourcePath, options.destinationPath, { errorOnExist: options.alreadyExistsBehavior === AlreadyExistsBehavior.Error, overwrite: options.alreadyExistsBehavior === AlreadyExistsBehavior.Overwrite }); @@ -1187,7 +1234,7 @@ export class FileSystem { }; _wrapException(() => { - fsx.copySync(options.sourcePath, options.destinationPath, { + _getFsx().copySync(options.sourcePath, options.destinationPath, { dereference: !!options.dereferenceSymlinks, errorOnExist: options.alreadyExistsBehavior === AlreadyExistsBehavior.Error, overwrite: options.alreadyExistsBehavior === AlreadyExistsBehavior.Overwrite, @@ -1207,7 +1254,7 @@ export class FileSystem { }; await _wrapExceptionAsync(async () => { - await fsx.copy(options.sourcePath, options.destinationPath, { + await _getFsx().copy(options.sourcePath, options.destinationPath, { dereference: !!options.dereferenceSymlinks, errorOnExist: options.alreadyExistsBehavior === AlreadyExistsBehavior.Error, overwrite: options.alreadyExistsBehavior === AlreadyExistsBehavior.Overwrite, @@ -1231,7 +1278,7 @@ export class FileSystem { }; try { - fsx.unlinkSync(filePath); + fs.unlinkSync(filePath); } catch (error) { if (options.throwIfNotExists || !FileSystem.isNotExistError(error as Error)) { throw error; @@ -1254,7 +1301,10 @@ export class FileSystem { }; try { - await fsx.unlink(filePath); + // This is exactly what fs-extra's unlink() does (graceful-fs does not wrap fs.unlink) + await new Promise((resolve: () => void, reject: (error: Error) => void) => { + fs.unlink(filePath, (error: Error | null) => (error != null ? reject(error) : resolve())); + }); } catch (error) { if (options.throwIfNotExists || !FileSystem.isNotExistError(error as Error)) { throw error; @@ -1329,7 +1379,7 @@ export class FileSystem { */ public static getLinkStatistics(path: string): FileSystemStats { return _wrapException(() => { - return fsx.lstatSync(path); + return _getFsx().lstatSync(path); }); } @@ -1338,7 +1388,7 @@ export class FileSystem { */ public static async getLinkStatisticsAsync(path: string): Promise { return await _wrapExceptionAsync(() => { - return fsx.lstat(path); + return _getFsx().lstat(path); }); } @@ -1355,7 +1405,7 @@ export class FileSystem { */ public static readLink(path: string): string { return _wrapException(() => { - return fsx.readlinkSync(path); + return _getFsx().readlinkSync(path); }); } @@ -1364,7 +1414,7 @@ export class FileSystem { */ public static async readLinkAsync(path: string): Promise { return await _wrapExceptionAsync(() => { - return fsx.readlink(path); + return _getFsx().readlink(path); }); } @@ -1389,7 +1439,7 @@ export class FileSystem { _wrapException(() => { return _handleLink(() => { // For directories, we use a Windows "junction". On POSIX operating systems, this produces a regular symlink. - return fsx.symlinkSync(options.linkTargetPath, options.newLinkPath, 'junction'); + return _getFsx().symlinkSync(options.linkTargetPath, options.newLinkPath, 'junction'); }, options); }); } @@ -1401,7 +1451,7 @@ export class FileSystem { await _wrapExceptionAsync(() => { return _handleLinkAsync(() => { // For directories, we use a Windows "junction". On POSIX operating systems, this produces a regular symlink. - return fsx.symlink(options.linkTargetPath, options.newLinkPath, 'junction'); + return _getFsx().symlink(options.linkTargetPath, options.newLinkPath, 'junction'); }, options); }); } @@ -1422,7 +1472,7 @@ export class FileSystem { public static createSymbolicLinkFile(options: IFileSystemCreateLinkOptions): void { _wrapException(() => { return _handleLink(() => { - return fsx.symlinkSync(options.linkTargetPath, options.newLinkPath, 'file'); + return _getFsx().symlinkSync(options.linkTargetPath, options.newLinkPath, 'file'); }, options); }); } @@ -1433,7 +1483,7 @@ export class FileSystem { public static async createSymbolicLinkFileAsync(options: IFileSystemCreateLinkOptions): Promise { await _wrapExceptionAsync(() => { return _handleLinkAsync(() => { - return fsx.symlink(options.linkTargetPath, options.newLinkPath, 'file'); + return _getFsx().symlink(options.linkTargetPath, options.newLinkPath, 'file'); }, options); }); } @@ -1454,7 +1504,7 @@ export class FileSystem { public static createSymbolicLinkFolder(options: IFileSystemCreateLinkOptions): void { _wrapException(() => { return _handleLink(() => { - return fsx.symlinkSync(options.linkTargetPath, options.newLinkPath, 'dir'); + return _getFsx().symlinkSync(options.linkTargetPath, options.newLinkPath, 'dir'); }, options); }); } @@ -1465,7 +1515,7 @@ export class FileSystem { public static async createSymbolicLinkFolderAsync(options: IFileSystemCreateLinkOptions): Promise { await _wrapExceptionAsync(() => { return _handleLinkAsync(() => { - return fsx.symlink(options.linkTargetPath, options.newLinkPath, 'dir'); + return _getFsx().symlink(options.linkTargetPath, options.newLinkPath, 'dir'); }, options); }); } @@ -1490,7 +1540,7 @@ export class FileSystem { _wrapException(() => { return _handleLink( () => { - return fsx.linkSync(options.linkTargetPath, options.newLinkPath); + return _getFsx().linkSync(options.linkTargetPath, options.newLinkPath); }, { ...options, linkTargetMustExist: true } ); @@ -1504,7 +1554,7 @@ export class FileSystem { await _wrapExceptionAsync(() => { return _handleLinkAsync( () => { - return fsx.link(options.linkTargetPath, options.newLinkPath); + return _getFsx().link(options.linkTargetPath, options.newLinkPath); }, { ...options, linkTargetMustExist: true } ); @@ -1518,7 +1568,7 @@ export class FileSystem { */ public static getRealPath(linkPath: string): string { return _wrapException(() => { - return fsx.realpathSync(linkPath); + return fs.realpathSync(linkPath); }); } @@ -1527,7 +1577,7 @@ export class FileSystem { */ public static async getRealPathAsync(linkPath: string): Promise { return await _wrapExceptionAsync(() => { - return fsx.realpath(linkPath); + return _getFsx().realpath(linkPath); }); } diff --git a/libraries/node-core-library/src/Import.ts b/libraries/node-core-library/src/Import.ts index 029fbe104e4..4fad60fc877 100644 --- a/libraries/node-core-library/src/Import.ts +++ b/libraries/node-core-library/src/Import.ts @@ -5,7 +5,7 @@ import * as path from 'node:path'; import nodeModule = require('module'); import importLazy = require('import-lazy'); -import * as Resolve from 'resolve'; +import type * as Resolve from 'resolve'; import { PackageJsonLookup } from './PackageJsonLookup'; import { FileSystem } from './FileSystem'; @@ -14,6 +14,20 @@ import { PackageName } from './PackageName'; type RealpathFnType = Parameters[1]['realpath']; +type ResolveFunction = typeof import('resolve'); + +let _resolveFunction: ResolveFunction | undefined; + +/** + * The "resolve" package is only needed by some of the Import APIs, so it is loaded on first use. + */ +function _getResolveFunction(): ResolveFunction { + if (!_resolveFunction) { + _resolveFunction = require('resolve') as ResolveFunction; + } + return _resolveFunction; +} + /** * Common options shared by {@link IImportResolveModuleOptions} and {@link IImportResolvePackageOptions} * @public @@ -297,7 +311,7 @@ export class Import { } try { - return Resolve.sync(modulePath, { + return _getResolveFunction().sync(modulePath, { basedir: normalizedRootPath, preserveSymlinks: false, realpathSync: getRealPath @@ -373,7 +387,7 @@ export class Import { } : undefined; - Resolve.default( + _getResolveFunction()( modulePath, { basedir: normalizedRootPath, @@ -450,7 +464,7 @@ export class Import { }) : // Append `/package.json` to ensure `resolve.sync` doesn't attempt to return a system package, and to avoid // having to mess with the `packageFilter` option. - Resolve.sync(`${packageName}/package.json`, { + _getResolveFunction().sync(`${packageName}/package.json`, { basedir: normalizedRootPath, preserveSymlinks: false, realpathSync: getRealPath @@ -514,7 +528,7 @@ export class Import { } : undefined; - Resolve.default( + _getResolveFunction()( // Append `/package.json` to ensure `resolve` doesn't attempt to return a system package, and to avoid // having to mess with the `packageFilter` option. `${packageName}/package.json`, diff --git a/libraries/node-core-library/src/JsonSchema.ts b/libraries/node-core-library/src/JsonSchema.ts index 8864740ef41..bf5b905120b 100644 --- a/libraries/node-core-library/src/JsonSchema.ts +++ b/libraries/node-core-library/src/JsonSchema.ts @@ -4,12 +4,36 @@ import * as os from 'node:os'; import * as path from 'node:path'; -import Ajv, { type Options as AjvOptions, type ErrorObject, type ValidateFunction } from 'ajv'; -import AjvDraft04 from 'ajv-draft-04'; -import addFormats from 'ajv-formats'; +import type { default as AjvType, Options as AjvOptions, ErrorObject, ValidateFunction } from 'ajv'; +import type AjvDraft04Type from 'ajv-draft-04'; +import type addFormatsType from 'ajv-formats'; import { JsonFile, type JsonObject } from './JsonFile'; import { FileSystem } from './FileSystem'; +import { analyzeSchemaForFastPath, isDefinitelyValid, type IJsonSchemaFastPathPlan } from './JsonSchemaFastPath'; + +interface IAjvModules { + Ajv: typeof AjvType; + AjvDraft04: typeof AjvDraft04Type; + addFormats: typeof addFormatsType; +} + +let _ajvModules: IAjvModules | undefined; + +/** + * The ajv packages are relatively expensive to load, so they are only loaded when a schema + * actually needs to be compiled. + */ +function _getAjvModules(): IAjvModules { + if (!_ajvModules) { + _ajvModules = { + Ajv: require('ajv').default, + AjvDraft04: require('ajv-draft-04').default, + addFormats: require('ajv-formats').default + }; + } + return _ajvModules; +} /** * Pattern matching JSON Schema vendor extension keywords in the form `x--`, @@ -210,6 +234,8 @@ export class JsonSchema { | Record | IJsonSchemaCustomFormat> | undefined = undefined; private _rejectVendorExtensionKeywords: boolean = false; + // undefined = not analyzed yet; false = the schema is not supported by the fast path + private _fastPathPlan: IJsonSchemaFastPathPlan | false | undefined = undefined; private constructor() {} @@ -297,7 +323,8 @@ export class JsonSchema { allowUnionTypes: true }; - let validator: Ajv; + const { Ajv, AjvDraft04, addFormats } = _getAjvModules(); + let validator: AjvType; // Keep legacy support for older draft-04 schema switch (targetSchemaVersion) { case 'draft-04': { @@ -392,6 +419,12 @@ export class JsonSchema { errorCallback: (errorInfo: IJsonSchemaErrorInfo) => void, options?: IJsonSchemaValidateObjectWithOptions ): void { + // Most validated objects are valid. If that can be proven without compiling the schema (which requires + // loading ajv and generating code), skip the compilation. Otherwise, ajv produces the canonical result. + if (this._isDefinitelyValidWithoutCompiling(jsonObject, options)) { + return; + } + this.ensureCompiled(); if (options?.ignoreSchemaField) { @@ -413,6 +446,53 @@ export class JsonSchema { } } + private _isDefinitelyValidWithoutCompiling( + jsonObject: JsonObject, + options: IJsonSchemaValidateObjectWithOptions | undefined + ): boolean { + if (this._validator || this._dependentSchemas.length > 0 || this._customFormats) { + return false; + } + + if (this._fastPathPlan === undefined) { + // Throws the same errors as ensureCompiled() would (for example if the schema file cannot be loaded) + this._ensureLoaded(); + try { + this._fastPathPlan = + analyzeSchemaForFastPath(this._schemaObject!, { + schemaVersion: this._schemaVersion, + rejectVendorExtensionKeywords: this._rejectVendorExtensionKeywords + }) ?? false; + } catch { + this._fastPathPlan = false; + } + } + + if (this._fastPathPlan === false) { + return false; + } + + let data: unknown = jsonObject; + if (options?.ignoreSchemaField) { + if (typeof jsonObject !== 'object' || jsonObject === null || Array.isArray(jsonObject)) { + return false; + } + + const { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + $schema, + ...remainder + } = jsonObject; + data = remainder; + } + + try { + return isDefinitelyValid(this._fastPathPlan, data); + } catch { + return false; + } + } + private _ensureLoaded(): string { if (!this._schemaObject) { this._schemaObject = JsonFile.load(this._filename); diff --git a/libraries/node-core-library/src/JsonSchemaFastPath.ts b/libraries/node-core-library/src/JsonSchemaFastPath.ts new file mode 100644 index 00000000000..0b8ddfe0ef4 --- /dev/null +++ b/libraries/node-core-library/src/JsonSchemaFastPath.ts @@ -0,0 +1,1280 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/** + * A *sound* fast path for {@link JsonSchema} validation that avoids compiling the schema with ajv. + * + * `JsonSchema` compiles every schema with ajv (`strictSchema: true, allowUnionTypes: true`, using `ajv-draft-04` + * for draft-04 schemas), which is expensive: loading ajv and generating code for a typical schema takes tens of + * milliseconds, and most validated objects are valid. + * + * This module answers a single question: "would ajv compile this schema without any error or warning, and accept + * this data?" It only returns `true` when it can prove that the answer is yes. In every other case (unsupported + * keyword, anything ajv's strict mode would log or reject, unusual data, or simply invalid data) it returns `false`, + * and the caller must fall back to the real ajv validation, which produces the canonical behavior and + * error messages. + * + * The supported subset intentionally mirrors ajv's semantics rather than the JSON schema specification where the + * two differ (for example, keywords next to `$ref` are applied, and `required`/`properties` use + * `data[key] !== undefined`). + */ + +type JsonSchemaDraft = 'draft-04' | 'draft-07'; + +type JsonTypeName = 'array' | 'boolean' | 'integer' | 'null' | 'number' | 'object' | 'string'; + +interface ISchemaObject { + [key: string]: unknown; +} + +interface ISchemaPlan { + readonly root: ISchemaObject; + readonly regExpCache: Map; + readonly refTargets: Map; + readonly compiledNodes: Map; +} + +const JSON_TYPE_NAMES: ReadonlySet = new Set([ + 'array', + 'boolean', + 'integer', + 'null', + 'number', + 'object', + 'string' +]); + +// See JsonSchema.ts in @rushstack/node-core-library +const VENDOR_EXTENSION_KEY_PATTERN: RegExp = /^x-[a-z0-9]+-[a-z0-9]+(-[a-z0-9]+)*$/; + +// ajv-formats' "regex" format rejects patterns that use the unsupported "\Z" anchor +const Z_ANCHOR_REGEXP: RegExp = /[^\\]\\Z/; + +const SIMPLE_DEFINITION_REF_REGEXP: RegExp = /^#\/definitions\/([A-Za-z0-9_.-]+)$/; + +/** + * The data type(s) that each type-specific keyword applies to (ajv's keyword definitions). Used to replicate ajv's + * `strictTypes` checks. + */ +const KEYWORD_APPLICABLE_TYPE: ReadonlyMap = new Map([ + ['properties', 'object'], + ['patternProperties', 'object'], + ['additionalProperties', 'object'], + ['required', 'object'], + ['minProperties', 'object'], + ['maxProperties', 'object'], + ['items', 'array'], + ['minItems', 'array'], + ['maxItems', 'array'], + ['uniqueItems', 'array'], + ['minLength', 'string'], + ['maxLength', 'string'], + ['pattern', 'string'], + ['minimum', 'number'], + ['maximum', 'number'], + ['exclusiveMinimum', 'number'], + ['exclusiveMaximum', 'number'] +]); + +class UnsupportedSchemaError extends Error {} + +function unsupported(): never { + throw new UnsupportedSchemaError(); +} + +function isPlainObject(value: unknown): value is ISchemaObject { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + + const prototype: unknown = Object.getPrototypeOf(value); + return prototype === Object.prototype; +} + +function isNonNegativeInteger(value: unknown): boolean { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} + +function isFiniteNumber(value: unknown): boolean { + return typeof value === 'number' && isFinite(value); +} + +function hasOwn(obj: object, key: string): boolean { + return Object.prototype.hasOwnProperty.call(obj, key); +} + +/** + * Checks that the value is JSON data that the validator can reason about exactly: `null`, booleans, finite numbers, + * strings, dense arrays, and objects with the standard prototype and without an own `__proto__` key. + * Symbol-keyed properties (such as configuration file annotations) are ignored, like they are by ajv. + */ +function isSimpleJsonData(value: unknown, depth: number): boolean { + if (depth > 256) { + return false; + } + + switch (typeof value) { + case 'string': + case 'boolean': + return true; + case 'number': + return isFinite(value); + case 'object': { + if (value === null) { + return true; + } + + if (Array.isArray(value)) { + if (Object.getPrototypeOf(value) !== Array.prototype) { + return false; + } + + for (let i: number = 0; i < value.length; i++) { + if (!(i in value) || !isSimpleJsonData(value[i], depth + 1)) { + return false; + } + } + + return true; + } + + if ( + Object.getPrototypeOf(value) !== Object.prototype || + // These own keys change the behavior of ajv's generated code or of fast-deep-equal + hasOwn(value, '__proto__') || + hasOwn(value, 'constructor') || + hasOwn(value, 'valueOf') || + hasOwn(value, 'toString') + ) { + return false; + } + + for (const key in value) { + if (hasOwn(value, key)) { + if (!isSimpleJsonData((value as Record)[key], depth + 1)) { + return false; + } + } else { + // Inherited enumerable property + return false; + } + } + + return true; + } + default: + return false; + } +} + +/** + * Deep equality with the semantics of `fast-deep-equal` (used by ajv), restricted to simple JSON data. + */ +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) { + return true; + } + + if (a && b && typeof a === 'object' && typeof b === 'object') { + if (Array.isArray(a) !== Array.isArray(b)) { + return false; + } + + if (Array.isArray(a)) { + const bArray: unknown[] = b as unknown[]; + if (a.length !== bArray.length) { + return false; + } + + for (let i: number = 0; i < a.length; i++) { + if (!deepEqual(a[i], bArray[i])) { + return false; + } + } + + return true; + } + + const aKeys: string[] = Object.keys(a); + if (aKeys.length !== Object.keys(b).length) { + return false; + } + + for (const key of aKeys) { + if (!hasOwn(b, key)) { + return false; + } + } + + for (const key of aKeys) { + if (!deepEqual((a as Record)[key], (b as Record)[key])) { + return false; + } + } + + return true; + } + + return false; +} + +function getSchemaTypes(schema: ISchemaObject): JsonTypeName[] { + const type: unknown = schema.type; + if (type === undefined) { + return []; + } + + return (Array.isArray(type) ? type : [type]) as JsonTypeName[]; +} + +// Replicates ajv's includesType() +function includesType(types: JsonTypeName[], type: JsonTypeName): boolean { + return types.includes(type) || (type === 'integer' && types.includes('number')); +} + +// Replicates ajv's hasApplicableType() +function hasApplicableType(schemaTypes: JsonTypeName[], keywordType: JsonTypeName): boolean { + return schemaTypes.includes(keywordType) || (keywordType === 'number' && schemaTypes.includes('integer')); +} + +class SchemaAnalyzer { + private readonly _root: ISchemaObject; + private readonly _draft: JsonSchemaDraft; + private readonly _vendorKeywords: ReadonlySet; + private readonly _regExpCache: Map = new Map(); + private readonly _refTargets: Map = new Map(); + // Schema objects analyzed with an empty type context (the root, $ref targets and definitions) + private readonly _analyzedWithEmptyContext: Set = new Set(); + + public constructor(root: ISchemaObject, draft: JsonSchemaDraft, rejectVendorExtensionKeywords: boolean) { + this._root = root; + this._draft = draft; + const vendorKeywords: Set = new Set(); + // JsonSchema registers the top-level vendor extension keywords with ajv, unless they are rejected + if (!rejectVendorExtensionKeywords) { + for (const key of Object.keys(root)) { + if (VENDOR_EXTENSION_KEY_PATTERN.test(key)) { + vendorKeywords.add(key); + } + } + } + + this._vendorKeywords = vendorKeywords; + } + + public analyze(): ISchemaPlan { + this._analyzeSchema(this._root, [], true); + return { + root: this._root, + regExpCache: this._regExpCache, + refTargets: this._refTargets, + compiledNodes: new Map() + }; + } + + private _getRegExp(pattern: unknown): RegExp { + if (typeof pattern !== 'string') { + unsupported(); + } + + let regExp: RegExp | undefined = this._regExpCache.get(pattern); + if (!regExp) { + if (Z_ANCHOR_REGEXP.test(pattern)) { + unsupported(); + } + + try { + // ajv uses the "u" flag (unicodeRegExp: true) + regExp = new RegExp(pattern, 'u'); + } catch { + unsupported(); + } + + this._regExpCache.set(pattern, regExp); + } + + return regExp; + } + + private _resolveRef(ref: unknown): ISchemaObject { + if (typeof ref !== 'string') { + unsupported(); + } + + let target: ISchemaObject | undefined = this._refTargets.get(ref); + if (target) { + return target; + } + + if (ref === '#') { + this._refTargets.set(ref, this._root); + return this._root; + } + + const match: RegExpExecArray | null = SIMPLE_DEFINITION_REF_REGEXP.exec(ref); + if (!match) { + unsupported(); + } + + const definitions: unknown = this._root.definitions; + if (!isPlainObject(definitions) || !hasOwn(definitions, match[1])) { + unsupported(); + } + + const definition: unknown = definitions[match[1]]; + if (!isPlainObject(definition)) { + unsupported(); + } + + target = definition; + this._refTargets.set(ref, target); + return target; + } + + private _analyzeSubschema(schema: unknown, contextTypes: JsonTypeName[]): void { + if (!isPlainObject(schema)) { + // Boolean schemas are not supported by draft-04, and are not used by Heft + unsupported(); + } + + this._analyzeSchema(schema, contextTypes, false); + } + + private _analyzeWithEmptyContext(schema: ISchemaObject): void { + if (!this._analyzedWithEmptyContext.has(schema)) { + this._analyzedWithEmptyContext.add(schema); + this._analyzeSchema(schema, [], schema === this._root); + } + } + + private _analyzeSchema(schema: ISchemaObject, contextTypes: JsonTypeName[], isRoot: boolean): void { + if (isRoot) { + this._analyzedWithEmptyContext.add(schema); + } + + const draft: JsonSchemaDraft = this._draft; + let hasRuleOtherThanRef: boolean = false; + + for (const key of Object.keys(schema)) { + const value: unknown = schema[key]; + if (key !== '$ref') { + hasRuleOtherThanRef = true; + } + + switch (key) { + case '$schema': { + if (!isRoot) { + unsupported(); + } + + break; + } + + case 'title': + case 'description': + case '$comment': { + if (typeof value !== 'string') { + unsupported(); + } + + break; + } + + case 'default': { + break; + } + + case 'examples': { + if (draft !== 'draft-07' || !Array.isArray(value)) { + unsupported(); + } + + break; + } + + case 'definitions': { + if (!isPlainObject(value)) { + unsupported(); + } + + for (const definitionName of Object.keys(value)) { + const definition: unknown = value[definitionName]; + if (!isPlainObject(definition)) { + unsupported(); + } + + this._analyzeWithEmptyContext(definition); + } + + break; + } + + case 'type': { + const types: unknown[] = Array.isArray(value) ? value : [value]; + if (types.length === 0 || new Set(types).size !== types.length) { + unsupported(); + } + + for (const type of types) { + if (typeof type !== 'string' || !JSON_TYPE_NAMES.has(type)) { + unsupported(); + } + } + + break; + } + + case 'enum': { + if (!Array.isArray(value) || value.length === 0 || !isSimpleJsonData(value, 0)) { + unsupported(); + } + + for (let i: number = 0; i < value.length; i++) { + for (let j: number = i + 1; j < value.length; j++) { + if (deepEqual(value[i], value[j])) { + unsupported(); + } + } + } + + break; + } + + case 'const': { + if (!isSimpleJsonData(value, 0)) { + unsupported(); + } + + break; + } + + case 'properties': { + if (!isPlainObject(value)) { + unsupported(); + } + + for (const propertyName of Object.keys(value)) { + if (propertyName in Object.prototype) { + // Includes "__proto__"; ajv's handling of these property names is unusual + unsupported(); + } + + this._analyzeSubschema(value[propertyName], []); + } + + break; + } + + case 'patternProperties': { + if (!isPlainObject(value)) { + unsupported(); + } + + for (const pattern of Object.keys(value)) { + if (pattern === '__proto__') { + unsupported(); + } + + this._getRegExp(pattern); + this._analyzeSubschema(value[pattern], []); + } + + break; + } + + case 'additionalProperties': { + if (typeof value !== 'boolean') { + this._analyzeSubschema(value, []); + } + + break; + } + + case 'required': { + if (!Array.isArray(value) || (draft === 'draft-04' && value.length === 0)) { + unsupported(); + } + + const seen: Set = new Set(); + for (const propertyName of value) { + if (typeof propertyName !== 'string' || seen.has(propertyName) || propertyName in Object.prototype) { + unsupported(); + } + + seen.add(propertyName); + } + + break; + } + + case 'items': { + // The array form (tuple validation) is not supported + this._analyzeSubschema(value, []); + break; + } + + case 'minItems': + case 'maxItems': + case 'minLength': + case 'maxLength': + case 'minProperties': + case 'maxProperties': { + if (!isNonNegativeInteger(value)) { + unsupported(); + } + + break; + } + + case 'uniqueItems': { + if (typeof value !== 'boolean') { + unsupported(); + } + + break; + } + + case 'pattern': { + this._getRegExp(value); + break; + } + + case 'minimum': + case 'maximum': { + if (!isFiniteNumber(value)) { + unsupported(); + } + + break; + } + + case 'exclusiveMinimum': + case 'exclusiveMaximum': { + // draft-04 uses a boolean modifier, which is not supported + if (draft !== 'draft-07' || !isFiniteNumber(value)) { + unsupported(); + } + + break; + } + + case 'allOf': + case 'anyOf': + case 'oneOf': { + // These are validated in place, so the subschemas inherit the type context; this is handled below + if (!Array.isArray(value) || value.length === 0) { + unsupported(); + } + + break; + } + + case 'not': { + // Validated in place; handled below + break; + } + + case '$ref': { + this._analyzeWithEmptyContext(this._resolveRef(value)); + break; + } + + default: { + if (!this._vendorKeywords.has(key)) { + // Unknown or unsupported keyword + unsupported(); + } + + break; + } + } + } + + // ajv's strict mode rejects a property that also matches a pattern property (allowMatchingProperties: false) + const properties: unknown = schema.properties; + const patternProperties: unknown = schema.patternProperties; + if (isPlainObject(properties) && isPlainObject(patternProperties)) { + for (const pattern of Object.keys(patternProperties)) { + const regExp: RegExp = this._getRegExp(pattern); + for (const propertyName of Object.keys(properties)) { + if (regExp.test(propertyName)) { + unsupported(); + } + } + } + } + + let dataTypes: JsonTypeName[] = contextTypes; + if (schema.$ref !== undefined && !hasRuleOtherThanRef) { + // ajv only evaluates the $ref, and skips the strictTypes checks for this schema object + return; + } + + // Replicate ajv's checkStrictTypes() (strictTypes: "log"). Any warning means the schema is not supported, + // so that the real ajv path logs it. + const types: JsonTypeName[] = getSchemaTypes(schema); + if (types.length) { + if (!contextTypes.length) { + dataTypes = types; + } else { + for (const type of types) { + if (!includesType(contextTypes, type)) { + unsupported(); + } + } + + // Replicates ajv's narrowSchemaTypes() + const narrowedTypes: JsonTypeName[] = []; + for (const contextType of contextTypes) { + if (includesType(types, contextType)) { + narrowedTypes.push(contextType); + } else if (types.includes('integer') && contextType === 'number') { + narrowedTypes.push('integer'); + } + } + + dataTypes = narrowedTypes; + } + } + + for (const key of Object.keys(schema)) { + const applicableType: JsonTypeName | undefined = KEYWORD_APPLICABLE_TYPE.get(key); + if (applicableType && !hasApplicableType(dataTypes, applicableType)) { + unsupported(); + } + } + + // In-place applicators inherit the (narrowed) type context + for (const key of ['allOf', 'anyOf', 'oneOf'] as const) { + const subschemas: unknown = schema[key]; + if (subschemas !== undefined) { + for (const subschema of subschemas as unknown[]) { + this._analyzeSubschema(subschema, dataTypes); + } + } + } + + if (schema.not !== undefined) { + this._analyzeSubschema(schema.not, dataTypes); + } + } +} + +function getSchemaDraft(schema: ISchemaObject, schemaVersion: JsonSchemaDraft | undefined): JsonSchemaDraft | undefined { + // Mirrors JsonSchema.ensureCompiled(): the schemaVersion option selects the ajv class, otherwise it is inferred + // from "$schema" (defaulting to draft-07). Only the meta-schema URLs that ajv resolves are supported. + const $schema: unknown = schema.$schema; + let inferred: JsonSchemaDraft | undefined; + switch ($schema) { + case undefined: + return schemaVersion ?? 'draft-07'; + case 'http://json-schema.org/draft-07/schema#': + case 'http://json-schema.org/draft-07/schema': + inferred = 'draft-07'; + break; + case 'http://json-schema.org/draft-04/schema#': + case 'http://json-schema.org/draft-04/schema': + inferred = 'draft-04'; + break; + default: + return undefined; + } + + return schemaVersion === undefined || schemaVersion === inferred ? inferred : undefined; +} + +/** + * A schema object, preprocessed for validation: only the keywords that are present are populated, and every + * instance has the same shape, which keeps property access fast even before the code is optimized. + */ +interface ICompiledNode { + types: JsonTypeName[] | undefined; + ref: ICompiledNode | undefined; + enumValues: unknown[] | undefined; + hasConst: boolean; + constValue: unknown; + allOf: ICompiledNode[] | undefined; + anyOf: ICompiledNode[] | undefined; + oneOf: ICompiledNode[] | undefined; + not: ICompiledNode | undefined; + stringChecks: IStringChecks | undefined; + numberChecks: INumberChecks | undefined; + arrayChecks: IArrayChecks | undefined; + objectChecks: IObjectChecks | undefined; +} + +interface IStringChecks { + minLength: number | undefined; + maxLength: number | undefined; + pattern: RegExp | undefined; +} + +interface INumberChecks { + minimum: number | undefined; + maximum: number | undefined; + exclusiveMinimum: number | undefined; + exclusiveMaximum: number | undefined; +} + +interface IArrayChecks { + minItems: number | undefined; + maxItems: number | undefined; + items: ICompiledNode | undefined; + uniqueItems: boolean; +} + +interface IObjectChecks { + required: string[] | undefined; + minProperties: number | undefined; + maxProperties: number | undefined; + properties: ISchemaObject | undefined; + propertyNodes: Map | undefined; + patternProperties: [RegExp, ICompiledNode][] | undefined; + // `true` if additional properties are allowed without validation + additionalProperties: boolean | ICompiledNode; +} + +function getRegExp(plan: ISchemaPlan, pattern: string): RegExp { + let regExp: RegExp | undefined = plan.regExpCache.get(pattern); + if (!regExp) { + // ajv uses the "u" flag (unicodeRegExp: true) + regExp = new RegExp(pattern, 'u'); + plan.regExpCache.set(pattern, regExp); + } + + return regExp; +} + +function compileNode(plan: ISchemaPlan, schema: ISchemaObject): ICompiledNode { + let node: ICompiledNode | undefined = plan.compiledNodes.get(schema); + if (node) { + return node; + } + + node = { + types: undefined, + ref: undefined, + enumValues: undefined, + hasConst: false, + constValue: undefined, + allOf: undefined, + anyOf: undefined, + oneOf: undefined, + not: undefined, + stringChecks: undefined, + numberChecks: undefined, + arrayChecks: undefined, + objectChecks: undefined + }; + // Register before compiling subschemas, to support recursive references + plan.compiledNodes.set(schema, node); + + const compileSubschemas: (value: unknown) => ICompiledNode[] = (value: unknown) => + (value as ISchemaObject[]).map((subschema: ISchemaObject) => compileNode(plan, subschema)); + + if (schema.type !== undefined) { + node.types = getSchemaTypes(schema); + } + + if (schema.$ref !== undefined) { + const ref: string = schema.$ref as string; + let target: ISchemaObject | undefined = plan.refTargets.get(ref); + if (!target) { + target = + ref === '#' + ? plan.root + : ((plan.root.definitions as ISchemaObject)[ + SIMPLE_DEFINITION_REF_REGEXP.exec(ref)![1] + ] as ISchemaObject); + plan.refTargets.set(ref, target); + } + + node.ref = compileNode(plan, target); + } + + if (schema.enum !== undefined) { + node.enumValues = schema.enum as unknown[]; + } + + if (schema.const !== undefined) { + node.hasConst = true; + node.constValue = schema.const; + } + + if (schema.allOf !== undefined) { + // The order of evaluation doesn't affect the result (and validation has no side effects), so evaluate + // subschemas without a $ref first: they are cheaper, and often reject the data. + const allOf: ICompiledNode[] = compileSubschemas(schema.allOf); + node.allOf = allOf + .filter((n: ICompiledNode) => n.ref === undefined) + .concat(allOf.filter((n: ICompiledNode) => n.ref !== undefined)); + } + + if (schema.anyOf !== undefined) { + node.anyOf = compileSubschemas(schema.anyOf); + } + + if (schema.oneOf !== undefined) { + node.oneOf = compileSubschemas(schema.oneOf); + } + + if (schema.not !== undefined) { + node.not = compileNode(plan, schema.not as ISchemaObject); + } + + if (schema.minLength !== undefined || schema.maxLength !== undefined || schema.pattern !== undefined) { + node.stringChecks = { + minLength: schema.minLength as number | undefined, + maxLength: schema.maxLength as number | undefined, + pattern: schema.pattern !== undefined ? getRegExp(plan, schema.pattern as string) : undefined + }; + } + + if ( + schema.minimum !== undefined || + schema.maximum !== undefined || + schema.exclusiveMinimum !== undefined || + schema.exclusiveMaximum !== undefined + ) { + node.numberChecks = { + minimum: schema.minimum as number | undefined, + maximum: schema.maximum as number | undefined, + exclusiveMinimum: schema.exclusiveMinimum as number | undefined, + exclusiveMaximum: schema.exclusiveMaximum as number | undefined + }; + } + + if ( + schema.minItems !== undefined || + schema.maxItems !== undefined || + schema.items !== undefined || + schema.uniqueItems === true + ) { + node.arrayChecks = { + minItems: schema.minItems as number | undefined, + maxItems: schema.maxItems as number | undefined, + items: schema.items !== undefined ? compileNode(plan, schema.items as ISchemaObject) : undefined, + uniqueItems: schema.uniqueItems === true + }; + } + + const additionalProperties: unknown = schema.additionalProperties; + if ( + schema.required !== undefined || + schema.minProperties !== undefined || + schema.maxProperties !== undefined || + schema.properties !== undefined || + schema.patternProperties !== undefined || + (additionalProperties !== undefined && additionalProperties !== true) + ) { + const properties: ISchemaObject | undefined = schema.properties as ISchemaObject | undefined; + let propertyNodes: Map | undefined; + if (properties) { + propertyNodes = new Map(); + for (const propertyName of Object.keys(properties)) { + propertyNodes.set(propertyName, compileNode(plan, properties[propertyName] as ISchemaObject)); + } + } + + const patternPropertiesSchema: ISchemaObject | undefined = schema.patternProperties as + | ISchemaObject + | undefined; + let patternProperties: [RegExp, ICompiledNode][] | undefined; + if (patternPropertiesSchema) { + patternProperties = []; + for (const pattern of Object.keys(patternPropertiesSchema)) { + patternProperties.push([ + getRegExp(plan, pattern), + compileNode(plan, patternPropertiesSchema[pattern] as ISchemaObject) + ]); + } + } + + node.objectChecks = { + required: schema.required as string[] | undefined, + minProperties: schema.minProperties as number | undefined, + maxProperties: schema.maxProperties as number | undefined, + properties, + propertyNodes, + patternProperties, + additionalProperties: + additionalProperties === undefined || additionalProperties === true + ? true + : additionalProperties === false + ? false + : compileNode(plan, additionalProperties as ISchemaObject) + }; + } + + return node; +} + +function matchesType(data: unknown, type: JsonTypeName): boolean { + switch (type) { + case 'string': + return typeof data === 'string'; + case 'number': + // Data is known to contain only finite numbers (strictNumbers: true) + return typeof data === 'number'; + case 'integer': + return typeof data === 'number' && Number.isInteger(data); + case 'boolean': + return typeof data === 'boolean'; + case 'null': + return data === null; + case 'array': + return Array.isArray(data); + case 'object': + return typeof data === 'object' && data !== null && !Array.isArray(data); + default: + return false; + } +} + +function countCodePoints(value: string): number { + let count: number = 0; + for (let i: number = 0; i < value.length; i++) { + const charCode: number = value.charCodeAt(i); + count++; + if (charCode >= 0xd800 && charCode <= 0xdbff && i + 1 < value.length) { + const nextCharCode: number = value.charCodeAt(i + 1); + if (nextCharCode >= 0xdc00 && nextCharCode <= 0xdfff) { + // Surrogate pair, counted as one character (ajv's ucs2length) + i++; + } + } + } + + return count; +} + +function validateNode(node: ICompiledNode, data: unknown): boolean { + const types: JsonTypeName[] | undefined = node.types; + if (types !== undefined) { + let typeMatches: boolean = false; + for (const type of types) { + if (matchesType(data, type)) { + typeMatches = true; + break; + } + } + + if (!typeMatches) { + return false; + } + } + + if (node.ref !== undefined && !validateNode(node.ref, data)) { + return false; + } + + const enumValues: unknown[] | undefined = node.enumValues; + if (enumValues !== undefined) { + let found: boolean = false; + for (const enumValue of enumValues) { + if (deepEqual(data, enumValue)) { + found = true; + break; + } + } + + if (!found) { + return false; + } + } + + if (node.hasConst && !deepEqual(data, node.constValue)) { + return false; + } + + const allOf: ICompiledNode[] | undefined = node.allOf; + if (allOf !== undefined) { + for (const subschema of allOf) { + if (!validateNode(subschema, data)) { + return false; + } + } + } + + const anyOf: ICompiledNode[] | undefined = node.anyOf; + if (anyOf !== undefined) { + let anyPassed: boolean = false; + for (const subschema of anyOf) { + if (validateNode(subschema, data)) { + anyPassed = true; + break; + } + } + + if (!anyPassed) { + return false; + } + } + + const oneOf: ICompiledNode[] | undefined = node.oneOf; + if (oneOf !== undefined) { + let passingCount: number = 0; + for (const subschema of oneOf) { + if (validateNode(subschema, data) && ++passingCount > 1) { + return false; + } + } + + if (passingCount !== 1) { + return false; + } + } + + if (node.not !== undefined && validateNode(node.not, data)) { + return false; + } + + switch (typeof data) { + case 'string': { + const stringChecks: IStringChecks | undefined = node.stringChecks; + if (stringChecks !== undefined) { + const { minLength, maxLength, pattern } = stringChecks; + if (minLength !== undefined || maxLength !== undefined) { + const length: number = countCodePoints(data); + if (minLength !== undefined && length < minLength) { + return false; + } + + if (maxLength !== undefined && length > maxLength) { + return false; + } + } + + if (pattern !== undefined && !pattern.test(data)) { + return false; + } + } + + break; + } + + case 'number': { + const numberChecks: INumberChecks | undefined = node.numberChecks; + if (numberChecks !== undefined) { + const { minimum, maximum, exclusiveMinimum, exclusiveMaximum } = numberChecks; + if (minimum !== undefined && data < minimum) { + return false; + } + + if (maximum !== undefined && data > maximum) { + return false; + } + + if (exclusiveMinimum !== undefined && data <= exclusiveMinimum) { + return false; + } + + if (exclusiveMaximum !== undefined && data >= exclusiveMaximum) { + return false; + } + } + + break; + } + + case 'object': { + if (data === null) { + break; + } + + if (Array.isArray(data)) { + const arrayChecks: IArrayChecks | undefined = node.arrayChecks; + if (arrayChecks !== undefined) { + const { minItems, maxItems, items, uniqueItems } = arrayChecks; + if (minItems !== undefined && data.length < minItems) { + return false; + } + + if (maxItems !== undefined && data.length > maxItems) { + return false; + } + + if (items !== undefined) { + for (const item of data) { + if (!validateNode(items, item)) { + return false; + } + } + } + + if (uniqueItems) { + for (let i: number = 1; i < data.length; i++) { + for (let j: number = 0; j < i; j++) { + if (deepEqual(data[i], data[j])) { + return false; + } + } + } + } + } + + break; + } + + const objectChecks: IObjectChecks | undefined = node.objectChecks; + if (objectChecks === undefined) { + break; + } + + const dataObject: Record = data as Record; + const { required, minProperties, maxProperties, properties, propertyNodes, patternProperties } = + objectChecks; + if (required !== undefined) { + for (const propertyName of required) { + if (dataObject[propertyName] === undefined) { + return false; + } + } + } + + const dataKeys: string[] = Object.keys(dataObject); + if (minProperties !== undefined && dataKeys.length < minProperties) { + return false; + } + + if (maxProperties !== undefined && dataKeys.length > maxProperties) { + return false; + } + + if (propertyNodes !== undefined) { + for (const [propertyName, propertyNode] of propertyNodes) { + const propertyValue: unknown = dataObject[propertyName]; + if (propertyValue !== undefined && !validateNode(propertyNode, propertyValue)) { + return false; + } + } + } + + if (patternProperties !== undefined) { + for (const [regExp, patternNode] of patternProperties) { + for (const key of dataKeys) { + if (regExp.test(key) && !validateNode(patternNode, dataObject[key])) { + return false; + } + } + } + } + + const additionalProperties: boolean | ICompiledNode = objectChecks.additionalProperties; + if (additionalProperties !== true) { + for (const key of dataKeys) { + if (properties !== undefined && hasOwn(properties, key)) { + continue; + } + + if (patternProperties !== undefined) { + let matchesPattern: boolean = false; + for (const [regExp] of patternProperties) { + if (regExp.test(key)) { + matchesPattern = true; + break; + } + } + + if (matchesPattern) { + continue; + } + } + + if (additionalProperties === false || !validateNode(additionalProperties, dataObject[key])) { + return false; + } + } + } + + break; + } + + default: { + break; + } + } + + return true; +} + +/** + * Deep-copies data that was verified by isSimpleJsonData(). Unlike structuredClone(), the copy is created in the + * current realm (structuredClone() may belong to another realm, for example in a vm context). + */ +function cloneJsonData(value: unknown): unknown { + if (typeof value !== 'object' || value === null) { + return value; + } + + if (Array.isArray(value)) { + return value.map(cloneJsonData); + } + + const result: Record = {}; + for (const key of Object.keys(value)) { + result[key] = cloneJsonData((value as Record)[key]); + } + + return result; +} + +/** + * Options for {@link analyzeSchemaForFastPath}. + */ +export interface IJsonSchemaFastPathOptions { + schemaVersion: JsonSchemaDraft | undefined; + rejectVendorExtensionKeywords: boolean; +} + +/** + * The result of analyzing a schema that is in the supported subset. + */ +export interface IJsonSchemaFastPathPlan { + readonly _plan: ISchemaPlan; +} + +/** + * Analyzes a schema for {@link isDefinitelyValid}. Returns `undefined` if the schema is not in the supported subset + * (or if ajv would reject it or log a warning while compiling it). + * + * @remarks + * The schema is copied, so later changes to `schemaObject` do not affect the returned plan (like a compiled ajv + * validator). + */ +export function analyzeSchemaForFastPath( + schemaObject: object, + options: IJsonSchemaFastPathOptions +): IJsonSchemaFastPathPlan | undefined { + if (!isPlainObject(schemaObject) || !isSimpleJsonData(schemaObject, 0)) { + return undefined; + } + + const root: ISchemaObject = cloneJsonData(schemaObject) as ISchemaObject; + const draft: JsonSchemaDraft | undefined = getSchemaDraft(root, options.schemaVersion); + if (!draft) { + return undefined; + } + + try { + return { _plan: new SchemaAnalyzer(root, draft, options.rejectVendorExtensionKeywords).analyze() }; + } catch (e) { + if (!(e instanceof UnsupportedSchemaError)) { + throw e; + } + + return undefined; + } +} + +/** + * Returns `true` only if validating `data` with ajv against the analyzed schema is guaranteed to succeed. A `false` + * result means "unknown": the caller must perform the real validation. + */ +export function isDefinitelyValid(plan: IJsonSchemaFastPathPlan, data: unknown): boolean { + if (!isSimpleJsonData(data, 0)) { + return false; + } + + return validateNode(compileNode(plan._plan, plan._plan.root), data); +} diff --git a/libraries/node-core-library/src/LockFile.ts b/libraries/node-core-library/src/LockFile.ts index c8c913056e4..7f1a94c73bd 100644 --- a/libraries/node-core-library/src/LockFile.ts +++ b/libraries/node-core-library/src/LockFile.ts @@ -2,6 +2,8 @@ // See LICENSE in the project root for license information. import * as path from 'node:path'; +// This import is only used for types (the module is loaded lazily); the value-style import keeps the API report stable. +// eslint-disable-next-line @typescript-eslint/consistent-type-imports import * as child_process from 'node:child_process'; import { FileSystem } from './FileSystem'; @@ -9,6 +11,13 @@ import { FileWriter } from './FileWriter'; import { Async } from './Async'; import { getWindowsLockFileDirtyPath, tryAcquireWindowsLockFile } from './WindowsLockFile'; +/** + * node:child_process (which also loads the net, dgram and stream implementations) is only loaded when needed. + */ +function _getChildProcess(): typeof child_process { + return require('node:child_process'); +} + /** * http://man7.org/linux/man-pages/man5/proc.5.html * (22) starttime %llu @@ -82,7 +91,7 @@ export function getProcessStartTime(pid: number): string | undefined { throw new Error(`Unsupported system: ${process.platform}`); } - const psResult: child_process.SpawnSyncReturns = child_process.spawnSync('ps', args, { + const psResult: child_process.SpawnSyncReturns = _getChildProcess().spawnSync('ps', args, { encoding: 'utf8' }); const psStdout: string = psResult.stdout; diff --git a/libraries/node-core-library/src/test/JsonSchemaFastPath.test.ts b/libraries/node-core-library/src/test/JsonSchemaFastPath.test.ts new file mode 100644 index 00000000000..758f9e5f62b --- /dev/null +++ b/libraries/node-core-library/src/test/JsonSchemaFastPath.test.ts @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { JsonObject } from '../JsonFile'; +import { JsonSchema, type IJsonSchemaFromObjectOptions } from '../JsonSchema'; +import { analyzeSchemaForFastPath, isDefinitelyValid } from '../JsonSchemaFastPath'; + +// The fast path must never change the observable behavior of JsonSchema: these tests compare +// a JsonSchema that can use the fast path with one that has already been compiled with ajv +// (a compiled validator disables the fast path). + +interface IOutcome { + thrown?: string; + errors: string[]; +} + +function validate( + schemaObject: JsonObject, + data: JsonObject, + compileFirst: boolean, + options?: IJsonSchemaFromObjectOptions, + ignoreSchemaField?: boolean +): IOutcome { + const outcome: IOutcome = { errors: [] }; + try { + const schema: JsonSchema = JsonSchema.fromLoadedObject(schemaObject, options); + if (compileFirst) { + schema.ensureCompiled(); + } + schema.validateObjectWithCallback(data, (errorInfo) => outcome.errors.push(errorInfo.details), { + ignoreSchemaField + }); + } catch (e) { + outcome.thrown = (e as Error).message; + } + return outcome; +} + +function expectSameAsAjv( + schemaObject: JsonObject, + data: JsonObject, + options?: IJsonSchemaFromObjectOptions, + ignoreSchemaField?: boolean +): IOutcome { + const withFastPath: IOutcome = validate(schemaObject, data, false, options, ignoreSchemaField); + const withAjv: IOutcome = validate(schemaObject, data, true, options, ignoreSchemaField); + expect(withFastPath).toEqual(withAjv); + return withFastPath; +} + +const DRAFT_04: string = 'http://json-schema.org/draft-04/schema#'; +const DRAFT_07: string = 'http://json-schema.org/draft-07/schema#'; + +const OBJECT_SCHEMA: JsonObject = { + $schema: DRAFT_04, + type: 'object', + additionalProperties: false, + required: ['name'], + properties: { + name: { type: 'string', minLength: 2 }, + list: { type: 'array', items: { type: 'string', pattern: '^[a-z]+$' } } + } +}; + +describe('JsonSchemaFastPath', () => { + it('proves simple valid objects without compiling', () => { + const plan = analyzeSchemaForFastPath(OBJECT_SCHEMA, { + schemaVersion: undefined, + rejectVendorExtensionKeywords: false + }); + expect(plan).toBeDefined(); + expect(isDefinitelyValid(plan!, { name: 'ab', list: ['x'] })).toBe(true); + // Invalid data is never "definitely valid" + expect(isDefinitelyValid(plan!, { name: 'a' })).toBe(false); + expect(isDefinitelyValid(plan!, { name: 'ab', extra: 1 })).toBe(false); + // Code points, not UTF-16 code units (like ajv) + expect(isDefinitelyValid(plan!, { name: '\ud83d\ude00' })).toBe(false); + }); + + it('matches ajv for valid and invalid data', () => { + expect(expectSameAsAjv(OBJECT_SCHEMA, { name: 'ab', list: ['x', 'y'] })).toEqual({ errors: [] }); + expect(expectSameAsAjv(OBJECT_SCHEMA, { name: 'a' }).errors).toHaveLength(1); + expect(expectSameAsAjv(OBJECT_SCHEMA, { name: 'ab', list: ['X'] }).errors).toHaveLength(1); + expectSameAsAjv(OBJECT_SCHEMA, { list: [] }); + expectSameAsAjv(OBJECT_SCHEMA, { name: 'ab', unexpected: true }); + }); + + it('honors the schemaVersion option', () => { + const noSchemaField: JsonObject = { ...OBJECT_SCHEMA, $schema: undefined }; + delete noSchemaField.$schema; + expectSameAsAjv(noSchemaField, { name: 'ab' }, { schemaVersion: 'draft-04' }); + expectSameAsAjv(noSchemaField, { name: 'ab' }, { schemaVersion: 'draft-07' }); + // Conflicting "$schema" and schemaVersion: ajv reports an error, which the fast path must not hide + expect(expectSameAsAjv(OBJECT_SCHEMA, { name: 'ab' }, { schemaVersion: 'draft-07' }).thrown).toBeDefined(); + expect( + analyzeSchemaForFastPath(OBJECT_SCHEMA, { schemaVersion: 'draft-07', rejectVendorExtensionKeywords: false }) + ).toBeUndefined(); + }); + + it('honors rejectVendorExtensionKeywords', () => { + const vendorSchema: JsonObject = { ...OBJECT_SCHEMA, 'x-tsdoc-release-tag': '@beta' }; + expect(expectSameAsAjv(vendorSchema, { name: 'ab' })).toEqual({ errors: [] }); + expect( + expectSameAsAjv(vendorSchema, { name: 'ab' }, { rejectVendorExtensionKeywords: true }).thrown + ).toBeDefined(); + }); + + it('does not hide schema errors or strict mode errors', () => { + expect( + expectSameAsAjv({ $schema: DRAFT_07, type: 'object', properties: { a: { type: 'strng' } } }, {}).thrown + ).toBeDefined(); + expect( + expectSameAsAjv({ $schema: DRAFT_07, type: 'object', properties: { a: { bogusKeyword: 1 } } }, {}).thrown + ).toBeDefined(); + expect(expectSameAsAjv({ $schema: DRAFT_04, type: 'object', required: [] }, {}).thrown).toBeDefined(); + }); + + it('strips the $schema field only when ignoreSchemaField is set', () => { + const data: JsonObject = { $schema: 'https://example.com/schema.json', name: 'ab' }; + expect(expectSameAsAjv(OBJECT_SCHEMA, data, undefined, true)).toEqual({ errors: [] }); + expect(expectSameAsAjv(OBJECT_SCHEMA, data, undefined, false).errors).toHaveLength(1); + }); + + it('is not affected by changes to the schema object after the first validation (like a compiled validator)', () => { + const schemaObject: JsonObject = JSON.parse(JSON.stringify(OBJECT_SCHEMA)); + const schema: JsonSchema = JsonSchema.fromLoadedObject(schemaObject); + const errors: string[] = []; + schema.validateObjectWithCallback({ name: 'ab' }, (errorInfo) => errors.push(errorInfo.details)); + schemaObject.properties.name.minLength = 5; + schema.validateObjectWithCallback({ name: 'ab' }, (errorInfo) => errors.push(errorInfo.details)); + expect(errors).toEqual([]); + }); +}); diff --git a/libraries/operation-graph/config/heft.json b/libraries/operation-graph/config/heft.json new file mode 100644 index 00000000000..860f73615d3 --- /dev/null +++ b/libraries/operation-graph/config/heft.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", + + "extends": "decoupled-local-node-rig/profiles/default/config/heft.json", + "phasesByName": { + "build": { + "tasksByName": { + // Make lib-commonjs/index.js load each re-exported module on first access, so that consumers + // only pay for the parts of this package that they actually use. + "lazy-barrel": { + "taskDependencies": ["typescript"], + "taskPlugin": { + "pluginPackage": "@rushstack/heft", + "pluginName": "run-script-plugin", + "options": { + "scriptPath": "./node_modules/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js" + } + } + } + } + } + } +} diff --git a/libraries/rig-package/src/Helpers.ts b/libraries/rig-package/src/Helpers.ts index 49457a4f7fd..af3061ac543 100644 --- a/libraries/rig-package/src/Helpers.ts +++ b/libraries/rig-package/src/Helpers.ts @@ -4,17 +4,29 @@ import * as path from 'node:path'; import * as fs from 'node:fs'; -import nodeResolve from 'resolve'; +import type nodeResolve from 'resolve'; // These helpers avoid taking dependencies on other NPM packages +let _nodeResolve: typeof nodeResolve | undefined; + // Based on Path.isDownwardRelative() from @rushstack/node-core-library const _upwardPathSegmentRegex: RegExp = /([\/\\]|^)\.\.([\/\\]|$)/; export class Helpers { + /** + * The "resolve" package is only needed when the rig package must be resolved, so it is loaded on first use. + */ + public static getNodeResolve(): typeof nodeResolve { + if (!_nodeResolve) { + _nodeResolve = require('resolve') as typeof nodeResolve; + } + return _nodeResolve; + } + public static async nodeResolveAsync(id: string, opts: nodeResolve.AsyncOpts): Promise { return await new Promise((resolve: (result: string) => void, reject: (error: Error) => void) => { - nodeResolve(id, opts, (error: Error | null, result: string | undefined) => { + Helpers.getNodeResolve()(id, opts, (error: Error | null, result: string | undefined) => { if (error) { reject(error); } else { diff --git a/libraries/rig-package/src/RigConfig.ts b/libraries/rig-package/src/RigConfig.ts index e3b5e928477..2e593373b7b 100644 --- a/libraries/rig-package/src/RigConfig.ts +++ b/libraries/rig-package/src/RigConfig.ts @@ -4,7 +4,7 @@ import * as path from 'node:path'; import * as fs from 'node:fs'; -import * as nodeResolve from 'resolve'; +import type * as nodeResolve from 'resolve'; import * as jju from 'jju'; import { Helpers } from './Helpers'; @@ -388,7 +388,7 @@ export class RigConfig implements IRigConfig { const rigPackageJsonModuleSpecifier: string = `${this.rigPackageName}/package.json`; const resolveOptions: nodeResolve.Opts = { basedir: this.projectFolderPath }; - const resolvedRigPackageJsonPath: string = nodeResolve.sync( + const resolvedRigPackageJsonPath: string = Helpers.getNodeResolve().sync( rigPackageJsonModuleSpecifier, resolveOptions ); diff --git a/libraries/terminal/config/heft.json b/libraries/terminal/config/heft.json new file mode 100644 index 00000000000..860f73615d3 --- /dev/null +++ b/libraries/terminal/config/heft.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", + + "extends": "decoupled-local-node-rig/profiles/default/config/heft.json", + "phasesByName": { + "build": { + "tasksByName": { + // Make lib-commonjs/index.js load each re-exported module on first access, so that consumers + // only pay for the parts of this package that they actually use. + "lazy-barrel": { + "taskDependencies": ["typescript"], + "taskPlugin": { + "pluginPackage": "@rushstack/heft", + "pluginName": "run-script-plugin", + "options": { + "scriptPath": "./node_modules/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js" + } + } + } + } + } + } +} diff --git a/libraries/ts-command-line/config/heft.json b/libraries/ts-command-line/config/heft.json new file mode 100644 index 00000000000..860f73615d3 --- /dev/null +++ b/libraries/ts-command-line/config/heft.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", + + "extends": "decoupled-local-node-rig/profiles/default/config/heft.json", + "phasesByName": { + "build": { + "tasksByName": { + // Make lib-commonjs/index.js load each re-exported module on first access, so that consumers + // only pay for the parts of this package that they actually use. + "lazy-barrel": { + "taskDependencies": ["typescript"], + "taskPlugin": { + "pluginPackage": "@rushstack/heft", + "pluginName": "run-script-plugin", + "options": { + "scriptPath": "./node_modules/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js" + } + } + } + } + } + } +} diff --git a/rigs/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js b/rigs/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js new file mode 100644 index 00000000000..fbe28851c90 --- /dev/null +++ b/rigs/decoupled-local-node-rig/profiles/default/includes/lazy-barrel/lazyBarrel.js @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +'use strict'; + +// A Heft "run-script-plugin" script that rewrites a TypeScript-emitted CommonJS barrel +// (lib-commonjs/index.js by default) so that each re-exported module is only loaded when one of +// its exports is first accessed. +// +// Only the `var X_1 = require("./X");` lines are changed. The `exports.A = ... = void 0;` lines and the +// `Object.defineProperty(exports, "A", { enumerable: true, get: function () { return X_1.A; } });` lines are +// kept as emitted, so the export names, property attributes and values are identical, and Node.js can still +// detect the named exports for ESM importers. Any line that does not match the expected TypeScript output +// fails the build rather than producing a partially transformed barrel. + +const fs = require('node:fs'); +const path = require('node:path'); + +const MARKER = '__lazyBarrelRequire'; + +const REQUIRE_LINE_REGEXP = /^var ([A-Za-z_$][\w$]*) = require\(("\.{1,2}\/[^"]+")\);$/; +const ALLOWED_LINE_REGEXPS = [ + /^$/, + /^"use strict";$/, + /^\/\/.*$/, + /^\s*\/?\*.*$/, + /^Object\.defineProperty\(exports, "__esModule", \{ value: true \}\);$/, + /^exports\.[A-Za-z_$][\w$]*( = exports\.[A-Za-z_$][\w$]*)* = void 0;$/, + /^Object\.defineProperty\(exports, "[A-Za-z_$][\w$]*", \{ enumerable: true, get: function \(\) \{ return [A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*; \} \}\);$/ +]; + +const HELPER = + `function ${MARKER}(load) { var m; ` + + 'return new Proxy({}, { get: function (_target, key) { return (m || (m = load()))[key]; } }); }'; + +/** + * @param {string} source - the emitted CommonJS barrel + * @param {string} filePath - used in error messages + * @returns {string} the lazy barrel + */ +function transformBarrel(source, filePath) { + const newline = source.includes('\r\n') ? '\r\n' : '\n'; + const lines = source.split(/\r?\n/); + let sourceMapLineIndex = -1; + let requireCount = 0; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const requireMatch = REQUIRE_LINE_REGEXP.exec(line); + if (requireMatch) { + lines[i] = `var ${requireMatch[1]} = ${MARKER}(function () { return require(${requireMatch[2]}); });`; + requireCount++; + } else if (line.startsWith('//# sourceMappingURL=')) { + sourceMapLineIndex = i; + } else if (!ALLOWED_LINE_REGEXPS.some((regexp) => regexp.test(line))) { + throw new Error(`${filePath}:${i + 1}: unexpected line in the barrel, cannot make it lazy: ${line}`); + } + } + + if (requireCount === 0) { + return source; + } + + // Function declarations are hoisted, so the helper can be placed at the end of the file. + // Keeping all other lines in place keeps the existing source map line numbers valid. + if (sourceMapLineIndex >= 0) { + lines.splice(sourceMapLineIndex, 0, HELPER); + } else { + lines.push(HELPER); + } + return lines.join(newline); +} + +async function runAsync({ heftConfiguration, heftTaskSession, scriptOptions }) { + const barrelPaths = (scriptOptions && scriptOptions.barrels) || ['lib-commonjs/index.js']; + for (const barrelPath of barrelPaths) { + const filePath = path.resolve(heftConfiguration.buildFolderPath, barrelPath); + const source = await fs.promises.readFile(filePath, 'utf8'); + if (source.includes(MARKER)) { + // Already transformed (for example, TypeScript did not re-emit the file in an incremental build) + continue; + } + + const result = transformBarrel(source, filePath); + if (result !== source) { + await fs.promises.writeFile(filePath, result, 'utf8'); + heftTaskSession.logger.terminal.writeVerboseLine(`Made ${barrelPath} lazy`); + } + } +} + +exports.runAsync = runAsync; +exports.transformBarrel = transformBarrel;