From ea8e3ab9d9c475a68eaed7e231f9720d1a90d74e Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:31:53 +0100 Subject: [PATCH] feat(lib)!: add asset pipeline foundation types and hash utilities BREAKING CHANGE: TerraformAsset.assetHash is now readonly (required by `implements IAsset`). The generated setter is removed in Java, C#, Python and Go. It was assigned once at construction, so this only removes an assignment that would have desynced `assetHash` from `path`. --- .gitignore | 2 +- packages/cdktn/src/asset-hash.ts | 131 ++++++++ packages/cdktn/src/assets.ts | 240 +++++++++++++++ packages/cdktn/src/errors.ts | 39 +++ packages/cdktn/src/ignore-strategy.ts | 103 +++++++ packages/cdktn/src/index.ts | 3 + packages/cdktn/src/private/fs.ts | 202 ++++++++++-- packages/cdktn/src/terraform-asset.ts | 143 ++++++--- packages/cdktn/test/asset-hash.test.ts | 291 ++++++++++++++++++ packages/cdktn/test/assets-types.test.ts | 182 +++++++++++ .../cdktn/test/canonical-asset-hash.test.ts | 221 ++++++++++++- 11 files changed, 1497 insertions(+), 60 deletions(-) create mode 100644 packages/cdktn/src/asset-hash.ts create mode 100644 packages/cdktn/src/assets.ts create mode 100644 packages/cdktn/src/ignore-strategy.ts create mode 100644 packages/cdktn/test/asset-hash.test.ts create mode 100644 packages/cdktn/test/assets-types.test.ts diff --git a/.gitignore b/.gitignore index ed9f99186..312579bb3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,7 @@ bootstrap.json cdk-terrain.github-issues .idea tsconfig.tsbuildinfo -examples/java/gradle-shared-module/.gradle/ +.gradle/ .nx/ diff --git a/packages/cdktn/src/asset-hash.ts b/packages/cdktn/src/asset-hash.ts new file mode 100644 index 000000000..a8fa7e6a5 --- /dev/null +++ b/packages/cdktn/src/asset-hash.ts @@ -0,0 +1,131 @@ +// Copyright (c) HashiCorp, Inc +// SPDX-License-Identifier: MPL-2.0 +import * as crypto from "crypto"; +import * as path from "path"; +import { hashPath, findFileAboveCwd } from "./private/fs"; +import { ExcludeIgnoreStrategy, IIgnoreStrategy } from "./ignore-strategy"; +import { + assetHashConflictingExcludeOptions, + assetHashOutOfScopeCdktnJson, +} from "./errors"; + +/** + * Options for {@link AssetHash.of}. + */ +export interface AssetHashOptions { + /** + * Paths to exclude, relative to the hashed path. Cannot be combined with + * `ignoreStrategy`, which replaces this matcher rather than layering on + * top of it. + * + * @default - nothing is excluded + */ + readonly exclude?: string[]; + + /** + * Extra information to fold into the hash. + * + * @default - no extra data + */ + readonly extraHash?: string; + + /** + * Exclusion matching, for callers that need `.gitignore` / `.dockerignore` + * parity rather than the built-in exact-path / suffix / directory matcher. + * + * @default - `exclude` is used with the built-in matcher + */ + readonly ignoreStrategy?: IIgnoreStrategy; +} + +/** + * Computes a content hash of a file or directory without staging it. + * + * Providers that read a local path directly — a Docker build `context`, for + * example — need a content hash to drive `triggers`, but have no use for a + * staged copy of the source. `Asset` and `TerraformAsset` always produce a + * staged copy; this is the identity half without the staging half. + */ +export class AssetHash { + /** + * Content hash of a file or directory, without staging it. + * + * This is a hash of the source tree, and it does not depend on how the + * source is later packaged. `hashPath` is always called with `archive` + * unset, so directory records are part of the digest; the hash of a given + * tree is therefore the same whether it is later copied, zipped, or packed + * by a custom `IAssetPackaging` such as `tar.bz2`. + * + * A consequence worth stating: this does not equal + * `TerraformAsset(dir, { type: ARCHIVE }).assetHash` for a directory with + * subdirectories. That asset frames its hash to the emitted ZIP, which has + * no directory entries (see #323), so directory-only changes move it and + * not this. It equals a `FILE` / `DIRECTORY` `TerraformAsset` hash only + * when the `canonicalAssetHashes` flag is enabled, since that flag is what + * puts the asset on this same canonical scheme. + * + * A relative `filePath` is resolved against the directory containing + * `cdktf.json`, the same base `TerraformAsset` uses, so both hash the same + * source regardless of the process working directory. Absolute paths are + * used as-is. Throws if `filePath` is relative and no `cdktf.json` is found + * above the working directory. + * @param filePath - path to a file or directory to hash + * @param options - see {@link AssetHashOptions} + */ + public static of(filePath: string, options: AssetHashOptions = {}): string { + if (options.exclude?.length && options.ignoreStrategy) { + throw assetHashConflictingExcludeOptions(); + } + + const resolved = AssetHash.resolvePath(filePath); + const strategy = + options.ignoreStrategy ?? + new ExcludeIgnoreStrategy(options.exclude ?? []); + + // Pinned to the canonical scheme: this is a brand-new API with no + // existing hashes to preserve, so it has no reason to start on the + // legacy scheme that `canonicalAssetHashes` exists to move away from. + const baseHash = hashPath(resolved, { + canonical: true, + shouldExclude: (relativePath, isDirectory) => + strategy.ignores({ relativePath, isDirectory }), + descendIntoExcludedDirectories: + strategy.pruneExcludedDirectories === false, + }); + + if (!options.extraHash) { + return baseHash; + } + + return crypto + .createHash("md5") + .update(baseHash) + .update(options.extraHash) + .digest("hex") + .slice(0, 32) + .toUpperCase(); + } + + /** + * Resolve `filePath` to an absolute path using the same base as + * `TerraformAsset`: relative paths are anchored to the directory holding + * `cdktf.json`, not `process.cwd()`, so a hash taken here matches the one + * `TerraformAsset` computes for the same source even when the app is run + * from a subdirectory or a test runner. Unlike `TerraformAsset`, there is + * no construct scope to read the `cdktfJsonPath` context from, so this uses + * the `findFileAboveCwd` fallback directly. + * @param filePath - path passed to {@link AssetHash.of} + */ + private static resolvePath(filePath: string): string { + if (path.isAbsolute(filePath)) { + return filePath; + } + const cdktfJsonPath = findFileAboveCwd("cdktf.json"); + if (!cdktfJsonPath) { + throw assetHashOutOfScopeCdktnJson(filePath); + } + return path.resolve(path.dirname(cdktfJsonPath), filePath); + } + + private constructor() {} +} diff --git a/packages/cdktn/src/assets.ts b/packages/cdktn/src/assets.ts new file mode 100644 index 000000000..2a379faf6 --- /dev/null +++ b/packages/cdktn/src/assets.ts @@ -0,0 +1,240 @@ +// Copyright (c) HashiCorp, Inc +// SPDX-License-Identifier: MPL-2.0 + +import * as fs from "fs"; +import { archiveSync, copySync } from "./private/fs"; +import { IIgnoreStrategy } from "./ignore-strategy"; + +/** + * Common interface for all assets. + */ +export interface IAsset { + /** + * A hash of this asset, which is available at construction time. As this is a plain string, it + * can be used in construct IDs in order to enforce creation of a new resource when the content + * hash has changed. + */ + readonly assetHash: string; +} + +/** + * Asset hash options + */ +export interface AssetOptions { + /** + * Specify a custom hash for this asset. If `assetHashType` is set it must + * be set to `AssetHashType.CUSTOM`. The value is used verbatim as the asset + * hash, and because it names the staged asset file it may only contain + * letters, digits, `_`, `.` and `-`. + * + * NOTE: the hash is used in order to identify a specific revision of the asset, and + * used for optimizing and caching deployment activities related to this asset such as + * packaging, uploading to cloud storage, etc. If you chose to customize the hash, you will + * need to make sure it is updated every time the asset changes, or otherwise it is + * possible that some deployments will not be invalidated. + * + * @default - based on `assetHashType` + */ + readonly assetHash?: string; + + /** + * Specifies the type of hash to calculate for this asset. + * + * If `assetHash` is configured, this option must be `undefined` or + * `AssetHashType.CUSTOM`. + * + * @default - the default is `AssetHashType.SOURCE`, but if `assetHash` is + * explicitly specified this value defaults to `AssetHashType.CUSTOM`. + */ + readonly assetHashType?: AssetHashType; +} + +/** + * The type of asset hash + * + * NOTE: the hash is used in order to identify a specific revision of the asset, and + * used for optimizing and caching deployment activities related to this asset such as + * packaging, uploading to cloud storage, etc. + */ +export enum AssetHashType { + /** + * Based on the content of the source path + * + * Use `SOURCE` when the content of the asset changes frequently or when + * you want to track changes to the source files directly. + */ + SOURCE = "source", + + /** + * Based on the content of the bundling output + * + * Use `OUTPUT` when the source of the asset is a top level folder containing + * code and/or dependencies that are not directly linked to the asset. + */ + OUTPUT = "output", + + /** + * Use a custom hash + */ + CUSTOM = "custom", +} + +/** + * How a staged asset is produced and what shape it takes on disk. + * + * Packaging answers two independent questions: how the artifact is produced + * (copy, zip, tar.gz, ...) and whether the result is a directory or a single + * file. A closed enum can only ever answer the second one, so it is an + * interface rather than an enum — custom formats (e.g. `tar.bz2`) need no + * core change. + */ +export interface IAssetPackaging { + /** + * Appended to the staged artifact name, e.g. ".zip", "", ".tar.bz2". + */ + readonly extension: string; + + /** + * Whether the staged result is a directory rather than a single file. + * + * Publishers branch on this to decide whether they upload one object or + * sync a tree. + */ + readonly producesDirectory: boolean; + + /** + * Perform the staging transformation, writing the packaged result to + * `options.target`. + * @param options - see {@link PackOptions} + */ + pack(options: PackOptions): void; +} + +/** + * Options for {@link IAssetPackaging.pack}. + * + * A struct rather than positional parameters: adding a struct field is + * additive, adding a method parameter is not, and `pack` is called through + * JSII where that distinction is a breaking-change boundary. + */ +export interface PackOptions { + /** + * Path to the resolved (already bundled, if applicable) source. + */ + readonly source: string; + + /** + * Path the packaged result should be written to. + */ + readonly target: string; + + /** + * Entries to omit from the packaged result. Must match the strategy used + * to hash the same source, or the hash and the artifact describe + * different sets of files. + * + * @default - nothing is excluded + */ + readonly ignoreStrategy?: IIgnoreStrategy; +} + +/** + * Copies a single file verbatim. + */ +class FilePackaging implements IAssetPackaging { + public readonly extension = ""; + public readonly producesDirectory = false; + public pack(options: PackOptions): void { + fs.copyFileSync(options.source, options.target); + } +} + +/** + * Copies a directory tree verbatim, without archiving it. + */ +class DirectoryPackaging implements IAssetPackaging { + public readonly extension = ""; + public readonly producesDirectory = true; + public pack(options: PackOptions): void { + copySync(options.source, options.target, { + shouldExclude: options.ignoreStrategy + ? (relativePath, isDirectory) => + options.ignoreStrategy!.ignores({ relativePath, isDirectory }) + : undefined, + descendIntoExcludedDirectories: + options.ignoreStrategy?.pruneExcludedDirectories === false, + }); + } +} + +/** + * Archives a directory tree into a single zip file. + */ +class ZipPackaging implements IAssetPackaging { + public readonly extension = ".zip"; + public readonly producesDirectory = false; + public pack(options: PackOptions): void { + archiveSync( + options.source, + options.target, + options.ignoreStrategy + ? (relativePath, isDirectory) => + options.ignoreStrategy!.ignores({ relativePath, isDirectory }) + : undefined, + options.ignoreStrategy?.pruneExcludedDirectories === false, + ); + } +} + +/** + * Built-in packaging strategies. Custom formats implement `IAssetPackaging` + * directly rather than extending this class. + */ +export class AssetPackaging { + /** + * Copy a single file as-is. + */ + public static readonly FILE: IAssetPackaging = new FilePackaging(); + + /** + * Copy a directory tree as-is, without archiving. + */ + public static readonly DIRECTORY: IAssetPackaging = new DirectoryPackaging(); + + /** + * Archive a directory tree into a single zip file. + */ + public static readonly ZIP: IAssetPackaging = new ZipPackaging(); + + private constructor() {} +} + +/** + * A staged artifact, ready to hand to an `IAssetPublisher`. + * + * Deliberately narrower than a location: `path` and `isDirectory` are known + * once staging runs, at synth time, before anything is published. Where an + * asset ends up — a bucket name, an object key, a URL — is resolved at + * apply time and belongs on the publisher's own reference type instead. + */ +export interface StagedAsset { + /** + * A hash on the content source. This hash is used to uniquely identify this + * asset throughout the system. If this value doesn't change, the asset will + * not be rebuilt or republished. + */ + readonly assetHash: string; + + /** + * The path to the staged artifact, relative to the stack directory. + */ + readonly path: string; + + /** + * Whether the staged artifact is a directory rather than a single file. + * + * Publishers branch on this to decide whether they upload one object or + * sync a tree; see `IAssetPackaging.producesDirectory`. + */ + readonly isDirectory: boolean; +} diff --git a/packages/cdktn/src/errors.ts b/packages/cdktn/src/errors.ts index 13a58c7c6..24f07101c 100644 --- a/packages/cdktn/src/errors.ts +++ b/packages/cdktn/src/errors.ts @@ -62,6 +62,45 @@ Take one of these values from the AssetType Enum. Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets `); +export const assetHashConflictingExcludeOptions = () => + new Error( + `Both 'exclude' and 'ignoreStrategy' were passed to AssetHash.of(), but 'ignoreStrategy' replaces 'exclude' rather than combining with it. Pass only one.`, + ); + +export const assetHashTypeOutputNotSupported = (id: string) => + new Error( + `TerraformAsset ${id} was configured with assetHashType 'OUTPUT', but bundling is not implemented yet, so there is no output to hash. Use 'SOURCE' (the default) to hash the source, or 'CUSTOM' with an explicit 'assetHash'. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + +export const assetHashTypeCustomRequiresHash = (id: string) => + new Error( + `TerraformAsset ${id} was configured with assetHashType 'CUSTOM' but no 'assetHash'. A custom hash type requires an explicit 'assetHash' value. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + +export const assetHashConflictingHashType = (id: string) => + new Error( + `TerraformAsset ${id} was configured with an explicit 'assetHash', so 'assetHashType' must be undefined or 'CUSTOM'. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + +export const assetHashTypeUnknown = (id: string, assetHashType: unknown) => + new Error( + `TerraformAsset ${id} was configured with an unknown assetHashType '${String(assetHashType)}'. Use one of AssetHashType.SOURCE, AssetHashType.OUTPUT, or AssetHashType.CUSTOM. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + +export const assetHashOutOfScopeCdktnJson = (configPath: string) => + new Error( + `AssetHash.of() was called with a relative path '${configPath}', but we cannot find the cdktf.json above your current working directory '${process.cwd()}' + +The cdktf.json file is needed to establish the base for the relative path (the '.' in './foo/bar'), so that the hash matches TerraformAsset for the same source. + +Place a cdktf.json at the root of your project, or pass an absolute path. Learn more: https://cdktn.io/docs/create-and-deploy/configuration-file +`, + ); + export const dynamicBlockNotSupported = (_foreachExpression: string) => new Error( `We do not support directly resolving a TerraformDynamicBlock. Dynamic blocks are only supported on block attributes of resources, data sources, and providers. diff --git a/packages/cdktn/src/ignore-strategy.ts b/packages/cdktn/src/ignore-strategy.ts new file mode 100644 index 000000000..9903b601b --- /dev/null +++ b/packages/cdktn/src/ignore-strategy.ts @@ -0,0 +1,103 @@ +// Copyright (c) HashiCorp, Inc +// SPDX-License-Identifier: MPL-2.0 +import { excludeMatcher } from "./private/fs"; + +/** + * A single entry presented to an {@link IIgnoreStrategy}. + * + * A struct rather than a positional `relativePath` string so the shape can + * evolve. `isDirectory` is carried from the outset because `.gitignore` / + * `.dockerignore` semantics turn on it: a `foo/` pattern matches a + * directory but not a file named `foo`, and a matcher cannot recover + * that distinction from the path text alone. + */ +export interface IgnoreQuery { + /** + * `/`-separated path relative to the asset root. + */ + readonly relativePath: string; + + /** + * Whether this entry is a directory rather than a file or symlink. + * + * The tree walkers set this from `lstat`, so a strategy can honor + * directory-only patterns without stat-ing the path itself (which it has + * no root to resolve against). Excluding a directory also excludes + * everything below it, since the walkers stop descending once a directory + * is excluded. + */ + readonly isDirectory: boolean; +} + +/** + * Decides whether a path relative to an asset root is excluded from staging + * and hashing. + * + * Core ships only the exact-path / `*.ext` / directory matcher used by + * `exclude` today (`ExcludeIgnoreStrategy`). Full glob, `.gitignore`, and + * `.dockerignore` parity can be implemented against this interface without + * core taking on a glob parser. + */ +export interface IIgnoreStrategy { + /** + * Whether the given entry should be excluded. + * @param query - the entry under consideration, see {@link IgnoreQuery} + */ + ignores(query: IgnoreQuery): boolean; + + /** + * Whether excluding a directory also excludes everything beneath it. + * + * When `true` (the default), the walkers stop descending as soon as a + * directory is excluded — cheaper, and correct for a strategy whose + * patterns never re-include a path below an excluded parent. + * + * A strategy with negation patterns must set this to `false`: + * `.gitignore` / `.dockerignore` allow `node_modules` followed by + * `!node_modules/keep`, which is only reachable if the walk descends into + * the excluded `node_modules` and asks about `node_modules/keep`. The + * excluded directory entry itself is still omitted; only the descent + * changes. Opting out costs a full walk of excluded subtrees. + * + * @default true + */ + readonly pruneExcludedDirectories?: boolean; + + /** + * A value identifying this strategy's exclusion behavior, suitable for + * folding into a cache key. Two strategies that return the same + * `cacheKey` must exclude the same paths. + * + * Callers such as `AssetStaging`'s result cache key on a JSON-serializable + * representation of their inputs, which a strategy instance is not. Omit + * this when the strategy's behavior can't be summarized this way; the + * caller then has to treat every call as uncacheable. + * + * @default - this strategy cannot be represented in a cache key + */ + readonly cacheKey?: string; +} + +/** + * The default ignore strategy: exact paths, `*.ext` suffixes, and + * directories (with everything inside them). + */ +export class ExcludeIgnoreStrategy implements IIgnoreStrategy { + private readonly matcher: (relativePath: string) => boolean; + + public readonly cacheKey?: string; + + // Undefined (the interface default) prunes excluded directories, which is + // correct here since the built-in patterns have no negation form. + public readonly pruneExcludedDirectories?: boolean; + + constructor(exclude: string[]) { + this.matcher = excludeMatcher(exclude); + this.cacheKey = `exclude:${JSON.stringify(exclude)}`; + } + + public ignores(query: IgnoreQuery): boolean { + // Path-based matching only; `isDirectory` is unused. + return this.matcher(query.relativePath); + } +} diff --git a/packages/cdktn/src/index.ts b/packages/cdktn/src/index.ts index 75ace0f6b..f56cbc04c 100644 --- a/packages/cdktn/src/index.ts +++ b/packages/cdktn/src/index.ts @@ -44,6 +44,9 @@ export * from "./importable-resource"; export * from "./terraform-resource-targets"; export * from "./upgrade-id-aspect"; export * from "./terraform-data-resource"; +export * from "./assets"; +export * from "./ignore-strategy"; +export * from "./asset-hash"; // required for JSII because Fn extends from it export * from "./functions/terraform-functions.generated"; export * from "./functions/provider-function"; diff --git a/packages/cdktn/src/private/fs.ts b/packages/cdktn/src/private/fs.ts index e33036c15..4f86ca102 100644 --- a/packages/cdktn/src/private/fs.ts +++ b/packages/cdktn/src/private/fs.ts @@ -30,49 +30,111 @@ function zipAttrs(mode: number): number { return (mode << 16) >>> 0; } +/** + * Predicate deciding whether a tree entry is skipped. + * `relPath` is always `/`-separated and relative to the walk root, so patterns + * behave identically on Windows. `isDirectory` is supplied from the walker's + * `lstat` so `.gitignore` / `.dockerignore`-style directory-only patterns can + * be honored without the predicate stat-ing the path itself. + */ +export type ExcludePredicate = ( + relPath: string, + isDirectory: boolean, +) => boolean; + +export interface CopySyncOptions { + /** + * Entries for which this returns true are not copied. Called with the + * entry's `/`-separated relative path and whether it is a directory. + * Excluding a directory also skips everything below it, unless + * {@link descendIntoExcludedDirectories} is set. + * + * @default - nothing is excluded + */ + readonly shouldExclude?: ExcludePredicate; + + /** + * Keep walking into an excluded directory instead of pruning it. The + * directory entry itself is still omitted; its children are visited and + * re-tested against {@link shouldExclude}, so a strategy with negation + * patterns (`node_modules` + `!node_modules/keep`) can re-include entries + * below an excluded parent. Costs a full walk of excluded subtrees, so it + * is off unless a strategy asks for it. + * + * @default false + */ + readonly descendIntoExcludedDirectories?: boolean; +} + // Full implementation at https://github.com/jprichardson/node-fs-extra/blob/master/lib/copy/copy-sync.js /** * Copy a file or directory. The directory can have contents and subfolders. + * Symlinks are recreated as symlinks rather than dereferenced, which keeps the + * copy consistent with {@link hashPath} (it hashes links by their target) and + * makes dangling links and link cycles harmless. * @param src - source path * @param dest - destination path + * @param options - copy behaviour, see {@link CopySyncOptions} */ -export function copySync(src: string, dest: string) { +export function copySync( + src: string, + dest: string, + options: CopySyncOptions = {}, +) { /** * Copies file if present otherwise walks subfolder. * @param p - path relative to src/dest + * @param relPath - `/`-separated path relative to the copy root */ - function copyItem(p: string) { + function copyItem(p: string, relPath: string) { const sourcePath = path.resolve(src, p); const stat = fs.lstatSync(sourcePath); + const excluded = !!options.shouldExclude?.(relPath, stat.isDirectory()); + // Skip, unless it is an excluded directory being descended into. + if ( + excluded && + !(stat.isDirectory() && options.descendIntoExcludedDirectories) + ) { + return; + } if (stat.isSymbolicLink()) { fs.symlinkSync(fs.readlinkSync(sourcePath), path.resolve(dest, p)); } else if (stat.isFile()) { fs.copyFileSync(sourcePath, path.resolve(dest, p)); } else if (stat.isDirectory()) { - walkSubfolder(p); + walkSubfolder(p, relPath); } } /** * Copies contents of subfolder. * @param p - path relative to src/dest + * @param relPath - `/`-separated path relative to the copy root */ - function walkSubfolder(p: string) { + function walkSubfolder(p: string, relPath: string) { const sourceDir = path.resolve(src, p); fs.mkdirSync(path.resolve(dest, p), { recursive: true }); fs.readdirSync(sourceDir).forEach((item: string) => - copyItem(path.join(p, item)), + copyItem(path.join(p, item), relPath ? `${relPath}/${item}` : item), ); } - walkSubfolder("."); + walkSubfolder(".", ""); } /** * Zips contents at src and places zip archive at dest. * @param src - directory to archive * @param dest - path to write the resulting zip to + * @param shouldExclude - entries to omit, see {@link CopySyncOptions.shouldExclude} + * @param descendIntoExcludedDirectories - keep walking excluded directories, + * see {@link CopySyncOptions.descendIntoExcludedDirectories} */ -export function archiveSync(src: string, dest: string) { +export function archiveSync( + src: string, + dest: string, + shouldExclude?: ExcludePredicate, + descendIntoExcludedDirectories = false, +) { try { const files: Record = {}; const walk = (dir: string, prefix: string) => { @@ -82,6 +144,13 @@ export function archiveSync(src: string, dest: string) { const full = path.join(dir, entry); const zipPath = prefix ? `${prefix}/${entry}` : entry; const stat = fs.lstatSync(full); + const excluded = !!shouldExclude?.(zipPath, stat.isDirectory()); + if ( + excluded && + !(stat.isDirectory() && descendIntoExcludedDirectories) + ) { + continue; + } if (stat.isSymbolicLink()) { // Store the link target as the entry data with S_IFLNK attrs so // extractors recreate the symlink instead of a copy of the target. @@ -132,6 +201,26 @@ export interface HashPathOptions { * the legacy scheme, which never records directories. */ readonly archive?: boolean; + /** + * Entries for which this returns true are omitted from the digest. Excluding + * a directory also omits everything below it, unless + * {@link descendIntoExcludedDirectories} is set. The same predicate must be + * given to {@link copySync} so the hash and the emitted artifact describe the + * same set of files. + * + * @default - nothing is excluded + */ + readonly shouldExclude?: ExcludePredicate; + + /** + * Keep walking into excluded directories, see + * {@link CopySyncOptions.descendIntoExcludedDirectories}. Must match the + * value given to {@link copySync} / {@link archiveSync} so the digest covers + * the same file set the artifact contains. + * + * @default false + */ + readonly descendIntoExcludedDirectories?: boolean; } /** @@ -146,8 +235,17 @@ export interface HashPathOptions { */ export function hashPath(src: string, options: HashPathOptions = {}): string { const digest = options.canonical - ? canonicalHashPath(src, !options.archive) - : legacyHashPath(src); + ? canonicalHashPath( + src, + !options.archive, + options.shouldExclude, + options.descendIntoExcludedDirectories, + ) + : legacyHashPath( + src, + options.shouldExclude, + options.descendIntoExcludedDirectories, + ); return digest.slice(0, HASH_LEN).toUpperCase(); } @@ -160,8 +258,15 @@ export function hashPath(src: string, options: HashPathOptions = {}): string { * bytes, so a file containing `foo` can never collide with a symlink * targeting `foo`. * @param src - path to a file or directory to hash + * @param shouldExclude - entries to omit, see {@link HashPathOptions.shouldExclude} + * @param descendIntoExcludedDirectories - keep walking excluded directories, + * see {@link HashPathOptions.descendIntoExcludedDirectories} */ -function legacyHashPath(src: string): string { +function legacyHashPath( + src: string, + shouldExclude?: ExcludePredicate, + descendIntoExcludedDirectories = false, +): string { const content = crypto.createHash("md5"); const links = crypto.createHash("md5"); let linkCount = 0; @@ -182,12 +287,18 @@ function legacyHashPath(src: string): string { } else if (stat.isFile()) { content.update(fs.readFileSync(p)); } else if (stat.isDirectory()) { - fs.readdirSync(p).forEach((filename) => - hashRecursion( - path.resolve(p, filename), - relPath ? `${relPath}/${filename}` : filename, - ), - ); + fs.readdirSync(p).forEach((filename) => { + const entryRelPath = relPath ? `${relPath}/${filename}` : filename; + const childPath = path.resolve(p, filename); + const childIsDir = fs.lstatSync(childPath).isDirectory(); + if (shouldExclude?.(entryRelPath, childIsDir)) { + // Skip, unless it is an excluded directory being descended into. + if (!(childIsDir && descendIntoExcludedDirectories)) { + return; + } + } + hashRecursion(childPath, entryRelPath); + }); } } @@ -220,8 +331,14 @@ function legacyHashPath(src: string): string { * @param src - path to a file or directory to hash * @param includeDirectories - record directory entries; false for archive * artifacts, where the emitted zip has no directory entries + * @param shouldExclude - entries to omit, see {@link HashPathOptions.shouldExclude} */ -function canonicalHashPath(src: string, includeDirectories: boolean): string { +function canonicalHashPath( + src: string, + includeDirectories: boolean, + shouldExclude?: ExcludePredicate, + descendIntoExcludedDirectories = false, +): string { const hash = crypto.createHash("md5"); /** @@ -243,14 +360,22 @@ function canonicalHashPath(src: string, includeDirectories: boolean): string { hash.update(`F ${mode} ${relPath}\0${data.length}\0`); hash.update(data); } else if (stat.isDirectory()) { + // Records the `D` entry even for a descended-into excluded directory, + // matching the empty directory copySync leaves on disk. if (relPath && includeDirectories) { hash.update(`D ${relPath}\0`); } for (const filename of fs.readdirSync(p).sort()) { - hashRecursion( - path.resolve(p, filename), - relPath ? `${relPath}/${filename}` : filename, - ); + const entryRelPath = relPath ? `${relPath}/${filename}` : filename; + const childPath = path.resolve(p, filename); + const childIsDir = fs.lstatSync(childPath).isDirectory(); + if (shouldExclude?.(entryRelPath, childIsDir)) { + // Skip, unless it is an excluded directory being descended into. + if (!(childIsDir && descendIntoExcludedDirectories)) { + continue; + } + } + hashRecursion(childPath, entryRelPath); } } } @@ -259,6 +384,41 @@ function canonicalHashPath(src: string, includeDirectories: boolean): string { return hash.digest("hex"); } +/** + * Build a predicate matching the exclusion forms accepted by + * `AssetHashOptions.exclude` (via `ExcludeIgnoreStrategy`): an exact relative + * path, a `*.ext` suffix, or a directory (with or without a trailing `/`), + * which also excludes its contents. + * Deliberately not a full glob implementation — `**`, `?`, character classes and + * `!` negation are not supported, and a pattern is never interpreted as + * anchoring to a subdirectory it does not name. + * The returned matcher looks only at the path, not at whether the entry is a + * directory: `dir` and `dir/` both match a directory named `dir` and its + * contents. It is therefore narrower than {@link ExcludePredicate}, which also + * receives an `isDirectory` flag for strategies that need it. + * @param exclude - patterns to exclude + * @returns predicate over `/`-separated paths relative to the asset root + */ +export function excludeMatcher( + exclude: string[], +): (relativePath: string) => boolean { + // `/`-separated throughout: relative paths are normalized before matching, so + // `dir/child` patterns work the same on Windows. + const patterns = exclude.map((p) => p.replace(/\\/g, "/")); + return (relativePath: string) => { + for (const pattern of patterns) { + if (pattern.startsWith("*.") && relativePath.endsWith(pattern.slice(1))) { + return true; + } + const dir = pattern.endsWith("/") ? pattern.slice(0, -1) : pattern; + if (relativePath === dir || relativePath.startsWith(`${dir}/`)) { + return true; + } + } + return false; + }; +} + /** * Walk upward from `rootPath` looking for a file with the given name. * Returns the absolute path of the first match, or `null` if the search diff --git a/packages/cdktn/src/terraform-asset.ts b/packages/cdktn/src/terraform-asset.ts index dd2358bcc..5e872d803 100644 --- a/packages/cdktn/src/terraform-asset.ts +++ b/packages/cdktn/src/terraform-asset.ts @@ -4,11 +4,12 @@ import { Construct } from "constructs"; import * as fs from "fs"; import * as path from "path"; import { - copySync, - archiveSync, - hashPath, - findFileAboveCwd, -} from "./private/fs"; + AssetPackaging, + AssetHashType, + IAsset, + IAssetPackaging, +} from "./assets"; +import { hashPath, findFileAboveCwd } from "./private/fs"; import { CANONICAL_ASSET_HASHES } from "./features"; import { ISynthesisSession } from "./synthesize"; import { addCustomSynthesis } from "./synthesize/synthesizer"; @@ -17,6 +18,10 @@ import { assetExpectsDirectory, assetOutOfScopeOfCDKTFJson, assetTypeNotImplemented, + assetHashTypeOutputNotSupported, + assetHashTypeCustomRequiresHash, + assetHashConflictingHashType, + assetHashTypeUnknown, } from "./errors"; export interface TerraformAssetConfig { @@ -26,6 +31,19 @@ export interface TerraformAssetConfig { readonly type?: AssetType; // hash value of the asset, if passed will be used as returned assetHash readonly assetHash?: string; + /** + * How the `assetHash` is derived. + * + * `SOURCE` (the default) hashes the source path. `CUSTOM` uses the + * `assetHash` value verbatim and requires it to be set. `OUTPUT` is not + * supported yet — there is no bundling step to produce an output to hash — + * and throws if requested. + * + * If `assetHash` is set, this must be `undefined` or `AssetHashType.CUSTOM`. + * + * @default AssetHashType.SOURCE + */ + readonly assetHashType?: AssetHashType; } export enum AssetType { @@ -34,15 +52,29 @@ export enum AssetType { ARCHIVE, } -const ARCHIVE_NAME = "archive.zip"; +// Base name for a packaged (non-directory, non-verbatim-file) artifact. The +// packaging's `extension` is appended, so a zip stays `archive.zip` and a +// future `tar.bz2` packaging would be `archive.tar.bz2` with no change here. +const ARCHIVE_BASENAME = "archive"; const ASSETS_DIRECTORY = "assets"; +/** + * How each `AssetType` is actually written to disk at synthesis time. + * Internal wiring only: swapping this map's values is how a future format + * would be added, without any change to the public `AssetType` surface. + */ +const PACKAGING_BY_TYPE: Record = { + [AssetType.FILE]: AssetPackaging.FILE, + [AssetType.DIRECTORY]: AssetPackaging.DIRECTORY, + [AssetType.ARCHIVE]: AssetPackaging.ZIP, +}; + // eslint-disable-next-line jsdoc/require-jsdoc -export class TerraformAsset extends Construct { +export class TerraformAsset extends Construct implements IAsset { private stack: TerraformStack; private sourcePath: string; // hash value of the asset that can be passed to consuming constructs (e.g. to not recreate a lambda function in case the underlying files did not change) - public assetHash: string; + public readonly assetHash: string; // file type of the asset, either AssetType.FILE, AssetType.DIRECTORY, AssetType.ARCHIVE public type: AssetType; @@ -79,12 +111,7 @@ export class TerraformAsset extends Construct { const stat = fs.statSync(this.sourcePath); const inferredType = stat.isFile() ? AssetType.FILE : AssetType.DIRECTORY; this.type = config.type ?? inferredType; - this.assetHash = - config.assetHash || - hashPath(this.sourcePath, { - canonical: !!this.node.tryGetContext(CANONICAL_ASSET_HASHES), - archive: this.type === AssetType.ARCHIVE, - }); + this.assetHash = this.resolveAssetHash(id, config); if (stat.isFile() && this.type !== AssetType.FILE) { throw assetExpectsDirectory(id, config.path); @@ -99,6 +126,46 @@ export class TerraformAsset extends Construct { }); } + /** + * Resolve the asset hash from `assetHash` and `assetHashType`. + * + * Honors the same contract `AssetOptions` documents: an explicit + * `assetHash` means the type is `CUSTOM`, `CUSTOM` requires a hash, and + * `OUTPUT` is rejected because there is no bundling step to hash yet. + * `SOURCE` (the default) hashes the source path as before. + * @param id - construct id, for error messages + * @param config - the asset configuration + */ + private resolveAssetHash(id: string, config: TerraformAssetConfig): string { + const { assetHash, assetHashType } = config; + + if (assetHash !== undefined) { + if ( + assetHashType !== undefined && + assetHashType !== AssetHashType.CUSTOM + ) { + throw assetHashConflictingHashType(id); + } + return assetHash; + } + + switch (assetHashType) { + case AssetHashType.CUSTOM: + throw assetHashTypeCustomRequiresHash(id); + case AssetHashType.OUTPUT: + throw assetHashTypeOutputNotSupported(id); + case AssetHashType.SOURCE: + case undefined: + return hashPath(this.sourcePath, { + canonical: !!this.node.tryGetContext(CANONICAL_ASSET_HASHES), + archive: this.type === AssetType.ARCHIVE, + }); + default: + // Out-of-range value from a non-TypeScript caller. + throw assetHashTypeUnknown(id, assetHashType); + } + } + private get namedFolder(): string { return path.posix.join( ASSETS_DIRECTORY, @@ -106,6 +173,19 @@ export class TerraformAsset extends Construct { ); } + /** + * How this asset is written to disk. The layout (directory vs single file) + * and the artifact name are derived from this rather than from `type` + * directly, so the two never disagree. + */ + private get packaging(): IAssetPackaging { + const packaging = PACKAGING_BY_TYPE[this.type]; + if (!packaging) { + throw assetTypeNotImplemented(); + } + return packaging; + } + /** * The path relative to the root of the terraform directory in posix format * Use this property to reference the asset @@ -114,7 +194,9 @@ export class TerraformAsset extends Construct { return path.posix.join( this.namedFolder, // readable name this.assetHash, // hash depending on content so that path changes if content changes - this.type === AssetType.DIRECTORY ? "" : this.fileName, + // A directory-producing packaging has no file segment; anything else + // contributes its artifact name. + this.packaging.producesDirectory ? "" : this.fileName, ); } @@ -122,12 +204,12 @@ export class TerraformAsset extends Construct { * Name of the asset */ public get fileName(): string { - switch (this.type) { - case AssetType.ARCHIVE: - return ARCHIVE_NAME; - default: - return path.basename(this.sourcePath); - } + const { extension } = this.packaging; + // Repackaged artifacts (extension set) get a stable base + extension; + // verbatim copies keep the source name. + return extension + ? `${ARCHIVE_BASENAME}${extension}` + : path.basename(this.sourcePath); } private _onSynthesize(session: ISynthesisSession) { @@ -145,27 +227,14 @@ export class TerraformAsset extends Construct { } const targetPath = path.join(basePath, this.path); + const packaging = this.packaging; - if (this.type === AssetType.DIRECTORY) { + if (packaging.producesDirectory) { fs.mkdirSync(targetPath, { recursive: true }); } else { fs.mkdirSync(path.dirname(targetPath), { recursive: true }); } - switch (this.type) { - case AssetType.FILE: - fs.copyFileSync(this.sourcePath, targetPath); - break; - - case AssetType.DIRECTORY: - copySync(this.sourcePath, targetPath); - break; - - case AssetType.ARCHIVE: - archiveSync(this.sourcePath, targetPath); - break; - default: - throw assetTypeNotImplemented(); - } + packaging.pack({ source: this.sourcePath, target: targetPath }); } } diff --git a/packages/cdktn/test/asset-hash.test.ts b/packages/cdktn/test/asset-hash.test.ts new file mode 100644 index 000000000..b8b09fd19 --- /dev/null +++ b/packages/cdktn/test/asset-hash.test.ts @@ -0,0 +1,291 @@ +// Copyright (c) HashiCorp, Inc +// SPDX-License-Identifier: MPL-2.0 +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { AssetHash, ExcludeIgnoreStrategy, IIgnoreStrategy } from "../lib"; + +describe("AssetHash", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "asset-hash-test-")); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + test("computes a stable hash without staging anything", () => { + fs.writeFileSync(path.join(tempDir, "a.txt"), "hello"); + + const hash1 = AssetHash.of(tempDir); + const hash2 = AssetHash.of(tempDir); + + expect(hash1).toBe(hash2); + // Checked against tempDir itself rather than the shared, OS-wide temp + // dir: tempDir is a uniquely-named mkdtemp directory nothing else in + // the process touches, so this is race-free, unlike scanning the + // shared dir, which every concurrent test file in this suite also + // creates and removes its own temp dirs in. What's actually worth + // guarding here is narrower anyway: a staged copy must not land back + // inside the very source tree being hashed. + expect(fs.readdirSync(tempDir)).toEqual(["a.txt"]); + }); + + test("changes when content changes", () => { + const file = path.join(tempDir, "a.txt"); + fs.writeFileSync(file, "hello"); + const original = AssetHash.of(tempDir); + + fs.writeFileSync(file, "goodbye"); + expect(AssetHash.of(tempDir)).not.toBe(original); + }); + + test("extraHash changes the digest", () => { + fs.writeFileSync(path.join(tempDir, "a.txt"), "hello"); + const withoutExtra = AssetHash.of(tempDir); + const withExtra = AssetHash.of(tempDir, { extraHash: "salt" }); + + expect(withExtra).not.toBe(withoutExtra); + }); + + test("exclude omits matched paths from the hash", () => { + fs.writeFileSync(path.join(tempDir, "a.txt"), "hello"); + fs.writeFileSync(path.join(tempDir, "b.log"), "noise"); + + const hashWithLog = AssetHash.of(tempDir, { exclude: [] }); + fs.writeFileSync(path.join(tempDir, "b.log"), "different noise"); + const hashWithChangedLog = AssetHash.of(tempDir, { exclude: [] }); + const hashExcludingLog = AssetHash.of(tempDir, { exclude: ["*.log"] }); + + expect(hashWithChangedLog).not.toBe(hashWithLog); + + fs.writeFileSync(path.join(tempDir, "b.log"), "noise"); + expect(AssetHash.of(tempDir, { exclude: ["*.log"] })).toBe( + hashExcludingLog, + ); + }); + + test("accepts a custom IIgnoreStrategy", () => { + fs.writeFileSync(path.join(tempDir, "a.txt"), "hello"); + fs.writeFileSync(path.join(tempDir, "ignored.tmp"), "noise"); + + const strategy: IIgnoreStrategy = { + ignores: ({ relativePath }) => relativePath.endsWith(".tmp"), + }; + + const withStrategy = AssetHash.of(tempDir, { ignoreStrategy: strategy }); + + fs.writeFileSync(path.join(tempDir, "ignored.tmp"), "different noise"); + expect(AssetHash.of(tempDir, { ignoreStrategy: strategy })).toBe( + withStrategy, + ); + }); + + test("descends into excluded directories when the strategy opts out of pruning", () => { + fs.mkdirSync(path.join(tempDir, "node_modules")); + fs.writeFileSync(path.join(tempDir, "node_modules", "junk.js"), "junk"); + fs.writeFileSync(path.join(tempDir, "node_modules", "keep.js"), "keep"); + + // A negation strategy: exclude the whole `node_modules` tree, but re-include + // `node_modules/keep.js`. Reachable only because pruning is off, so the walk + // descends into the excluded directory and asks about its children. + const strategy: IIgnoreStrategy = { + pruneExcludedDirectories: false, + ignores: ({ relativePath }) => + (relativePath === "node_modules" || + relativePath.startsWith("node_modules/")) && + relativePath !== "node_modules/keep.js", + }; + + const withStrategy = AssetHash.of(tempDir, { ignoreStrategy: strategy }); + + // The re-included file is part of the hash: changing it moves the digest. + fs.writeFileSync(path.join(tempDir, "node_modules", "keep.js"), "changed"); + expect(AssetHash.of(tempDir, { ignoreStrategy: strategy })).not.toBe( + withStrategy, + ); + + // A sibling still under exclusion is not: changing it does not. + fs.writeFileSync(path.join(tempDir, "node_modules", "keep.js"), "keep"); + fs.writeFileSync( + path.join(tempDir, "node_modules", "junk.js"), + "more junk", + ); + expect(AssetHash.of(tempDir, { ignoreStrategy: strategy })).toBe( + withStrategy, + ); + }); + + test("prunes excluded directories by default", () => { + fs.mkdirSync(path.join(tempDir, "node_modules")); + fs.writeFileSync(path.join(tempDir, "node_modules", "keep.js"), "keep"); + + // Same negation shape, but pruning left at its default. The walk never + // descends into the excluded directory, so the `!keep.js` re-include is + // unreachable and the file does not affect the hash. + const strategy: IIgnoreStrategy = { + ignores: ({ relativePath }) => + (relativePath === "node_modules" || + relativePath.startsWith("node_modules/")) && + relativePath !== "node_modules/keep.js", + }; + + const withStrategy = AssetHash.of(tempDir, { ignoreStrategy: strategy }); + + fs.writeFileSync(path.join(tempDir, "node_modules", "keep.js"), "changed"); + expect(AssetHash.of(tempDir, { ignoreStrategy: strategy })).toBe( + withStrategy, + ); + }); + + test("ExcludeIgnoreStrategy matches the same rules as `exclude`", () => { + const strategy = new ExcludeIgnoreStrategy(["*.log", "node_modules"]); + + expect( + strategy.ignores({ relativePath: "a.log", isDirectory: false }), + ).toBe(true); + expect( + strategy.ignores({ + relativePath: "node_modules/foo/index.js", + isDirectory: false, + }), + ).toBe(true); + expect( + strategy.ignores({ relativePath: "src/index.ts", isDirectory: false }), + ).toBe(false); + }); + + test("passes isDirectory through to a custom strategy", () => { + fs.mkdirSync(path.join(tempDir, "keep")); + fs.writeFileSync(path.join(tempDir, "keep", "a.txt"), "hello"); + fs.mkdirSync(path.join(tempDir, "drop")); + fs.writeFileSync(path.join(tempDir, "drop", "b.txt"), "noise"); + + // A directory-only strategy: excludes the `drop` directory but would keep + // a file of the same name. It can only make that call because the walker + // tells it the entry is a directory. + const strategy: IIgnoreStrategy = { + ignores: ({ relativePath, isDirectory }) => + isDirectory && relativePath === "drop", + }; + + const withStrategy = AssetHash.of(tempDir, { ignoreStrategy: strategy }); + + // Changing content under the excluded directory does not move the hash. + fs.writeFileSync(path.join(tempDir, "drop", "b.txt"), "different noise"); + expect(AssetHash.of(tempDir, { ignoreStrategy: strategy })).toBe( + withStrategy, + ); + + // Changing content under the kept directory does move it. + fs.writeFileSync(path.join(tempDir, "keep", "a.txt"), "changed"); + expect(AssetHash.of(tempDir, { ignoreStrategy: strategy })).not.toBe( + withStrategy, + ); + }); + + test("ExcludeIgnoreStrategy exposes a cacheKey", () => { + const strategy = new ExcludeIgnoreStrategy(["*.log"]); + expect(strategy.cacheKey).toBeDefined(); + expect(new ExcludeIgnoreStrategy(["*.log"]).cacheKey).toBe( + strategy.cacheKey, + ); + expect(new ExcludeIgnoreStrategy(["*.tmp"]).cacheKey).not.toBe( + strategy.cacheKey, + ); + }); + + test("throws when both exclude and ignoreStrategy are given", () => { + fs.writeFileSync(path.join(tempDir, "a.txt"), "hello"); + + expect(() => + AssetHash.of(tempDir, { + exclude: ["*.log"], + ignoreStrategy: new ExcludeIgnoreStrategy(["*.tmp"]), + }), + ).toThrow(/exclude.*ignoreStrategy|ignoreStrategy.*exclude/i); + }); + + test("hashes on the canonical scheme regardless of the feature flag", () => { + // Canonical hashing frames every entry with its path; legacy hashes only + // file bytes with no path recorded. So identical bytes under different + // names collide on legacy and diverge on canonical -- which is the one + // property that actually distinguishes the two schemes. + const dirA = path.join(tempDir, "a"); + const dirB = path.join(tempDir, "b"); + fs.mkdirSync(dirA); + fs.mkdirSync(dirB); + fs.writeFileSync(path.join(dirA, "a.txt"), "hello"); + fs.writeFileSync(path.join(dirB, "b.txt"), "hello"); + + expect(AssetHash.of(dirA)).not.toBe(AssetHash.of(dirB)); + }); + + describe("relative path resolution", () => { + let projectRoot: string; + let originalCwd: string; + + beforeEach(() => { + // realpath so comparisons hold on macOS, where os.tmpdir() lives under + // the /var -> /private/var symlink. + projectRoot = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), "asset-hash-project-")), + ); + fs.writeFileSync(path.join(projectRoot, "cdktf.json"), "{}"); + fs.mkdirSync(path.join(projectRoot, "assets", "thing"), { + recursive: true, + }); + fs.writeFileSync( + path.join(projectRoot, "assets", "thing", "a.txt"), + "hello", + ); + originalCwd = process.cwd(); + }); + + afterEach(() => { + process.chdir(originalCwd); + fs.rmSync(projectRoot, { recursive: true, force: true }); + }); + + test("resolves a relative path against cdktf.json, not the cwd", () => { + // Run from a nested subdirectory so cwd-based resolution would look in + // the wrong place; only cdktf.json-based resolution finds the source. + const nested = path.join(projectRoot, "assets", "thing"); + process.chdir(nested); + + const viaRelative = AssetHash.of("./assets/thing"); + const viaAbsolute = AssetHash.of(path.join(projectRoot, "assets/thing")); + + expect(viaRelative).toBe(viaAbsolute); + }); + + test("throws when a relative path has no cdktf.json above the cwd", () => { + // A sibling temp tree with no cdktf.json anywhere above it. + const orphan = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), "asset-hash-orphan-")), + ); + process.chdir(orphan); + try { + expect(() => AssetHash.of("./whatever")).toThrow(/cdktf\.json/); + } finally { + fs.rmSync(orphan, { recursive: true, force: true }); + } + }); + + test("uses an absolute path as-is without needing cdktf.json", () => { + const orphan = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), "asset-hash-orphan-")), + ); + fs.writeFileSync(path.join(orphan, "a.txt"), "hello"); + process.chdir(orphan); + try { + // No cdktf.json above orphan, but an absolute path never consults it. + expect(() => AssetHash.of(orphan)).not.toThrow(); + } finally { + fs.rmSync(orphan, { recursive: true, force: true }); + } + }); + }); +}); diff --git a/packages/cdktn/test/assets-types.test.ts b/packages/cdktn/test/assets-types.test.ts new file mode 100644 index 000000000..7a9386d56 --- /dev/null +++ b/packages/cdktn/test/assets-types.test.ts @@ -0,0 +1,182 @@ +// Copyright (c) HashiCorp, Inc +// SPDX-License-Identifier: MPL-2.0 + +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + AssetHash, + AssetHashType, + AssetPackaging, + ExcludeIgnoreStrategy, + type IIgnoreStrategy, + type StagedAsset, + type AssetOptions, +} from "../lib"; + +describe("Assets Types", () => { + describe("AssetHashType", () => { + test("has expected values", () => { + expect(AssetHashType.SOURCE).toBe("source"); + expect(AssetHashType.OUTPUT).toBe("output"); + expect(AssetHashType.CUSTOM).toBe("custom"); + }); + }); + + describe("AssetPackaging", () => { + test("has expected shapes", () => { + expect(AssetPackaging.FILE.producesDirectory).toBe(false); + expect(AssetPackaging.DIRECTORY.producesDirectory).toBe(true); + expect(AssetPackaging.ZIP.producesDirectory).toBe(false); + expect(AssetPackaging.ZIP.extension).toBe(".zip"); + }); + + describe("pack", () => { + let tempDir: string; + let source: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync( + path.join(os.tmpdir(), "asset-packaging-test-"), + ); + source = path.join(tempDir, "source"); + fs.mkdirSync(source); + fs.writeFileSync(path.join(source, "a.txt"), "keep"); + fs.writeFileSync(path.join(source, "b.log"), "drop"); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + test("DIRECTORY omits paths the ignoreStrategy excludes", () => { + const target = path.join(tempDir, "target"); + AssetPackaging.DIRECTORY.pack({ + source, + target, + ignoreStrategy: new ExcludeIgnoreStrategy(["*.log"]), + }); + + expect(fs.existsSync(path.join(target, "a.txt"))).toBe(true); + expect(fs.existsSync(path.join(target, "b.log"))).toBe(false); + }); + + test("ZIP omits paths the ignoreStrategy excludes", () => { + const target = path.join(tempDir, "target.zip"); + AssetPackaging.ZIP.pack({ + source, + target, + ignoreStrategy: new ExcludeIgnoreStrategy(["*.log"]), + }); + + const zipped = fs.readFileSync(target); + expect(zipped.toString("latin1")).toContain("a.txt"); + expect(zipped.toString("latin1")).not.toContain("b.log"); + }); + + test("DIRECTORY re-includes a file below an excluded dir when pruning is off", () => { + fs.mkdirSync(path.join(source, "node_modules")); + fs.writeFileSync(path.join(source, "node_modules", "junk.js"), "junk"); + fs.writeFileSync(path.join(source, "node_modules", "keep.js"), "keep"); + + const strategy: IIgnoreStrategy = { + pruneExcludedDirectories: false, + ignores: ({ relativePath }) => + (relativePath === "node_modules" || + relativePath.startsWith("node_modules/")) && + relativePath !== "node_modules/keep.js", + }; + + const target = path.join(tempDir, "target"); + AssetPackaging.DIRECTORY.pack({ + source, + target, + ignoreStrategy: strategy, + }); + + expect( + fs.existsSync(path.join(target, "node_modules", "keep.js")), + ).toBe(true); + expect( + fs.existsSync(path.join(target, "node_modules", "junk.js")), + ).toBe(false); + + // The packed tree and a hash of the source under the same strategy + // describe the same file set: hashing the staged copy with no + // exclusions matches hashing the source through the strategy. + expect(AssetHash.of(target)).toBe( + AssetHash.of(source, { ignoreStrategy: strategy }), + ); + }); + + test("passes isDirectory to the strategy so directory-only patterns work", () => { + // A file and a directory share the name `build`. A directory-only + // strategy must drop the directory while keeping the file, which is + // only possible because pack tells it which entry is which. + fs.mkdirSync(path.join(source, "build")); + fs.writeFileSync(path.join(source, "build", "out.o"), "artifact"); + fs.writeFileSync(path.join(source, "build.txt"), "not the dir"); + + const strategy: IIgnoreStrategy = { + ignores: ({ relativePath, isDirectory }) => + isDirectory && relativePath === "build", + }; + + const target = path.join(tempDir, "target"); + AssetPackaging.DIRECTORY.pack({ + source, + target, + ignoreStrategy: strategy, + }); + + expect(fs.existsSync(path.join(target, "build"))).toBe(false); + expect(fs.existsSync(path.join(target, "build.txt"))).toBe(true); + }); + }); + }); + + describe("StagedAsset", () => { + test("can represent a staged file", () => { + const asset: StagedAsset = { + assetHash: "abc123", + path: "assets/asset.abc123.zip", + isDirectory: false, + }; + + expect(asset.assetHash).toBe("abc123"); + expect(asset.path).toBe("assets/asset.abc123.zip"); + expect(asset.isDirectory).toBe(false); + }); + + test("can represent a staged directory", () => { + const asset: StagedAsset = { + assetHash: "def456", + path: "assets/asset.def456", + isDirectory: true, + }; + + expect(asset.isDirectory).toBe(true); + }); + }); + + describe("AssetOptions", () => { + test("can specify custom hash", () => { + const options: AssetOptions = { + assetHash: "my-custom-hash", + assetHashType: AssetHashType.CUSTOM, + }; + + expect(options.assetHash).toBe("my-custom-hash"); + expect(options.assetHashType).toBe(AssetHashType.CUSTOM); + }); + + test("can specify hash type without custom hash", () => { + const options: AssetOptions = { + assetHashType: AssetHashType.SOURCE, + }; + + expect(options.assetHashType).toBe(AssetHashType.SOURCE); + expect(options.assetHash).toBeUndefined(); + }); + }); +}); diff --git a/packages/cdktn/test/canonical-asset-hash.test.ts b/packages/cdktn/test/canonical-asset-hash.test.ts index fe26ac08f..b82730a72 100644 --- a/packages/cdktn/test/canonical-asset-hash.test.ts +++ b/packages/cdktn/test/canonical-asset-hash.test.ts @@ -5,7 +5,15 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { Testing, TerraformStack, TerraformAsset, AssetType } from "../src"; +import { + Testing, + TerraformStack, + TerraformAsset, + AssetType, + AssetHash, + AssetHashType, + IAsset, +} from "../src"; import { CANONICAL_ASSET_HASHES } from "../src/features"; import { TerraformModuleAsset } from "../src/terraform-module-asset"; import { archiveSync, hashPath } from "../src/private/fs"; @@ -293,6 +301,217 @@ describe("TerraformAsset with the canonicalAssetHashes flag", () => { expect(asset.assetHash).toBe(canonicalArchive(srcDir)); expect(asset.assetHash).not.toBe(canonical(srcDir)); }); + + // Pins the relationship between AssetHash.of (a packaging-independent + // source-tree identity) and TerraformAsset (whose framing depends on type). + // The subdirectory matters: it is what makes DIRECTORY and ARCHIVE framing + // diverge, since ARCHIVE omits directory records. + describe("AssetHash.of relationship to TerraformAsset", () => { + beforeEach(() => { + fs.mkdirSync(path.join(srcDir, "sub")); + fs.writeFileSync(path.join(srcDir, "sub", "b.txt"), "nested"); + }); + + test("equals a DIRECTORY asset hash when the canonical flag is on", () => { + const stack = new TerraformStack( + Testing.app({ context: { [CANONICAL_ASSET_HASHES]: "true" } }), + "on", + ); + const asset = new TerraformAsset(stack, "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + }); + + expect(AssetHash.of(srcDir)).toBe(asset.assetHash); + }); + + test("does not equal an ARCHIVE asset hash (archive omits directory records)", () => { + const stack = new TerraformStack( + Testing.app({ context: { [CANONICAL_ASSET_HASHES]: "true" } }), + "on", + ); + const asset = new TerraformAsset(stack, "asset", { + path: srcDir, + type: AssetType.ARCHIVE, + }); + + expect(AssetHash.of(srcDir)).not.toBe(asset.assetHash); + }); + + test("does not equal a DIRECTORY asset hash on the legacy scheme", () => { + // AssetHash.of is always canonical; a project that has not opted into + // the flag hashes its TerraformAsset the legacy way, so the two differ. + const stack = new TerraformStack( + Testing.app({ enableFutureFlags: false }), + "off", + ); + const asset = new TerraformAsset(stack, "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + }); + + expect(AssetHash.of(srcDir)).not.toBe(asset.assetHash); + }); + }); +}); + +describe("TerraformAsset assetHashType", () => { + let srcDir: string; + + beforeEach(() => { + srcDir = createTempDir(); + fs.writeFileSync(path.join(srcDir, "a.txt"), "content"); + }); + + afterEach(() => { + fs.rmSync(srcDir, { recursive: true, force: true }); + }); + + const stack = () => + new TerraformStack( + Testing.app({ context: { [CANONICAL_ASSET_HASHES]: "true" } }), + "s", + ); + + test("implements IAsset", () => { + const asset: IAsset = new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + }); + + expect(typeof asset.assetHash).toBe("string"); + }); + + test("SOURCE (the default) hashes the source", () => { + const asset = new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHashType: AssetHashType.SOURCE, + }); + + expect(asset.assetHash).toBe(hashPath(srcDir, { canonical: true })); + }); + + test("CUSTOM uses the provided assetHash verbatim", () => { + const asset = new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHash: "my-custom-hash", + assetHashType: AssetHashType.CUSTOM, + }); + + expect(asset.assetHash).toBe("my-custom-hash"); + }); + + test("an explicit assetHash implies CUSTOM without stating the type", () => { + const asset = new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHash: "my-custom-hash", + }); + + expect(asset.assetHash).toBe("my-custom-hash"); + }); + + test("CUSTOM without an assetHash throws", () => { + expect( + () => + new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHashType: AssetHashType.CUSTOM, + }), + ).toThrow(/CUSTOM.*assetHash|assetHash/i); + }); + + test("an assetHash with a non-CUSTOM type throws", () => { + expect( + () => + new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHash: "my-custom-hash", + assetHashType: AssetHashType.SOURCE, + }), + ).toThrow(/assetHashType.*CUSTOM|CUSTOM/i); + }); + + test("OUTPUT is rejected until bundling exists", () => { + expect( + () => + new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + }), + ).toThrow(/OUTPUT/); + }); + + test("an out-of-range hash type throws instead of returning undefined", () => { + // Models a value another jsii language could pass that TypeScript's type + // system would reject; the switch's default guards it at runtime. + expect( + () => + new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + assetHashType: "bogus" as unknown as AssetHashType, + }), + ).toThrow(/unknown assetHashType/i); + }); +}); + +describe("TerraformAsset artifact layout derives from the packaging", () => { + let srcDir: string; + let srcFile: string; + + beforeEach(() => { + srcDir = createTempDir(); + fs.writeFileSync(path.join(srcDir, "a.txt"), "content"); + srcFile = path.join(srcDir, "a.txt"); + }); + + afterEach(() => { + fs.rmSync(srcDir, { recursive: true, force: true }); + }); + + const stack = () => + new TerraformStack( + Testing.app({ context: { [CANONICAL_ASSET_HASHES]: "true" } }), + "s", + ); + + test("DIRECTORY has no file segment (producesDirectory)", () => { + const asset = new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.DIRECTORY, + }); + + // The path ends at the hash directory, with nothing appended. + expect(asset.path.endsWith(asset.assetHash)).toBe(true); + }); + + test("FILE keeps the source filename (no extension packaging)", () => { + const asset = new TerraformAsset(stack(), "asset", { + path: srcFile, + type: AssetType.FILE, + }); + + expect(asset.fileName).toBe("a.txt"); + expect(asset.path.endsWith("/a.txt")).toBe(true); + }); + + test("ARCHIVE names the artifact from the packaging extension", () => { + const asset = new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.ARCHIVE, + }); + + // ZipPackaging.extension is ".zip", so the artifact is archive.zip - + // unchanged from before the wiring, but now derived rather than hardcoded. + expect(asset.fileName).toBe("archive.zip"); + expect(asset.path.endsWith("/archive.zip")).toBe(true); + }); }); describe("TerraformModuleAsset with the canonicalAssetHashes flag", () => {