From 2979c4c21178bc3cb95d51e9d1c8985507c117ad Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 24 Jul 2026 17:19:35 +0530 Subject: [PATCH] fix(import): preserve env-locale pairing for AM asset publish (DX-9772) publishAmSpaces flattened each asset's publish_details into independent environments[] and locales[] arrays, so the CMA republished the cartesian product. For a ragged AM publish state (different locales per environment) this over-published to env-locale pairs that never existed on the source. Add buildPublishGroups: group publish_details by environment, coalesce environments with an identical locale set, and emit one publish call per group so each call is a single rectangle the CMA reproduces exactly. A rectangular asset still collapses to one call (unchanged behavior). Scope: AM (publishAmSpaces) only. The legacy publish() path carries the same flatten and will be fixed on development, then back-merged. The DX-1656 invalid-environment guard is preserved (envs absent from the destination are still skipped). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/import/modules/assets.ts | 97 ++++++++++++------- .../test/unit/import/modules/assets.test.ts | 61 ++++++++++++ 2 files changed, 124 insertions(+), 34 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index fb875091b..b2afa06b8 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -273,6 +273,48 @@ export default class ImportAssets extends BaseClass { ); } + /** + * Groups already-filtered `publish_details` into publish payloads that preserve the source + * env↔locale pairing. Entries are grouped by environment, then environments sharing an + * identical locale set are coalesced into one payload. Each returned group is a rectangle + * (its environments × its locales), so the CMA env×locale cross-product reproduces exactly + * the source pairs — never a phantom combination (the DX-9772 over-publish). + * + * A fully-rectangular input (every env published to the same locales) collapses to a single + * group, so behavior is unchanged for the common case; a ragged input fans out into one group + * per distinct locale set. Only environments present in `this.environments` are kept, so the + * env-name lookup is always safe — this preserves the DX-1656 invalid-environment guard. + * Callers apply any further scoping (e.g. AM's `api_key`) before calling. + */ + private buildPublishGroups( + publishDetails: Record[], + ): { environments: string[]; locales: string[] }[] { + const localesByEnv = new Map>(); + for (const { environment, locale } of publishDetails || []) { + if (!locale || !this.environments?.hasOwnProperty(environment)) continue; + let set = localesByEnv.get(environment); + if (!set) { + set = new Set(); + localesByEnv.set(environment, set); + } + set.add(locale); + } + + // Coalesce environments with an identical locale set into a single payload. + const groups = new Map(); + for (const [envUid, localeSet] of localesByEnv) { + const locales = [...localeSet].sort(); + const signature = locales.join(' '); + const existing = groups.get(signature); + if (existing) { + existing.environments.push(this.environments[envUid].name); + } else { + groups.set(signature, { environments: [this.environments[envUid].name], locales }); + } + } + return [...groups.values()]; + } + /** * Publishes imported AM (Contentstack Assets / spaces) assets, mirroring the legacy `publish()` * but re-pointed at each space's chunk store under `spaces/{oldSpaceUid}/assets`. @@ -330,9 +372,13 @@ export default class ImportAssets extends BaseClass { const fsUtil = new FsUtility({ basePath: assetsDir, indexFileName: assetsFileName }); for (const _ of values(fsUtil.indexFileContent)) { const chunkData = await fsUtil.readChunkFiles.next().catch(() => ({})); - publishableCount += filter(values(chunkData as Record[]), (asset) => - this.isAmAssetPublishable(asset, sourceStack), - ).length; + for (const asset of values(chunkData as Record[])) { + if (!this.isAmAssetPublishable(asset, sourceStack)) continue; + // Count publish calls (one per env-locale-set group), not assets, so ticks stay 1:1. + publishableCount += this.buildPublishGroups( + filter(asset.publish_details, (pd: any) => pd?.api_key === sourceStack), + ).length; + } } } @@ -362,37 +408,15 @@ export default class ImportAssets extends BaseClass { handleAndLogError(error, { ...this.importConfig.context, uid, title }); }; + // apiData is a pre-expanded sub-item ({ uid, title, publishDetails }); one per env-locale-set + // group (see Pass 2). Pairing is already preserved, so this only resolves the destination UID. const serializeData = (apiOptions: ApiOptions) => { - const { apiData: asset } = apiOptions; - const publishDetails = filter( - asset.publish_details, - (pd: any) => pd?.api_key === sourceStack && this.environments?.hasOwnProperty(pd?.environment), - ); - - if (!publishDetails.length) { - apiOptions.entity = undefined; - return apiOptions; - } - - const environments = uniq(map(publishDetails, ({ environment }) => this.environments[environment].name)); - const locales = uniq(map(publishDetails, 'locale')); - - if (environments.length === 0 || locales.length === 0) { - log.debug(`Skipping publish for asset ${asset.uid} - no valid environments/locales`, this.importConfig.context); - apiOptions.entity = undefined; - return apiOptions; - } - - asset.locales = locales; - asset.environments = environments; - apiOptions.apiData.publishDetails = { locales, environments }; - - apiOptions.uid = this.assetsUidMap[asset.uid] as string; + const { apiData } = apiOptions; + apiOptions.uid = this.assetsUidMap[apiData.uid] as string; if (!apiOptions.uid) { - log.debug(`Skipping publish for asset ${asset.uid} - no UID mapping found`, this.importConfig.context); + log.debug(`Skipping publish for asset ${apiData.uid} - no UID mapping found`, this.importConfig.context); apiOptions.entity = undefined; } - return apiOptions; }; @@ -404,10 +428,15 @@ export default class ImportAssets extends BaseClass { const indexerCount = values(indexer).length; for (const index in indexer) { - const apiContent = filter(values(await fsUtil.readChunkFiles.next()), (asset) => - this.isAmAssetPublishable(asset, sourceStack), - ); - log.debug(`Found ${apiContent.length} publishable CS Assets in chunk ${index}`, this.importConfig.context); + // Expand each publishable asset into one sub-item per env-locale-set group, so each + // makeConcurrentCall item is a single-rectangle publish (preserves env↔locale pairing). + const apiContent = values(await fsUtil.readChunkFiles.next()).flatMap((asset: Record) => { + if (!this.isAmAssetPublishable(asset, sourceStack)) return []; + return this.buildPublishGroups( + filter(asset.publish_details, (pd: any) => pd?.api_key === sourceStack), + ).map((publishDetails) => ({ uid: asset.uid, title: asset.title, publishDetails })); + }); + log.debug(`Found ${apiContent.length} CS Asset publish calls in chunk ${index}`, this.importConfig.context); await this.makeConcurrentCall({ apiContent, diff --git a/packages/contentstack-import/test/unit/import/modules/assets.test.ts b/packages/contentstack-import/test/unit/import/modules/assets.test.ts index 895216076..c33b37854 100644 --- a/packages/contentstack-import/test/unit/import/modules/assets.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/assets.test.ts @@ -770,6 +770,67 @@ describe('ImportAssets', () => { }); }); + describe('buildPublishGroups() method', () => { + it('collapses a RECTANGULAR asset to a single publish call (behavior-preserving)', () => { + importAssets['environments'] = { + 'e1': { name: 'production' }, + 'e2': { name: 'preview' }, + 'e3': { name: 'development' }, + }; + const pd = ['e1', 'e2', 'e3'].flatMap((environment) => [ + { environment, locale: 'en-us' }, + { environment, locale: 'fr-fr' }, + ]); + + const groups = (importAssets as any).buildPublishGroups(pd); + + expect(groups).to.have.lengthOf(1); + expect(groups[0].locales).to.deep.equal(['en-us', 'fr-fr']); + expect(groups[0].environments).to.have.members(['production', 'preview', 'development']); + }); + + it('fans a RAGGED asset out into one group per distinct locale set (DX-9772)', () => { + importAssets['environments'] = { 'e1': { name: 'new' }, 'e2': { name: 'blt5795' } }; + const pd = [ + { environment: 'e2', locale: 'en-us' }, + { environment: 'e1', locale: 'en-us' }, + { environment: 'e1', locale: 'ar' }, + ]; + + const groups = (importAssets as any).buildPublishGroups(pd); + + expect(groups).to.have.lengthOf(2); + expect(groups).to.deep.include.members([ + { environments: ['new'], locales: ['ar', 'en-us'] }, + { environments: ['blt5795'], locales: ['en-us'] }, + ]); + // blt5795 must never be paired with ar (the phantom the old flatten produced). + const blt5795 = groups.find((g: any) => g.environments.includes('blt5795')); + expect(blt5795.locales).to.not.include('ar'); + }); + + it('handles a single env-locale pair (1×1) as one group', () => { + importAssets['environments'] = { 'e1': { name: 'production' } }; + const groups = (importAssets as any).buildPublishGroups([{ environment: 'e1', locale: 'en-us' }]); + expect(groups).to.deep.equal([{ environments: ['production'], locales: ['en-us'] }]); + }); + + it('drops environments absent from the destination (preserves DX-1656 guard)', () => { + importAssets['environments'] = { 'e1': { name: 'production' } }; + const groups = (importAssets as any).buildPublishGroups([ + { environment: 'e1', locale: 'en-us' }, + { environment: 'gone', locale: 'en-us' }, + ]); + expect(groups).to.deep.equal([{ environments: ['production'], locales: ['en-us'] }]); + }); + + it('returns [] for empty publish_details', () => { + importAssets['environments'] = { 'e1': { name: 'production' } }; + expect((importAssets as any).buildPublishGroups([])).to.deep.equal([]); + expect((importAssets as any).buildPublishGroups(undefined)).to.deep.equal([]); + }); + }); + describe('constructFolderImportOrder() method', () => { it('should order folders with null parent_uid first', () => { const folders = [