feat: TransportProvider abstraction and DeployPipeline scaffolding @W-24222968@

package.json

@@ -25,19 +25,21 @@
     "node": ">=22.0.0"
   },
   "dependencies": {
-    "@salesforce/core": "^9.2.2",
+    "@salesforce/core": "^9.3.2",
     "@salesforce/kit": "^4.0.0",
     "@salesforce/ts-types": "^3.2.0",
     "@salesforce/types": "^1.6.0",
     "fast-levenshtein": "^3.0.0",
     "fast-xml-parser": "^5.7.3",
+    "form-data": "^4.0.6",
     "got": "^11.8.6",
     "graceful-fs": "^4.2.11",
     "ignore": "^5.3.2",
     "jszip": "^3.10.1",
     "mime": "2.6.0",
     "minimatch": "^9.0.9",
     "proxy-agent": "^6.5.0",
+    "tar-stream": "^3.2.1",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
@@ -53,6 +55,7 @@
     "@types/mocha": "^10.0.10",
     "@types/node": "^18",
     "@types/sinon": "^10.0.20",
+    "@types/tar-stream": "^3.1.4",
     "deep-equal-in-any-order": "^1.1.19",
     "esbuild": "^0.28.0",
     "eslint": "^10.4.0",

src/client/index.ts

@@ -22,6 +22,7 @@ export {
   ScopedPostDeploy,
 } from './metadataApiDeploy';
 export { MetadataApiRetrieve, RetrieveResult, ScopedPostRetrieve, ScopedPreRetrieve } from './metadataApiRetrieve';
+export type { TransportStageEvent } from './transports';
 export {
   ComponentDeployment,
   ComponentRetrieval,

src/client/metadataApiDeploy.ts

@@ -28,6 +28,7 @@ import { ReplacementEvent } from '../convert/types';
 import { MetadataConverter } from '../convert';
 import { ComponentSet } from '../collections';
 import { MetadataTransfer, MetadataTransferOptions } from './metadataTransfer';
+import { AsyncTransportHandle } from './transports/types';
 import {
   AsyncResult,
   ComponentStatus,
@@ -56,6 +57,8 @@ const messages = Messages.loadMessages('@salesforce/source-deploy-retrieve', 'sd
 // TODO: (NEXT MAJOR) this should just be a readonly object and not a class.
 export class DeployResult implements MetadataTransferResult {
   private fileResponses?: FileResponse[];
+  private transportFileResponses: FileResponse[] = [];
+  private asyncTransportHandles: AsyncTransportHandle[] = [];
 
   public constructor(
     public readonly response: MetadataApiDeployStatus,
@@ -69,19 +72,29 @@ export class DeployResult implements MetadataTransferResult {
     if (!this.fileResponses) {
       this.fileResponses = [
         ...new Map(
-          (this.components
-            ? buildFileResponsesFromComponentSet(this.components)(this.response)
-            : buildFileResponses(this.response)
-          ).map((v) => {
-            const base = `${v.type}#${v.fullName}#${v.filePath ?? ''}#${v.state}`;
-            const key = 'error' in v ? `${base}#${v.error}#${String(v.lineNumber)}#${String(v.columnNumber)}` : base;
-            return [key, v] as const;
-          })
+          // MDAPI and required transport steps can report the same source file.
+          // A serialized structural key preserves one response without adding a dependency.
+          [
+            ...(this.components
+              ? buildFileResponsesFromComponentSet(this.components)(this.response)
+              : buildFileResponses(this.response)),
+            ...this.transportFileResponses,
+          ].map((v) => [JSON.stringify(v), v])
         ).values(),
       ];
     }
     return this.fileResponses;
   }
+
+  public addTransportResults(fileResponses: FileResponse[], asyncHandles: AsyncTransportHandle[] = []): void {
+    this.transportFileResponses.push(...fileResponses);
+    this.asyncTransportHandles.push(...asyncHandles);
+    this.fileResponses = undefined;
+  }
+
+  public getAsyncTransportHandles(): AsyncTransportHandle[] {
+    return this.asyncTransportHandles;
+  }
 }
 
 export type MetadataApiDeployOptions = {

src/client/metadataApiRetrieve.ts

@@ -38,6 +38,7 @@ import {
 import { extract } from './retrieveExtract';
 import { getPackageOptions } from './retrieveExtract';
 import { MetadataApiRetrieveOptions } from './types';
+import { AsyncTransportHandle } from './transports/types';
 
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/source-deploy-retrieve', 'sdr');
@@ -48,6 +49,8 @@ export class RetrieveResult implements MetadataTransferResult {
   // rather than "Changed".
   private localComponents: ComponentSet;
   private fileResponses?: FileResponse[];
+  private transportFileResponses: FileResponse[] = [];
+  private asyncTransportHandles: AsyncTransportHandle[] = [];
 
   /**
    * @param response The metadata retrieve response from the server
@@ -69,7 +72,7 @@ export class RetrieveResult implements MetadataTransferResult {
       return this.fileResponses;
     }
 
-    this.fileResponses = [];
+    const responses: FileResponse[] = [];
 
     // construct failures
     if (this.response.messages) {
@@ -80,15 +83,15 @@ export class RetrieveResult implements MetadataTransferResult {
         const matches = new RegExp(/.+'(.+)'.+'(.+)'/).exec(message.problem);
         if (matches) {
           const [typeName, fullName] = matches.slice(1);
-          this.fileResponses.push({
+          responses.push({
             fullName,
             type: typeName,
             state: ComponentStatus.Failed,
             error: message.problem,
             problemType: 'Error',
           });
         } else {
-          this.fileResponses.push({
+          responses.push({
             fullName: '',
             type: '',
             problemType: 'Error',
@@ -110,21 +113,37 @@ export class RetrieveResult implements MetadataTransferResult {
 
       if (!type.children || Object.values(type.children.types).some((t) => t.unaddressableWithoutParent)) {
         for (const filePath of retrievedComponent.walkContent()) {
-          this.fileResponses.push({ ...baseResponse, filePath } satisfies FileResponseSuccess);
+          responses.push({ ...baseResponse, filePath } satisfies FileResponseSuccess);
         }
       }
 
       if (xml) {
-        this.fileResponses.push({ ...baseResponse, filePath: xml } satisfies FileResponseSuccess);
+        responses.push({ ...baseResponse, filePath: xml } satisfies FileResponseSuccess);
       }
     }
 
     // Add file responses for components that support partial delete (e.g., DigitalExperience)
     // where pieces of the component were deleted in the org, then retrieved.
-    this.fileResponses.push(...(this.partialDeleteFileResponses ?? []));
+    responses.push(...(this.partialDeleteFileResponses ?? []));
+
+    responses.push(...this.transportFileResponses);
+
+    // MDAPI extraction and required transport retrieval can report the same source file.
+    // A serialized structural key preserves one response without adding a dependency.
+    this.fileResponses = [...new Map(responses.map((v) => [JSON.stringify(v), v])).values()];
 
     return this.fileResponses;
   }
+
+  public addTransportResults(fileResponses: FileResponse[], asyncHandles: AsyncTransportHandle[] = []): void {
+    this.transportFileResponses.push(...fileResponses);
+    this.asyncTransportHandles.push(...asyncHandles);
+    this.fileResponses = undefined;
+  }
+
+  public getAsyncTransportHandles(): AsyncTransportHandle[] {
+    return this.asyncTransportHandles;
+  }
 }
 
 export class MetadataApiRetrieve extends MetadataTransfer<

src/client/metadataTransfer.ts

@@ -54,6 +54,7 @@ export abstract class MetadataTransfer<
   protected mdapiTempDir?: string;
   private transferId: Options['id'];
   private event = new EventEmitter();
+  private readonly resultProcessors: Array<(result: Result, status: Status) => Promise<void>> = [];
   private usernameOrConnection: string | Connection;
   private apiVersion?: string;
   private consecutiveErrorRetries = 0;
@@ -148,6 +149,11 @@ export abstract class MetadataTransfer<
       }
 
       const result = await this.post(completedMdapiStatus);
+      for (const processor of this.resultProcessors) {
+        // Processors extend the operation result before it is returned to callers.
+        // eslint-disable-next-line no-await-in-loop
+        await processor(result, completedMdapiStatus);
+      }
       if (completedMdapiStatus.status === RequestStatus.Canceled) {
         this.event.emit('cancel', completedMdapiStatus);
       } else {
@@ -176,7 +182,7 @@ export abstract class MetadataTransfer<
           // this keeps SfError data for failures in post deploy/retrieve.
           error.setData({
             id: this.id,
-            causeErrorData: error.data,
+            causeErrorData: err.data as AnyJson,
           });
 
           error.actions = err.actions;
@@ -201,6 +207,11 @@ export abstract class MetadataTransfer<
     this.event.on('finish', subscriber);
   }
 
+  /** @internal Register work that must complete before pollStatus returns its result. */
+  public addResultProcessor(processor: (result: Result, status: Status) => Promise<void>): void {
+    this.resultProcessors.push(processor);
+  }
+
   public onCancel(subscriber: (result: Status | undefined) => void): void {
     this.event.on('cancel', subscriber);
   }