feat: forward ACP prompt content to Muse

README.md

@@ -133,3 +133,8 @@ The work board lives in `.pm/` (workstream w1).
 Apache-2.0. Portions derived from
 [claude-agent-acp](https://github.com/agentclientprotocol/claude-agent-acp)
 (Zed Industries) — see `NOTICE`.
+
+Prompt images (PNG, JPEG, GIF, WebP) are supported: the SDK receives inline image
+parts; legacy exec uses private per-turn files removed during cleanup. Exec
+requires accompanying text or a resource link. Audio and embedded resources
+remain unsupported.

docs/sdk-migration.md

@@ -26,19 +26,19 @@ MUSE_CODE_ACP_BACKEND=exec muse-code-acp
 
 ## ACP surface (advertised)
 
-| Capability                                | Advertised?                                    | Contract owner                                               |
-| ----------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------ |
-| Protocol major 1                          | yes (always returned as our supported version) | `src/acp-agent.ts` initialize + `src/tests/acp-wire.test.ts` |
-| Prompt: text + resource_link              | baseline (empty `promptCapabilities`)          | `src/prompt-content.ts`                                      |
-| Prompt: image / audio / embedded resource | **no**                                         | rejected with invalid params                                 |
-| MCP stdio                                 | yes (baseline; http/sse not advertised)        | `docs/mcp-passthrough.md`                                    |
-| `session/load`, `session/list`            | yes                                            | session store + export helpers                               |
-| Auth logout                               | yes                                            | `src/auth.ts`                                                |
-| Terminal auth method                      | only if `clientCapabilities.auth.terminal`     | `src/auth.ts`                                                |
-| Interactive permissions (SDK backend)     | yes                                            | `src/muse-permissions.ts` + live approval suite              |
-| Form elicitation (SDK user input)         | yes when client advertises `elicitation.form`  | `src/muse-user-input.ts`                                     |
-| fs / terminal RPC                         | **no**                                         | omitted client caps never invoked                            |
-| Session fork/delete/close                 | **no**                                         | unadvertised                                                 |
+| Capability                            | Advertised?                                    | Contract owner                                               |
+| ------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------ |
+| Protocol major 1                      | yes (always returned as our supported version) | `src/acp-agent.ts` initialize + `src/tests/acp-wire.test.ts` |
+| Prompt: text + resource_link          | baseline (empty `promptCapabilities`)          | `src/prompt-content.ts`                                      |
+| Prompt: audio / embedded resource     | **no**                                         | rejected with invalid params                                 |
+| MCP stdio                             | yes (baseline; http/sse not advertised)        | `docs/mcp-passthrough.md`                                    |
+| `session/load`, `session/list`        | yes                                            | session store + export helpers                               |
+| Auth logout                           | yes                                            | `src/auth.ts`                                                |
+| Terminal auth method                  | only if `clientCapabilities.auth.terminal`     | `src/auth.ts`                                                |
+| Interactive permissions (SDK backend) | yes                                            | `src/muse-permissions.ts` + live approval suite              |
+| Form elicitation (SDK user input)     | yes when client advertises `elicitation.form`  | `src/muse-user-input.ts`                                     |
+| fs / terminal RPC                     | **no**                                         | omitted client caps never invoked                            |
+| Session fork/delete/close             | **no**                                         | unadvertised                                                 |
 
 ## Public SDK API map
 
@@ -137,3 +137,8 @@ conversation and that the saved model/effort survive the ACP process restart.
 Publishing resolves the release ref to an immutable commit, runs this same CI
 workflow on that commit, and only publishes after all checks succeed. Manual
 publishing follows the same checks.
+
+Prompt images are advertised and sent as ordered MSP `image` parts with
+`mediaType` and `base64Data`. PNG, JPEG, GIF, and WebP are accepted; malformed
+base64 is rejected before a turn starts. Legacy exec stages private temporary
+files and requires text or a resource link alongside images.

src/acp-agent.ts

@@ -59,6 +59,7 @@ import { MuseSdkHandle, spawnMuseSdkTurn, readMuseSdkSession } from "./muse-sdk.
 import { readSessionEffort, writeSessionEffort } from "./session-preferences.js";
 import { createMuseMcpOverlay, MuseMcpOverlay } from "./mcp-overlay.js";
 import { readMuseSettings } from "./muse-settings.js";
+import { compileMusePrompt, type CompiledMusePrompt } from "./prompt-files.js";
 import { convertPromptContent } from "./prompt-content.js";
 import { exportToUpdates, runMuseExport } from "./session-export.js";
 import { listStoredSessions } from "./session-store.js";
@@ -110,6 +111,7 @@ export interface SessionState {
   activeTurn: MuseExecHandle | MuseSdkHandle | null;
   /** Set by `session/cancel`; forces the turn to settle with `cancelled`. */
   cancelRequested: boolean;
+  turnFinished: Promise<void> | null;
   /** Active ACP session mode; decides the safety flags of the next spawn. */
   modeId: MuseModeId;
   /** Model + reasoning effort applied to every spawn for this session. */
@@ -173,10 +175,9 @@ export class MuseAcpAgent {
     return {
       protocolVersion: PROTOCOL_VERSION,
       // Only advertise what is actually implemented; capabilities grow with
-      // the milestones that ship them. Empty promptCapabilities = baseline
-      // text + resource_link only (no image/audio/embedded).
+      // the milestones that ship them. Images work on both backends.
       agentCapabilities: {
-        promptCapabilities: {},
+        promptCapabilities: { image: true },
         mcpCapabilities: {},
         loadSession: true,
         sessionCapabilities: { list: {} },
@@ -240,6 +241,7 @@ export class MuseAcpAgent {
       cwd: params.cwd,
       museSessionId: sessionId,
       activeTurn: null,
+      turnFinished: null,
       cancelRequested: false,
       modeId: "default",
       config,
@@ -358,6 +360,7 @@ export class MuseAcpAgent {
       cwd: params.cwd,
       museSessionId: params.sessionId,
       activeTurn: null,
+      turnFinished: null,
       cancelRequested: false,
       modeId: "default",
       config,
@@ -415,7 +418,7 @@ export class MuseAcpAgent {
 
   async prompt(params: PromptRequest): Promise<PromptResponse> {
     const session = this.requireSession(params.sessionId);
-    if (session.activeTurn) {
+    if (session.turnFinished) {
       throw RequestError.invalidRequest(
         undefined,
         `session ${params.sessionId} already has a prompt turn in flight`,
@@ -427,18 +430,22 @@ export class MuseAcpAgent {
       throw converted.error;
     }
 
+    const finished = Promise.withResolvers<void>();
+    session.turnFinished = finished.promise;
     session.cancelRequested = false;
-    const baseEnv = this.options.env ?? process.env;
-    const mcpOverlay =
-      this.backend === "sdk" || session.mcpServers.length > 0
-        ? createMuseMcpOverlay(
-            session.mcpServers,
-            baseEnv,
-            this.backend === "sdk" ? session.config : undefined,
-          )
-        : null;
-    session.activeMcpOverlay = mcpOverlay;
+    let compiledPrompt: CompiledMusePrompt | undefined;
+    let mcpOverlay: MuseMcpOverlay | null = null;
     try {
+      const baseEnv = this.options.env ?? process.env;
+      mcpOverlay =
+        this.backend === "sdk" || session.mcpServers.length > 0
+          ? createMuseMcpOverlay(
+              session.mcpServers,
+              baseEnv,
+              this.backend === "sdk" ? session.config : undefined,
+            )
+          : null;
+      session.activeMcpOverlay = mcpOverlay;
       if (this.backend === "sdk") {
         if (this.options.provider === "echo") {
           throw RequestError.invalidParams(
@@ -473,9 +480,12 @@ export class MuseAcpAgent {
           await handle.done.catch(() => {});
         }
       }
+      compiledPrompt = await compileMusePrompt(params.prompt);
+      if (session.cancelRequested) return { stopReason: "cancelled" };
       const translator = new TurnTranslator(params.sessionId, this.logger);
       const handle = spawnMuseExec({
-        prompt: converted.text,
+        prompt: compiledPrompt.prompt,
+        imagePaths: compiledPrompt.imagePaths,
         sessionId: session.museSessionId,
         cwd: session.cwd,
         museBinary: this.options.museBinary,
@@ -533,10 +543,19 @@ export class MuseAcpAgent {
           return (unreachable(outcome, this.logger), { stopReason: "end_turn" });
       }
     } finally {
-      session.activeTurn = null;
-      mcpOverlay?.cleanup();
-      if (session.activeMcpOverlay === mcpOverlay) {
-        session.activeMcpOverlay = null;
+      try {
+        session.activeTurn?.kill();
+        await session.activeTurn?.done.catch(() => {});
+        mcpOverlay?.cleanup();
+      } finally {
+        try {
+          await compiledPrompt?.cleanup();
+        } finally {
+          session.activeTurn = null;
+          session.activeMcpOverlay = null;
+          session.turnFinished = null;
+          finished.resolve();
+        }
       }
     }
   }

src/muse-exec.ts

@@ -16,6 +16,8 @@ export interface MuseExecOptions {
   provider?: "meta" | "echo";
   model?: string;
   reasoningEffort?: string;
+  /** Turn-scoped images forwarded through Muse's repeatable `--image` flag. */
+  imagePaths?: string[];
   /** Extra CLI flags appended verbatim (e.g. `--echo-delay-ms` in tests). */
   extraArgs?: string[];
   env?: Record<string, string | undefined>;
@@ -59,6 +61,9 @@ export function spawnMuseExec(options: MuseExecOptions): MuseExecHandle {
   if (options.reasoningEffort) {
     args.push("--reasoning-effort", options.reasoningEffort);
   }
+  for (const imagePath of options.imagePaths ?? []) {
+    args.push("--image", imagePath);
+  }
   // Note: no `--no-session-log` ever — muse rejects it alongside
   // `--session-id` ("a session id needs retained logging"), and session
   // continuity/resume depend on the retained log. Tests isolate the store via

src/muse-sdk.ts

@@ -28,7 +28,7 @@ import {
   resolvePermissionChoice,
 } from "./muse-permissions.js";
 import { MuseSdkTranslator } from "./muse-sdk-events.js";
-import type { MuseTextInputPart } from "./prompt-content.js";
+import type { MuseInputPart } from "./prompt-content.js";
 import {
   MuseUserInputRequest,
   settleUserInput,
@@ -41,7 +41,7 @@ export interface MuseSdkOptions {
   sessionId: string;
   cwd: string;
   /** Ordered Muse turn input parts (text encodings of ACP content). */
-  input: MuseTextInputPart[];
+  input: MuseInputPart[];
   model: string;
   reasoningEffort: string;
   readOnly: boolean;

src/prompt-content.ts

@@ -1,7 +1,10 @@
+import { decodeImage, IMAGE_EXTENSIONS } from "./prompt-images.js";
 import { ContentBlock, PromptRequest, RequestError } from "@agentclientprotocol/sdk";
 
-/** Muse turn input text part (MSP declares only text | image; we use text). */
+/** Muse turn input text part. */
 export type MuseTextInputPart = { type: "text"; text: string };
+export type MuseInputPart =
+  MuseTextInputPart | { type: "image"; base64Data: string; mediaType: string };
 
 /**
  * Lossless text encoding for ACP `resource_link` blocks. Muse's turn input
@@ -25,12 +28,11 @@ export function formatResourceLink(
 }
 
 export type PromptConversion =
-  { ok: true; parts: MuseTextInputPart[]; text: string } | { ok: false; error: RequestError };
+  { ok: true; parts: MuseInputPart[]; text: string } | { ok: false; error: RequestError };
 
 /**
  * Convert ACP prompt content into Muse turn input and a legacy exec string.
- * Baseline ACP requires text + resource_link; optional image/audio/resource
- * are rejected when present because this adapter does not advertise them.
+ * Baseline ACP requires text + resource_link; images use inline MSP parts; audio and embedded resources are rejected.
  */
 export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptConversion {
   if (blocks.length === 0) {
@@ -40,7 +42,7 @@ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptCon
     };
   }
 
-  const parts: MuseTextInputPart[] = [];
+  const parts: MuseInputPart[] = [];
   for (const block of blocks) {
     switch (block.type) {
       case "text":
@@ -49,14 +51,34 @@ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptCon
       case "resource_link":
         parts.push({ type: "text", text: formatResourceLink(block) });
         break;
-      case "image":
+      case "image": {
+        const mediaType = block.mimeType.trim().toLowerCase();
+        if (!IMAGE_EXTENSIONS.has(mediaType))
+          return {
+            ok: false,
+            error: RequestError.invalidParams(
+              undefined,
+              "supported MIME types: image/png, image/jpeg, image/gif, image/webp",
+            ),
+          };
+        try {
+          parts.push({
+            type: "image",
+            mediaType,
+            base64Data: decodeImage(block.data).toString("base64"),
+          });
+        } catch (error) {
+          return { ok: false, error: error as RequestError };
+        }
+        break;
+      }
       case "audio":
       case "resource":
         return {
           ok: false,
           error: RequestError.invalidParams(
             undefined,
-            `unsupported prompt content type: ${block.type}; this agent advertises only text and resource_link`,
+            `unsupported prompt content type: ${block.type}; this agent advertises text, resource_link and image; send embedded resources as resource_link blocks instead`,
           ),
         };
       default:
@@ -71,10 +93,10 @@ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptCon
   }
 
   const text = parts
-    .map((part) => part.text)
+    .flatMap((part) => (part.type === "text" ? [part.text] : []))
     .join("\n\n")
     .trim();
-  if (text.length === 0) {
+  if (text.length === 0 && !parts.some((part) => part.type === "image")) {
     return {
       ok: false,
       error: RequestError.invalidParams(

src/prompt-files.ts

@@ -0,0 +1,54 @@
+import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { RequestError, type ContentBlock } from "@agentclientprotocol/sdk";
+import { convertPromptContent } from "./prompt-content.js";
+import { IMAGE_EXTENSIONS } from "./prompt-images.js";
+export type CompiledMusePrompt = {
+  prompt: string;
+  imagePaths: string[];
+  cleanup(): Promise<void>;
+};
+
+export async function compileMusePrompt(blocks: ContentBlock[]): Promise<CompiledMusePrompt> {
+  const converted = convertPromptContent(blocks);
+  if (!converted.ok) throw converted.error;
+  const images = converted.parts.flatMap((part) =>
+    part.type === "image"
+      ? [
+          {
+            bytes: Buffer.from(part.base64Data, "base64"),
+            extension: IMAGE_EXTENSIONS.get(part.mediaType)!,
+          },
+        ]
+      : [],
+  );
+  const prompt = converted.text;
+  if (!prompt)
+    throw RequestError.invalidParams(
+      undefined,
+      "Muse Code requires text or a resource link alongside image content",
+    );
+  if (images.length === 0) {
+    return { prompt, imagePaths: [], cleanup: async () => {} };
+  }
+
+  const directory = await mkdtemp(join(tmpdir(), "muse-code-acp-images-"));
+  try {
+    await chmod(directory, 0o700);
+    const imagePaths: string[] = [];
+    for (const [index, image] of images.entries()) {
+      const imagePath = join(directory, `image-${String(index + 1)}.${image.extension}`);
+      await writeFile(imagePath, image.bytes, { mode: 0o600 });
+      imagePaths.push(imagePath);
+    }
+    return {
+      prompt,
+      imagePaths,
+      cleanup: async () => await rm(directory, { recursive: true, force: true }),
+    };
+  } catch (error) {
+    await rm(directory, { recursive: true, force: true });
+    throw error;
+  }
+}

src/prompt-images.ts

@@ -0,0 +1,30 @@
+import { RequestError } from "@agentclientprotocol/sdk";
+export const IMAGE_EXTENSIONS = new Map([
+  ["image/gif", "gif"],
+  ["image/jpeg", "jpg"],
+  ["image/png", "png"],
+  ["image/webp", "webp"],
+]);
+
+function unsupportedContent(type: string, detail?: string): RequestError {
+  const suffix = detail ? ` (${detail})` : "";
+  return RequestError.invalidParams(
+    undefined,
+    `unsupported ACP prompt content: ${type}${suffix}. ` +
+      "Muse Code accepts text, resource links, and PNG/JPEG/GIF/WebP images; " +
+      "send embedded resources as resource_link blocks instead.",
+  );
+}
+
+export function decodeImage(data: string): Buffer {
+  const normalized = data.replace(/\s/gu, "");
+  if (!normalized || normalized.length % 4 === 1 || !/^[A-Za-z0-9+/]*={0,2}$/u.test(normalized)) {
+    throw unsupportedContent("image", "invalid base64 data");
+  }
+  const decoded = Buffer.from(normalized, "base64");
+  const canonical = normalized.replace(/=+$/u, "");
+  if (!decoded.length || decoded.toString("base64").replace(/=+$/u, "") !== canonical) {
+    throw unsupportedContent("image", "invalid base64 data");
+  }
+  return decoded;
+}

src/tests/fixtures/cat-image.ts

@@ -0,0 +1,3 @@
+/** PNG containing the black word CAT on a white background. */
+export const CAT_IMAGE_BASE64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAIAAABhdOiYAAAACXBIWXMAAAABAAAAAQBPJcTWAAAC8ElEQVR4nO2YMUhqURjHlUDMFMSlEDQ3EQIhpKYgwUVxMBoCh9xsaCjEFmsKgohocFMi3FwincUgHB2KBGsMESGQFAchI/F9vODjvnu17xo+3jvy/SY75/zvPefH6bv3XM2A+RbNv57A/w4LImBBBCyIgAURsCACFkTAgghYEAELImBBBCyIgAURsCCCvyKo2WxeXFwEg8HFxUWj0TgzM2OxWFZXV/f39+/u7kaldnZ2NH/y9PSEvS8vL5oxmchaJizo8/Pz6Ohodnb2m3kfHBwogx8fHyBRNjKRSOCAaRD0/v7u9XrJee/u7iqzuVxOOdJut+OAaRC0tbUlW97l5WW9Xu/1eq+vr6AgFAqNErS5uTl0kaVSadTtbm5upCMfHh4muBZkYoKKxaJ0um63++3tTTkMFnx2diZr7HQ6Op0Os/Pz8/g7Go2OuqNggvx+P85Vq9VWq1X1WdhomDWZTKlUCv80m82wAYemRBIE1Uev1+NcfT7fWPH19XXMhsNh2Hrw4MMWEDE0JZKgSqUinevx8bH6LBQp2HGYvb6+hkZpsd/Y2BgaFEnQ7e2tdK7pdFp99vT0FIOwDbvdLjQmk0lshPLUbreVQYEFQU1Rn11aWsIgPOa+GmFbSS8IVUkZFEnQj//FHh8fpcFMJoNdHo8H29fW1pRZkQT9uEjH43FMQWFutVrYdXJyIl1/rVaTZUUSNFA85p+fn8lIv9+3Wq0adYAvWVwwQbIXxeXl5aGVVfqiKIt8j8vlkl1KMEEDxVHD4XBcXV01Gg04iMJRI5/Py44akUhEvSDg/v5eejvxBI11WIXB8NIsa5QBj/y5uTkcE4vFpL3iCRr8/txxeHio5nNHNpuVNpbL5aEX3N7exjELCwtQtrBLSEFfNJvN8/PzQCBgs9kMBsPXB7OVlZW9vT38YBYMBnFtTqdz1KVkdapQKGCXwIKmCRZEwIIIWBABCyJgQQQsiIAFEbAgAhZEwIIIWBABCyJgQQQsiOAXJ2r353yEvnMAAAAASUVORK5CYII=";