diff --git a/README.md b/README.md index 69b968b..b45d154 100644 --- a/README.md +++ b/README.md @@ -133,3 +133,8 @@ The work board lives in `.pm/` (workstream w1). Apache-2.0. Portions derived from [claude-agent-acp](https://github.com/agentclientprotocol/claude-agent-acp) (Zed Industries) — see `NOTICE`. + +Prompt images (PNG, JPEG, GIF, WebP) are supported: the SDK receives inline image +parts; legacy exec uses private per-turn files removed during cleanup. Exec +requires accompanying text or a resource link. Audio and embedded resources +remain unsupported. diff --git a/docs/sdk-migration.md b/docs/sdk-migration.md index c8dd647..21bf582 100644 --- a/docs/sdk-migration.md +++ b/docs/sdk-migration.md @@ -26,19 +26,19 @@ MUSE_CODE_ACP_BACKEND=exec muse-code-acp ## ACP surface (advertised) -| Capability | Advertised? | Contract owner | -| ----------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------ | -| Protocol major 1 | yes (always returned as our supported version) | `src/acp-agent.ts` initialize + `src/tests/acp-wire.test.ts` | -| Prompt: text + resource_link | baseline (empty `promptCapabilities`) | `src/prompt-content.ts` | -| Prompt: image / audio / embedded resource | **no** | rejected with invalid params | -| MCP stdio | yes (baseline; http/sse not advertised) | `docs/mcp-passthrough.md` | -| `session/load`, `session/list` | yes | session store + export helpers | -| Auth logout | yes | `src/auth.ts` | -| Terminal auth method | only if `clientCapabilities.auth.terminal` | `src/auth.ts` | -| Interactive permissions (SDK backend) | yes | `src/muse-permissions.ts` + live approval suite | -| Form elicitation (SDK user input) | yes when client advertises `elicitation.form` | `src/muse-user-input.ts` | -| fs / terminal RPC | **no** | omitted client caps never invoked | -| Session fork/delete/close | **no** | unadvertised | +| Capability | Advertised? | Contract owner | +| ------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------ | +| Protocol major 1 | yes (always returned as our supported version) | `src/acp-agent.ts` initialize + `src/tests/acp-wire.test.ts` | +| Prompt: text + resource_link | baseline (empty `promptCapabilities`) | `src/prompt-content.ts` | +| Prompt: audio / embedded resource | **no** | rejected with invalid params | +| MCP stdio | yes (baseline; http/sse not advertised) | `docs/mcp-passthrough.md` | +| `session/load`, `session/list` | yes | session store + export helpers | +| Auth logout | yes | `src/auth.ts` | +| Terminal auth method | only if `clientCapabilities.auth.terminal` | `src/auth.ts` | +| Interactive permissions (SDK backend) | yes | `src/muse-permissions.ts` + live approval suite | +| Form elicitation (SDK user input) | yes when client advertises `elicitation.form` | `src/muse-user-input.ts` | +| fs / terminal RPC | **no** | omitted client caps never invoked | +| Session fork/delete/close | **no** | unadvertised | ## Public SDK API map @@ -137,3 +137,8 @@ conversation and that the saved model/effort survive the ACP process restart. Publishing resolves the release ref to an immutable commit, runs this same CI workflow on that commit, and only publishes after all checks succeed. Manual publishing follows the same checks. + +Prompt images are advertised and sent as ordered MSP `image` parts with +`mediaType` and `base64Data`. PNG, JPEG, GIF, and WebP are accepted; malformed +base64 is rejected before a turn starts. Legacy exec stages private temporary +files and requires text or a resource link alongside images. diff --git a/src/acp-agent.ts b/src/acp-agent.ts index f651bb7..6144462 100644 --- a/src/acp-agent.ts +++ b/src/acp-agent.ts @@ -59,6 +59,7 @@ import { MuseSdkHandle, spawnMuseSdkTurn, readMuseSdkSession } from "./muse-sdk. import { readSessionEffort, writeSessionEffort } from "./session-preferences.js"; import { createMuseMcpOverlay, MuseMcpOverlay } from "./mcp-overlay.js"; import { readMuseSettings } from "./muse-settings.js"; +import { compileMusePrompt, type CompiledMusePrompt } from "./prompt-files.js"; import { convertPromptContent } from "./prompt-content.js"; import { exportToUpdates, runMuseExport } from "./session-export.js"; import { listStoredSessions } from "./session-store.js"; @@ -110,6 +111,7 @@ export interface SessionState { activeTurn: MuseExecHandle | MuseSdkHandle | null; /** Set by `session/cancel`; forces the turn to settle with `cancelled`. */ cancelRequested: boolean; + turnFinished: Promise | null; /** Active ACP session mode; decides the safety flags of the next spawn. */ modeId: MuseModeId; /** Model + reasoning effort applied to every spawn for this session. */ @@ -173,10 +175,9 @@ export class MuseAcpAgent { return { protocolVersion: PROTOCOL_VERSION, // Only advertise what is actually implemented; capabilities grow with - // the milestones that ship them. Empty promptCapabilities = baseline - // text + resource_link only (no image/audio/embedded). + // the milestones that ship them. Images work on both backends. agentCapabilities: { - promptCapabilities: {}, + promptCapabilities: { image: true }, mcpCapabilities: {}, loadSession: true, sessionCapabilities: { list: {} }, @@ -240,6 +241,7 @@ export class MuseAcpAgent { cwd: params.cwd, museSessionId: sessionId, activeTurn: null, + turnFinished: null, cancelRequested: false, modeId: "default", config, @@ -358,6 +360,7 @@ export class MuseAcpAgent { cwd: params.cwd, museSessionId: params.sessionId, activeTurn: null, + turnFinished: null, cancelRequested: false, modeId: "default", config, @@ -415,7 +418,7 @@ export class MuseAcpAgent { async prompt(params: PromptRequest): Promise { const session = this.requireSession(params.sessionId); - if (session.activeTurn) { + if (session.turnFinished) { throw RequestError.invalidRequest( undefined, `session ${params.sessionId} already has a prompt turn in flight`, @@ -427,18 +430,22 @@ export class MuseAcpAgent { throw converted.error; } + const finished = Promise.withResolvers(); + session.turnFinished = finished.promise; session.cancelRequested = false; - const baseEnv = this.options.env ?? process.env; - const mcpOverlay = - this.backend === "sdk" || session.mcpServers.length > 0 - ? createMuseMcpOverlay( - session.mcpServers, - baseEnv, - this.backend === "sdk" ? session.config : undefined, - ) - : null; - session.activeMcpOverlay = mcpOverlay; + let compiledPrompt: CompiledMusePrompt | undefined; + let mcpOverlay: MuseMcpOverlay | null = null; try { + const baseEnv = this.options.env ?? process.env; + mcpOverlay = + this.backend === "sdk" || session.mcpServers.length > 0 + ? createMuseMcpOverlay( + session.mcpServers, + baseEnv, + this.backend === "sdk" ? session.config : undefined, + ) + : null; + session.activeMcpOverlay = mcpOverlay; if (this.backend === "sdk") { if (this.options.provider === "echo") { throw RequestError.invalidParams( @@ -473,9 +480,12 @@ export class MuseAcpAgent { await handle.done.catch(() => {}); } } + compiledPrompt = await compileMusePrompt(params.prompt); + if (session.cancelRequested) return { stopReason: "cancelled" }; const translator = new TurnTranslator(params.sessionId, this.logger); const handle = spawnMuseExec({ - prompt: converted.text, + prompt: compiledPrompt.prompt, + imagePaths: compiledPrompt.imagePaths, sessionId: session.museSessionId, cwd: session.cwd, museBinary: this.options.museBinary, @@ -533,10 +543,19 @@ export class MuseAcpAgent { return (unreachable(outcome, this.logger), { stopReason: "end_turn" }); } } finally { - session.activeTurn = null; - mcpOverlay?.cleanup(); - if (session.activeMcpOverlay === mcpOverlay) { - session.activeMcpOverlay = null; + try { + session.activeTurn?.kill(); + await session.activeTurn?.done.catch(() => {}); + mcpOverlay?.cleanup(); + } finally { + try { + await compiledPrompt?.cleanup(); + } finally { + session.activeTurn = null; + session.activeMcpOverlay = null; + session.turnFinished = null; + finished.resolve(); + } } } } diff --git a/src/muse-exec.ts b/src/muse-exec.ts index 925420f..963abee 100644 --- a/src/muse-exec.ts +++ b/src/muse-exec.ts @@ -16,6 +16,8 @@ export interface MuseExecOptions { provider?: "meta" | "echo"; model?: string; reasoningEffort?: string; + /** Turn-scoped images forwarded through Muse's repeatable `--image` flag. */ + imagePaths?: string[]; /** Extra CLI flags appended verbatim (e.g. `--echo-delay-ms` in tests). */ extraArgs?: string[]; env?: Record; @@ -59,6 +61,9 @@ export function spawnMuseExec(options: MuseExecOptions): MuseExecHandle { if (options.reasoningEffort) { args.push("--reasoning-effort", options.reasoningEffort); } + for (const imagePath of options.imagePaths ?? []) { + args.push("--image", imagePath); + } // Note: no `--no-session-log` ever — muse rejects it alongside // `--session-id` ("a session id needs retained logging"), and session // continuity/resume depend on the retained log. Tests isolate the store via diff --git a/src/muse-sdk.ts b/src/muse-sdk.ts index 15ea9cc..e0689af 100644 --- a/src/muse-sdk.ts +++ b/src/muse-sdk.ts @@ -28,7 +28,7 @@ import { resolvePermissionChoice, } from "./muse-permissions.js"; import { MuseSdkTranslator } from "./muse-sdk-events.js"; -import type { MuseTextInputPart } from "./prompt-content.js"; +import type { MuseInputPart } from "./prompt-content.js"; import { MuseUserInputRequest, settleUserInput, @@ -41,7 +41,7 @@ export interface MuseSdkOptions { sessionId: string; cwd: string; /** Ordered Muse turn input parts (text encodings of ACP content). */ - input: MuseTextInputPart[]; + input: MuseInputPart[]; model: string; reasoningEffort: string; readOnly: boolean; diff --git a/src/prompt-content.ts b/src/prompt-content.ts index 9428b7b..05545a2 100644 --- a/src/prompt-content.ts +++ b/src/prompt-content.ts @@ -1,7 +1,10 @@ +import { decodeImage, IMAGE_EXTENSIONS } from "./prompt-images.js"; import { ContentBlock, PromptRequest, RequestError } from "@agentclientprotocol/sdk"; -/** Muse turn input text part (MSP declares only text | image; we use text). */ +/** Muse turn input text part. */ export type MuseTextInputPart = { type: "text"; text: string }; +export type MuseInputPart = + MuseTextInputPart | { type: "image"; base64Data: string; mediaType: string }; /** * Lossless text encoding for ACP `resource_link` blocks. Muse's turn input @@ -25,12 +28,11 @@ export function formatResourceLink( } export type PromptConversion = - { ok: true; parts: MuseTextInputPart[]; text: string } | { ok: false; error: RequestError }; + { ok: true; parts: MuseInputPart[]; text: string } | { ok: false; error: RequestError }; /** * Convert ACP prompt content into Muse turn input and a legacy exec string. - * Baseline ACP requires text + resource_link; optional image/audio/resource - * are rejected when present because this adapter does not advertise them. + * Baseline ACP requires text + resource_link; images use inline MSP parts; audio and embedded resources are rejected. */ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptConversion { if (blocks.length === 0) { @@ -40,7 +42,7 @@ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptCon }; } - const parts: MuseTextInputPart[] = []; + const parts: MuseInputPart[] = []; for (const block of blocks) { switch (block.type) { case "text": @@ -49,14 +51,34 @@ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptCon case "resource_link": parts.push({ type: "text", text: formatResourceLink(block) }); break; - case "image": + case "image": { + const mediaType = block.mimeType.trim().toLowerCase(); + if (!IMAGE_EXTENSIONS.has(mediaType)) + return { + ok: false, + error: RequestError.invalidParams( + undefined, + "supported MIME types: image/png, image/jpeg, image/gif, image/webp", + ), + }; + try { + parts.push({ + type: "image", + mediaType, + base64Data: decodeImage(block.data).toString("base64"), + }); + } catch (error) { + return { ok: false, error: error as RequestError }; + } + break; + } case "audio": case "resource": return { ok: false, error: RequestError.invalidParams( undefined, - `unsupported prompt content type: ${block.type}; this agent advertises only text and resource_link`, + `unsupported prompt content type: ${block.type}; this agent advertises text, resource_link and image; send embedded resources as resource_link blocks instead`, ), }; default: @@ -71,10 +93,10 @@ export function convertPromptContent(blocks: PromptRequest["prompt"]): PromptCon } const text = parts - .map((part) => part.text) + .flatMap((part) => (part.type === "text" ? [part.text] : [])) .join("\n\n") .trim(); - if (text.length === 0) { + if (text.length === 0 && !parts.some((part) => part.type === "image")) { return { ok: false, error: RequestError.invalidParams( diff --git a/src/prompt-files.ts b/src/prompt-files.ts new file mode 100644 index 0000000..750778d --- /dev/null +++ b/src/prompt-files.ts @@ -0,0 +1,54 @@ +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { RequestError, type ContentBlock } from "@agentclientprotocol/sdk"; +import { convertPromptContent } from "./prompt-content.js"; +import { IMAGE_EXTENSIONS } from "./prompt-images.js"; +export type CompiledMusePrompt = { + prompt: string; + imagePaths: string[]; + cleanup(): Promise; +}; + +export async function compileMusePrompt(blocks: ContentBlock[]): Promise { + const converted = convertPromptContent(blocks); + if (!converted.ok) throw converted.error; + const images = converted.parts.flatMap((part) => + part.type === "image" + ? [ + { + bytes: Buffer.from(part.base64Data, "base64"), + extension: IMAGE_EXTENSIONS.get(part.mediaType)!, + }, + ] + : [], + ); + const prompt = converted.text; + if (!prompt) + throw RequestError.invalidParams( + undefined, + "Muse Code requires text or a resource link alongside image content", + ); + if (images.length === 0) { + return { prompt, imagePaths: [], cleanup: async () => {} }; + } + + const directory = await mkdtemp(join(tmpdir(), "muse-code-acp-images-")); + try { + await chmod(directory, 0o700); + const imagePaths: string[] = []; + for (const [index, image] of images.entries()) { + const imagePath = join(directory, `image-${String(index + 1)}.${image.extension}`); + await writeFile(imagePath, image.bytes, { mode: 0o600 }); + imagePaths.push(imagePath); + } + return { + prompt, + imagePaths, + cleanup: async () => await rm(directory, { recursive: true, force: true }), + }; + } catch (error) { + await rm(directory, { recursive: true, force: true }); + throw error; + } +} diff --git a/src/prompt-images.ts b/src/prompt-images.ts new file mode 100644 index 0000000..86a12b1 --- /dev/null +++ b/src/prompt-images.ts @@ -0,0 +1,30 @@ +import { RequestError } from "@agentclientprotocol/sdk"; +export const IMAGE_EXTENSIONS = new Map([ + ["image/gif", "gif"], + ["image/jpeg", "jpg"], + ["image/png", "png"], + ["image/webp", "webp"], +]); + +function unsupportedContent(type: string, detail?: string): RequestError { + const suffix = detail ? ` (${detail})` : ""; + return RequestError.invalidParams( + undefined, + `unsupported ACP prompt content: ${type}${suffix}. ` + + "Muse Code accepts text, resource links, and PNG/JPEG/GIF/WebP images; " + + "send embedded resources as resource_link blocks instead.", + ); +} + +export function decodeImage(data: string): Buffer { + const normalized = data.replace(/\s/gu, ""); + if (!normalized || normalized.length % 4 === 1 || !/^[A-Za-z0-9+/]*={0,2}$/u.test(normalized)) { + throw unsupportedContent("image", "invalid base64 data"); + } + const decoded = Buffer.from(normalized, "base64"); + const canonical = normalized.replace(/=+$/u, ""); + if (!decoded.length || decoded.toString("base64").replace(/=+$/u, "") !== canonical) { + throw unsupportedContent("image", "invalid base64 data"); + } + return decoded; +} diff --git a/src/tests/fixtures/cat-image.ts b/src/tests/fixtures/cat-image.ts new file mode 100644 index 0000000..cc4a878 --- /dev/null +++ b/src/tests/fixtures/cat-image.ts @@ -0,0 +1,3 @@ +/** PNG containing the black word CAT on a white background. */ +export const CAT_IMAGE_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAIAAABhdOiYAAAACXBIWXMAAAABAAAAAQBPJcTWAAAC8ElEQVR4nO2YMUhqURjHlUDMFMSlEDQ3EQIhpKYgwUVxMBoCh9xsaCjEFmsKgohocFMi3FwincUgHB2KBGsMESGQFAchI/F9vODjvnu17xo+3jvy/SY75/zvPefH6bv3XM2A+RbNv57A/w4LImBBBCyIgAURsCACFkTAgghYEAELImBBBCyIgAURsCCCvyKo2WxeXFwEg8HFxUWj0TgzM2OxWFZXV/f39+/u7kaldnZ2NH/y9PSEvS8vL5oxmchaJizo8/Pz6Ohodnb2m3kfHBwogx8fHyBRNjKRSOCAaRD0/v7u9XrJee/u7iqzuVxOOdJut+OAaRC0tbUlW97l5WW9Xu/1eq+vr6AgFAqNErS5uTl0kaVSadTtbm5upCMfHh4muBZkYoKKxaJ0um63++3tTTkMFnx2diZr7HQ6Op0Os/Pz8/g7Go2OuqNggvx+P85Vq9VWq1X1WdhomDWZTKlUCv80m82wAYemRBIE1Uev1+NcfT7fWPH19XXMhsNh2Hrw4MMWEDE0JZKgSqUinevx8bH6LBQp2HGYvb6+hkZpsd/Y2BgaFEnQ7e2tdK7pdFp99vT0FIOwDbvdLjQmk0lshPLUbreVQYEFQU1Rn11aWsIgPOa+GmFbSS8IVUkZFEnQj//FHh8fpcFMJoNdHo8H29fW1pRZkQT9uEjH43FMQWFutVrYdXJyIl1/rVaTZUUSNFA85p+fn8lIv9+3Wq0adYAvWVwwQbIXxeXl5aGVVfqiKIt8j8vlkl1KMEEDxVHD4XBcXV01Gg04iMJRI5/Py44akUhEvSDg/v5eejvxBI11WIXB8NIsa5QBj/y5uTkcE4vFpL3iCRr8/txxeHio5nNHNpuVNpbL5aEX3N7exjELCwtQtrBLSEFfNJvN8/PzQCBgs9kMBsPXB7OVlZW9vT38YBYMBnFtTqdz1KVkdapQKGCXwIKmCRZEwIIIWBABCyJgQQQsiIAFEbAgAhZEwIIIWBABCyJgQQQsiOAXJ2r353yEvnMAAAAASUVORK5CYII="; diff --git a/src/tests/fixtures/fake-muse.cjs b/src/tests/fixtures/fake-muse.cjs index efd8ea2..a3fdd10 100755 --- a/src/tests/fixtures/fake-muse.cjs +++ b/src/tests/fixtures/fake-muse.cjs @@ -46,6 +46,25 @@ if (process.argv.includes("exec") && process.env.FAKE_MUSE_SETTINGS_CAPTURE) { ); } +if (process.argv.includes("exec") && process.env.FAKE_MUSE_ARGV_CAPTURE) { + const fs = require("node:fs"); + const imagePaths = process.argv.flatMap((arg, index, argv) => + arg === "--image" && argv[index + 1] ? [argv[index + 1]] : [], + ); + fs.writeFileSync( + process.env.FAKE_MUSE_ARGV_CAPTURE, + JSON.stringify({ + argv: process.argv.slice(2), + directoryMode: imagePaths[0] ? fs.statSync(require("node:path").dirname(imagePaths[0])).mode & 0o777 : null, + images: imagePaths.map((imagePath) => ({ + path: imagePath, + data: fs.readFileSync(imagePath).toString("base64"), + mode: fs.statSync(imagePath).mode & 0o777, + })), + }), + ); +} + const mode = process.env.FAKE_MUSE_MODE ?? "block"; if (mode === "exit0") { process.exit(0); diff --git a/src/tests/muse-sdk-live.test.ts b/src/tests/muse-sdk-live.test.ts index 1c49c4b..5b8bf0f 100644 --- a/src/tests/muse-sdk-live.test.ts +++ b/src/tests/muse-sdk-live.test.ts @@ -1,3 +1,5 @@ +import { startLoopbackProvider } from "./loopback-provider.js"; +import { CAT_IMAGE_BASE64 } from "./fixtures/cat-image.js"; import { methods } from "@agentclientprotocol/sdk"; import { spawnSync } from "node:child_process"; import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; @@ -179,3 +181,49 @@ describe.skipIf(!available)("SDK live host (no external API)", () => { } }, 60_000); }); + +describe.skipIf(!available)("SDK image provider input", () => { + it("forwards image bytes through the real host", async () => { + const provider = await startLoopbackProvider({ + scriptedToolCallWhen: ["never-call-tools"], + scriptedToolCallCommand: "", + holdMs: 20, + }); + const cwd = join(provider.root, "workspace"); + mkdirSync(cwd); + const client = connectTestClient({ + backend: "sdk", + env: { + HOME: provider.home, + PATH: process.env.PATH, + XDG_CONFIG_HOME: join(provider.root, "config"), + XDG_DATA_HOME: join(provider.root, "data"), + TBH_DISABLE_TELEMETRY: "1", + TBH_CREDENTIAL_BACKEND: "file", + }, + }); + try { + const ctx = await initialized(client); + const { sessionId } = await ctx.request(methods.agent.session.new, { cwd, mcpServers: [] }); + await expect( + ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [ + { type: "text", text: "describe image-marker" }, + { type: "image", mimeType: "image/png", data: CAT_IMAGE_BASE64 }, + ], + }), + ).resolves.toEqual({ stopReason: "end_turn" }); + expect( + provider.requests().some((r) => { + const input = JSON.stringify(r.input); + return input.includes("image-marker") && input.includes(CAT_IMAGE_BASE64); + }), + ).toBe(true); + } finally { + await client.agent.dispose(); + await provider.close(); + await rm(provider.root, { recursive: true, force: true }); + } + }, 60000); +}); diff --git a/src/tests/muse-sdk.test.ts b/src/tests/muse-sdk.test.ts index 34bcbb5..e566b29 100644 --- a/src/tests/muse-sdk.test.ts +++ b/src/tests/muse-sdk.test.ts @@ -32,6 +32,24 @@ function sdkClient(mode = "complete") { } describe("SDK backend over ACP", () => { + it("preserves image order and bytes through MSP", async () => { + const client = sdkClient(); + const { ctx, sessionId } = await newTestSession(client); + await ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [ + { type: "text", text: "before" }, + { type: "image", mimeType: "image/png", data: "YQ==" }, + { type: "text", text: "after" }, + ], + }); + expect(client.requests().find((r) => r.method === "turn/start").params.input).toEqual([ + { type: "text", text: "before" }, + { type: "image", mediaType: "image/png", base64Data: "YQ==" }, + { type: "text", text: "after" }, + ]); + }); + it("uses UUIDv7 sessions and streams each text segment once, including completion before ack", async () => { const client = sdkClient(); const { ctx, sessionId, modes } = await newTestSession(client); diff --git a/src/tests/prompt-content.test.ts b/src/tests/prompt-content.test.ts index f8d037e..f472040 100644 --- a/src/tests/prompt-content.test.ts +++ b/src/tests/prompt-content.test.ts @@ -36,8 +36,8 @@ describe("prompt content conversion", () => { return; } expect(converted.parts).toHaveLength(2); - expect(converted.parts[1].text).toContain("URI: file:///tmp/日本語.md"); - expect(converted.parts[1].text).toContain("café"); + expect((converted.parts[1] as { text: string }).text).toContain("URI: file:///tmp/日本語.md"); + expect((converted.parts[1] as { text: string }).text).toContain("café"); expect( formatResourceLink({ type: "resource_link", diff --git a/src/tests/prompt-images.test.ts b/src/tests/prompt-images.test.ts new file mode 100644 index 0000000..2fd3987 --- /dev/null +++ b/src/tests/prompt-images.test.ts @@ -0,0 +1,229 @@ +import { methods, PROTOCOL_VERSION, type ContentBlock } from "@agentclientprotocol/sdk"; +import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { capturingLogger, connectTestClient, fakeMuseBinary, newTestSession } from "./helpers.js"; + +type MuseCapture = { + argv: string[]; + directoryMode: number | null; + images: Array<{ path: string; data: string; mode: number }>; +}; + +function capturePath(): string { + return join(mkdtempSync(join(tmpdir(), "muse-content-capture-")), "capture.json"); +} + +function fakeClient(capture: string, mode = "exit0") { + return connectTestClient({ + backend: "exec", + museBinary: fakeMuseBinary(), + env: { + ...process.env, + FAKE_MUSE_ARGV_CAPTURE: capture, + FAKE_MUSE_MODE: mode, + }, + }); +} + +function readCapture(path: string): MuseCapture { + return JSON.parse(readFileSync(path, "utf8")) as MuseCapture; +} + +describe("ACP prompt content", () => { + it("advertises image support without claiming audio or embedded resources", async () => { + const testClient = fakeClient(capturePath()); + const ctx = await testClient.connect(); + + const response = await ctx.request(methods.agent.initialize, { + protocolVersion: PROTOCOL_VERSION, + }); + + expect(response.agentCapabilities?.promptCapabilities).toEqual({ image: true }); + }); + + it("stages supported images privately and removes them after the turn", async () => { + const capture = capturePath(); + const testClient = fakeClient(capture); + const { ctx, sessionId } = await newTestSession(testClient); + const data = Buffer.from("image bytes").toString("base64"); + + await ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [ + { type: "image", data, mimeType: "image/png" }, + { type: "image", data, mimeType: "image/jpeg" }, + { type: "image", data, mimeType: "image/gif" }, + { type: "image", data, mimeType: "image/webp" }, + { type: "text", text: "Describe the images." }, + ], + }); + + const result = readCapture(capture); + expect(result.directoryMode).toBe(0o700); + expect(result.images.map((image) => image.mode)).toEqual([0o600, 0o600, 0o600, 0o600]); + expect(result.images.map((image) => image.data)).toEqual([data, data, data, data]); + expect(result.images.map((image) => image.path.slice(image.path.lastIndexOf(".")))).toEqual([ + ".png", + ".jpg", + ".gif", + ".webp", + ]); + expect(result.images.every((image) => !existsSync(image.path))).toBe(true); + expect(result.argv.at(-1)).toBe("Describe the images."); + }); + + it("reserves the session while image files are being staged", async () => { + const testClient = fakeClient(capturePath()); + const { sessionId } = await newTestSession(testClient); + const first = testClient.agent.prompt({ + sessionId, + prompt: [ + { type: "image", data: "YQ==", mimeType: "image/png" }, + { type: "text", text: "First prompt." }, + ], + }); + + await expect( + testClient.agent.prompt({ + sessionId, + prompt: [{ type: "text", text: "Second prompt." }], + }), + ).rejects.toMatchObject({ + code: -32600, + message: expect.stringMatching(/already has a prompt/), + }); + await expect(first).resolves.toEqual({ stopReason: "end_turn" }); + }); + + it("honors cancellation while image files are being staged", async () => { + const capture = capturePath(); + const testClient = fakeClient(capture); + const { sessionId } = await newTestSession(testClient); + const prompt = testClient.agent.prompt({ + sessionId, + prompt: [ + { type: "image", data: "YQ==", mimeType: "image/png" }, + { type: "text", text: "Cancel before spawn." }, + ], + }); + + await testClient.agent.cancel({ sessionId }); + + await expect(prompt).resolves.toEqual({ stopReason: "cancelled" }); + expect(existsSync(capture)).toBe(false); + }); + + it.each([ + { + name: "audio", + block: { type: "audio", data: "YQ==", mimeType: "audio/wav" } satisfies ContentBlock, + message: /unsupported prompt content type: audio/, + }, + { + name: "embedded resource", + block: { + type: "resource", + resource: { uri: "file:///context.txt", text: "context" }, + } satisfies ContentBlock, + message: /send embedded resources as resource_link blocks/, + }, + { + name: "unsupported image MIME", + block: { type: "image", data: "YQ==", mimeType: "image/bmp" } satisfies ContentBlock, + message: /supported MIME types/, + }, + { + name: "invalid image base64", + block: { type: "image", data: "not base64!", mimeType: "image/png" } satisfies ContentBlock, + message: /invalid base64 data/, + }, + ])("rejects $name before spawning Muse", async ({ block, message }) => { + const lines: string[] = []; + const testClient = connectTestClient( + { + backend: "exec", + museBinary: fakeMuseBinary(), + env: { ...process.env, FAKE_MUSE_MODE: "exit0" }, + }, + capturingLogger(lines), + ); + const { ctx, sessionId } = await newTestSession(testClient); + + await expect( + ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [{ type: "text", text: "Use this." }, block], + }), + ).rejects.toMatchObject({ code: -32602, message }); + expect(lines.some((line) => line.includes("muse-exec spawn"))).toBe(false); + }); + + it("rejects image-only prompts with the native Muse requirement", async () => { + const testClient = fakeClient(capturePath()); + const { ctx, sessionId } = await newTestSession(testClient); + + await expect( + ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [{ type: "image", data: "YQ==", mimeType: "image/png" }], + }), + ).rejects.toMatchObject({ + code: -32602, + message: expect.stringMatching(/requires text or a resource link alongside image content/), + }); + }); + + it("removes staged images when the Muse child fails", async () => { + const capture = capturePath(); + const testClient = fakeClient(capture, "exit1"); + const { ctx, sessionId } = await newTestSession(testClient); + + await expect( + ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [ + { type: "image", data: "YQ==", mimeType: "image/png" }, + { type: "text", text: "Fail after staging." }, + ], + }), + ).rejects.toMatchObject({ code: -32603 }); + + const [image] = readCapture(capture).images; + if (!image) { + throw new Error("fake Muse did not capture the staged image"); + } + expect(existsSync(dirname(image.path))).toBe(false); + }); + + it("removes staged images when the Muse process cannot spawn", async () => { + const lines: string[] = []; + const testClient = connectTestClient( + { + backend: "exec", + museBinary: join(tmpdir(), "missing-muse-binary"), + env: { ...process.env }, + }, + capturingLogger(lines), + ); + const { ctx, sessionId } = await newTestSession(testClient); + + await expect( + ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [ + { type: "image", data: "YQ==", mimeType: "image/png" }, + { type: "text", text: "Fail before starting." }, + ], + }), + ).rejects.toMatchObject({ code: -32603 }); + + const spawnLine = lines.find((line) => line.includes("muse-exec spawn")); + const imagePath = spawnLine?.match(/--image (\S+)/u)?.[1]; + if (!imagePath) { + throw new Error("spawn diagnostics did not include the staged image path"); + } + expect(existsSync(dirname(imagePath))).toBe(false); + }); +}); diff --git a/src/tests/prompt.test.ts b/src/tests/prompt.test.ts index c08975f..553bf54 100644 --- a/src/tests/prompt.test.ts +++ b/src/tests/prompt.test.ts @@ -113,7 +113,7 @@ describe("session/prompt (fake muse)", () => { ).rejects.toMatchObject({ code: -32602 }); }); - it("accepts resource_link-only prompts and rejects unsupported image content", async () => { + it("accepts resource_link-only prompts and requires text alongside exec images", async () => { const testClient = connectTestClient({ backend: "exec", museBinary: fakeMuseBinary() }); const { ctx, sessionId } = await newTestSession(testClient); @@ -143,7 +143,7 @@ describe("session/prompt (fake muse)", () => { ], }), ).rejects.toMatchObject({ - message: expect.stringMatching(/unsupported prompt content type: image/), + message: expect.stringMatching(/requires text or a resource link alongside image content/), }); }); }); diff --git a/src/tests/real-provider.integration.test.ts b/src/tests/real-provider.integration.test.ts index 20ac570..4000ca7 100644 --- a/src/tests/real-provider.integration.test.ts +++ b/src/tests/real-provider.integration.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { connectTestClient, museAvailable } from "./helpers.js"; import { initialized } from "./helpers.js"; +import { CAT_IMAGE_BASE64 } from "./fixtures/cat-image.js"; /** * The one real-model run in the suite (m2 definition of done). Gated behind @@ -51,4 +52,38 @@ describe.skipIf(!enabled)("real provider integration", () => { expect(JSON.stringify(toolUpdates)).toContain("marker.txt"); expect(JSON.stringify(toolUpdates)).toContain("beacon"); }, 300_000); + + it("forwards an ACP image through Muse's native vision path", async () => { + const xdg = mkdtempSync(join(tmpdir(), "muse-image-integration-xdg-")); + const testClient = connectTestClient({ + env: { ...process.env, XDG_DATA_HOME: xdg }, + }); + const ctx = await initialized(testClient); + const cwd = mkdtempSync(join(tmpdir(), "muse-image-integration-cwd-")); + const { sessionId } = await ctx.request(methods.agent.session.new, { cwd, mcpServers: [] }); + await ctx.request(methods.agent.session.setConfigOption, { + sessionId, + configId: "reasoningEffort", + value: "low", + }); + + const response = await ctx.request(methods.agent.session.prompt, { + sessionId, + prompt: [ + { type: "image", data: CAT_IMAGE_BASE64, mimeType: "image/png" }, + { + type: "text", + text: "Read the single uppercase word in the attached image. Reply with only that word.", + }, + ], + }); + + expect(response.stopReason).toBe("end_turn"); + const text = testClient.updates + .map((update) => update.update) + .filter((update) => update.sessionUpdate === "agent_message_chunk") + .map((update) => (update.content.type === "text" ? update.content.text : "")) + .join(""); + expect(text.toUpperCase()).toContain("CAT"); + }, 300_000); });