From 1beaf589369530e6cc9669c187a80795e5ec6e8f Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Tue, 4 Aug 2026 18:55:57 -0500 Subject: [PATCH] =?UTF-8?q?feat(text=20chat):=20add=20repeatable=20--image?= =?UTF-8?q?=20flag=20for=20multimodal=20M3=20=F0=9F=96=BC=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MiniMax-M3 is multimodal, but `text chat` had no way to send an image — users had to hand-write a base64 messages JSON file, and the obvious OpenAI `image_url` shape is rejected because the CLI posts to the Anthropic-compatible /messages endpoint. - `--image ` on `text chat`, repeatable, so multi-image compare/diff works in one call - new `toImageBlock()` in utils/image reuses `toDataUri()` (local paths, http(s) URLs, existing data URIs) and emits the Anthropic block shape `{ type: 'image', source: { type: 'base64', media_type, data } }` - images append to the last user message, promoting string content to a block array; with no `--message` they become the user message - images force `MiniMax-M3` when `--model` is unset, so a text-only `defaultTextModel` in config can't silently break the request - docs: README, README_CN, skill/SKILL.md (incl. the OpenAI-vs-Anthropic block-shape gotcha) Closes #224 --- README.md | 2 + README_CN.md | 2 + skill/SKILL.md | 10 +++ src/commands/text/chat.ts | 35 +++++++++- src/types/api.ts | 3 +- src/utils/image.ts | 20 ++++++ test/commands/text/chat.test.ts | 110 ++++++++++++++++++++++++++++++++ 7 files changed, 179 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e032623..f203a03 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ mmx text chat --model MiniMax-M3 --message "Hello" --stream mmx text chat --system "You are a coding assistant" --message "Fizzbuzz in Go" mmx text chat --message "user:Hi" --message "assistant:Hey!" --message "How are you?" cat messages.json | mmx text chat --messages-file - --output json +mmx text chat --image photo.jpg --message "What breed is this dog?" +mmx text chat --image before.png --image after.png --message "What changed?" ``` ### `mmx image` diff --git a/README_CN.md b/README_CN.md index 0b7c5f0..3c8ea97 100644 --- a/README_CN.md +++ b/README_CN.md @@ -72,6 +72,8 @@ mmx text chat --model MiniMax-M3 --message "你好" --stream mmx text chat --system "你是编程助手" --message "用 Go 写 Fizzbuzz" mmx text chat --message "user:你好" --message "assistant:嗨！" --message "你叫什么名字？" cat messages.json | mmx text chat --messages-file - --output json +mmx text chat --image photo.jpg --message "这是什么品种的狗？" +mmx text chat --image before.png --image after.png --message "这两张图有什么不同？" ``` ### `mmx image` diff --git a/skill/SKILL.md b/skill/SKILL.md index 5c7c3cf..9c34eec 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -57,6 +57,7 @@ mmx text chat --message [flags] | `--message ` | string, **required**, repeatable | Message text. Prefix with `role:` to set role (e.g. `"system:You are helpful"`, `"user:Hello"`) | | `--messages-file ` | string | JSON file with messages array. Use `-` for stdin | | `--system ` | string | System prompt | +| `--image ` | string, repeatable | Image to send with the message (auto base64-encoded). Forces `MiniMax-M3` unless `--model` is set | | `--model ` | string | Model ID (default: `MiniMax-M3`) | | `--max-tokens ` | number | Max tokens (default: 4096) | | `--temperature ` | number | Sampling temperature (0.0, 1.0] | @@ -76,8 +77,17 @@ mmx text chat \ # From file cat conversation.json | mmx text chat --messages-file - --output json + +# With images (M3 is multimodal; --image is repeatable) +mmx text chat --image photo.jpg --message "What breed is this dog?" --quiet +mmx text chat --image before.png --image after.png \ + --message "List every visual difference between these two." --quiet ``` +`text chat` posts to the Anthropic-compatible `/messages` endpoint, so hand-written +`--messages-file` image blocks must use `{"type":"image","source":{"type":"base64",...}}`. +The OpenAI `image_url` shape is rejected. `--image` emits the correct shape for you. + **stdout**: response text (text mode) or full response object (json mode). --- diff --git a/src/commands/text/chat.ts b/src/commands/text/chat.ts index fe2702d..d6897f6 100644 --- a/src/commands/text/chat.ts +++ b/src/commands/text/chat.ts @@ -17,6 +17,7 @@ import type { import { readFileSync } from 'fs'; import { isInteractive } from '../../utils/env'; import { promptText, failIfMissing } from '../../utils/prompt'; +import { toImageBlock } from '../../utils/image'; // --------------------------------------------------------------------------- // Thinking indicator — dynamic spinner + color-cycling label @@ -146,6 +147,27 @@ function parseMessages(flags: GlobalFlags): ParsedMessages { return { system, messages }; } +/** + * Attach image blocks to the last user message, promoting its content from a + * bare string to a block array. Images land after the text so the model reads + * the instruction first. + */ +function attachImages(messages: ChatMessage[], images: ContentBlock[]): void { + let idx = messages.length - 1; + while (idx >= 0 && messages[idx]!.role !== 'user') idx--; + + if (idx < 0) { + messages.push({ role: 'user', content: images }); + return; + } + + const target = messages[idx]!; + const content = typeof target.content === 'string' + ? (target.content ? [{ type: 'text' as const, text: target.content }] : []) + : target.content; + target.content = [...content, ...images]; +} + function extractText(content: ContentBlock[]): string { return content .filter((b): b is Extract => b.type === 'text') @@ -163,6 +185,7 @@ export default defineCommand({ { flag: '--message ', description: 'Message text (repeatable, prefix role: to set role)', required: true, type: 'array' }, { flag: '--messages-file ', description: 'JSON file with messages array (use - for stdin)' }, { flag: '--system ', description: 'System prompt' }, + { flag: '--image ', description: 'Image to send with the message (repeatable, base64 encoded automatically)', type: 'array' }, { flag: '--max-tokens ', description: 'Maximum tokens to generate (default: 4096)', type: 'number' }, { flag: '--temperature ', description: 'Sampling temperature (0.0, 1.0]', type: 'number' }, { flag: '--top-p ', description: 'Nucleus sampling threshold', type: 'number' }, @@ -173,14 +196,17 @@ export default defineCommand({ 'mmx text chat --message "What is MiniMax?"', 'mmx text chat --model MiniMax-M3 --system "You are a coding assistant." --message "Write fizzbuzz in Python"', 'mmx text chat --message "Hello" --message "assistant:Hi!" --message "How are you?"', + 'mmx text chat --image photo.jpg --message "What breed is this dog?"', + 'mmx text chat --image before.png --image after.png --message "List every visual difference."', 'cat conversation.json | mmx text chat --messages-file - --stream', 'mmx text chat --message "Hello" --output json', ], async run(config: Config, flags: GlobalFlags) { const { system, messages: parsedMessages } = parseMessages(flags); let messages = parsedMessages; + const imageInputs = (flags.image as string[] | undefined) ?? []; - if (messages.length === 0) { + if (messages.length === 0 && imageInputs.length === 0) { if (isInteractive({ nonInteractive: config.nonInteractive })) { const hint = await promptText({ message: 'Enter your message:', @@ -195,8 +221,13 @@ export default defineCommand({ } } + if (imageInputs.length > 0) { + attachImages(messages, await Promise.all(imageInputs.map(toImageBlock))); + } + + // Images require a multimodal model, so they override a text-only config default. const model = (flags.model as string) - || config.defaultTextModel + || (imageInputs.length > 0 ? 'MiniMax-M3' : config.defaultTextModel) || 'MiniMax-M3'; const format = detectOutputFormat(config.output); const shouldStream = flags.stream === true || ( diff --git a/src/types/api.ts b/src/types/api.ts index badf507..8538e03 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -4,7 +4,8 @@ export type ContentBlock = | { type: 'text'; text: string } | { type: 'thinking'; thinking: string } | { type: 'tool_use'; id: string; name: string; input: Record } - | { type: 'tool_result'; tool_use_id: string; content: string }; + | { type: 'tool_result'; tool_use_id: string; content: string } + | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }; export interface ChatMessage { role: 'user' | 'assistant'; diff --git a/src/utils/image.ts b/src/utils/image.ts index ab53b63..7d146d7 100644 --- a/src/utils/image.ts +++ b/src/utils/image.ts @@ -2,6 +2,9 @@ import { readFileSync, existsSync, statSync } from 'fs'; import { extname } from 'path'; import { CLIError } from '../errors/base'; import { ExitCode } from '../errors/codes'; +import type { ContentBlock } from '../types/api'; + +type ImageBlock = Extract; export const IMAGE_MIME_TYPES: Record = { '.jpg': 'image/jpeg', @@ -60,3 +63,20 @@ export async function toDataUri(image: string): Promise { if (!IMAGE_MIME_TYPES[ext]) throw new CLIError(`Unsupported image format "${ext}". Supported: jpg, jpeg, png, webp`, ExitCode.USAGE); return localFileToDataUri(image); } + +/** + * Convert a path / URL / data URI into an Anthropic-shaped image content block. + * The Messages API rejects the OpenAI `image_url` shape, so callers targeting + * `/anthropic/v1/messages` must use this instead of a raw data URI. + */ +export async function toImageBlock(image: string): Promise { + const uri = await toDataUri(image); + const match = /^data:([^;,]+);base64,(.*)$/s.exec(uri); + if (!match) { + throw new CLIError( + `Unsupported image source "${image}": expected a base64 data URI, file path, or http(s) URL.`, + ExitCode.USAGE, + ); + } + return { type: 'image', source: { type: 'base64', media_type: match[1]!, data: match[2]! } }; +} diff --git a/test/commands/text/chat.test.ts b/test/commands/text/chat.test.ts index 2c592b6..8f32635 100644 --- a/test/commands/text/chat.test.ts +++ b/test/commands/text/chat.test.ts @@ -1,4 +1,7 @@ import { describe, it, expect, afterEach } from 'bun:test'; +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; import { createMockServer, jsonResponse, sseResponse, type MockServer } from '../../helpers/mock-server'; import textChatResponse from '../../fixtures/text-chat-response.json'; import type { Config } from '../../../src/config/schema'; @@ -302,4 +305,111 @@ describe('text chat command', () => { console.log = originalLog; } }); + + describe('--image', () => { + // 1x1 transparent PNG + const PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + const dir = mkdtempSync(join(tmpdir(), 'mmx-chat-image-')); + const imgA = join(dir, 'a.png'); + const imgB = join(dir, 'b.png'); + writeFileSync(imgA, Buffer.from(PNG_BASE64, 'base64')); + writeFileSync(imgB, Buffer.from(PNG_BASE64, 'base64')); + + const baseConfig: Config = { + apiKey: 'test-key', + region: 'global' as const, + baseUrl: 'https://api.mmx.io', + output: 'json', + timeout: 10, + verbose: false, + quiet: false, + noColor: true, + yes: false, + dryRun: true, + nonInteractive: true, + async: false, + }; + + const baseFlags = { + quiet: false, + verbose: false, + noColor: true, + yes: false, + dryRun: true, + help: false, + nonInteractive: true, + async: false, + }; + + async function dryRunRequest(config: Config, flags: Record) { + const { default: chatCommand } = await import('../../../src/commands/text/chat'); + const originalLog = console.log; + let output = ''; + console.log = (msg: string) => { output += msg; }; + try { + await chatCommand.execute(config, { ...baseFlags, ...flags } as never); + } finally { + console.log = originalLog; + } + return JSON.parse(output).request; + } + + it('appends Anthropic-shaped image blocks to the user message', async () => { + const request = await dryRunRequest(baseConfig, { + message: ['What is this?'], + image: [imgA], + }); + + expect(request.messages).toHaveLength(1); + expect(request.messages[0].content).toEqual([ + { type: 'text', text: 'What is this?' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG_BASE64 } }, + ]); + }); + + it('supports multiple images in one message', async () => { + const request = await dryRunRequest(baseConfig, { + message: ['Compare these.'], + image: [imgA, imgB], + }); + + const blocks = request.messages[0].content; + expect(blocks).toHaveLength(3); + expect(blocks.filter((b: { type: string }) => b.type === 'image')).toHaveLength(2); + }); + + it('sends images with no --message', async () => { + const request = await dryRunRequest(baseConfig, { image: [imgA] }); + + expect(request.messages).toHaveLength(1); + expect(request.messages[0].role).toBe('user'); + expect(request.messages[0].content).toEqual([ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG_BASE64 } }, + ]); + }); + + it('overrides a text-only defaultTextModel with MiniMax-M3', async () => { + const request = await dryRunRequest( + { ...baseConfig, defaultTextModel: 'MiniMax-Text-01' }, + { message: ['What is this?'], image: [imgA] }, + ); + + expect(request.model).toBe('MiniMax-M3'); + }); + + it('still honours an explicit --model', async () => { + const request = await dryRunRequest( + { ...baseConfig, defaultTextModel: 'MiniMax-Text-01' }, + { message: ['What is this?'], image: [imgA], model: 'MiniMax-VL-01' }, + ); + + expect(request.model).toBe('MiniMax-VL-01'); + }); + + it('errors on a missing image file', async () => { + await expect( + dryRunRequest(baseConfig, { message: ['hi'], image: [join(dir, 'nope.png')] }), + ).rejects.toThrow(/File not found/); + }); + }); });