From 5cf79539fcf0b3b9437f34a82190f85909d44a85 Mon Sep 17 00:00:00 2001 From: zhangqingkun976 <1047045074@QQ.COM> Date: Tue, 22 Sep 2026 09:34:47 +0800 Subject: [PATCH] feat(host-core): let Read return an image the model can be shown A session asked to look at an image on disk could only be told the file was binary content and had no text to read (issue #711), while the same app happily sent an image the user attached. The runtime already turned a host tool result shaped `{text, images: [{data, mimeType}]}` into a model-visible image block; no tool produced that shape. - `Read` now returns it for `.png`, `.jpg`, `.jpeg`, `.gif` and `.webp`. The block is attached only when the active model accepts images, and the text is kept either way, so a model without vision reports that it cannot see the image instead of inventing its contents. - The magic number decides the type and has to agree with the extension: a text file renamed `.png` is still `TOOL_BINARY_CONTENT`, because a request carrying a broken image fails as a whole. - An image above 3 MB raw (base64 inflates by four thirds, and the strictest per-image ceiling in use is 5 MB encoded) is refused with the new `TOOL_IMAGE_TOO_LARGE`, naming the size and the alternative, rather than being cut into an unreadable fragment. - The base64 payload never enters a persisted UI message or transcript record; the stored detail keeps `imageCount`, as the attachment rule already required. C:\Users\10470\.pi-desktop\scratch\cb9e2d41-8a55-4a18-899d-92ca2791ab51\commit-711-r2.txt --- crates/host-core/src/tools/mod.rs | 179 ++++++++++++++++++ docs/spec/03-runtime/08-error-codes.md | 3 +- docs/spec/03-runtime/16-tool-result-limits.md | 31 ++- docs/spec/06-delivery/04-e2e-test-plan.md | 27 +++ docs/spec/08-meta/decisions-log.md | 19 ++ docs/zh-CN/spec/03-runtime/08-error-codes.md | 3 +- .../spec/03-runtime/16-tool-result-limits.md | 18 +- .../spec/06-delivery/04-e2e-test-plan.md | 15 ++ docs/zh-CN/spec/08-meta/decisions-log.md | 10 + packages/agent-runtime/src/runtime.test.ts | 79 ++++++++ packages/shared/src/errors.ts | 1 + 11 files changed, 378 insertions(+), 7 deletions(-) diff --git a/crates/host-core/src/tools/mod.rs b/crates/host-core/src/tools/mod.rs index 0479585e2..a2b770da6 100644 --- a/crates/host-core/src/tools/mod.rs +++ b/crates/host-core/src/tools/mod.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use base64::{engine::general_purpose::STANDARD as B64, Engine}; use ignore::WalkBuilder; use regex::RegexBuilder; use serde::{Deserialize, Serialize}; @@ -549,6 +550,50 @@ const BINARY_EXTENSIONS: &[&str] = &[ "wasm", "war", "webp", "xls", "xlsx", "zip", ]; +/// Image types this build will inline for a model that can view images. Both +/// Anthropic-family and OpenAI-family endpoints accept these four. +const IMAGE_EXTENSION_MIME: &[(&str, &str)] = &[ + ("png", "image/png"), + ("jpg", "image/jpeg"), + ("jpeg", "image/jpeg"), + ("gif", "image/gif"), + ("webp", "image/webp"), +]; + +/// Raw bytes above this are not inlined. Base64 inflates by four thirds, and +/// the strictest per-image ceiling this build talks to is 5 MB encoded, so the +/// bound has to leave room for the encoding. +const MAX_INLINE_IMAGE_BYTES: usize = 3 * 1024 * 1024; + +/// Public code for an image that is real but too large to inline. +const TOOL_IMAGE_TOO_LARGE: &str = "TOOL_IMAGE_TOO_LARGE"; + +fn image_mime_for_extension(extension: &str) -> Option<&'static str> { + IMAGE_EXTENSION_MIME + .iter() + .find(|(ext, _)| *ext == extension) + .map(|(_, mime)| *mime) +} + +/// The declared type has to agree with the bytes. A text file renamed `.png` +/// is not an image, and a request carrying one fails as a whole, so the magic +/// number decides and the extension is only a hint. +fn sniff_image_mime(bytes: &[u8]) -> Option<&'static str> { + if bytes.starts_with(&[0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A]) { + return Some("image/png"); + } + if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + return Some("image/jpeg"); + } + if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + return Some("image/gif"); + } + if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WEBP" { + return Some("image/webp"); + } + None +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ToolsExecuteParams { @@ -1209,6 +1254,48 @@ fn root_label(root_kind: ToolRoot) -> &'static str { } } +/// One image, in the shape the runtime turns into a model-visible image block: +/// `{text, images: [{data, mimeType}]}`. The runtime attaches the image only +/// when the active model accepts images, and keeps the text either way, so the +/// text has to stand on its own for a model that cannot see it. +fn read_image( + resolved: &Path, + display: &str, + root_kind: ToolRoot, + size: u64, +) -> Result { + if size > MAX_INLINE_IMAGE_BYTES as u64 { + return Err(( + TOOL_IMAGE_TOO_LARGE.into(), + format!( + "{display} is {:.1} MB, above the {} MB bound for reading an image into the model context; ask the user to attach it to the prompt instead, or shrink it first", + size as f64 / (1024.0 * 1024.0), + MAX_INLINE_IMAGE_BYTES / (1024 * 1024) + ), + )); + } + let bytes = + std::fs::read(resolved).map_err(|e| ("TOOL_FAILED".into(), format!("read failed: {e}")))?; + let Some(mime) = sniff_image_mime(&bytes) else { + return Err(( + "TOOL_BINARY_CONTENT".into(), + format!( + "{display} has an image extension but its bytes are not a PNG, JPEG, GIF or WebP image" + ), + )); + }; + Ok(json!({ + "path": display, + "root": root_label(root_kind), + "text": format!( + "{display} is a {mime} image ({} bytes). It is attached for models that can view images; if you cannot see it, say so rather than guessing at its contents.", + bytes.len() + ), + "images": [{ "data": B64.encode(&bytes), "mimeType": mime }], + "fileBytes": bytes.len(), + })) +} + fn tool_read( workspace: Option<&Path>, scratch: Option<&Path>, @@ -1261,6 +1348,13 @@ fn tool_read( .extension() .map(|ext| ext.to_string_lossy().to_lowercase()); if let Some(ext) = &extension { + // An image the model may be shown is a different outcome from a binary + // it cannot read: the file has no text, but it does have content, and + // the runtime turns this shape into an image block for a model that + // accepts one (issue #711). + if image_mime_for_extension(ext).is_some() { + return read_image(&resolved, &display, root_kind, meta.len()); + } if BINARY_EXTENSIONS.contains(&ext.as_str()) { return Err(( "TOOL_BINARY_CONTENT".into(), @@ -3249,6 +3343,91 @@ mod tests { } } + /// A 1x1 PNG, so the fixture is a real image and not a renamed text file. + fn tiny_png() -> Vec { + const B64_PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8AAAwAB/AL+2w0AAAAASUVORK5CYII="; + B64.decode(B64_PNG).expect("fixture decodes") + } + + #[tokio::test] + async fn read_returns_an_image_for_the_model_to_view() { + let dir = tempfile::tempdir().unwrap(); + let png = tiny_png(); + std::fs::write(dir.path().join("shot.png"), &png).unwrap(); + + let result = execute_tool( + Some(dir.path()), + None, + "Read", + &serde_json::json!({ "path": "shot.png" }), + 5_000, + ) + .await; + + assert!( + result.ok, + "an image read is not a failure: {:?}", + result.error_code + ); + // The shape the runtime turns into a model-visible image block. + let images = result.content["images"] + .as_array() + .expect("images array is present"); + assert_eq!(images.len(), 1); + assert_eq!(images[0]["mimeType"], "image/png"); + assert_eq!( + images[0]["data"].as_str().expect("base64 payload"), + B64.encode(&png) + ); + assert_eq!(result.content["fileBytes"], png.len()); + // The text stands on its own for a model that cannot see images. + let text = result.content["text"].as_str().expect("text is present"); + assert!(text.contains("shot.png"), "{text}"); + assert!(text.contains("image/png"), "{text}"); + assert!(text.contains("cannot see it"), "{text}"); + } + + #[tokio::test] + async fn read_refuses_an_image_extension_whose_bytes_are_not_an_image() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("notes.png"), "just text, renamed\n").unwrap(); + + let result = execute_tool( + Some(dir.path()), + None, + "Read", + &serde_json::json!({ "path": "notes.png" }), + 5_000, + ) + .await; + + assert!(!result.ok); + assert_eq!(result.error_code.as_deref(), Some("TOOL_BINARY_CONTENT")); + } + + #[tokio::test] + async fn read_refuses_an_image_too_large_to_inline() { + let dir = tempfile::tempdir().unwrap(); + let mut big = tiny_png(); + big.resize(MAX_INLINE_IMAGE_BYTES + 1, 0); + std::fs::write(dir.path().join("huge.png"), &big).unwrap(); + + let result = execute_tool( + Some(dir.path()), + None, + "Read", + &serde_json::json!({ "path": "huge.png" }), + 5_000, + ) + .await; + + assert!(!result.ok); + assert_eq!(result.error_code.as_deref(), Some("TOOL_IMAGE_TOO_LARGE")); + let error = result.content["error"].as_str().expect("error text"); + assert!(error.contains("attach it to the prompt"), "{error}"); + assert!(error.contains("3 MB"), "{error}"); + } + #[tokio::test] async fn security_denylist_blocks_read_write_edit_and_hides_search_results() { let dir = tempfile::tempdir().unwrap(); diff --git a/docs/spec/03-runtime/08-error-codes.md b/docs/spec/03-runtime/08-error-codes.md index 4468fd325..cfb204f8f 100644 --- a/docs/spec/03-runtime/08-error-codes.md +++ b/docs/spec/03-runtime/08-error-codes.md @@ -111,7 +111,8 @@ does not turn temporary thread pressure into a host process exit. | `PATH_OUTSIDE_WORKSPACE` | no | path escapes sandbox before an explicit outside-path permission decision, or a prompt attachment is outside its session scratch/project/attachment roots | | `WORKSPACE_PATH_DENIED` | no | an explicit `Read`/`Write`/`Edit` path hit the always-on security denylist (private keys, `.env` files, credential bundles, `.git/objects`); an outside-path grant does not lift it (spec 15 §3) | | `READ_PATH_IS_DIRECTORY` | no | `Read` was given a directory; the result carries a `Glob` suggestion | -| `TOOL_BINARY_CONTENT` | no | `Read` refused to dump a binary file into the model context | +| `TOOL_BINARY_CONTENT` | no | `Read` refused to dump a binary file into the model context, or a file whose image extension does not match its bytes | +| `TOOL_IMAGE_TOO_LARGE` | no | `Read` found a real image above the inline bound (3 MB raw); the message carries the size and the alternative | | `TOOL_NOT_FOUND` | no | unknown tool | | `TOOL_DENIED` | no | permission denied / mode forbidden | | `TOOL_TIMEOUT` | yes | tool execution timeout | diff --git a/docs/spec/03-runtime/16-tool-result-limits.md b/docs/spec/03-runtime/16-tool-result-limits.md index 2582b4f36..e1d4c55bb 100644 --- a/docs/spec/03-runtime/16-tool-result-limits.md +++ b/docs/spec/03-runtime/16-tool-result-limits.md @@ -121,6 +121,26 @@ ignore files from applying — the same rule that lets `path` reach into or its UI/diagnostic result; it changes only future reconstructed model context +**Image reads (issue #711).** A file whose extension is an image type +(`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`) is not a binary the model cannot +read: `Read` returns it in the shape the runtime turns into a model-visible +image block — `{path, root, text, images: [{data, mimeType}], fileBytes}` — +where `data` is standard base64. The runtime attaches the image only when the +active model accepts images (`input`/`modalities` includes `image`) and keeps +the text either way, so a model that cannot see images still learns that an +image is there instead of inventing its contents; the base64 payload never +enters a persisted UI message or transcript record, only `imageCount` does. + +- the magic number decides the type, and it has to agree with the extension: a + text file renamed `.png` is refused as `TOOL_BINARY_CONTENT` rather than sent + as a broken image, which would fail the whole request; +- an image above 3 MB raw (base64 inflates by four thirds, and the strictest + per-image ceiling in use is 5 MB encoded) is refused as + `TOOL_IMAGE_TOO_LARGE` with the size and the actionable alternative, instead + of being truncated into an unreadable fragment; +- every other binary extension keeps the existing `TOOL_BINARY_CONTENT` + outcome, and the byte sniff after this point is unchanged for text. + ## 5. Partial result flags Every bounded tool reports `truncated: boolean`. For Read, that flag is true @@ -174,8 +194,10 @@ counts are the stable signals. The UI truncated chip follows `truncated`. 96KB of progress noise is what makes the model retry blindly 3. binary files: do not dump raw binary into model; return metadata error `TOOL_BINARY_CONTENT`. Detected by extension blacklist plus a sniff of the - first 4KB (any NUL byte, or >30% non-printable). Grep skips binary files - silently rather than matching lossily-decoded bytes + first 4KB (any NUL byte, or >30% non-printable). An image type is the + exception and is inlined as an image block (§4a); a file whose bytes are + not the image its extension claims is still refused. Grep skips binary + files silently rather than matching lossily-decoded bytes 4. a single line longer than the whole budget yields a char-boundary-safe prefix (or suffix, for a tail cut), never an empty payload 5. aggregate checkpoint truncation must preserve every provider-valid assistant @@ -194,7 +216,10 @@ counts are the stable signals. The UI truncated chip follows `truncated`. from the `Edit` provenance set - [x] Read paginates a multi-megabyte file instead of refusing it, reports the next offset, and does not set `truncated` when the requested window was filled -- [x] Read refuses binary content with `TOOL_BINARY_CONTENT` +- [x] Read refuses binary content with `TOOL_BINARY_CONTENT`, except an image + type, which returns `{text, images}` for a model that accepts images +- [x] an image above the inline bound is refused with `TOOL_IMAGE_TOO_LARGE`, + and an `.png` that is not a PNG is refused as binary - [x] an explicit `path` reaches into an ignored tree (`node_modules`, spill dir) - [x] Glob and Grep order results by modification time, newest first - [x] truncated results still valid UTF-8 text diff --git a/docs/spec/06-delivery/04-e2e-test-plan.md b/docs/spec/06-delivery/04-e2e-test-plan.md index 39a771835..cfc5f1040 100644 --- a/docs/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/spec/06-delivery/04-e2e-test-plan.md @@ -8112,6 +8112,8 @@ identify the platform validation still needed. | F — Persistence (stored model binding array) | E2E-PROVIDER-stored-binding-array-reads-entry-by-entry | | Quality (stored model binding array) | E2E-PROVIDER-stored-binding-array-reads-entry-by-entry | | Quality (unique tool-call ids) | E2E-RUNTIME-unique-tool-call-ids-per-request | +| E — Tools & permissions (image read) | E2E-TOOL-read-returns-an-image-the-model-can-see | +| C — Conversation & stream (image read) | E2E-TOOL-read-returns-an-image-the-model-can-see | | G — Plugin host lifecycle (crash report) | E2E-PLUGIN-crash-report-names-the-exit-code | | Quality (crash report) | E2E-PLUGIN-crash-report-names-the-exit-code | @@ -14351,6 +14353,31 @@ the latest destination. These assertions measure work counts, not device FPS. and a unique one (identity, no log line). - **Status:** Unit-covered; no end-to-end driver issues a real provider request against a duplicated transcript. +### E2E-TOOL-read-returns-an-image-the-model-can-see + +- **Preconditions:** A workspace containing a real PNG, a text file renamed + `.png`, and an image above the inline bound; a configured model that declares + image input, plus one that does not. Deterministic provider fixture. +- **Steps:** Ask the agent to read the PNG, then the renamed file, then the + oversized image. Repeat the first step after switching to the model without + image input. +- **Expected:** The PNG read succeeds and the request carries an `image` block + with the matching `mimeType`; the renamed file fails with + `TOOL_BINARY_CONTENT`; the oversized image fails with `TOOL_IMAGE_TOO_LARGE` + naming its size and the alternative; with the non-vision model the request + carries the text alone and the model reports it cannot see the image. No + base64 payload appears in the transcript or in a persisted UI message. +- **Specs:** `03-runtime/16-tool-result-limits.md` §4/§6, + `03-runtime/08-error-codes.md`, `08-meta/decisions-log.md` D609. +- **Acceptance:** E (tools), C (conversation and stream), Quality. +- **Milestone:** Post-MVP regression coverage. +- **Automation:** host-core `tools::tests::read_returns_an_image_for_the_model_to_view`, + `read_refuses_an_image_extension_whose_bytes_are_not_an_image` and + `read_refuses_an_image_too_large_to_inline`; agent-runtime + `runtime.test.ts` covers both halves of the model-side bridge (attached with + vision, dropped without). +- **Status:** Unit-covered end to end for the mapping; no desktop E2E driver + reads a real image into a live provider request. ### E2E-MCP-HTTP-ACK — HTTP acknowledgement and authorization status diff --git a/docs/spec/08-meta/decisions-log.md b/docs/spec/08-meta/decisions-log.md index 4cc7e2031..78cd78b11 100644 --- a/docs/spec/08-meta/decisions-log.md +++ b/docs/spec/08-meta/decisions-log.md @@ -6514,6 +6514,25 @@ that was sitting at the bottom — including after the turn had finished. - The guard is deliberately the request boundary rather than the history rebuild: it also covers a duplicate that appears while the session runs, which a rebuild-time filter cannot see. +## 2026-09-21 — `Read` returns an image the model can be shown (D609, issue #711) + +- A file whose extension is an image type (`.png`, `.jpg`, `.jpeg`, `.gif`, + `.webp`) is no longer refused as binary content. `Read` returns + `{path, root, text, images: [{data, mimeType}], fileBytes}`, the shape the + runtime already turned into a model-visible image block but which no tool + produced until now; the block is attached only when the active model accepts + images, and the text is kept either way so a model that cannot see images + says so instead of inventing contents. +- The magic number decides the type and has to agree with the extension: a text + file renamed `.png` is still refused with `TOOL_BINARY_CONTENT`, because a + request carrying a broken image fails as a whole. An image above 3 MB raw + (base64 inflates by four thirds; the strictest per-image ceiling in use is + 5 MB encoded) is refused with the new `TOOL_IMAGE_TOO_LARGE`, carrying the + size and the alternative. +- The base64 payload never enters a persisted UI message or transcript record; + the stored detail keeps `imageCount` only, which is what the existing + attachment rule already required. See `03-runtime/16-tool-result-limits.md` + §4 and `03-runtime/08-error-codes.md`. ## 2026-09-21 — A plugin crash reports its exit code without copying raw output (D607, issue #747) diff --git a/docs/zh-CN/spec/03-runtime/08-error-codes.md b/docs/zh-CN/spec/03-runtime/08-error-codes.md index 204b27dec..af3c36965 100644 --- a/docs/zh-CN/spec/03-runtime/08-error-codes.md +++ b/docs/zh-CN/spec/03-runtime/08-error-codes.md @@ -113,7 +113,8 @@ stdio 与 Tokio 的动态阻塞池隔离，因此后一种情况 | `PATH_OUTSIDE_WORKSPACE` | 不 | 在显式外部路径权限决策之前路径逃逸沙箱，或提示词附件位于其会话 scratch/project/attachment 根目录之外 | | `WORKSPACE_PATH_DENIED` | 不 | 显式的 `Read`/`Write`/`Edit` 路径命中了始终开启的安全拒绝名单（私钥、`.env` 文件、凭证包、`.git/objects`）；外部路径授权不会解除它（规格 15 §3） | | `READ_PATH_IS_DIRECTORY` | 不 | `Read` 拿到的是目录；结果附带一条 `Glob` 建议 | -| `TOOL_BINARY_CONTENT` | 不 | `Read` 拒绝把二进制文件倾倒进模型上下文 | +| `TOOL_BINARY_CONTENT` | 不 | `Read` 拒绝把二进制文件倾倒进模型上下文，或图片扩展名与实际字节不符 | +| `TOOL_IMAGE_TOO_LARGE` | 不 | `Read` 读到真实图片但超过内联上限（原始 3 MB）；消息带上大小与替代做法 | | `TOOL_NOT_FOUND` | 不 | 未知工具 | | `TOOL_DENIED` | 不 | 权限被拒绝/模式被禁止 | | `TOOL_TIMEOUT` | 是的 | 工具执行超时 | diff --git a/docs/zh-CN/spec/03-runtime/16-tool-result-limits.md b/docs/zh-CN/spec/03-runtime/16-tool-result-limits.md index ded53cfb3..1dc7e28e0 100644 --- a/docs/zh-CN/spec/03-runtime/16-tool-result-limits.md +++ b/docs/zh-CN/spec/03-runtime/16-tool-result-limits.md @@ -166,8 +166,21 @@ type GlobResult = { matches: string[]; count: number; truncated: boolean; notice 96KB 的进度噪声导致模型盲目重试 3. 二进制文件：不要将原始二进制文件转储到模型中；返回元数据错误 `TOOL_BINARY_CONTENT`。通过扩展黑名单加上嗅探来检测 - 第一个 4KB（任何 NUL 字节，或 >30% 不可打印）。 Grep 跳过二进制文件 + 第一个 4KB（任何 NUL 字节，或 >30% 不可打印）。图片类型是例外，会以内联图片块返回（§4a）； + 扩展名声称是图片但字节并不是图片的文件仍被拒绝。 Grep 跳过二进制文件 静默地而不是匹配有损解码的字节 + +**图片读取（issue #711）。** 扩展名为图片类型（`.png`、`.jpg`、`.jpeg`、`.gif`、`.webp`）的文件并不是模型读不了的 +二进制：`Read` 以运行时能转成模型可见图片块的形状返回——`{path, root, text, images: [{data, mimeType}], fileBytes}`， +其中 `data` 是标准 base64。运行时只在当前模型接受图片（`input`/`modalities` 含 `image`）时附带图片，文本两种情况都保留， +因此看不到图片的模型也会知道「这里有一张图」，而不是凭空编造内容；base64 载荷绝不进入持久化的 UI 消息或转录记录，只记录 +`imageCount`。 + +- 由幻数决定类型，且必须与扩展名一致：把文本文件改名为 `.png` 会按 `TOOL_BINARY_CONTENT` 拒绝，而不是当作坏图片发出去 + （那会让整个请求失败）； +- 原始大小超过 3 MB 的图片（base64 会膨胀 4/3，而最严格的单图上限是编码后 5 MB）会以 `TOOL_IMAGE_TOO_LARGE` 拒绝， + 消息带上大小与可执行的替代做法，而不是截成无法阅读的碎片； +- 其它二进制扩展名保持原有 `TOOL_BINARY_CONTENT` 结果，此后的文本字节嗅探不变。 4. 比整个预算长的单行会产生字符边界安全前缀 （或后缀，用于尾部切割），绝不是空的有效负载 5. 聚合检查点截断必须保留每个提供商有效的助手 @@ -184,7 +197,8 @@ type GlobResult = { matches: string[]; count: number; truncated: boolean; notice - [x] Grep 在 `headLimit` 和 `truncated: true` 处停止 - [x] Grep 和 Read 在 16,384 个字符处剪辑行，且被剪辑的行被排除在 `Edit` 来源集之外 - [x] Read 对多兆字节文件进行分页而不是拒绝它，报告下一个偏移量，并在填满请求窗口时不设 `truncated` -- [x] 读取拒绝带有 `TOOL_BINARY_CONTENT` 的二进制内容 +- [x] 读取拒绝带有 `TOOL_BINARY_CONTENT` 的二进制内容；图片类型除外，会为接受图片的模型返回 `{text, images}` +- [x] 超过内联上限的图片以 `TOOL_IMAGE_TOO_LARGE` 拒绝，扩展名是 `.png` 而字节不是 PNG 时按二进制拒绝 - [x] 显式 `path` 到达被忽略的树（`node_modules`，溢出目录） - [x] Glob 和 Grep 按修改时间对结果进行排序，最新的在前 - [x] 截断结果仍然有效 UTF-8 文本 diff --git a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md index 3015ed50a..c891a8fab 100644 --- a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md @@ -5337,6 +5337,8 @@ eleven-tool-round desktop paths are verified by | C / F / 品质 —— 上下文估算保持安全（校准） | E2E-CONTEXT-estimate-calibration-stays-safe | | C — 对话与流式（工具调用 id 唯一） | E2E-RUNTIME-unique-tool-call-ids-per-request | | 品质（工具调用 id 唯一） | E2E-RUNTIME-unique-tool-call-ids-per-request | +| E — 工具与权限（图片读取） | E2E-TOOL-read-returns-an-image-the-model-can-see | +| C — 对话与流式（图片读取） | E2E-TOOL-read-returns-an-image-the-model-can-see | | G — 插件宿主生命周期（崩溃上报） | E2E-PLUGIN-crash-report-names-the-exit-code | | 品质（崩溃上报） | E2E-PLUGIN-crash-report-names-the-exit-code | | F — 持久化（存储的模型绑定数组） | E2E-PROVIDER-stored-binding-array-reads-entry-by-entry | @@ -8447,6 +8449,19 @@ the latest destination. These assertions measure work counts, not device FPS. - **自动化：** `packages/agent-runtime/src/runtime.test.ts` 用真实的运行时覆盖两半：重复历史（丢弃 + 一行日志）与唯一历史 （同一对象、无日志）。 - **状态：** 单元测试覆盖；没有端到端驱动对重复转录发出真实提供商请求。 +### E2E-TOOL-read-returns-an-image-the-model-can-see + +- **先决条件：** 工作区里有一张真实 PNG、一个改名为 `.png` 的文本文件、一张超过内联上限的图片；配置一个声明图片输入的模型， + 以及一个不声明图片输入的模型。确定性提供商夹具。 +- **步骤：** 让 agent 依次读 PNG、改名文件、超限图片。切到不支持图片输入的模型后重复第一步。 +- **预期：** 读 PNG 成功，请求带上 `mimeType` 匹配的 `image` 块；改名文件以 `TOOL_BINARY_CONTENT` 失败；超限图片以 + `TOOL_IMAGE_TOO_LARGE` 失败并给出大小与替代做法；不支持图片时请求只带文本，模型如实说明看不到图片。转录或持久化的 UI 消息 + 中不出现任何 base64 载荷。 +- **规格：** 03-runtime/16-tool-result-limits §4/§6、03-runtime/08-error-codes、08-meta/decisions-log D609。 + **验收：** E（工具）、C（对话与流）、品质。**里程碑：** Post-MVP 回归覆盖。 +- **自动化：** host-core `read_returns_an_image_for_the_model_to_view`、`read_refuses_an_image_extension_whose_bytes_are_not_an_image`、 + `read_refuses_an_image_too_large_to_inline`；agent-runtime `runtime.test.ts` 覆盖模型侧桥接的两半（有视觉则附带、无视觉则丢弃）。 +- **状态：** 映射链路单元测试端到端覆盖；没有桌面 E2E 驱动把真实图片读进实况提供商请求。 ### E2E-MCP-HTTP-ACK — HTTP acknowledgement and authorization status diff --git a/docs/zh-CN/spec/08-meta/decisions-log.md b/docs/zh-CN/spec/08-meta/decisions-log.md index cb3a1f969..f111e3670 100644 --- a/docs/zh-CN/spec/08-meta/decisions-log.md +++ b/docs/zh-CN/spec/08-meta/decisions-log.md @@ -4619,6 +4619,16 @@ that amendment are retired by ADR 0268; the upstream work-panel lifecycle stays. - 一旦发生丢弃，会在 `agent` 日志通道上报告一次，带上会话与 id，使下一次同类报障能指向写入方而不只是提供商那句话。 见 `03-runtime/02-agent-runtime §5`。 - 守卫刻意放在请求边界而不是历史重建处：这样也能覆盖**会话运行期间**产生的重复，而重建期的过滤看不到它。 +## 2026-09-21 —— `Read` 返回模型可查看的图片（D609，issue #711） + +- 扩展名为图片类型（`.png`、`.jpg`、`.jpeg`、`.gif`、`.webp`）的文件不再被当作二进制内容拒绝。`Read` 返回 + `{path, root, text, images: [{data, mimeType}], fileBytes}`——这正是运行时能转成模型可见图片块的形状，此前却没有 + 任何工具产出它；图片块只在当前模型接受图片时附带，文本两种情况都保留，因此看不到图片的模型会如实说明，而不是编造内容。 +- 由幻数决定类型并必须与扩展名一致：把文本文件改名为 `.png` 仍按 `TOOL_BINARY_CONTENT` 拒绝，因为带着坏图片的请求会整体 + 失败。原始大小超过 3 MB 的图片（base64 膨胀 4/3，最严格的单图上限是编码后 5 MB）以新增的 `TOOL_IMAGE_TOO_LARGE` 拒绝， + 消息带上大小与替代做法。 +- base64 载荷绝不进入持久化的 UI 消息或转录记录；落库的细节只保留 `imageCount`，与既有附件规则一致。 + 见 `03-runtime/16-tool-result-limits.md` §4 与 `03-runtime/08-error-codes.md`。 ## 2026-09-21 —— 插件崩溃上报带上退出码但不复制原始输出（D607，issue #747） diff --git a/packages/agent-runtime/src/runtime.test.ts b/packages/agent-runtime/src/runtime.test.ts index cc0346e2b..fba2d1bb6 100644 --- a/packages/agent-runtime/src/runtime.test.ts +++ b/packages/agent-runtime/src/runtime.test.ts @@ -568,6 +568,85 @@ describe("DesktopAgentRuntime configuration matching", () => { await runtime.dispose(); }); + it("turns a host image read into an image block for a model that can see", async () => { + + const host = { + call: vi.fn(async (method: string) => + method === "tools.execute" + ? { + ok: true, + content: { + path: "shot.png", + root: "workspace", + text: "shot.png is a image/png image (68 bytes).", + images: [{ data: "iVBORw0KGgo=", mimeType: "image/png" }], + fileBytes: 68, + }, + } + : undefined, + ), + }; + // The default fixture model declares `input: ["text", "image"]`. + const runtime = createRuntime({ host }); + const read = (runtime as any).agent.state.tools.find( + (tool: any) => tool.name === "Read", + ); + + const result = await read.execute("read-image", { path: "shot.png" }); + + expect(result.content[0]).toMatchObject({ type: "text" }); + expect(result.content[0].text).toContain("image/png"); + // The base64 payload crosses as an image block, which is what pi-ai sends. + expect(result.content[1]).toEqual({ + type: "image", + data: "iVBORw0KGgo=", + mimeType: "image/png", + }); + // The ref stays out of the persisted detail; only its count is recorded. + expect(result.details).toMatchObject({ path: "shot.png", imageCount: 1 }); + expect(result.details).not.toHaveProperty("images"); + + await runtime.dispose(); + }); + + it("keeps only the text when the model cannot see images", async () => { + const host = { + call: vi.fn(async (method: string) => + method === "tools.execute" + ? { + ok: true, + content: { + path: "shot.png", + root: "workspace", + text: "shot.png is a image/png image (68 bytes).", + images: [{ data: "iVBORw0KGgo=", mimeType: "image/png" }], + fileBytes: 68, + }, + } + : undefined, + ), + }; + const runtime = createRuntime({ + host, + provider: { + ...provider, + modelConfig: { ...provider.modelConfig!, input: ["text"] }, + }, + }); + const read = (runtime as any).agent.state.tools.find( + (tool: any) => tool.name === "Read", + ); + + const result = await read.execute("read-image", { path: "shot.png" }); + + // The text says an image is there, so the model can tell the user it cannot + // see it instead of inventing its contents. + expect(result.content).toHaveLength(1); + expect(result.content[0].text).toContain("image/png"); + expect(result.details).toMatchObject({ imageCount: 0 }); + + await runtime.dispose(); + }); it("terminates a repeated Edit mismatch on the third failed attempt", async () => { const host = { diff --git a/packages/shared/src/errors.ts b/packages/shared/src/errors.ts index 02a097b3f..92febdd37 100644 --- a/packages/shared/src/errors.ts +++ b/packages/shared/src/errors.ts @@ -237,6 +237,7 @@ export const ErrorCodes = { WORKSPACE_PATH_DENIED: "WORKSPACE_PATH_DENIED", READ_PATH_IS_DIRECTORY: "READ_PATH_IS_DIRECTORY", TOOL_BINARY_CONTENT: "TOOL_BINARY_CONTENT", + TOOL_IMAGE_TOO_LARGE: "TOOL_IMAGE_TOO_LARGE", // Plan/Goal host-side codes that reach the sidecar as `errorCode`. PLAN_SESSION_NOT_FOUND: "PLAN_SESSION_NOT_FOUND", PLAN_WORKSPACE_REQUIRED: "PLAN_WORKSPACE_REQUIRED",