diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ed95ba8f..4459b9a8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- **The post-tool-use hook no longer collapses escape-heavy tool outputs to `{}`.** The client-side 32k wire bound sliced strings by raw character count, ignoring that JSON escaping (quotes, backslashes, control characters) can double or triple the serialized form. An oversized truncated string then failed the object accounting's cost check, and the loop's `break` discarded every remaining field — outputs like `{stdout: "", file_path: ..., exit_code: 0}` arrived as `{}`, losing exactly the small fields the structure-preserving bound exists to keep. The hook's truncation (now in `src/hooks/truncate.ts`, shared by the built copies in `dist/hooks/` and `plugin/scripts/`) mirrors the server's `boundValue`: strings are cut by binary search on serialized length, object fields are taken smallest-first so a huge `stdout` cannot crowd out `file_path`/`exit_code`, and a single unfittable entry is skipped instead of discarding the rest. Field order of over-budget objects changes in transit only; the server re-bounds and re-orders anyway. - **Fresh installs are portable and persistent (aligned with upstream #892).** The bundled `iii-config.yaml` uses cwd-relative paths and the engine was spawned without a cwd, so on global and npx installs `./data/state_store.db` and `./data/stream_store` landed in whatever directory the user ran the CLI from, and the `iii-exec` supervision block never resolved — nothing respawned the worker after it died, which surfaced as "all data gone" reports against a live REST port. `agentmemory start` now regenerates a runtime config each boot (`~/.agentmemory/data/iii-config.runtime.yaml`) with absolute data paths and an absolute worker exec line, anchors the engine cwd at `~/.agentmemory`, and copies any legacy `./data` stores from the invocation directory on first run. Repo checkouts keep the cwd config and repo-root cwd, so dev behavior is unchanged; user overrides via env or `~/.agentmemory/iii-config.yaml` are passed through verbatim. The viewer port is now configurable (`III_VIEWER_PORT`, default REST + 2), worker pidfile paths honor `AGENTMEMORY_RUNTIME_DIR`, Docker installs gain uid/gid/chown controls, and `agentmemory remove` covers the generated runtime config. `agentmemory doctor` and startup-failure reporting were expanded accordingly. - **Observation capture no longer grows the persisted live-stream store.** Raw and compressed observation events use `stream::send` instead of `stream::set`; the viewer still receives live updates and REST remains the history source, but `data/stream_store` no longer accumulates a write-only second and third copy of every observation. - **Server-side capture payloads are bounded without flattening structured tool data.** `tool_input`, `tool_output`, prompts and the raw envelope are trimmed to configurable character budgets (`AGENTMEMORY_OBSERVE_*`); object fields such as `file_path` survive truncation, `0` disables an individual bound, and positive budgets below 2 are clamped to the smallest serialized string. diff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjs index 90435754e9..5975dbc775 100755 --- a/plugin/scripts/post-tool-use.mjs +++ b/plugin/scripts/post-tool-use.mjs @@ -31,6 +31,62 @@ function hookCwd(data) { if (projectDir && projectDir.trim()) return projectDir; } //#endregion +//#region src/hooks/truncate.ts +const TRUNCATION_MARKER = "...[truncated]"; +function serializedLength(value) { + return JSON.stringify(value)?.length ?? 0; +} +function truncate(value, max) { + if (typeof value === "string") { + if (serializedLength(value) <= max) return value; + let keep = 0; + let hi = value.length; + while (keep < hi) { + const mid = Math.ceil((keep + hi) / 2); + if (serializedLength(value.slice(0, mid) + "...[truncated]") <= max) keep = mid; + else hi = mid - 1; + } + return value.slice(0, keep) + TRUNCATION_MARKER; + } + if (Array.isArray(value)) { + if (serializedLength(value) <= max) return value; + const bounded = []; + let remaining = Math.max(0, max - 2); + for (const item of value) { + const comma = bounded.length > 0 ? 1 : 0; + if (remaining <= comma) break; + const next = truncate(item, remaining - comma); + const cost = comma + serializedLength(next); + if (cost > remaining) continue; + bounded.push(next); + remaining -= cost; + } + return bounded; + } + if (typeof value === "object" && value !== null) { + if (serializedLength(value) <= max) return value; + const bounded = {}; + let remaining = Math.max(0, max - 2); + const sized = Object.entries(value).map((entry) => ({ + entry, + size: serializedLength(entry[1]) + })); + sized.sort((a, b) => a.size - b.size); + for (const { entry } of sized) { + const [key, item] = entry; + const overhead = (Object.keys(bounded).length > 0 ? 1 : 0) + JSON.stringify(key).length + 1; + if (remaining <= overhead) continue; + const next = truncate(item, remaining - overhead); + const cost = overhead + serializedLength(next); + if (cost > remaining) continue; + bounded[key] = next; + remaining -= cost; + } + return bounded; + } + return value; +} +//#endregion //#region src/hooks/post-tool-use.ts function isSdkChildContext(payload) { if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true; @@ -116,44 +172,6 @@ function extractImageData(output) { cleanOutput: output }; } -const TRUNCATION_MARKER = "...[truncated]"; -function truncate(value, max) { - if (typeof value === "string") { - if (value.length + 14 <= max) return value; - return value.slice(0, Math.max(0, max - 14)) + TRUNCATION_MARKER; - } - if (Array.isArray(value)) { - if (JSON.stringify(value).length <= max) return value; - const bounded = []; - let remaining = Math.max(0, max - 2); - for (const item of value) { - const comma = bounded.length > 0 ? 1 : 0; - if (remaining <= comma) break; - const next = truncate(item, remaining - comma); - const cost = comma + JSON.stringify(next).length; - if (cost > remaining) break; - bounded.push(next); - remaining -= cost; - } - return bounded; - } - if (typeof value === "object" && value !== null) { - if (JSON.stringify(value).length <= max) return value; - const bounded = {}; - let remaining = Math.max(0, max - 2); - for (const [key, item] of Object.entries(value)) { - const overhead = (Object.keys(bounded).length > 0 ? 1 : 0) + JSON.stringify(key).length + 1; - if (remaining <= overhead) break; - const next = truncate(item, remaining - overhead); - const cost = overhead + JSON.stringify(next).length; - if (cost > remaining) break; - bounded[key] = next; - remaining -= cost; - } - return bounded; - } - return value; -} main().catch(() => process.exit(0)); //#endregion export {}; diff --git a/src/hooks/post-tool-use.ts b/src/hooks/post-tool-use.ts index ef23f34de4..abb1e48e39 100644 --- a/src/hooks/post-tool-use.ts +++ b/src/hooks/post-tool-use.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node import { resolveProject, hookCwd } from "./_project.js"; +import { truncate } from "./truncate.js"; function isSdkChildContext(payload: unknown): boolean { if (process.env["AGENTMEMORY_SDK_CHILD"] === "1") return true; @@ -105,52 +106,4 @@ function extractImageData(output: unknown): { imageData: string | undefined; cle return { imageData: undefined, cleanOutput: output }; } -const TRUNCATION_MARKER = "...[truncated]"; - -// Structure-preserving client-side bound. A flat JSON.stringify+slice kept -// the wire small but destroyed object shape for outputs over the cap — the -// largest payloads lost the very fields (file_path etc.) the server's -// structure-preserving truncation exists to keep. Walk the value and keep as -// many whole entries as fit the budget; strings get sliced. The server bound -// (AGENTMEMORY_OBSERVE_TOOL_OUTPUT_CHARS) remains authoritative — this cap -// only limits wire size and ingress parse memory. -function truncate(value: unknown, max: number): unknown { - if (typeof value === "string") { - if (value.length + TRUNCATION_MARKER.length <= max) return value; - return value.slice(0, Math.max(0, max - TRUNCATION_MARKER.length)) + TRUNCATION_MARKER; - } - if (Array.isArray(value)) { - if (JSON.stringify(value).length <= max) return value; - const bounded: unknown[] = []; - let remaining = Math.max(0, max - 2); - for (const item of value) { - const comma = bounded.length > 0 ? 1 : 0; - if (remaining <= comma) break; - const next = truncate(item, remaining - comma); - const cost = comma + JSON.stringify(next).length; - if (cost > remaining) break; - bounded.push(next); - remaining -= cost; - } - return bounded; - } - if (typeof value === "object" && value !== null) { - if (JSON.stringify(value).length <= max) return value; - const bounded: Record = {}; - let remaining = Math.max(0, max - 2); - for (const [key, item] of Object.entries(value as Record)) { - const overhead = - (Object.keys(bounded).length > 0 ? 1 : 0) + JSON.stringify(key).length + 1; - if (remaining <= overhead) break; - const next = truncate(item, remaining - overhead); - const cost = overhead + JSON.stringify(next).length; - if (cost > remaining) break; - bounded[key] = next; - remaining -= cost; - } - return bounded; - } - return value; -} - main().catch(() => process.exit(0)); diff --git a/src/hooks/truncate.ts b/src/hooks/truncate.ts new file mode 100644 index 0000000000..b83816de2e --- /dev/null +++ b/src/hooks/truncate.ts @@ -0,0 +1,81 @@ +// Structure-preserving client-side bound for hook payloads. A flat +// JSON.stringify+slice kept the wire small but destroyed object shape for +// outputs over the cap — the largest payloads lost the very fields +// (file_path etc.) the server's structure-preserving truncation exists to +// keep. Walk the value and keep as many whole entries as fit the budget; +// strings get sliced. The server bound +// (AGENTMEMORY_OBSERVE_TOOL_OUTPUT_CHARS) remains authoritative — this cap +// only limits wire size and ingress parse memory. + +export const TRUNCATION_MARKER = "...[truncated]"; + +function serializedLength(value: unknown): number { + return JSON.stringify(value)?.length ?? 0; +} + +export function truncate(value: unknown, max: number): unknown { + if (typeof value === "string") { + if (serializedLength(value) <= max) return value; + // Escape expansion (quotes, backslashes, control characters) makes the + // serialized form up to six times the raw character count, so slicing by + // character count alone overshoots the budget — and inside an object that + // overshoot made the caller's cost check reject the field and drop every + // field after it, so escape-heavy tool outputs arrived as {}. Find the + // longest prefix whose serialized form still fits by binary search, the + // same way the server's boundString does. + let keep = 0; + let hi = value.length; + while (keep < hi) { + const mid = Math.ceil((keep + hi) / 2); + const candidate = value.slice(0, mid) + TRUNCATION_MARKER; + if (serializedLength(candidate) <= max) keep = mid; + else hi = mid - 1; + } + return value.slice(0, keep) + TRUNCATION_MARKER; + } + if (Array.isArray(value)) { + if (serializedLength(value) <= max) return value; + const bounded: unknown[] = []; + let remaining = Math.max(0, max - 2); + for (const item of value) { + const comma = bounded.length > 0 ? 1 : 0; + if (remaining <= comma) break; + const next = truncate(item, remaining - comma); + const cost = comma + serializedLength(next); + // Skip, not stop: one entry that cannot fit (long primitives, a + // string against a sub-budget smaller than the marker) must not + // discard later entries that still fit. + if (cost > remaining) continue; + bounded.push(next); + remaining -= cost; + } + return bounded; + } + if (typeof value === "object" && value !== null) { + if (serializedLength(value) <= max) return value; + const bounded: Record = {}; + let remaining = Math.max(0, max - 2); + // Smallest fields first, mirroring the server's boundValue: a huge + // stdout must not crowd out file_path / exit_code when the budget runs + // out. Field order is not semantic for the wire — the server re-bounds + // and re-orders anyway. + const sized = Object.entries(value as Record).map( + (entry) => ({ entry, size: serializedLength(entry[1]) }), + ); + sized.sort((a, b) => a.size - b.size); + for (const { entry } of sized) { + const [key, item] = entry; + const overhead = + (Object.keys(bounded).length > 0 ? 1 : 0) + JSON.stringify(key).length + 1; + // Long keys can outlive the budget while later short keys still fit. + if (remaining <= overhead) continue; + const next = truncate(item, remaining - overhead); + const cost = overhead + serializedLength(next); + if (cost > remaining) continue; + bounded[key] = next; + remaining -= cost; + } + return bounded; + } + return value; +} diff --git a/test/post-tool-use-truncation.test.ts b/test/post-tool-use-truncation.test.ts new file mode 100644 index 0000000000..b29ba0ec70 --- /dev/null +++ b/test/post-tool-use-truncation.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { truncate, TRUNCATION_MARKER } from "../src/hooks/truncate.js"; + +// The hook caps tool_output at 32000 characters on the wire, but every case +// here uses smaller budgets so the arithmetic stays auditable. + +describe("truncate — serialized-length string bound", () => { + it("returns a string unchanged when it fits", () => { + expect(truncate("hello", 100)).toBe("hello"); + }); + + it("returns a string unchanged when it fits exactly", () => { + const s = "a".repeat(50); + expect(JSON.stringify(s).length).toBe(52); + expect(truncate(s, 52)).toBe(s); + }); + + it("cuts so the serialized form fits even when escapes double the length", () => { + const quoteHeavy = '"'.repeat(2000); + const result = truncate(quoteHeavy, 300) as string; + expect(result.endsWith(TRUNCATION_MARKER)).toBe(true); + // 2 quotes + 2 per escaped quote + marker must fit the budget. + expect(JSON.stringify(result).length).toBeLessThanOrEqual(300); + }); + + it("handles backslash and control-character expansion", () => { + const s = "\n".repeat(500); + const result = truncate(s, 100) as string; + expect(result.endsWith(TRUNCATION_MARKER)).toBe(true); + expect(JSON.stringify(result).length).toBeLessThanOrEqual(100); + }); + + it("keeps as many whole characters as the budget allows", () => { + const s = "a".repeat(1000); + const result = truncate(s, 100) as string; + // 2 quotes + 84 a's + 14 marker chars = 100 serialized. + expect(result).toBe("a".repeat(84) + TRUNCATION_MARKER); + }); + + it("does not throw on budgets smaller than the marker", () => { + const result = truncate("abcdef", 0) as string; + expect(typeof result).toBe("string"); + }); +}); + +describe("truncate — smallest fields survive a huge sibling", () => { + it("keeps file_path and exit_code when an escape-heavy stdout cannot fit", () => { + const output = { + stdout: '"'.repeat(600), + file_path: "/very/important/path.ts", + exit_code: 0, + }; + const result = truncate(output, 60) as Record; + expect(result["file_path"]).toBe("/very/important/path.ts"); + expect(result["exit_code"]).toBe(0); + expect("stdout" in result).toBe(false); + expect(JSON.stringify(result).length).toBeLessThanOrEqual(60); + }); + + it("keeps small fields and a truncated stdout when the budget allows both", () => { + const output = { + stdout: '"'.repeat(600), + file_path: "/very/important/path.ts", + exit_code: 0, + }; + const result = truncate(output, 300) as Record; + expect(result["file_path"]).toBe("/very/important/path.ts"); + expect(result["exit_code"]).toBe(0); + expect((result["stdout"] as string).endsWith(TRUNCATION_MARKER)).toBe(true); + expect(JSON.stringify(result).length).toBeLessThanOrEqual(300); + }); + + it("keeps later fields when a long primitive cannot fit", () => { + const output = { + a: 1234567890123456789012345678901234567890, + keep_me: "yes", + }; + const result = truncate(output, 40) as Record; + expect(result["keep_me"]).toBe("yes"); + }); + + it("keeps later fields when a long key cannot fit", () => { + const output = { + "an extremely long field name that outlives the budget": "v", + k: 1, + }; + const result = truncate(output, 40) as Record; + expect(result["k"]).toBe(1); + }); + + it("keeps later array elements when an unfittable element is skipped", () => { + // JSON.stringify renders this as the 22-char "1.2345678901234567e+49". + const big = 1.2345678901234567e49; + const arr = [big, "x"]; + const result = truncate(arr, 23) as unknown[]; + expect(result).toEqual(["x"]); + }); + + it("keeps array order for elements that fit", () => { + // Serializes to the 21-char "1.2345678901234567e+39". + const big = 1.2345678901234567e39; + const arr = [big, "tail"]; + const result = truncate(arr, 60) as unknown[]; + expect(result).toEqual([big, "tail"]); + }); +}); + +describe("truncate — whole-value passthrough", () => { + it("returns an object unchanged when it fits", () => { + const value = { a: 1, b: [1, 2, 3], c: "text" }; + expect(truncate(value, 1000)).toEqual(value); + }); + + it("returns an array unchanged when it fits", () => { + const value = [1, "two", { three: 3 }]; + expect(truncate(value, 1000)).toEqual(value); + }); + + it("passes primitives through", () => { + expect(truncate(42, 0)).toBe(42); + expect(truncate(null, 0)).toBeNull(); + expect(truncate(true, 0)).toBe(true); + }); + + it("bounds nested structures within the overall budget", () => { + const value = { + files: Array.from({ length: 50 }, (_, i) => ({ + path: `/repo/dir-${i}/file-${i}.ts`, + content: "line\n".repeat(20), + })), + summary: "done", + }; + const result = truncate(value, 500); + expect(JSON.stringify(result).length).toBeLessThanOrEqual(500); + expect((result as Record)["summary"]).toBe("done"); + }); + + it("never returns an empty object for a non-empty over-budget input", () => { + const output = { stdout: '"'.repeat(60000), file_path: "/p", exit_code: 0 }; + const result = truncate(output, 32000) as Record; + expect(Object.keys(result).length).toBeGreaterThan(0); + expect(result["file_path"]).toBe("/p"); + expect(result["exit_code"]).toBe(0); + expect(JSON.stringify(result).length).toBeLessThanOrEqual(32000); + }); +});