From 19173770542c4efd8a2c9f6d1450d27379c952f9 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:23:15 +0800 Subject: [PATCH] fix(audit): make index_persist audit entries opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of upstream rohitg00/agentmemory#1182 (39658a9) onto the v0.9.29-patch line, adapted only for this branch's ledger-era index-persistence (same gate, same vocabulary, same tests). Index shard writes, manifest publishes, and generation reclaims record no user-data deletions, yet a single save() emits several index_persist audit rows; on a real store they reached 59876 of 84028 entries (71%). On this branch they would also dominate the new AGENTMEMORY_AUDIT_MAX retention window within hours of sustained capture (~4 rows per 5s debounced save) and evict genuine deletion records, so they are now opt-in via AGENTMEMORY_AUDIT_INDEX_PERSIST=1 (trimmed, case-insensitive 1/true; 0/false/anything else stays off). - src/state/index-persistence.ts: auditIndexPersistEnabled() gate, verbatim from upstream - test/index-persistence.test.ts: gating suite ported verbatim (default-off, affirmative vocabulary, off vocabulary, persistence unaffected) — 12 new tests, suite now 1756 passing - plugin/skills/agentmemory-config/REFERENCE.md: regenerated (52 vars) - CHANGELOG.md: document the new default under [Unreleased] --- CHANGELOG.md | 1 + plugin/skills/agentmemory-config/REFERENCE.md | 3 +- src/state/index-persistence.ts | 15 +++ test/index-persistence.test.ts | 91 +++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60ef9c9712..76d330e5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - **Server-side capture payloads are bounded without flattening structured tool data.** `tool_input`, `tool_output`, prompts and the raw envelope are trimmed to configurable character budgets (`AGENTMEMORY_OBSERVE_*`); object fields such as `file_path` survive truncation, `0` disables an individual bound, and positive budgets below 2 are clamped to the smallest serialized string. - **Graph and semantic provenance are bounded.** Graph nodes, edges, insights and semantic session provenance keep only the freshest source ids instead of growing for the life of the session. - **Audit retention runs in a background sweep with bounded deletes.** `recordAudit` stays a single KV write. `mem::audit-sweep` trims on a timer, caps deletes per pass, retains rows whose delete failed, and never evicts rows that share the retention boundary's millisecond. The pinned iii engine only supports whole-scope `state::list`, so enabled sweeps and queries still load the entire audit log into memory before applying limits; large existing logs require a future paginated engine scan or separate storage migration. `AGENTMEMORY_AUDIT_MAX=0` and `AGENTMEMORY_AUDIT_SWEEP_DELETE_BATCH=0` skip the sweep scan entirely; query results are capped at 1000, with explicit `limit=0` retaining the historical 100-row default. +- **Index-persistence audit rows are opt-in (aligned with upstream #1182).** Shard writes, manifest publishes, and generation reclaims remove no user rows, yet a single save emits several `index_persist` audit rows — on a real store they reached 59876 of 84028 entries (71%). Left on by default they would dominate the new `AGENTMEMORY_AUDIT_MAX` window within hours of sustained capture and evict genuine deletion records, so they are now gated behind `AGENTMEMORY_AUDIT_INDEX_PERSIST=1` (trimmed, case-insensitive `1`/`true`; everything else, including `0` and `false`, stays off). - **Stranded index generations are reclaimed safely (#1115).** Every published generation is recorded in a retry-preserving ledger before its first shard write. Boot reclaims at most `AGENTMEMORY_INDEX_RECLAIM_BOOT_MAX_SHARDS` shards in a background pass, later saves reclaim the remainder, and a failed delete keeps its ledger entry for the next attempt. - **Index saves are serialised instead of interleaved.** Concurrent save callers collapse into one running save plus one queued save rather than several writers racing on the same shards. - **Eviction removes observations from BM25 and vector search.** Deleted observations no longer remain as in-memory postings that are re-serialized on every index save; dry runs leave the index untouched. diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index a742643c2b..491108a5dc 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,10 +3,11 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 51 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 52 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` +- `AGENTMEMORY_AUDIT_INDEX_PERSIST` - `AGENTMEMORY_AUDIT_MAX` - `AGENTMEMORY_AUDIT_SWEEP_CONCURRENCY` - `AGENTMEMORY_AUDIT_SWEEP_DELETE_BATCH` diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 8d0369382d..0970137d5a 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -18,6 +18,20 @@ const INDEX_SHARD_KEY = "data"; const DEFAULT_INDEX_SHARD_CHARS = 2_000_000; const DEFAULT_BOOT_RECLAIM_MAX_DELETES = 200; +// mem:audit exists to record structural deletions of user data — that is +// the policy stated at the top of src/functions/audit.ts. Index shard +// writes and manifest publishes remove no user rows, so they fall outside +// it, yet a single save() emits three of them: on a real store they +// reached 59876 of 84028 entries (71%), which is what makes the audit log +// slow to query and bloats startup. Off by default; set +// AGENTMEMORY_AUDIT_INDEX_PERSIST=1 when debugging index persistence. +function auditIndexPersistEnabled(): boolean { + const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; + if (!raw) return false; + const normalized = raw.trim().toLowerCase(); + return normalized === "1" || normalized === "true"; +} + type IndexShardManifest = { v: 1; generation?: string; @@ -608,6 +622,7 @@ export class IndexPersistence { targetIds: string[], details: Record, ): Promise { + if (!auditIndexPersistEnabled()) return; await safeAudit( this.kv, "index_persist", diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index ec4478c7a4..34f72bebe3 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1274,3 +1274,94 @@ describe("IndexPersistence save coalescing", () => { expect(manifestWrites).toBeLessThanOrEqual(2); }); }); + +describe("index_persist audit gating", () => { + let kv: ReturnType; + let previousFlag: string | undefined; + + beforeEach(() => { + // AGENTMEMORY_* variables are documented as living in + // ~/.agentmemory/.env, so a developer running the suite on a + // configured machine can inherit this one. Clear it going in and put + // whatever was there back on the way out. + previousFlag = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; + delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; + vi.useFakeTimers(); + kv = mockKV(); + }); + + afterEach(() => { + vi.useRealTimers(); + if (previousFlag === undefined) { + delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; + } else { + process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previousFlag; + } + }); + + async function indexPersistEntries(): Promise> { + const entries = await kv.list<{ operation: string }>("mem:audit"); + return entries.filter((entry) => entry.operation === "index_persist"); + } + + it("writes no index_persist audit entries by default", async () => { + const persistence = new IndexPersistence( + kv as never, + makeBm25("obs_1", "auth handler"), + null, + ); + + await persistence.save(); + + expect(await indexPersistEntries()).toEqual([]); + }); + + it.each(["1", " 1 ", "true", "TRUE", " true "])( + "writes index_persist audit entries when set to %j", + async (value) => { + process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value; + const persistence = new IndexPersistence( + kv as never, + makeBm25("obs_1", "auth handler"), + null, + ); + + await persistence.save(); + + expect((await indexPersistEntries()).length).toBeGreaterThan(0); + }, + ); + + // Anything that is not an affirmative stays off. "0" and "false" are the + // ones an operator is likely to reach for to disable it, and they must + // not read as "present, therefore enabled". + it.each(["0", "false", "yes", "", " "])( + "keeps auditing off when set to %j", + async (value) => { + process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value; + const persistence = new IndexPersistence( + kv as never, + makeBm25("obs_1", "auth handler"), + null, + ); + + await persistence.save(); + + expect(await indexPersistEntries()).toEqual([]); + }, + ); + + it("still persists the index when auditing is off", async () => { + const persistence = new IndexPersistence( + kv as never, + makeBm25("obs_1", "auth handler"), + null, + ); + + await persistence.save(); + + const loaded = await persistence.load(); + expect(loaded.bm25).not.toBeNull(); + expect(loaded.bm25!.size).toBe(1); + }); +});