SpillwaveSolutions · RichardHightower · Aug 30, 2026 · Aug 30, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -193,7 +193,6 @@ jobs:
   benchmark-smoke:
     name: Benchmark Suite Smoke Test
     runs-on: ubuntu-24.04
-    continue-on-error: true
     needs: [test]
     steps:
       - uses: actions/checkout@v4
@@ -213,19 +212,27 @@ jobs:
         with:
           shared-key: "bench-smoke"
 
-      - name: Build memory-bench
-        run: cargo build -p memory-bench
+      - name: Test memory-bench
+        run: cargo test -p memory-bench
 
-      - name: Smoke test (help only — no daemon required)
+      - name: Smoke locomo adapter (1 conversation, mock judge)
         run: |
-          cargo run -p memory-bench -- --help
-          cargo run -p memory-bench -- all --help
-          cargo run -p memory-bench -- locomo --help
+          cargo run -p memory-bench -- smoke --output /tmp/locomo-smoke.json
+          python3 - <<'PY'
+          import json, sys
+          d = json.load(open("/tmp/locomo-smoke.json"))
+          assert d["conversations"] == 1, d
+          assert d["total_questions"] >= 1, d
+          assert d["metric"] == "context_hit_rate", d
+          assert "locomo_llm_judge" != d["metric"]
+          print("smoke ok", d["metric"], "questions", d["total_questions"])
+          PY
+
 
   # Summary job that depends on all other jobs
   ci-success:
     name: CI Success
-    needs: [fmt, clippy, test, build, doc, e2e]
+    needs: [fmt, clippy, test, build, doc, e2e, benchmark-smoke]
     runs-on: ubuntu-24.04
     if: always()
     steps:
@@ -236,7 +243,8 @@ jobs:
              [[ "${{ needs.test.result }}" != "success" ]] || \
              [[ "${{ needs.build.result }}" != "success" ]] || \
              [[ "${{ needs.doc.result }}" != "success" ]] || \
-             [[ "${{ needs.e2e.result }}" != "success" ]]; then
+             [[ "${{ needs.e2e.result }}" != "success" ]] || \
+             [[ "${{ needs.benchmark-smoke.result }}" != "success" ]]; then
             echo "One or more jobs failed"
             exit 1
           fi

diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md
@@ -12,7 +12,7 @@
 - ✅ **v2.6 Cognitive Retrieval** — Phases 39-44 (shipped 2026-03-16)
 - ✅ **v2.7 Multi-Runtime Portability** — Phases 45-50 (shipped 2026-03-22)
 - **v3.0 Competitive Parity & Benchmarks** — Phases 51-53 + Phase 51.5 (in progress; Phase 51.5 merged 2026-04-28)
-- **v3.1 Make It True** — Phases 54-58 (in progress; Phase 54 merged 2026-08-30, Phase 55 merged 2026-08-30, Phase 54.5 executing)
+- **v3.1 Make It True** — Phases 54-58 (in progress; Phases 54, 54.5, 55 merged 2026-08-30, Phase 56 executing)
 
 ## Phases
 
@@ -267,20 +267,27 @@ Close the claim/reality gap, then open the shop window. No new capabilities.
 - [x] 54-05: Honest daemon flags + attach indexes
 - [x] 54-06: Lock-poisoning recover_lock policy
 
-### Phase 54.5: Truth leaks + CI pin (in execution 2026-08-30)
+### Phase 54.5: Truth leaks + CI pin — COMPLETE 2026-08-30 (PR #35)
 
-- [ ] CI: pin rust-toolchain 1.97; allow `result_large_err` on generated proto
-- [ ] Explainability reports what actually ran
-- [ ] LLM rerank order survives salience; BM25 events carry text
-- [ ] Shared HNSW handle; concurrent fan-out; no per-event grip scan
+- [x] CI: pin rust-toolchain 1.97; allow `result_large_err` on generated proto
+- [x] Explainability reports what actually ran
+- [x] LLM rerank order survives salience; BM25 events carry text
+- [x] Shared HNSW handle; concurrent fan-out; no per-event grip scan
 
-### Phase 55: Performance Truth — COMPLETE 2026-08-30 (PR #33)
+### Phase 55: Performance Truth (2/2 plans) — COMPLETE 2026-08-30 (PR #33)
 
-### Phase 56: Honest Benchmarks (PR #34 open)
+- [x] 55-01: Split setup vs query (64.6s was toc_build)
+- [x] 55-02: Honest percentiles (n≥10 / n≥30)
+
+### Phase 56: Honest Benchmarks (3/3 plans) — IN EXECUTION 2026-08-30
+
+- [x] 56-01: Custom harness (real recall@k, content compression, fail-loud, isolation, ≥25 fixtures)
+- [x] 56-02: LOCOMO adapter v2 (real schema, mock vs llm-judge)
+- [x] 56-03: Smoke artifacts + HOLD comparison marketing
 
 ### Phase 57: Shop Window & Positioning (0/3)
 
 ### Phase 58: Launch (side quest)
 
-*Updated: 2026-08-30 — Phase 54.5 truth-leaks + CI pin in execution*
+*Updated: 2026-08-30 — Phase 54.5 merged (#35); Phase 56 Honest Benchmarks in execution*
 
diff --git a/.planning/STATE.md b/.planning/STATE.md
@@ -3,14 +3,14 @@ gsd_state_version: 1.0
 milestone_name: Make It True
 status: in_progress
 stopped_at: null
-last_updated: "2026-08-30T19:10:00.000Z"
-last_activity: 2026-08-30 — Phase 54.5 truth-leaks + CI toolchain pin
+last_updated: "2026-08-30T20:45:00.000Z"
+last_activity: 2026-08-30 — Phase 54.5 merged (#35); Phase 56 Honest Benchmarks rebasing
 progress:
   total_phases: 6
-  completed_phases: 2
+  completed_phases: 3
   total_plans: 14
-  completed_plans: 8
-  percent: 57
+  completed_plans: 11
+  percent: 79
 ---
 
 # Project State
@@ -20,29 +20,30 @@ progress:
 See: .planning/PROJECT.md (updated 2026-03-22)
 
 **Core value:** Agent can answer "what were we talking about last week?" without scanning everything
-**Current focus:** v3.1 Phase 54.5 — close residual Phase 54 honesty leaks and pin CI toolchain
+**Current focus:** v3.1 Phase 56 — Honest Benchmarks (real recall@k, locomo10.json schema, HOLD comparison marketing)
 
 ## Current Position
 
-Phase: 54.5 of 58 (Truth leaks + CI pin)
-Plan: implementing on `feature/phase-54.5-truth-leaks`
-Status: Phase 54 merged (#32); Phase 55 merged (#33); Phase 56 PR #34 open (clippy red from toolchain drift)
-Last activity: 2026-08-30 — explainability truth, shared HNSW, rust-toolchain.toml 1.97
+Phase: 56 of 58 (Honest Benchmarks)
+Plan: 01-03 implemented on `feature/phase-56-honest-benchmarks` (PR #34)
+Status: Phase 54 + 54.5 + 55 merged; Phase 56 rebase onto #35
+Last activity: 2026-08-30 — #35 merged (clippy pin); rebasing #34
 
-Progress: [██████░░░░] ~57% (8/14 plans; Phase 54.5 cleanup)
+Progress: [████████░░] ~79% (11/14 plans; Phase 56 of 54-58)
 
 ## Out-of-band Work
 
 ### Open PRs
 
 | PR | What | Status |
 |---|---|---|
-| #34 | Phase 56 Honest Benchmarks | Open; Clippy red (1.98 `result_large_err` on generated tonic stubs) |
+| #34 | Phase 56 Honest Benchmarks | Open; rebasing onto #35 |
 
 ### Recently Merged
 
 | PR | What | Merged |
 |---|---|---|
+| #35 | Phase 54.5 truth leaks + rustc 1.97 pin | 2026-08-30 |
 | #33 | Phase 55 Performance Truth | 2026-08-30 |
 | #32 | Phase 54 Integration Truth | 2026-08-30 |
 | #31 | v3.1 Make It True design spec | 2026-08-30 |
@@ -54,5 +55,7 @@ Progress: [██████░░░░] ~57% (8/14 plans; Phase 54.5 cleanup)
 ## Decisions
 
 - v3.1 scope: Make It True — no new capabilities; close claim/reality gap (Phases 54-58)
-- Phase 54.5 before more measurement: explainability must report what ran; shared HNSW handle
-- CI pins `rust-toolchain.toml` to 1.97 so floating stable cannot redden main
+- Phase 54.5: explainability reports what ran; shared HNSW handle; CI pins rust-toolchain.toml to 1.97
+- Phase 55: split setup vs query in `perf_bench`; p90/p99 withheld below 10/30 samples
+- Warm = one setup + N query samples; cold = new store per iteration
+- Phase 56: substring metric is `context_hit_rate`; HOLD LOCOMO comparison marketing until `locomo_llm_judge` artifact exists
diff --git a/.planning/phases/56-honest-benchmarks/56-01-PLAN.md b/.planning/phases/56-honest-benchmarks/56-01-PLAN.md
@@ -0,0 +1,19 @@
+---
+phase: 56-honest-benchmarks
+plan: 01
+type: execute
+wave: 1
+depends_on: []
+files_modified:
+  - crates/memory-bench/src/scorer.rs
+  - crates/memory-bench/src/runner.rs
+  - crates/memory-bench/src/fixture.rs
+  - crates/memory-bench/src/report.rs
+  - crates/memory-bench/src/main.rs
+  - benchmarks/fixtures/
+autonomous: true
+---
+
+<objective>
+Fix the custom harness: real recall@k, content-based compression, fail-loud CLI, per-test isolation, ≥25 fixtures.
+</objective>
diff --git a/.planning/phases/56-honest-benchmarks/56-02-PLAN.md b/.planning/phases/56-honest-benchmarks/56-02-PLAN.md
@@ -0,0 +1,18 @@
+---
+phase: 56-honest-benchmarks
+plan: 02
+type: execute
+wave: 1
+depends_on: [01]
+files_modified:
+  - crates/memory-bench/src/locomo.rs
+  - crates/memory-bench/src/judge.rs
+  - benchmarks/scripts/download-locomo.sh
+  - benchmarks/fixtures/locomo-smoke.json
+  - crates/memory-cli/src/commands/add.rs
+autonomous: true
+---
+
+<objective>
+LOCOMO adapter v2: real locomo10.json schema, isolated store per conversation, mock vs llm-judge, honest metric names.
+</objective>
diff --git a/.planning/phases/56-honest-benchmarks/56-03-PLAN.md b/.planning/phases/56-honest-benchmarks/56-03-PLAN.md
@@ -0,0 +1,17 @@
+---
+phase: 56-honest-benchmarks
+plan: 03
+type: execute
+wave: 1
+depends_on: [01, 02]
+files_modified:
+  - benchmarks/results/
+  - docs/benchmarks.md
+  - .github/workflows/ci.yml
+  - benchmarks/baselines.toml
+autonomous: true
+---
+
+<objective>
+Run smoke, commit artifacts, document methodology, HOLD comparison marketing until llm-judge exists.
+</objective>
diff --git a/.planning/phases/56-honest-benchmarks/56-CONTEXT.md b/.planning/phases/56-honest-benchmarks/56-CONTEXT.md
@@ -0,0 +1,34 @@
+# Phase 56: Honest Benchmarks - Context
+
+**Gathered:** 2026-08-30
+**Status:** In execution
+**Source:** docs/plans/v3.1-make-it-true-plan.md
+
+## Phase Boundary
+
+A benchmark story that survives ten minutes of hostile review — or no
+benchmark story at all. No new retrieval capabilities.
+
+## What was wrong (v3.0)
+
+- `recall_at_5` == `accuracy` (one bool per test / test count)
+- `compression_ratio` summed **path-string** lengths, not file contents
+- `memory add`/`search` failures swallowed (`let _ =`); dead daemon → accuracy 0.0
+- Shared store across tests and LOCOMO conversations
+- 4 tests / ~60 lines
+- LOCOMO adapter invented `conversation_id`/`turns`/`questions`; download URL 404s
+- Substring scoring labeled as if it were LOCOMO
+
+## Decisions
+
+- Real recall@k against `relevant` labels; omit the metric when empty
+- Compression reads setup file contents (`ceil(chars/4)`)
+- Fail loud on CLI errors
+- Mock backend: isolated store per test / per LOCOMO conversation (tested)
+- CLI backend: fail loud; isolation is "fresh daemon" (operator); documented
+- ≥25 fixtures across temporal / multi / compress with distractor sessions
+- Real locomo10.json schema; numeric answers; category 1–5 map
+- Substring metric named `context_hit_rate`; never `--compare`
+- LLM-as-judge is the only `locomo_llm_judge` path (temp 0, model recorded)
+- Decision gate: HOLD comparison marketing until a real llm-judge artifact exists
+- CI smoke runs the 1-conversation fixture with mock judge (not `--help`)
diff --git a/.planning/phases/56-honest-benchmarks/56-VERIFICATION.md b/.planning/phases/56-honest-benchmarks/56-VERIFICATION.md
@@ -0,0 +1,38 @@
+---
+phase: 56-honest-benchmarks
+verified: 2026-08-30
+status: passed
+---
+
+# Phase 56: Honest Benchmarks Verification
+
+**Phase Goal:** a benchmark story that survives hostile review — or no story at all.
+
+## Execution evidence
+
+| # | Truth | Status | Evidence |
+|---|-------|--------|----------|
+| 1 | recall@k ≠ accuracy | RUN | `compute_recall_at_k` unit test; committed `custom-harness-mock.json`: accuracy=0.88, recall_at_k=0.86 |
+| 2 | compression uses file contents | UNIT | `estimate_raw_tokens_reads_file_contents_not_paths`; report notes "raw = setup file *contents*" |
+| 3 | CLI failures abort | UNIT | `cli_ingest_fails_loud_when_binary_missing` |
+| 4 | Per-test / per-conversation isolation | UNIT + RUN | `mock_stores_do_not_bleed`; `isolation_no_cross_conversation_bleed`; smoke `isolation` field |
+| 5 | ≥25 fixtures | RUN | 25 tests in `custom-harness-mock.json`; `committed_fixtures_are_at_least_25` |
+| 6 | Real locomo10.json schema | UNIT | `real_schema_parses_including_numeric_answer`; `invented_v1_schema_is_rejected` |
+| 7 | Substring is not a LOCOMO score | RUN | smoke `metric=context_hit_rate`; `--compare` exits non-zero for mock scorer |
+| 8 | CI smoke executes the pipeline | CODE + RUN | `memory-bench smoke` writes results; local run 1 conversation / 4 questions |
+| 9 | Download URL is GitHub locomo10.json | CODE | `benchmarks/scripts/download-locomo.sh` fetches LICENSE.txt then data/locomo10.json |
+
+## Committed artifacts
+
+- [`benchmarks/results/custom-harness-mock.json`](../../../benchmarks/results/custom-harness-mock.json) — 25 tests, backend=mock, 22/25, failed=`compress-001,compress-003,multi-004`. **Not a production quality number.**
+- [`benchmarks/results/locomo-smoke.json`](../../../benchmarks/results/locomo-smoke.json) — 1 conversation, `metric=context_hit_rate`, overall_score=0.5 (2/4). **Not a LOCOMO score.**
+
+## Decision gate
+
+**HOLD comparison marketing.** No `locomo_llm_judge` artifact. Full `locomo10.json` + API judge was not run (no key). README must not claim a LOCOMO score.
+
+## Human verification (blockers)
+
+- [x] Committed results JSON produced by actually running the adapter
+- [x] Metric field is `context_hit_rate`, not an unlabeled LOCOMO score
+- [x] `--compare` refused for mock scorer
diff --git a/benchmarks/baselines.toml b/benchmarks/baselines.toml
@@ -1,14 +1,17 @@
-# Manually-maintained competitor benchmark scores.
-# Sources listed per entry.
+# Manually-maintained competitor numbers.
+# Each row names its metric. These are NOT commensurable with each other
+# or with Agent-Memory's custom-harness expected_contains accuracy.
 
 [memmachine]
 # Source: https://memmachine.ai/blog/2025/12/memmachine-v0.2-delivers-top-scores-and-efficiency-on-locomo-benchmark/
 locomo_score = 0.91
 token_reduction = 0.80
 latency_improvement = 0.75
+metric = "LOCOMO LLM-judge (their paper)"
 
 [mem0]
 # Source: https://mem0.ai/research
 accuracy_vs_openai_memory = 0.26
 token_reduction = 0.90
 latency_reduction = 0.91
+metric = "relative delta vs OpenAI memory"
diff --git a/benchmarks/fixtures/compression-001.toml b/benchmarks/fixtures/compression-001.toml
@@ -1,7 +1,54 @@
 [[test]]
 id = "compress-001"
-description = "Verify context is compressed vs raw session dump"
+category = "compress"
+description = "Verify context is compressed vs raw long-session dump"
 setup = ["sessions/long-session.jsonl"]
 query = "summarize the key decisions from this project"
-expected_contains = ["decision", "architecture"]
+expected_contains = ["RabbitMQ"]
+relevant = ["RabbitMQ", "Handlebars", "digest mode"]
+k = 5
 max_tokens = 800
+
+[[test]]
+id = "compress-002"
+category = "compress"
+description = "Long session plus distractors still compresses vs full file contents"
+setup = ["sessions/long-session.jsonl", "sessions/distractor-cooking.jsonl", "sessions/distractor-travel.jsonl"]
+query = "how do we handle notification delivery failures?"
+expected_contains = ["Dead letter"]
+relevant = ["Dead letter queue", "circuit breakers"]
+k = 5
+max_tokens = 800
+
+[[test]]
+id = "compress-003"
+category = "compress"
+description = "Auth + cache + k8s corpus is larger than retrieved snippets"
+setup = ["sessions/auth-decision.jsonl", "sessions/session-a.jsonl", "sessions/k8s-migration.jsonl", "sessions/long-session.jsonl"]
+query = "what did we decide about JWT expiry?"
+expected_contains = ["15 minutes"]
+relevant = ["15 minutes", "7 days"]
+k = 5
+max_tokens = 600
+
+[[test]]
+id = "compress-004"
+category = "compress"
+description = "Incident writeup retrieved from a mixed ops corpus"
+setup = ["sessions/incident.jsonl", "sessions/k8s-migration.jsonl", "sessions/oncall.jsonl", "sessions/long-session.jsonl"]
+query = "what PDB did we add after the checkout outage?"
+expected_contains = ["minAvailable"]
+relevant = ["minAvailable=2", "WhenEmpty"]
+k = 5
+max_tokens = 600
+
+[[test]]
+id = "compress-005"
+category = "compress"
+description = "Feature-flag decision retrieved from a fat mixed corpus"
+setup = ["sessions/feature-flags.jsonl", "sessions/job-queue.jsonl", "sessions/tracing.jsonl", "sessions/long-session.jsonl", "sessions/distractor-sports.jsonl"]
+query = "why did we reject LaunchDarkly?"
+expected_contains = ["cost"]
+relevant = ["8x the cost", "Unleash"]
+k = 5
+max_tokens = 600