diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dfce9ab..62b454b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -193,7 +193,6 @@ jobs:
benchmark-smoke:
name: Benchmark Suite Smoke Test
runs-on: ubuntu-24.04
- continue-on-error: true
needs: [test]
steps:
- uses: actions/checkout@v4
@@ -213,19 +212,27 @@ jobs:
with:
shared-key: "bench-smoke"
- - name: Build memory-bench
- run: cargo build -p memory-bench
+ - name: Test memory-bench
+ run: cargo test -p memory-bench
- - name: Smoke test (help only — no daemon required)
+ - name: Smoke locomo adapter (1 conversation, mock judge)
run: |
- cargo run -p memory-bench -- --help
- cargo run -p memory-bench -- all --help
- cargo run -p memory-bench -- locomo --help
+ cargo run -p memory-bench -- smoke --output /tmp/locomo-smoke.json
+ python3 - <<'PY'
+ import json, sys
+ d = json.load(open("/tmp/locomo-smoke.json"))
+ assert d["conversations"] == 1, d
+ assert d["total_questions"] >= 1, d
+ assert d["metric"] == "context_hit_rate", d
+ assert "locomo_llm_judge" != d["metric"]
+ print("smoke ok", d["metric"], "questions", d["total_questions"])
+ PY
+
# Summary job that depends on all other jobs
ci-success:
name: CI Success
- needs: [fmt, clippy, test, build, doc, e2e]
+ needs: [fmt, clippy, test, build, doc, e2e, benchmark-smoke]
runs-on: ubuntu-24.04
if: always()
steps:
@@ -236,7 +243,8 @@ jobs:
[[ "${{ needs.test.result }}" != "success" ]] || \
[[ "${{ needs.build.result }}" != "success" ]] || \
[[ "${{ needs.doc.result }}" != "success" ]] || \
- [[ "${{ needs.e2e.result }}" != "success" ]]; then
+ [[ "${{ needs.e2e.result }}" != "success" ]] || \
+ [[ "${{ needs.benchmark-smoke.result }}" != "success" ]]; then
echo "One or more jobs failed"
exit 1
fi
diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md
index 9a164ef..dae9a06 100644
--- a/.planning/ROADMAP.md
+++ b/.planning/ROADMAP.md
@@ -12,7 +12,7 @@
- ✅ **v2.6 Cognitive Retrieval** — Phases 39-44 (shipped 2026-03-16)
- ✅ **v2.7 Multi-Runtime Portability** — Phases 45-50 (shipped 2026-03-22)
- **v3.0 Competitive Parity & Benchmarks** — Phases 51-53 + Phase 51.5 (in progress; Phase 51.5 merged 2026-04-28)
-- **v3.1 Make It True** — Phases 54-58 (in progress; Phase 54 merged 2026-08-30, Phase 55 merged 2026-08-30, Phase 54.5 executing)
+- **v3.1 Make It True** — Phases 54-58 (in progress; Phases 54, 54.5, 55 merged 2026-08-30, Phase 56 executing)
## Phases
@@ -267,20 +267,27 @@ Close the claim/reality gap, then open the shop window. No new capabilities.
- [x] 54-05: Honest daemon flags + attach indexes
- [x] 54-06: Lock-poisoning recover_lock policy
-### Phase 54.5: Truth leaks + CI pin (in execution 2026-08-30)
+### Phase 54.5: Truth leaks + CI pin — COMPLETE 2026-08-30 (PR #35)
-- [ ] CI: pin rust-toolchain 1.97; allow `result_large_err` on generated proto
-- [ ] Explainability reports what actually ran
-- [ ] LLM rerank order survives salience; BM25 events carry text
-- [ ] Shared HNSW handle; concurrent fan-out; no per-event grip scan
+- [x] CI: pin rust-toolchain 1.97; allow `result_large_err` on generated proto
+- [x] Explainability reports what actually ran
+- [x] LLM rerank order survives salience; BM25 events carry text
+- [x] Shared HNSW handle; concurrent fan-out; no per-event grip scan
-### Phase 55: Performance Truth — COMPLETE 2026-08-30 (PR #33)
+### Phase 55: Performance Truth (2/2 plans) — COMPLETE 2026-08-30 (PR #33)
-### Phase 56: Honest Benchmarks (PR #34 open)
+- [x] 55-01: Split setup vs query (64.6s was toc_build)
+- [x] 55-02: Honest percentiles (n≥10 / n≥30)
+
+### Phase 56: Honest Benchmarks (3/3 plans) — IN EXECUTION 2026-08-30
+
+- [x] 56-01: Custom harness (real recall@k, content compression, fail-loud, isolation, ≥25 fixtures)
+- [x] 56-02: LOCOMO adapter v2 (real schema, mock vs llm-judge)
+- [x] 56-03: Smoke artifacts + HOLD comparison marketing
### Phase 57: Shop Window & Positioning (0/3)
### Phase 58: Launch (side quest)
-*Updated: 2026-08-30 — Phase 54.5 truth-leaks + CI pin in execution*
+*Updated: 2026-08-30 — Phase 54.5 merged (#35); Phase 56 Honest Benchmarks in execution*
diff --git a/.planning/STATE.md b/.planning/STATE.md
index b69e950..d7c6b2d 100644
--- a/.planning/STATE.md
+++ b/.planning/STATE.md
@@ -3,14 +3,14 @@ gsd_state_version: 1.0
milestone_name: Make It True
status: in_progress
stopped_at: null
-last_updated: "2026-08-30T19:10:00.000Z"
-last_activity: 2026-08-30 — Phase 54.5 truth-leaks + CI toolchain pin
+last_updated: "2026-08-30T20:45:00.000Z"
+last_activity: 2026-08-30 — Phase 54.5 merged (#35); Phase 56 Honest Benchmarks rebasing
progress:
total_phases: 6
- completed_phases: 2
+ completed_phases: 3
total_plans: 14
- completed_plans: 8
- percent: 57
+ completed_plans: 11
+ percent: 79
---
# Project State
@@ -20,16 +20,16 @@ progress:
See: .planning/PROJECT.md (updated 2026-03-22)
**Core value:** Agent can answer "what were we talking about last week?" without scanning everything
-**Current focus:** v3.1 Phase 54.5 — close residual Phase 54 honesty leaks and pin CI toolchain
+**Current focus:** v3.1 Phase 56 — Honest Benchmarks (real recall@k, locomo10.json schema, HOLD comparison marketing)
## Current Position
-Phase: 54.5 of 58 (Truth leaks + CI pin)
-Plan: implementing on `feature/phase-54.5-truth-leaks`
-Status: Phase 54 merged (#32); Phase 55 merged (#33); Phase 56 PR #34 open (clippy red from toolchain drift)
-Last activity: 2026-08-30 — explainability truth, shared HNSW, rust-toolchain.toml 1.97
+Phase: 56 of 58 (Honest Benchmarks)
+Plan: 01-03 implemented on `feature/phase-56-honest-benchmarks` (PR #34)
+Status: Phase 54 + 54.5 + 55 merged; Phase 56 rebase onto #35
+Last activity: 2026-08-30 — #35 merged (clippy pin); rebasing #34
-Progress: [██████░░░░] ~57% (8/14 plans; Phase 54.5 cleanup)
+Progress: [████████░░] ~79% (11/14 plans; Phase 56 of 54-58)
## Out-of-band Work
@@ -37,12 +37,13 @@ Progress: [██████░░░░] ~57% (8/14 plans; Phase 54.5 cleanup)
| PR | What | Status |
|---|---|---|
-| #34 | Phase 56 Honest Benchmarks | Open; Clippy red (1.98 `result_large_err` on generated tonic stubs) |
+| #34 | Phase 56 Honest Benchmarks | Open; rebasing onto #35 |
### Recently Merged
| PR | What | Merged |
|---|---|---|
+| #35 | Phase 54.5 truth leaks + rustc 1.97 pin | 2026-08-30 |
| #33 | Phase 55 Performance Truth | 2026-08-30 |
| #32 | Phase 54 Integration Truth | 2026-08-30 |
| #31 | v3.1 Make It True design spec | 2026-08-30 |
@@ -54,5 +55,7 @@ Progress: [██████░░░░] ~57% (8/14 plans; Phase 54.5 cleanup)
## Decisions
- v3.1 scope: Make It True — no new capabilities; close claim/reality gap (Phases 54-58)
-- Phase 54.5 before more measurement: explainability must report what ran; shared HNSW handle
-- CI pins `rust-toolchain.toml` to 1.97 so floating stable cannot redden main
+- Phase 54.5: explainability reports what ran; shared HNSW handle; CI pins rust-toolchain.toml to 1.97
+- Phase 55: split setup vs query in `perf_bench`; p90/p99 withheld below 10/30 samples
+- Warm = one setup + N query samples; cold = new store per iteration
+- Phase 56: substring metric is `context_hit_rate`; HOLD LOCOMO comparison marketing until `locomo_llm_judge` artifact exists
diff --git a/.planning/phases/56-honest-benchmarks/56-01-PLAN.md b/.planning/phases/56-honest-benchmarks/56-01-PLAN.md
new file mode 100644
index 0000000..4fb5776
--- /dev/null
+++ b/.planning/phases/56-honest-benchmarks/56-01-PLAN.md
@@ -0,0 +1,19 @@
+---
+phase: 56-honest-benchmarks
+plan: 01
+type: execute
+wave: 1
+depends_on: []
+files_modified:
+ - crates/memory-bench/src/scorer.rs
+ - crates/memory-bench/src/runner.rs
+ - crates/memory-bench/src/fixture.rs
+ - crates/memory-bench/src/report.rs
+ - crates/memory-bench/src/main.rs
+ - benchmarks/fixtures/
+autonomous: true
+---
+
+
+Fix the custom harness: real recall@k, content-based compression, fail-loud CLI, per-test isolation, ≥25 fixtures.
+
diff --git a/.planning/phases/56-honest-benchmarks/56-02-PLAN.md b/.planning/phases/56-honest-benchmarks/56-02-PLAN.md
new file mode 100644
index 0000000..2bd9e39
--- /dev/null
+++ b/.planning/phases/56-honest-benchmarks/56-02-PLAN.md
@@ -0,0 +1,18 @@
+---
+phase: 56-honest-benchmarks
+plan: 02
+type: execute
+wave: 1
+depends_on: [01]
+files_modified:
+ - crates/memory-bench/src/locomo.rs
+ - crates/memory-bench/src/judge.rs
+ - benchmarks/scripts/download-locomo.sh
+ - benchmarks/fixtures/locomo-smoke.json
+ - crates/memory-cli/src/commands/add.rs
+autonomous: true
+---
+
+
+LOCOMO adapter v2: real locomo10.json schema, isolated store per conversation, mock vs llm-judge, honest metric names.
+
diff --git a/.planning/phases/56-honest-benchmarks/56-03-PLAN.md b/.planning/phases/56-honest-benchmarks/56-03-PLAN.md
new file mode 100644
index 0000000..44c4135
--- /dev/null
+++ b/.planning/phases/56-honest-benchmarks/56-03-PLAN.md
@@ -0,0 +1,17 @@
+---
+phase: 56-honest-benchmarks
+plan: 03
+type: execute
+wave: 1
+depends_on: [01, 02]
+files_modified:
+ - benchmarks/results/
+ - docs/benchmarks.md
+ - .github/workflows/ci.yml
+ - benchmarks/baselines.toml
+autonomous: true
+---
+
+
+Run smoke, commit artifacts, document methodology, HOLD comparison marketing until llm-judge exists.
+
diff --git a/.planning/phases/56-honest-benchmarks/56-CONTEXT.md b/.planning/phases/56-honest-benchmarks/56-CONTEXT.md
new file mode 100644
index 0000000..e7690ad
--- /dev/null
+++ b/.planning/phases/56-honest-benchmarks/56-CONTEXT.md
@@ -0,0 +1,34 @@
+# Phase 56: Honest Benchmarks - Context
+
+**Gathered:** 2026-08-30
+**Status:** In execution
+**Source:** docs/plans/v3.1-make-it-true-plan.md
+
+## Phase Boundary
+
+A benchmark story that survives ten minutes of hostile review — or no
+benchmark story at all. No new retrieval capabilities.
+
+## What was wrong (v3.0)
+
+- `recall_at_5` == `accuracy` (one bool per test / test count)
+- `compression_ratio` summed **path-string** lengths, not file contents
+- `memory add`/`search` failures swallowed (`let _ =`); dead daemon → accuracy 0.0
+- Shared store across tests and LOCOMO conversations
+- 4 tests / ~60 lines
+- LOCOMO adapter invented `conversation_id`/`turns`/`questions`; download URL 404s
+- Substring scoring labeled as if it were LOCOMO
+
+## Decisions
+
+- Real recall@k against `relevant` labels; omit the metric when empty
+- Compression reads setup file contents (`ceil(chars/4)`)
+- Fail loud on CLI errors
+- Mock backend: isolated store per test / per LOCOMO conversation (tested)
+- CLI backend: fail loud; isolation is "fresh daemon" (operator); documented
+- ≥25 fixtures across temporal / multi / compress with distractor sessions
+- Real locomo10.json schema; numeric answers; category 1–5 map
+- Substring metric named `context_hit_rate`; never `--compare`
+- LLM-as-judge is the only `locomo_llm_judge` path (temp 0, model recorded)
+- Decision gate: HOLD comparison marketing until a real llm-judge artifact exists
+- CI smoke runs the 1-conversation fixture with mock judge (not `--help`)
diff --git a/.planning/phases/56-honest-benchmarks/56-VERIFICATION.md b/.planning/phases/56-honest-benchmarks/56-VERIFICATION.md
new file mode 100644
index 0000000..b74cf9d
--- /dev/null
+++ b/.planning/phases/56-honest-benchmarks/56-VERIFICATION.md
@@ -0,0 +1,38 @@
+---
+phase: 56-honest-benchmarks
+verified: 2026-08-30
+status: passed
+---
+
+# Phase 56: Honest Benchmarks Verification
+
+**Phase Goal:** a benchmark story that survives hostile review — or no story at all.
+
+## Execution evidence
+
+| # | Truth | Status | Evidence |
+|---|-------|--------|----------|
+| 1 | recall@k ≠ accuracy | RUN | `compute_recall_at_k` unit test; committed `custom-harness-mock.json`: accuracy=0.88, recall_at_k=0.86 |
+| 2 | compression uses file contents | UNIT | `estimate_raw_tokens_reads_file_contents_not_paths`; report notes "raw = setup file *contents*" |
+| 3 | CLI failures abort | UNIT | `cli_ingest_fails_loud_when_binary_missing` |
+| 4 | Per-test / per-conversation isolation | UNIT + RUN | `mock_stores_do_not_bleed`; `isolation_no_cross_conversation_bleed`; smoke `isolation` field |
+| 5 | ≥25 fixtures | RUN | 25 tests in `custom-harness-mock.json`; `committed_fixtures_are_at_least_25` |
+| 6 | Real locomo10.json schema | UNIT | `real_schema_parses_including_numeric_answer`; `invented_v1_schema_is_rejected` |
+| 7 | Substring is not a LOCOMO score | RUN | smoke `metric=context_hit_rate`; `--compare` exits non-zero for mock scorer |
+| 8 | CI smoke executes the pipeline | CODE + RUN | `memory-bench smoke` writes results; local run 1 conversation / 4 questions |
+| 9 | Download URL is GitHub locomo10.json | CODE | `benchmarks/scripts/download-locomo.sh` fetches LICENSE.txt then data/locomo10.json |
+
+## Committed artifacts
+
+- [`benchmarks/results/custom-harness-mock.json`](../../../benchmarks/results/custom-harness-mock.json) — 25 tests, backend=mock, 22/25, failed=`compress-001,compress-003,multi-004`. **Not a production quality number.**
+- [`benchmarks/results/locomo-smoke.json`](../../../benchmarks/results/locomo-smoke.json) — 1 conversation, `metric=context_hit_rate`, overall_score=0.5 (2/4). **Not a LOCOMO score.**
+
+## Decision gate
+
+**HOLD comparison marketing.** No `locomo_llm_judge` artifact. Full `locomo10.json` + API judge was not run (no key). README must not claim a LOCOMO score.
+
+## Human verification (blockers)
+
+- [x] Committed results JSON produced by actually running the adapter
+- [x] Metric field is `context_hit_rate`, not an unlabeled LOCOMO score
+- [x] `--compare` refused for mock scorer
diff --git a/benchmarks/baselines.toml b/benchmarks/baselines.toml
index 4d16044..6d47101 100644
--- a/benchmarks/baselines.toml
+++ b/benchmarks/baselines.toml
@@ -1,14 +1,17 @@
-# Manually-maintained competitor benchmark scores.
-# Sources listed per entry.
+# Manually-maintained competitor numbers.
+# Each row names its metric. These are NOT commensurable with each other
+# or with Agent-Memory's custom-harness expected_contains accuracy.
[memmachine]
# Source: https://memmachine.ai/blog/2025/12/memmachine-v0.2-delivers-top-scores-and-efficiency-on-locomo-benchmark/
locomo_score = 0.91
token_reduction = 0.80
latency_improvement = 0.75
+metric = "LOCOMO LLM-judge (their paper)"
[mem0]
# Source: https://mem0.ai/research
accuracy_vs_openai_memory = 0.26
token_reduction = 0.90
latency_reduction = 0.91
+metric = "relative delta vs OpenAI memory"
diff --git a/benchmarks/fixtures/compression-001.toml b/benchmarks/fixtures/compression-001.toml
index c4313e1..543a374 100644
--- a/benchmarks/fixtures/compression-001.toml
+++ b/benchmarks/fixtures/compression-001.toml
@@ -1,7 +1,54 @@
[[test]]
id = "compress-001"
-description = "Verify context is compressed vs raw session dump"
+category = "compress"
+description = "Verify context is compressed vs raw long-session dump"
setup = ["sessions/long-session.jsonl"]
query = "summarize the key decisions from this project"
-expected_contains = ["decision", "architecture"]
+expected_contains = ["RabbitMQ"]
+relevant = ["RabbitMQ", "Handlebars", "digest mode"]
+k = 5
max_tokens = 800
+
+[[test]]
+id = "compress-002"
+category = "compress"
+description = "Long session plus distractors still compresses vs full file contents"
+setup = ["sessions/long-session.jsonl", "sessions/distractor-cooking.jsonl", "sessions/distractor-travel.jsonl"]
+query = "how do we handle notification delivery failures?"
+expected_contains = ["Dead letter"]
+relevant = ["Dead letter queue", "circuit breakers"]
+k = 5
+max_tokens = 800
+
+[[test]]
+id = "compress-003"
+category = "compress"
+description = "Auth + cache + k8s corpus is larger than retrieved snippets"
+setup = ["sessions/auth-decision.jsonl", "sessions/session-a.jsonl", "sessions/k8s-migration.jsonl", "sessions/long-session.jsonl"]
+query = "what did we decide about JWT expiry?"
+expected_contains = ["15 minutes"]
+relevant = ["15 minutes", "7 days"]
+k = 5
+max_tokens = 600
+
+[[test]]
+id = "compress-004"
+category = "compress"
+description = "Incident writeup retrieved from a mixed ops corpus"
+setup = ["sessions/incident.jsonl", "sessions/k8s-migration.jsonl", "sessions/oncall.jsonl", "sessions/long-session.jsonl"]
+query = "what PDB did we add after the checkout outage?"
+expected_contains = ["minAvailable"]
+relevant = ["minAvailable=2", "WhenEmpty"]
+k = 5
+max_tokens = 600
+
+[[test]]
+id = "compress-005"
+category = "compress"
+description = "Feature-flag decision retrieved from a fat mixed corpus"
+setup = ["sessions/feature-flags.jsonl", "sessions/job-queue.jsonl", "sessions/tracing.jsonl", "sessions/long-session.jsonl", "sessions/distractor-sports.jsonl"]
+query = "why did we reject LaunchDarkly?"
+expected_contains = ["cost"]
+relevant = ["8x the cost", "Unleash"]
+k = 5
+max_tokens = 600
diff --git a/benchmarks/fixtures/locomo-smoke.json b/benchmarks/fixtures/locomo-smoke.json
new file mode 100644
index 0000000..270d98f
--- /dev/null
+++ b/benchmarks/fixtures/locomo-smoke.json
@@ -0,0 +1,71 @@
+[
+ {
+ "sample_id": "smoke-001",
+ "qa": [
+ {
+ "question": "When did Caroline go to the LGBTQ support group?",
+ "answer": "7 May 2023",
+ "evidence": ["D1:3"],
+ "category": 2
+ },
+ {
+ "question": "What caching layer did they choose?",
+ "answer": "Redis",
+ "evidence": ["D1:5"],
+ "category": 4
+ },
+ {
+ "question": "What is Melanie's dog named?",
+ "answer": "Biscuit",
+ "evidence": [],
+ "category": 5
+ },
+ {
+ "question": "Which two systems did they combine for auth and cache?",
+ "answer": "JWT and Redis",
+ "evidence": ["D1:4", "D1:5"],
+ "category": 1
+ }
+ ],
+ "conversation": {
+ "speaker_a": "Caroline",
+ "speaker_b": "Melanie",
+ "session_1_date_time": "1:56 pm on 8 May, 2023",
+ "session_1": [
+ {
+ "speaker": "Caroline",
+ "dia_id": "D1:1",
+ "text": "Hey Mel! Good to see you! How have you been?"
+ },
+ {
+ "speaker": "Melanie",
+ "dia_id": "D1:2",
+ "text": "Hey Caroline! Swamped with the kids and work. What's new with you?"
+ },
+ {
+ "speaker": "Caroline",
+ "dia_id": "D1:3",
+ "text": "I went to a LGBTQ support group on 7 May 2023 and it was so powerful."
+ },
+ {
+ "speaker": "Melanie",
+ "dia_id": "D1:4",
+ "text": "That's amazing. At work we finally picked JWT with refresh token rotation for the API."
+ },
+ {
+ "speaker": "Caroline",
+ "dia_id": "D1:5",
+ "text": "Nice. We chose Redis for the caching layer with a 15-minute TTL."
+ }
+ ],
+ "session_2_date_time": "4:04 pm on 20 January, 2023",
+ "session_2": [
+ {
+ "speaker": "Melanie",
+ "dia_id": "D2:1",
+ "text": "I painted a sunrise back in 2022. Nothing to do with dogs."
+ }
+ ]
+ }
+ }
+]
diff --git a/benchmarks/fixtures/multisession-001.toml b/benchmarks/fixtures/multisession-001.toml
index bfe377f..f6794d1 100644
--- a/benchmarks/fixtures/multisession-001.toml
+++ b/benchmarks/fixtures/multisession-001.toml
@@ -1,7 +1,109 @@
[[test]]
id = "multi-001"
+category = "multi"
description = "Connect a decision from session A with an outcome from session B"
-setup = ["sessions/session-a.jsonl", "sessions/session-b.jsonl", "sessions/session-c.jsonl"]
+setup = ["sessions/session-a.jsonl", "sessions/session-b.jsonl", "sessions/session-c.jsonl", "sessions/distractor-cooking.jsonl"]
query = "what was the outcome of the approach we chose last week?"
-expected_contains = ["performance", "latency"]
+expected_contains = ["latency"]
+relevant = ["95% cache hit", "3ms", "Redis"]
+k = 5
max_tokens = 600
+
+[[test]]
+id = "multi-002"
+category = "multi"
+description = "Connect the Kubernetes migration plan with the later outage"
+setup = ["sessions/k8s-migration.jsonl", "sessions/incident.jsonl", "sessions/distractor-sports.jsonl"]
+query = "why did the Kubernetes cutover hurt checkout?"
+expected_contains = ["PodDisruptionBudget"]
+relevant = ["PodDisruptionBudget", "Karpenter", "WhenEmpty"]
+k = 5
+max_tokens = 600
+
+[[test]]
+id = "multi-003"
+category = "multi"
+description = "Connect the bug fix with the follow-up deploy"
+setup = ["sessions/bug-fix.jsonl", "sessions/follow-up.jsonl", "sessions/distractor-travel.jsonl"]
+query = "did the avatar null pointer fix actually ship?"
+expected_contains = ["deployed"]
+relevant = ["deployed yesterday", "Option"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-004"
+category = "multi"
+description = "Auth decision still findable among distractors and a later cache session"
+setup = ["sessions/auth-decision.jsonl", "sessions/session-a.jsonl", "sessions/distractor-cooking.jsonl", "sessions/distractor-sports.jsonl"]
+query = "what did we decide for API authentication?"
+expected_contains = ["JWT"]
+relevant = ["JWT", "refresh token rotation"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-005"
+category = "multi"
+description = "Tracing RFC plus later incident should not confuse Tempo with Karpenter"
+setup = ["sessions/tracing.jsonl", "sessions/incident.jsonl", "sessions/distractor-travel.jsonl"]
+query = "where do we send OpenTelemetry traces?"
+expected_contains = ["Tempo"]
+relevant = ["Grafana Tempo", "OTLP"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-006"
+category = "multi"
+description = "Feature flags vs job queue — keep Unleash distinct from SKIP LOCKED"
+setup = ["sessions/feature-flags.jsonl", "sessions/job-queue.jsonl", "sessions/distractor-cooking.jsonl"]
+query = "what did we buy instead of building feature flags?"
+expected_contains = ["Unleash"]
+relevant = ["Unleash self-hosted"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-007"
+category = "multi"
+description = "On-call plus rate limits — Avery is not a rate-limit number"
+setup = ["sessions/oncall.jsonl", "sessions/rate-limit.jsonl", "sessions/distractor-sports.jsonl"]
+query = "when is the on-call handoff?"
+expected_contains = ["Friday"]
+relevant = ["Friday 16:00 CT", "api-primary"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-008"
+category = "multi"
+description = "Redis cache outcome still beats the original Postgres UNLOGGED alternative"
+setup = ["sessions/session-a.jsonl", "sessions/session-c.jsonl", "sessions/distractor-travel.jsonl"]
+query = "was Redis the right caching call versus PostgreSQL UNLOGGED tables?"
+expected_contains = ["14x"]
+relevant = ["14x latency improvement", "UNLOGGED"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-009"
+category = "multi"
+description = "Job queue throughput numbers survive a distractor-heavy setup"
+setup = ["sessions/job-queue.jsonl", "sessions/distractor-cooking.jsonl", "sessions/distractor-sports.jsonl", "sessions/distractor-travel.jsonl"]
+query = "how many jobs per second did SKIP LOCKED measure?"
+expected_contains = ["4k"]
+relevant = ["4k jobs/sec", "11k"]
+k = 5
+max_tokens = 500
+
+[[test]]
+id = "multi-010"
+category = "multi"
+description = "Checkout rate limit is not the default 120 rpm"
+setup = ["sessions/rate-limit.jsonl", "sessions/session-b.jsonl", "sessions/distractor-cooking.jsonl"]
+query = "what is the checkout endpoint rate limit?"
+expected_contains = ["20"]
+relevant = ["20/min", "checkout"]
+k = 5
+max_tokens = 500
diff --git a/benchmarks/fixtures/sessions/distractor-cooking.jsonl b/benchmarks/fixtures/sessions/distractor-cooking.jsonl
new file mode 100644
index 0000000..954daab
--- /dev/null
+++ b/benchmarks/fixtures/sessions/distractor-cooking.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"What's a good weeknight pasta?"}
+{"role":"assistant","content":"Cacio e pepe. Toast pepper in olive oil, melt pecorino with pasta water, toss spaghetti. Twelve minutes."}
+{"role":"user","content":"Can I use parmesan instead?"}
+{"role":"assistant","content":"Yes, but pecorino is sharper. Add a knob of butter if the sauce seizes."}
+{"role":"user","content":"Great, I'll cook that Thursday."}
+{"role":"assistant","content":"Buy pecorino on the way home. Nothing here is about authentication, Kubernetes, or Redis."}
diff --git a/benchmarks/fixtures/sessions/distractor-sports.jsonl b/benchmarks/fixtures/sessions/distractor-sports.jsonl
new file mode 100644
index 0000000..df96ffc
--- /dev/null
+++ b/benchmarks/fixtures/sessions/distractor-sports.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"Did the Longhorns cover last Saturday?"}
+{"role":"assistant","content":"Yes, Texas 31–17. The run game went over 200 yards. Quinn threw one pick."}
+{"role":"user","content":"Any injury news?"}
+{"role":"assistant","content":"Left tackle is questionable next week. Nothing about JWT, Redis, or Kubernetes in this thread."}
+{"role":"user","content":"Got it, just sports."}
+{"role":"assistant","content":"Sports only. Distractor session for retrieval tests."}
diff --git a/benchmarks/fixtures/sessions/distractor-travel.jsonl b/benchmarks/fixtures/sessions/distractor-travel.jsonl
new file mode 100644
index 0000000..0c662c7
--- /dev/null
+++ b/benchmarks/fixtures/sessions/distractor-travel.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"Should we fly or drive to Tucson next month?"}
+{"role":"assistant","content":"Drive. Two days each way, stop in Las Cruces. Flights into TUS are $420 this month."}
+{"role":"user","content":"Where do we stay?"}
+{"role":"assistant","content":"The usual casita near Campbell. Pack hiking shoes for Sabino Canyon."}
+{"role":"user","content":"Remind me this is a family trip, not a work decision."}
+{"role":"assistant","content":"Family trip to Tucson, drive, casita, Sabino Canyon. No bearing on the API stack."}
diff --git a/benchmarks/fixtures/sessions/feature-flags.jsonl b/benchmarks/fixtures/sessions/feature-flags.jsonl
new file mode 100644
index 0000000..e27bc9d
--- /dev/null
+++ b/benchmarks/fixtures/sessions/feature-flags.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"Do we build our own feature-flag service or buy one?"}
+{"role":"assistant","content":"Buy Unleash self-hosted. Building our own delayed the last two launches. Budget is $400/month."}
+{"role":"user","content":"What about LaunchDarkly?"}
+{"role":"assistant","content":"LaunchDarkly is 8x the cost at our seat count. Unleash self-hosted is the call."}
+{"role":"user","content":"Decision: Unleash self-hosted for feature flags, not LaunchDarkly, not a homegrown service."}
+{"role":"assistant","content":"I'll provision the Unleash Postgres and wire the SDK this week."}
diff --git a/benchmarks/fixtures/sessions/incident.jsonl b/benchmarks/fixtures/sessions/incident.jsonl
new file mode 100644
index 0000000..ab2ed6b
--- /dev/null
+++ b/benchmarks/fixtures/sessions/incident.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"What caused yesterday's 14-minute checkout outage?"}
+{"role":"assistant","content":"A Karpenter node roll deleted the checkout pods before the new nodes were Ready. We had no PodDisruptionBudget."}
+{"role":"user","content":"Fix?"}
+{"role":"assistant","content":"Add a PDB minAvailable=2 on checkout, and pin Karpenter disruption to `WhenEmpty`. Also page on PDB-blocked evictions."}
+{"role":"user","content":"This is why the Kubernetes migration needed a PDB before the EKS cutover."}
+{"role":"assistant","content":"Agreed. The outage is the evidence that Karpenter without a PDB is unsafe."}
diff --git a/benchmarks/fixtures/sessions/job-queue.jsonl b/benchmarks/fixtures/sessions/job-queue.jsonl
new file mode 100644
index 0000000..2696256
--- /dev/null
+++ b/benchmarks/fixtures/sessions/job-queue.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"Redis lists or Postgres SKIP LOCKED for the background job queue?"}
+{"role":"assistant","content":"Postgres SKIP LOCKED. We already have Postgres, and we need transactional enqueue with the order row."}
+{"role":"user","content":"What about throughput?"}
+{"role":"assistant","content":"We measured 4k jobs/sec on SKIP LOCKED vs 11k on Redis lists. We do 200 jobs/sec peak, so Postgres wins on operational simplicity."}
+{"role":"user","content":"Decision: Postgres SKIP LOCKED for the job queue, not Redis lists."}
+{"role":"assistant","content":"I'll add the jobs table and a worker in the API repo."}
diff --git a/benchmarks/fixtures/sessions/k8s-migration.jsonl b/benchmarks/fixtures/sessions/k8s-migration.jsonl
new file mode 100644
index 0000000..5effe42
--- /dev/null
+++ b/benchmarks/fixtures/sessions/k8s-migration.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"Should we migrate the API from ECS to Kubernetes this quarter?"}
+{"role":"assistant","content":"Yes. The plan is a blue-green cutover to EKS in the week of 12 February 2026. We'll keep ECS as fallback for two weeks."}
+{"role":"user","content":"What about cluster autoscaling?"}
+{"role":"assistant","content":"Use Karpenter on EKS with a 30-second node provision target. Cap the node pool at 40 m6i.xlarge instances."}
+{"role":"user","content":"Decision: we are migrating to Kubernetes / EKS with Karpenter, not staying on ECS."}
+{"role":"assistant","content":"Logged. Migration owner is platform-team; rollback is `kubectl` scale to zero plus ECS service restore."}
diff --git a/benchmarks/fixtures/sessions/oncall.jsonl b/benchmarks/fixtures/sessions/oncall.jsonl
new file mode 100644
index 0000000..01b5f7b
--- /dev/null
+++ b/benchmarks/fixtures/sessions/oncall.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"How should we rotate on-call starting next month?"}
+{"role":"assistant","content":"One week primary, one week secondary, handoff Friday 16:00 CT. PagerDuty schedule name is `api-primary`."}
+{"role":"user","content":"Who is first up?"}
+{"role":"assistant","content":"Avery is primary the week of 3 March 2026. Jordan is secondary."}
+{"role":"user","content":"Decision: weekly PagerDuty rotation, Avery first, Friday 16:00 CT handoff."}
+{"role":"assistant","content":"Calendar invite sent."}
diff --git a/benchmarks/fixtures/sessions/rate-limit.jsonl b/benchmarks/fixtures/sessions/rate-limit.jsonl
new file mode 100644
index 0000000..17ce227
--- /dev/null
+++ b/benchmarks/fixtures/sessions/rate-limit.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"What rate limits did we set on the public API?"}
+{"role":"assistant","content":"120 requests per minute per API key, burst 30. Checkout endpoints are 20/min. Health checks are unlimited."}
+{"role":"user","content":"Where is it enforced?"}
+{"role":"assistant","content":"Envoy local rate-limit filter plus a Redis token bucket for the checkout path. 429 with Retry-After."}
+{"role":"user","content":"Decision: 120 rpm default, 20 rpm checkout, Envoy plus Redis token bucket."}
+{"role":"assistant","content":"Config is in infra/envoy/ratelimit.yaml."}
diff --git a/benchmarks/fixtures/sessions/tracing.jsonl b/benchmarks/fixtures/sessions/tracing.jsonl
new file mode 100644
index 0000000..d79045d
--- /dev/null
+++ b/benchmarks/fixtures/sessions/tracing.jsonl
@@ -0,0 +1,6 @@
+{"role":"user","content":"Which tracing library should we standardize on?"}
+{"role":"assistant","content":"OpenTelemetry SDK with the OTLP exporter to Grafana Tempo. Do not add a second vendor SDK."}
+{"role":"user","content":"Sampling?"}
+{"role":"assistant","content":"Parent-based sampler, 5% head sample in prod, 100% on the checkout service."}
+{"role":"user","content":"Decision: OpenTelemetry plus Tempo, 5% head sample."}
+{"role":"assistant","content":"I'll write the tracing RFC tonight."}
diff --git a/benchmarks/fixtures/temporal-001.toml b/benchmarks/fixtures/temporal-001.toml
index 978c164..e272992 100644
--- a/benchmarks/fixtures/temporal-001.toml
+++ b/benchmarks/fixtures/temporal-001.toml
@@ -1,15 +1,109 @@
[[test]]
id = "temporal-001"
+category = "temporal"
description = "Recall an architectural decision made in a prior session"
-setup = ["sessions/auth-decision.jsonl"]
+setup = ["sessions/auth-decision.jsonl", "sessions/distractor-cooking.jsonl"]
query = "what authentication approach did we decide on?"
-expected_contains = ["JWT", "token"]
+expected_contains = ["JWT"]
+relevant = ["JWT", "refresh token rotation"]
+k = 5
max_tokens = 500
[[test]]
id = "temporal-002"
+category = "temporal"
description = "Recall a specific bug fix from two sessions ago"
-setup = ["sessions/bug-fix.jsonl", "sessions/follow-up.jsonl"]
+setup = ["sessions/bug-fix.jsonl", "sessions/follow-up.jsonl", "sessions/distractor-sports.jsonl"]
query = "how did we fix the null pointer exception?"
-expected_contains = ["null check", "Option"]
+expected_contains = ["Option"]
+relevant = ["Option", "DEFAULT_AVATAR_URL"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-003"
+category = "temporal"
+description = "Recall the Kubernetes migration date"
+setup = ["sessions/k8s-migration.jsonl", "sessions/distractor-travel.jsonl"]
+query = "when are we cutting over to Kubernetes?"
+expected_contains = ["EKS"]
+relevant = ["EKS", "Karpenter", "12 February 2026"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-004"
+category = "temporal"
+description = "Recall the tracing library decision"
+setup = ["sessions/tracing.jsonl", "sessions/distractor-cooking.jsonl"]
+query = "which tracing library did we standardize on?"
+expected_contains = ["OpenTelemetry"]
+relevant = ["OpenTelemetry", "Grafana Tempo", "5%"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-005"
+category = "temporal"
+description = "Recall the feature-flag vendor choice"
+setup = ["sessions/feature-flags.jsonl", "sessions/distractor-sports.jsonl"]
+query = "did we buy LaunchDarkly or something else for flags?"
+expected_contains = ["Unleash"]
+relevant = ["Unleash", "LaunchDarkly"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-006"
+category = "temporal"
+description = "Recall who is first on-call"
+setup = ["sessions/oncall.jsonl", "sessions/distractor-travel.jsonl"]
+query = "who is primary on-call the week of 3 March?"
+expected_contains = ["Avery"]
+relevant = ["Avery", "PagerDuty", "Friday 16:00"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-007"
+category = "temporal"
+description = "Recall the job-queue storage decision"
+setup = ["sessions/job-queue.jsonl", "sessions/distractor-cooking.jsonl"]
+query = "are we using Redis lists for the job queue?"
+expected_contains = ["SKIP LOCKED"]
+relevant = ["Postgres SKIP LOCKED", "Redis lists"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-008"
+category = "temporal"
+description = "Recall public API rate limits"
+setup = ["sessions/rate-limit.jsonl", "sessions/distractor-sports.jsonl"]
+query = "what is the default API rate limit per key?"
+expected_contains = ["120"]
+relevant = ["120 requests per minute", "20/min"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-009"
+category = "temporal"
+description = "Recall Redis caching TTL from the original decision"
+setup = ["sessions/session-a.jsonl", "sessions/distractor-travel.jsonl"]
+query = "what TTL did we pick for Redis cache?"
+expected_contains = ["15-minute"]
+relevant = ["15-minute default TTL", "cache-aside"]
+k = 5
+max_tokens = 400
+
+[[test]]
+id = "temporal-010"
+category = "temporal"
+description = "Recall the avatar null-pointer root cause"
+setup = ["sessions/bug-fix.jsonl", "sessions/distractor-cooking.jsonl"]
+query = "why did the user profile handler crash?"
+expected_contains = ["avatar"]
+relevant = ["get_avatar_url", "Option"]
+k = 5
max_tokens = 400
diff --git a/benchmarks/results/custom-harness-mock.json b/benchmarks/results/custom-harness-mock.json
new file mode 100644
index 0000000..09afcf6
--- /dev/null
+++ b/benchmarks/results/custom-harness-mock.json
@@ -0,0 +1,24 @@
+{
+ "backend": "mock",
+ "accuracy": 0.88,
+ "recall_at_k": 0.86,
+ "k": 5,
+ "token_usage_avg": 146,
+ "latency_p50_ms": 0,
+ "latency_p95_ms": 0,
+ "compression_ratio": 0.7112538219110017,
+ "test_count": 25,
+ "pass_count": 22,
+ "failed_ids": [
+ "compress-001",
+ "compress-003",
+ "multi-004"
+ ],
+ "caveats": [
+ "accuracy is expected_contains over this fixture suite, not LOCOMO",
+ "recall@k uses labeled relevant items in top-k, not accuracy under another name",
+ "compression_ratio compares retrieved tokens to setup file contents (not path strings)",
+ "each test ran against a fresh store (mock isolation; cli shares the daemon unless you restart it)",
+ "backend=mock uses in-process token-overlap retrieval; not a production quality number"
+ ]
+}
\ No newline at end of file
diff --git a/benchmarks/results/locomo-smoke.json b/benchmarks/results/locomo-smoke.json
new file mode 100644
index 0000000..6e3bc6c
--- /dev/null
+++ b/benchmarks/results/locomo-smoke.json
@@ -0,0 +1,115 @@
+{
+ "metric": "context_hit_rate",
+ "judge": "mock",
+ "temperature": null,
+ "model": null,
+ "dataset": "benchmarks/fixtures/locomo-smoke.json",
+ "isolation": "per-conversation temp store",
+ "conversations": 1,
+ "total_questions": 4,
+ "overall_score": 0.5,
+ "by_type": {
+ "temporal": {
+ "total": 1,
+ "correct": 1,
+ "score": 1.0
+ },
+ "adversarial": {
+ "total": 1,
+ "correct": 0,
+ "score": 0.0
+ },
+ "multi_hop": {
+ "total": 1,
+ "correct": 0,
+ "score": 0.0
+ },
+ "single_hop": {
+ "total": 1,
+ "correct": 1,
+ "score": 1.0
+ }
+ },
+ "per_conversation": [
+ {
+ "sample_id": "smoke-001",
+ "total_questions": 4,
+ "correct": 2,
+ "score": 0.5,
+ "by_type": {
+ "single_hop": {
+ "total": 1,
+ "correct": 1,
+ "score": 1.0
+ },
+ "adversarial": {
+ "total": 1,
+ "correct": 0,
+ "score": 0.0
+ },
+ "multi_hop": {
+ "total": 1,
+ "correct": 0,
+ "score": 0.0
+ },
+ "temporal": {
+ "total": 1,
+ "correct": 1,
+ "score": 1.0
+ }
+ },
+ "questions": [
+ {
+ "question": "When did Caroline go to the LGBTQ support group?",
+ "gold": "7 May 2023",
+ "category": "temporal",
+ "category_id": 2,
+ "predicted": "[D1:3 @ 1:56 pm on 8 May, 2023] Caroline: I went to a LGBTQ support group on 7 May 2023 and it was so powerful.\n[D1:1 @ 1:56 pm on 8 May, 2023] Caroline: Hey Mel! Good to see you! How have you been?\n[D1:2 @ 1:56 pm on 8 May, 2023] Melanie: Hey Caroline! Swamped with the kids and work. What's new with you?\n[D1:4 @ 1:56 pm on 8 May, 2023] Melanie: That's amazing. At work we finally picked JWT with refresh token rotation for the API.\n[D1:5 @ 1:56 pm on 8 May, 2023] Caroline: Nice. We chose Redis fo",
+ "correct": true,
+ "rationale": "gold substring present in context or predicted text (context_hit_rate)",
+ "latency_ms": 0,
+ "context_tokens": 136
+ },
+ {
+ "question": "What caching layer did they choose?",
+ "gold": "Redis",
+ "category": "single_hop",
+ "category_id": 4,
+ "predicted": "[D1:5 @ 1:56 pm on 8 May, 2023] Caroline: Nice. We chose Redis for the caching layer with a 15-minute TTL.\n[D1:2 @ 1:56 pm on 8 May, 2023] Melanie: Hey Caroline! Swamped with the kids and work. What's new with you?\n[D1:1 @ 1:56 pm on 8 May, 2023] Caroline: Hey Mel! Good to see you! How have you been?\n[D1:3 @ 1:56 pm on 8 May, 2023] Caroline: I went to a LGBTQ support group on 7 May 2023 and it was so powerful.\n[D1:4 @ 1:56 pm on 8 May, 2023] Melanie: That's amazing. At work we finally picked JWT",
+ "correct": true,
+ "rationale": "gold substring present in context or predicted text (context_hit_rate)",
+ "latency_ms": 0,
+ "context_tokens": 136
+ },
+ {
+ "question": "What is Melanie's dog named?",
+ "gold": "Biscuit",
+ "category": "adversarial",
+ "category_id": 5,
+ "predicted": "[D2:1 @ 4:04 pm on 20 January, 2023] Melanie: I painted a sunrise back in 2022. Nothing to do with dogs.\n[D1:2 @ 1:56 pm on 8 May, 2023] Melanie: Hey Caroline! Swamped with the kids and work. What's new with you?\n[D1:4 @ 1:56 pm on 8 May, 2023] Melanie: That's amazing. At work we finally picked JWT with refresh token rotation for the API.\n[D1:5 @ 1:56 pm on 8 May, 2023] Caroline: Nice. We chose Redis for the caching layer with a 15-minute TTL.\n[D1:1 @ 1:56 pm on 8 May, 2023] Caroline: Hey Mel! G",
+ "correct": false,
+ "rationale": "gold substring absent (context_hit_rate miss)",
+ "latency_ms": 0,
+ "context_tokens": 134
+ },
+ {
+ "question": "Which two systems did they combine for auth and cache?",
+ "gold": "JWT and Redis",
+ "category": "multi_hop",
+ "category_id": 1,
+ "predicted": "[D1:2 @ 1:56 pm on 8 May, 2023] Melanie: Hey Caroline! Swamped with the kids and work. What's new with you?\n[D1:3 @ 1:56 pm on 8 May, 2023] Caroline: I went to a LGBTQ support group on 7 May 2023 and it was so powerful.\n[D1:4 @ 1:56 pm on 8 May, 2023] Melanie: That's amazing. At work we finally picked JWT with refresh token rotation for the API.\n[D1:5 @ 1:56 pm on 8 May, 2023] Caroline: Nice. We chose Redis for the caching layer with a 15-minute TTL.\n[D1:1 @ 1:56 pm on 8 May, 2023] Caroline: Hey",
+ "correct": false,
+ "rationale": "gold substring absent (context_hit_rate miss)",
+ "latency_ms": 0,
+ "context_tokens": 136
+ }
+ ]
+ }
+ ],
+ "caveats": [
+ "one isolated mock store per conversation (no cross-conversation bleed)",
+ "metric is 'context_hit_rate' — do not quote as a published LOCOMO leaderboard number unless scorer is llm-judge with a pinned model",
+ "dataset license is CC BY-NC 4.0; verify LICENSE.txt before commercial use",
+ "mock scorer is substring context_hit_rate over token-overlap retrieval; not comparable to Mem0/MemMachine LLM-judge numbers"
+ ]
+}
\ No newline at end of file
diff --git a/benchmarks/scripts/download-locomo.sh b/benchmarks/scripts/download-locomo.sh
index f788983..6dd5584 100755
--- a/benchmarks/scripts/download-locomo.sh
+++ b/benchmarks/scripts/download-locomo.sh
@@ -1,9 +1,43 @@
#!/usr/bin/env bash
+# Fetch the real LoCoMo dataset (locomo10.json) from snap-research/locomo.
+# Prints LICENSE.txt before the data file so a human can confirm CC BY-NC 4.0.
set -euo pipefail
+
DEST="${1:-locomo-data}"
+REPO="https://raw.githubusercontent.com/snap-research/locomo/main"
mkdir -p "$DEST"
-echo "Downloading LOCOMO dataset to $DEST ..."
-curl -L "https://snap-research.github.io/locomo/data/locomo_v1.zip" -o "$DEST/locomo_v1.zip"
-unzip -q "$DEST/locomo_v1.zip" -d "$DEST"
-echo "Done. Dataset at: $DEST"
-echo "NOTE: Verify license terms at https://snap-research.github.io/locomo/ before publishing scores."
+
+echo "Fetching LICENSE.txt from snap-research/locomo ..."
+curl -fsSL "$REPO/LICENSE.txt" -o "$DEST/LICENSE.txt"
+echo
+echo "========== LICENSE.txt (verify before publishing any score) =========="
+sed -n '1,40p' "$DEST/LICENSE.txt"
+echo "======================================================================"
+echo
+echo "LoCoMo is licensed Creative Commons BY-NC 4.0."
+echo "Non-commercial research use only. Do not publish a score for commercial marketing"
+echo "without reading the full license at $DEST/LICENSE.txt"
+echo
+
+echo "Downloading data/locomo10.json ..."
+curl -fL "$REPO/data/locomo10.json" -o "$DEST/locomo10.json"
+
+python3 - "$DEST/locomo10.json" <<'PY' || {
+ echo "python3 sanity check failed; file is at $DEST/locomo10.json — parse it with memory-bench instead."
+ exit 0
+}
+import json, sys
+path = sys.argv[1]
+with open(path) as f:
+ data = json.load(f)
+assert isinstance(data, list), f"expected top-level array, got {type(data)}"
+assert data, "empty dataset"
+sample = data[0]
+assert "qa" in sample and "conversation" in sample, sample.keys()
+print(f"OK: {len(data)} conversations, sample keys={list(sample.keys())}")
+PY
+
+echo "Done. Dataset at: $DEST/locomo10.json"
+echo "Run: cargo run -p memory-bench -- locomo --dataset $DEST --scorer mock"
+echo "LLM-judge (requires OPENAI_API_KEY or ANTHROPIC_API_KEY):"
+echo " cargo run -p memory-bench -- locomo --dataset $DEST --scorer llm-judge"
diff --git a/crates/memory-bench/Cargo.toml b/crates/memory-bench/Cargo.toml
index effb861..3223cf4 100644
--- a/crates/memory-bench/Cargo.toml
+++ b/crates/memory-bench/Cargo.toml
@@ -17,6 +17,8 @@ toml = { workspace = true }
thiserror = { workspace = true }
anyhow = { workspace = true }
tracing = { workspace = true }
+chrono = { workspace = true }
+reqwest = { workspace = true, features = ["blocking"] }
[dependencies.tempfile]
workspace = true
diff --git a/crates/memory-bench/src/baseline.rs b/crates/memory-bench/src/baseline.rs
index 4986dca..ee9a9d2 100644
--- a/crates/memory-bench/src/baseline.rs
+++ b/crates/memory-bench/src/baseline.rs
@@ -9,7 +9,7 @@ pub struct Baselines {
pub mem0: Option,
}
-/// Scores for a single competitor.
+/// Scores for a single competitor. Each row must name its metric.
#[derive(Debug, Deserialize)]
pub struct CompetitorScore {
pub locomo_score: Option,
@@ -17,6 +17,8 @@ pub struct CompetitorScore {
pub latency_improvement: Option,
pub accuracy_vs_openai_memory: Option,
pub latency_reduction: Option,
+ /// What the number actually measures. Required for honest --compare.
+ pub metric: Option,
}
impl Baselines {
@@ -38,11 +40,13 @@ mod tests {
locomo_score = 0.91
token_reduction = 0.80
latency_improvement = 0.75
+metric = "LOCOMO LLM-judge (their paper)"
[mem0]
accuracy_vs_openai_memory = 0.26
token_reduction = 0.90
latency_reduction = 0.91
+metric = "relative delta vs OpenAI memory"
"#;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("baselines.toml");
@@ -52,12 +56,13 @@ latency_reduction = 0.91
let mm = baselines.memmachine.unwrap();
assert_eq!(mm.locomo_score, Some(0.91));
- assert_eq!(mm.token_reduction, Some(0.80));
- assert_eq!(mm.latency_improvement, Some(0.75));
+ assert_eq!(mm.metric.as_deref(), Some("LOCOMO LLM-judge (their paper)"));
let m0 = baselines.mem0.unwrap();
assert_eq!(m0.accuracy_vs_openai_memory, Some(0.26));
- assert_eq!(m0.token_reduction, Some(0.90));
- assert_eq!(m0.latency_reduction, Some(0.91));
+ assert_eq!(
+ m0.metric.as_deref(),
+ Some("relative delta vs OpenAI memory")
+ );
}
}
diff --git a/crates/memory-bench/src/cli.rs b/crates/memory-bench/src/cli.rs
index e56de27..8635af9 100644
--- a/crates/memory-bench/src/cli.rs
+++ b/crates/memory-bench/src/cli.rs
@@ -10,6 +10,14 @@ pub struct Cli {
/// Path to memory binary (default: searches PATH).
#[arg(long, global = true, default_value = "memory")]
pub memory_bin: String,
+
+ /// Retrieval backend: `mock` (isolated in-process) or `cli` (running daemon).
+ #[arg(long, global = true, default_value = "mock")]
+ pub backend: String,
+
+ /// gRPC endpoint for `--backend cli`.
+ #[arg(long, global = true, default_value = "http://127.0.0.1:50051")]
+ pub endpoint: String,
}
/// Available benchmark subcommands.
@@ -50,26 +58,41 @@ pub enum Commands {
/// Output file for JSON results.
#[arg(long)]
output: Option,
- /// Compare against competitor baselines.
+ /// Compare against competitor baselines (labeled, incommensurable).
#[arg(long)]
compare: bool,
/// Path to baselines TOML file.
#[arg(long, default_value = "benchmarks/baselines.toml")]
baselines: String,
},
- /// Run LOCOMO adapter benchmark.
+ /// Run LOCOMO adapter. Substring mode is `context_hit_rate`, not a LOCOMO score.
Locomo {
- /// Path to LOCOMO dataset directory.
+ /// Path to LOCOMO dataset file or directory (`locomo10.json`).
#[arg(long)]
dataset: String,
/// Output file for JSON results.
#[arg(long)]
output: Option,
- /// Compare against competitor baselines.
+ /// `mock` (context_hit_rate) or `llm-judge` (locomo_llm_judge).
+ #[arg(long, default_value = "mock")]
+ scorer: String,
+ /// Top-k retrieved snippets passed to the generator/judge.
+ #[arg(long, default_value_t = 5)]
+ top: usize,
+ /// Compare against competitor baselines. Refused for mock scorer.
#[arg(long)]
compare: bool,
/// Path to baselines TOML file.
#[arg(long, default_value = "benchmarks/baselines.toml")]
baselines: String,
},
+ /// CI smoke: 1-conversation fixture + mock backend + mock judge.
+ Smoke {
+ /// Path to the 1-conversation fixture (real locomo10.json shape).
+ #[arg(long, default_value = "benchmarks/fixtures/locomo-smoke.json")]
+ dataset: String,
+ /// Output file for JSON results.
+ #[arg(long)]
+ output: Option,
+ },
}
diff --git a/crates/memory-bench/src/fixture.rs b/crates/memory-bench/src/fixture.rs
index 5913daf..f0c859e 100644
--- a/crates/memory-bench/src/fixture.rs
+++ b/crates/memory-bench/src/fixture.rs
@@ -20,12 +20,30 @@ pub struct TestCase {
pub setup: Vec,
/// The query to run against the memory system.
pub query: String,
- /// Case-insensitive substrings that the response should contain.
+ /// Case-insensitive substrings that the response should contain (accuracy).
+ #[serde(default)]
pub expected_contains: Vec,
+ /// Labeled relevant items for recall@k. Independent of `expected_contains`.
+ #[serde(default)]
+ pub relevant: Vec,
+ /// k for recall@k (default 5).
+ #[serde(default = "default_k")]
+ pub k: usize,
+ /// Optional category label (temporal / multi / compress).
+ #[serde(default)]
+ pub category: Option,
/// Maximum token budget for the response.
+ #[serde(default = "default_max_tokens")]
pub max_tokens: usize,
}
+fn default_k() -> usize {
+ 5
+}
+fn default_max_tokens() -> usize {
+ 500
+}
+
impl Fixture {
/// Load and validate a fixture from a TOML file.
pub fn load(path: &Path) -> Result {
@@ -87,24 +105,16 @@ description = "recall a decision"
setup = ["sessions/auth.jsonl"]
query = "what auth did we pick?"
expected_contains = ["JWT"]
+relevant = ["JWT", "refresh token rotation"]
max_tokens = 500
-
-[[test]]
-id = "t-002"
-description = "recall a bug fix"
-setup = ["sessions/bug.jsonl"]
-query = "how was the bug fixed?"
-expected_contains = ["Option"]
-max_tokens = 400
"#
)
.unwrap();
let fixture = Fixture::load(&path).unwrap();
- assert_eq!(fixture.tests.len(), 2);
- assert_eq!(fixture.tests[0].id, "t-001");
- assert_eq!(fixture.tests[1].id, "t-002");
- assert_eq!(fixture.tests[0].max_tokens, 500);
+ assert_eq!(fixture.tests.len(), 1);
+ assert_eq!(fixture.tests[0].relevant.len(), 2);
+ assert_eq!(fixture.tests[0].k, 5);
}
#[test]
@@ -165,7 +175,6 @@ max_tokens = 100
fn test_load_dir_collects_all_fixtures() {
let dir = tempfile::tempdir().unwrap();
- // Create first fixture file
let path1 = dir.path().join("a.toml");
let mut f1 = std::fs::File::create(&path1).unwrap();
write!(
@@ -182,7 +191,6 @@ max_tokens = 100
)
.unwrap();
- // Create second fixture file
let path2 = dir.path().join("b.toml");
let mut f2 = std::fs::File::create(&path2).unwrap();
write!(
diff --git a/crates/memory-bench/src/judge.rs b/crates/memory-bench/src/judge.rs
new file mode 100644
index 0000000..0321a48
--- /dev/null
+++ b/crates/memory-bench/src/judge.rs
@@ -0,0 +1,294 @@
+//! Answer generation + judging for LOCOMO.
+//!
+//! Substring containment is `context_hit_rate` — never labeled a LOCOMO score.
+//! LLM-as-judge is the only path that may be labeled `locomo_llm_judge`.
+
+use anyhow::{bail, Context, Result};
+use serde::{Deserialize, Serialize};
+
+/// How an answer was scored.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ScorerKind {
+ /// Case-insensitive gold-answer substring in retrieved context or predicted text.
+ /// Named `context_hit_rate`. Not comparable to published LOCOMO numbers.
+ Mock,
+ /// Generate an answer then judge it with a pinned LLM at temperature 0.
+ LlmJudge,
+}
+
+impl ScorerKind {
+ pub fn parse(s: &str) -> Result {
+ match s {
+ "mock" | "context_hit_rate" => Ok(Self::Mock),
+ "llm-judge" | "llm_judge" => Ok(Self::LlmJudge),
+ other => bail!("unknown scorer '{other}' (expected mock|llm-judge)"),
+ }
+ }
+
+ pub fn metric_name(self) -> &'static str {
+ match self {
+ Self::Mock => "context_hit_rate",
+ Self::LlmJudge => "locomo_llm_judge",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct JudgeVerdict {
+ pub correct: bool,
+ pub rationale: String,
+}
+
+pub trait Judge: Send + Sync {
+ fn kind(&self) -> ScorerKind;
+ fn generate_answer(&self, question: &str, context: &str) -> Result;
+ fn judge(
+ &self,
+ question: &str,
+ gold: &str,
+ predicted: &str,
+ context: &str,
+ ) -> Result;
+}
+
+/// Substring judge. Predicted answer is the retrieved context itself.
+pub struct MockJudge;
+
+impl Judge for MockJudge {
+ fn kind(&self) -> ScorerKind {
+ ScorerKind::Mock
+ }
+
+ fn generate_answer(&self, _question: &str, context: &str) -> Result {
+ Ok(context.to_string())
+ }
+
+ fn judge(
+ &self,
+ _question: &str,
+ gold: &str,
+ predicted: &str,
+ context: &str,
+ ) -> Result {
+ let gold_l = gold.to_lowercase();
+ let hay_pred = predicted.to_lowercase();
+ let hay_ctx = context.to_lowercase();
+ let hit = !gold_l.is_empty() && (hay_pred.contains(&gold_l) || hay_ctx.contains(&gold_l));
+ Ok(JudgeVerdict {
+ correct: hit,
+ rationale: if hit {
+ "gold substring present in context or predicted text (context_hit_rate)".into()
+ } else {
+ "gold substring absent (context_hit_rate miss)".into()
+ },
+ })
+ }
+}
+
+/// LLM-as-judge using the same OpenAI/Anthropic wire protocols as
+/// `memory_toc::summarizer::api::ApiSummarizer`. Temperature is pinned at 0.
+pub struct ApiJudge {
+ pub model: String,
+ pub base_url: String,
+ pub api_key: String,
+ /// 0.0 — recorded in results.json.
+ pub temperature: f64,
+}
+
+impl ApiJudge {
+ pub fn from_env() -> Result {
+ let model =
+ std::env::var("MEMORY_BENCH_JUDGE_MODEL").unwrap_or_else(|_| "gpt-4o-mini".to_string());
+ if let Ok(key) = std::env::var("OPENAI_API_KEY") {
+ if !key.is_empty() {
+ return Ok(Self {
+ model,
+ base_url: std::env::var("OPENAI_BASE_URL")
+ .unwrap_or_else(|_| "https://api.openai.com/v1".into()),
+ api_key: key,
+ temperature: 0.0,
+ });
+ }
+ }
+ if let Ok(key) = std::env::var("ANTHROPIC_API_KEY") {
+ if !key.is_empty() {
+ return Ok(Self {
+ model: std::env::var("MEMORY_BENCH_JUDGE_MODEL")
+ .unwrap_or_else(|_| "claude-3-haiku-20240307".into()),
+ base_url: "https://api.anthropic.com/v1".into(),
+ api_key: key,
+ temperature: 0.0,
+ });
+ }
+ }
+ bail!(
+ "llm-judge requires OPENAI_API_KEY or ANTHROPIC_API_KEY; \
+ use --scorer mock for the CI smoke path"
+ );
+ }
+
+ fn is_anthropic(&self) -> bool {
+ self.base_url.contains("anthropic.com")
+ }
+
+ fn complete(&self, prompt: &str) -> Result {
+ let client = reqwest::blocking::Client::builder()
+ .timeout(std::time::Duration::from_secs(60))
+ .build()
+ .context("building judge HTTP client")?;
+ if self.is_anthropic() {
+ let url = format!("{}/messages", self.base_url.trim_end_matches('/'));
+ let body = serde_json::json!({
+ "model": self.model,
+ "max_tokens": 512,
+ "temperature": self.temperature,
+ "messages": [{"role": "user", "content": prompt}],
+ });
+ let resp = client
+ .post(url)
+ .header("x-api-key", &self.api_key)
+ .header("anthropic-version", "2023-06-01")
+ .json(&body)
+ .send()
+ .context("anthropic judge request")?;
+ let status = resp.status();
+ let v: serde_json::Value = resp.json().context("anthropic judge json")?;
+ if !status.is_success() {
+ bail!("anthropic judge HTTP {status}: {v}");
+ }
+ let text = v
+ .get("content")
+ .and_then(|c| c.as_array())
+ .and_then(|a| a.first())
+ .and_then(|m| m.get("text"))
+ .and_then(|t| t.as_str())
+ .unwrap_or("")
+ .to_string();
+ return Ok(text);
+ }
+ let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/'));
+ let body = serde_json::json!({
+ "model": self.model,
+ "temperature": self.temperature,
+ "messages": [{"role": "user", "content": prompt}],
+ });
+ let resp = client
+ .post(url)
+ .bearer_auth(&self.api_key)
+ .json(&body)
+ .send()
+ .context("openai judge request")?;
+ let status = resp.status();
+ let v: serde_json::Value = resp.json().context("openai judge json")?;
+ if !status.is_success() {
+ bail!("openai judge HTTP {status}: {v}");
+ }
+ let text = v
+ .get("choices")
+ .and_then(|c| c.as_array())
+ .and_then(|a| a.first())
+ .and_then(|m| m.get("message")?.get("content"))
+ .and_then(|t| t.as_str())
+ .unwrap_or("")
+ .to_string();
+ Ok(text)
+ }
+}
+
+const GENERATE_PROMPT: &str = "You answer questions using ONLY the retrieved context. \
+If the context is insufficient, say you don't know. Do not invent facts.\n\n\
+Context:\n{context}\n\nQuestion: {question}\n\nAnswer:";
+
+const JUDGE_PROMPT: &str = "You are a strict binary grader. Compare the predicted answer to the gold answer.\n\
+The predicted answer is correct if it contains the same key facts as the gold answer, allowing for \
+paraphrase and extra context. Reply with JSON only: {\"correct\": true|false, \"rationale\": \"...\"}\n\n\
+Question: {question}\nGold: {gold}\nPredicted: {predicted}\n";
+
+impl Judge for ApiJudge {
+ fn kind(&self) -> ScorerKind {
+ ScorerKind::LlmJudge
+ }
+
+ fn generate_answer(&self, question: &str, context: &str) -> Result {
+ let prompt = GENERATE_PROMPT
+ .replace("{context}", context)
+ .replace("{question}", question);
+ self.complete(&prompt)
+ }
+
+ fn judge(
+ &self,
+ question: &str,
+ gold: &str,
+ predicted: &str,
+ _context: &str,
+ ) -> Result {
+ let prompt = JUDGE_PROMPT
+ .replace("{question}", question)
+ .replace("{gold}", gold)
+ .replace("{predicted}", predicted);
+ let raw = self.complete(&prompt)?;
+ parse_verdict(&raw)
+ }
+}
+
+fn parse_verdict(raw: &str) -> Result {
+ let start = raw.find('{');
+ let end = raw.rfind('}');
+ if let (Some(s), Some(e)) = (start, end) {
+ if e >= s {
+ if let Ok(v) = serde_json::from_str::(&raw[s..=e]) {
+ let correct = v.get("correct").and_then(|c| c.as_bool()).unwrap_or(false);
+ let rationale = v
+ .get("rationale")
+ .and_then(|r| r.as_str())
+ .unwrap_or("")
+ .to_string();
+ return Ok(JudgeVerdict { correct, rationale });
+ }
+ }
+ }
+ let lower = raw.to_lowercase();
+ Ok(JudgeVerdict {
+ correct: lower.contains("\"correct\": true") || lower.contains("correct: true"),
+ rationale: raw.chars().take(240).collect(),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn mock_judge_hits_on_substring() {
+ let j = MockJudge;
+ let v = j
+ .judge("when?", "7 May 2023", "irrelevant", "Went on 7 May 2023")
+ .unwrap();
+ assert!(v.correct);
+ assert!(j.kind().metric_name() == "context_hit_rate");
+ }
+
+ #[test]
+ fn mock_judge_misses_when_absent() {
+ let j = MockJudge;
+ let v = j
+ .judge("when?", "7 May 2023", "no date here", "also no date")
+ .unwrap();
+ assert!(!v.correct);
+ }
+
+ #[test]
+ fn parse_verdict_json() {
+ let v = parse_verdict("sure\n{\"correct\": true, \"rationale\": \"matches\"}\n").unwrap();
+ assert!(v.correct);
+ assert_eq!(v.rationale, "matches");
+ }
+
+ #[test]
+ fn scorer_kind_never_aliases_substring_as_locomo() {
+ assert_eq!(ScorerKind::Mock.metric_name(), "context_hit_rate");
+ assert_eq!(ScorerKind::LlmJudge.metric_name(), "locomo_llm_judge");
+ }
+}
diff --git a/crates/memory-bench/src/lib.rs b/crates/memory-bench/src/lib.rs
index 81083fd..64759cf 100644
--- a/crates/memory-bench/src/lib.rs
+++ b/crates/memory-bench/src/lib.rs
@@ -1,5 +1,6 @@
pub mod baseline;
pub mod fixture;
+pub mod judge;
pub mod locomo;
pub mod report;
pub mod runner;
diff --git a/crates/memory-bench/src/locomo.rs b/crates/memory-bench/src/locomo.rs
index 7499eaf..c686e89 100644
--- a/crates/memory-bench/src/locomo.rs
+++ b/crates/memory-bench/src/locomo.rs
@@ -1,168 +1,451 @@
-//! LOCOMO dataset adapter for benchmark evaluation.
+//! LOCOMO adapter v2 — real `locomo10.json` schema, isolated stores, honest metrics.
//!
-//! Loads conversations from the Snap Research LOCOMO dataset format,
-//! scores answers against gold-standard questions, and aggregates
-//! results with per-question-type breakdowns (single_hop, multi_hop,
-//! temporal, open_domain).
+//! Dataset: (`data/locomo10.json`)
+//! License: CC BY-NC 4.0 (verified at download time).
+//!
+//! Category IDs (from `task_eval/evaluation.py` + data inspection of "When did…"
+//! questions labeled `category: 2`):
+//! 1 = multi_hop, 2 = temporal, 3 = open_domain, 4 = single_hop, 5 = adversarial
-use serde::{Deserialize, Serialize};
+use anyhow::{bail, Context, Result};
+use chrono::{DateTime, NaiveDate, NaiveTime, TimeZone, Utc};
+use serde::Serialize;
+use serde_json::Value;
use std::collections::HashMap;
use std::path::Path;
-use anyhow::Result;
-
-/// A single LOCOMO conversation containing turns and evaluation questions.
-#[derive(Debug, Deserialize, Clone)]
-pub struct LocomoConversation {
- /// Unique identifier for the conversation.
- pub conversation_id: String,
- /// Ordered dialogue turns.
- pub turns: Vec,
- /// Gold-standard questions for evaluation.
- pub questions: Vec,
+use crate::judge::{Judge, JudgeVerdict, ScorerKind};
+use crate::runner::{MockStore, QueryResult, RankedHit, RunConfig};
+
+/// Integer category → stable name used in results.json.
+pub fn category_name(id: i64) -> &'static str {
+ match id {
+ 1 => "multi_hop",
+ 2 => "temporal",
+ 3 => "open_domain",
+ 4 => "single_hop",
+ 5 => "adversarial",
+ _ => "unknown",
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct LocomoSample {
+ pub sample_id: String,
+ pub speaker_a: String,
+ pub speaker_b: String,
+ pub sessions: Vec,
+ pub qa: Vec,
+}
+
+#[derive(Debug, Clone)]
+pub struct LocomoSession {
+ pub index: usize,
+ pub date_time_raw: String,
+ pub timestamp: DateTime,
+ pub turns: Vec,
}
-/// A single dialogue turn in a LOCOMO conversation.
-#[derive(Debug, Deserialize, Clone)]
-pub struct Turn {
- /// Speaker role (e.g., "user", "assistant").
- pub role: String,
- /// Text content of the turn.
- pub content: String,
+#[derive(Debug, Clone)]
+pub struct LocomoTurn {
+ pub speaker: String,
+ pub dia_id: String,
+ pub text: String,
}
-/// A gold-standard evaluation question with expected answer and type.
-#[derive(Debug, Deserialize, Clone)]
-pub struct Question {
- /// The question text.
+#[derive(Debug, Clone)]
+pub struct LocomoQa {
pub question: String,
- /// The expected gold-standard answer.
pub answer: String,
- /// Question type: single_hop, multi_hop, temporal, or open_domain.
- #[serde(rename = "type")]
- pub question_type: String,
+ pub category: i64,
+ pub evidence: Vec,
}
-/// Evaluation result for a single conversation.
-#[derive(Debug, Serialize, Clone)]
-pub struct LocomoResult {
- /// Conversation identifier.
- pub conversation_id: String,
- /// Total number of questions evaluated.
- pub total_questions: usize,
- /// Number of correct answers.
- pub correct: usize,
- /// Overall score (correct / total).
- pub score: f64,
- /// Scores broken down by question type.
- pub by_type: HashMap,
+/// Per-question record written to results.json.
+#[derive(Debug, Clone, Serialize)]
+pub struct QuestionResult {
+ pub question: String,
+ pub gold: String,
+ pub category: String,
+ pub category_id: i64,
+ pub predicted: String,
+ pub correct: bool,
+ pub rationale: String,
+ pub latency_ms: u64,
+ pub context_tokens: usize,
}
-/// Score for a specific question type.
-#[derive(Debug, Serialize, Clone)]
+#[derive(Debug, Clone, Serialize)]
pub struct TypeScore {
- /// Total questions of this type.
pub total: usize,
- /// Correct answers of this type.
pub correct: usize,
- /// Score for this type (correct / total).
pub score: f64,
}
-/// Aggregate result across all conversations.
-#[derive(Debug, Serialize)]
+#[derive(Debug, Clone, Serialize)]
+pub struct LocomoConversationResult {
+ pub sample_id: String,
+ pub total_questions: usize,
+ pub correct: usize,
+ pub score: f64,
+ pub by_type: HashMap,
+ pub questions: Vec,
+}
+
+/// Aggregate across conversations. `metric` is the only name that may be
+/// quoted; it is never a bare "Accuracy" or unlabeled "LOCOMO score".
+#[derive(Debug, Clone, Serialize)]
pub struct LocomoAggregateResult {
- /// Number of conversations evaluated.
+ pub metric: String,
+ pub judge: String,
+ pub temperature: Option,
+ pub model: Option,
+ pub dataset: String,
+ pub isolation: String,
pub conversations: usize,
- /// Total questions across all conversations.
pub total_questions: usize,
- /// Overall score across all conversations.
pub overall_score: f64,
- /// Aggregated scores by question type.
pub by_type: HashMap,
- /// Per-conversation results.
- pub per_conversation: Vec,
+ pub per_conversation: Vec,
+ pub caveats: Vec,
}
-/// Load all LOCOMO conversations from a dataset directory.
+/// Load conversations from a file or directory.
///
-/// Reads all `.json` files in the directory and deserializes them
-/// into `LocomoConversation` structs.
-pub fn load_dataset(dir: &Path) -> Result> {
+/// * File: `locomo10.json` (top-level array) or a single sample object.
+/// * Directory: `locomo10.json` if present, otherwise every `*.json`.
+pub fn load_dataset(path: &Path) -> Result> {
+ if path.is_file() {
+ return load_json_file(path);
+ }
+ let locomo10 = path.join("locomo10.json");
+ if locomo10.is_file() {
+ return load_json_file(&locomo10);
+ }
let mut conversations = Vec::new();
- for entry in std::fs::read_dir(dir)? {
- let path = entry?.path();
- if path.extension().map(|e| e == "json").unwrap_or(false) {
- let content = std::fs::read_to_string(&path)?;
- let conv: LocomoConversation = serde_json::from_str(&content)?;
- conversations.push(conv);
- }
+ let mut entries: Vec<_> = std::fs::read_dir(path)
+ .with_context(|| format!("reading dataset dir {}", path.display()))?
+ .filter_map(|e| e.ok())
+ .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
+ .collect();
+ entries.sort_by_key(|e| e.path());
+ for entry in entries {
+ conversations.extend(load_json_file(&entry.path())?);
}
Ok(conversations)
}
-/// Score a single conversation's questions against retrieved answers.
-///
-/// Uses case-insensitive substring matching: an answer is correct if
-/// the retrieved text contains the gold answer (case-insensitive).
-pub fn score_conversation(conv: &LocomoConversation, answers: &[String]) -> LocomoResult {
- let mut by_type: HashMap = HashMap::new();
- let mut total_correct = 0;
+fn load_json_file(path: &Path) -> Result> {
+ let content =
+ std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
+ let value: Value =
+ serde_json::from_str(&content).with_context(|| format!("parsing {}", path.display()))?;
+ parse_dataset_value(&value)
+}
+
+fn parse_dataset_value(value: &Value) -> Result> {
+ match value {
+ Value::Array(items) => {
+ let mut out = Vec::new();
+ for (i, item) in items.iter().enumerate() {
+ out.push(parse_sample(item, format!("conv-{i}"))?);
+ }
+ Ok(out)
+ }
+ Value::Object(_) => Ok(vec![parse_sample(value, "conv-0".into())?]),
+ other => bail!("LOCOMO dataset must be a JSON array or object, got {other}"),
+ }
+}
- for (i, q) in conv.questions.iter().enumerate() {
- let answer = answers.get(i).map(|s| s.as_str()).unwrap_or("");
- let is_correct = answer.to_lowercase().contains(&q.answer.to_lowercase());
+/// Parse one real LOCOMO sample. Rejects the v3.0 invented schema.
+pub fn parse_sample(value: &Value, fallback_id: String) -> Result {
+ let obj = value
+ .as_object()
+ .ok_or_else(|| anyhow::anyhow!("sample is not an object"))?;
- if is_correct {
- total_correct += 1;
+ // The invented v1 schema used conversation_id / turns / questions.
+ if obj.contains_key("conversation_id") || obj.contains_key("turns") {
+ bail!(
+ "refusing invented v1 LOCOMO schema (conversation_id/turns/questions); \
+ expected locomo10.json sample_id + conversation.session_N + qa"
+ );
+ }
+ let conversation = obj
+ .get("conversation")
+ .and_then(|c| c.as_object())
+ .ok_or_else(|| anyhow::anyhow!("sample missing conversation object"))?;
+
+ let sample_id = obj
+ .get("sample_id")
+ .and_then(|s| s.as_str())
+ .map(|s| s.to_string())
+ .unwrap_or(fallback_id);
+
+ let speaker_a = conversation
+ .get("speaker_a")
+ .and_then(|s| s.as_str())
+ .unwrap_or("")
+ .to_string();
+ let speaker_b = conversation
+ .get("speaker_b")
+ .and_then(|s| s.as_str())
+ .unwrap_or("")
+ .to_string();
+
+ let mut sessions = Vec::new();
+ for n in 1..=64 {
+ let key = format!("session_{n}");
+ let Some(arr) = conversation.get(&key).and_then(|v| v.as_array()) else {
+ continue;
+ };
+ let date_key = format!("session_{n}_date_time");
+ let date_time_raw = conversation
+ .get(&date_key)
+ .and_then(|v| v.as_str())
+ .unwrap_or("")
+ .to_string();
+ let timestamp = parse_locomo_datetime(&date_time_raw).unwrap_or_else(|| {
+ Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0)
+ .single()
+ .unwrap_or_else(Utc::now)
+ });
+ let mut turns = Vec::new();
+ for t in arr {
+ turns.push(LocomoTurn {
+ speaker: t
+ .get("speaker")
+ .and_then(|s| s.as_str())
+ .unwrap_or("")
+ .to_string(),
+ dia_id: t
+ .get("dia_id")
+ .and_then(|s| s.as_str())
+ .unwrap_or("")
+ .to_string(),
+ text: t
+ .get("text")
+ .and_then(|s| s.as_str())
+ .unwrap_or("")
+ .to_string(),
+ });
}
+ sessions.push(LocomoSession {
+ index: n,
+ date_time_raw,
+ timestamp,
+ turns,
+ });
+ }
+ if sessions.is_empty() {
+ bail!("sample {sample_id} has no session_N arrays");
+ }
+
+ let qa_arr = obj
+ .get("qa")
+ .and_then(|v| v.as_array())
+ .ok_or_else(|| anyhow::anyhow!("sample {sample_id} missing qa array"))?;
+ let mut qa = Vec::new();
+ for item in qa_arr {
+ qa.push(LocomoQa {
+ question: item
+ .get("question")
+ .and_then(|s| s.as_str())
+ .unwrap_or("")
+ .to_string(),
+ answer: stringify_answer(item.get("answer")),
+ category: item.get("category").and_then(|c| c.as_i64()).unwrap_or(0),
+ evidence: item
+ .get("evidence")
+ .and_then(|e| e.as_array())
+ .map(|arr| {
+ arr.iter()
+ .filter_map(|x| x.as_str().map(|s| s.to_string()))
+ .collect()
+ })
+ .unwrap_or_default(),
+ });
+ }
- let entry = by_type.entry(q.question_type.clone()).or_insert((0, 0));
- entry.0 += 1; // total
- if is_correct {
- entry.1 += 1; // correct
+ Ok(LocomoSample {
+ sample_id,
+ speaker_a,
+ speaker_b,
+ sessions,
+ qa,
+ })
+}
+
+fn stringify_answer(v: Option<&Value>) -> String {
+ match v {
+ Some(Value::String(s)) => s.clone(),
+ Some(Value::Number(n)) => n.to_string(),
+ Some(Value::Bool(b)) => b.to_string(),
+ Some(Value::Null) | None => String::new(),
+ Some(other) => other.to_string(),
+ }
+}
+
+/// Parse `"1:56 pm on 8 May, 2023"` (LOCOMO session timestamps).
+pub fn parse_locomo_datetime(s: &str) -> Option> {
+ let s = s.trim();
+ if s.is_empty() {
+ return None;
+ }
+ let (time_part, date_part) = s.split_once(" on ")?;
+ let time_part = time_part.trim().to_lowercase();
+ let date_part = date_part.trim().trim_end_matches(',').replace(',', "");
+
+ let mut time_bits = time_part.split_whitespace();
+ let hm = time_bits.next()?;
+ let ampm = time_bits.next().unwrap_or("");
+ let mut hm_bits = hm.split(':');
+ let mut hour: u32 = hm_bits.next()?.parse().ok()?;
+ let minute: u32 = hm_bits.next().unwrap_or("0").parse().ok()?;
+ if ampm.starts_with('p') && hour != 12 {
+ hour += 12;
+ }
+ if ampm.starts_with('a') && hour == 12 {
+ hour = 0;
+ }
+
+ let mut date_bits = date_part.split_whitespace();
+ let day: u32 = date_bits.next()?.parse().ok()?;
+ let month_name = date_bits.next()?.to_lowercase();
+ let year: i32 = date_bits.next()?.parse().ok()?;
+ let month = month_from_name(&month_name)?;
+ let date = NaiveDate::from_ymd_opt(year, month, day)?;
+ let time = NaiveTime::from_hms_opt(hour, minute, 0)?;
+ Some(DateTime::::from_naive_utc_and_offset(
+ date.and_time(time),
+ Utc,
+ ))
+}
+
+fn month_from_name(name: &str) -> Option {
+ Some(match name {
+ "january" | "jan" => 1,
+ "february" | "feb" => 2,
+ "march" | "mar" => 3,
+ "april" | "apr" => 4,
+ "may" => 5,
+ "june" | "jun" => 6,
+ "july" | "jul" => 7,
+ "august" | "aug" => 8,
+ "september" | "sep" | "sept" => 9,
+ "october" | "oct" => 10,
+ "november" | "nov" => 11,
+ "december" | "dec" => 12,
+ _ => return None,
+ })
+}
+
+/// Ingest one conversation into a fresh mock store (isolation).
+pub fn ingest_sample_mock(sample: &LocomoSample) -> MockStore {
+ let mut store = MockStore::new();
+ for session in &sample.sessions {
+ let sid = format!("{}-session-{}", sample.sample_id, session.index);
+ for turn in &session.turns {
+ let text = format!(
+ "[{} @ {}] {}: {}",
+ turn.dia_id, session.date_time_raw, turn.speaker, turn.text
+ );
+ store.ingest_text(&sid, text);
}
}
+ store
+}
+
+/// Retrieve → generate → judge one conversation against an isolated store.
+pub fn evaluate_sample(
+ sample: &LocomoSample,
+ store: &MockStore,
+ judge: &dyn Judge,
+ top_k: usize,
+) -> LocomoConversationResult {
+ let mut questions = Vec::new();
+ let mut by_type: HashMap = HashMap::new();
+ let mut correct_n = 0usize;
- let total = conv.questions.len();
+ for qa in &sample.qa {
+ let retrieved: QueryResult = store.search(&qa.question, top_k);
+ let context = retrieved
+ .ranked
+ .iter()
+ .map(|h: &RankedHit| h.text.as_str())
+ .collect::>()
+ .join("\n");
+ let predicted = judge
+ .generate_answer(&qa.question, &context)
+ .unwrap_or_default();
+ let verdict: JudgeVerdict = judge
+ .judge(&qa.question, &qa.answer, &predicted, &context)
+ .unwrap_or(JudgeVerdict {
+ correct: false,
+ rationale: "judge error".into(),
+ });
+ if verdict.correct {
+ correct_n += 1;
+ }
+ let cat = category_name(qa.category).to_string();
+ let entry = by_type.entry(cat.clone()).or_insert((0, 0));
+ entry.0 += 1;
+ if verdict.correct {
+ entry.1 += 1;
+ }
+ questions.push(QuestionResult {
+ question: qa.question.clone(),
+ gold: qa.answer.clone(),
+ category: cat,
+ category_id: qa.category,
+ predicted: predicted.chars().take(500).collect(),
+ correct: verdict.correct,
+ rationale: verdict.rationale,
+ latency_ms: retrieved.latency_ms,
+ context_tokens: retrieved.tokens_estimated,
+ });
+ }
+
+ let total = sample.qa.len();
let score = if total == 0 {
0.0
} else {
- total_correct as f64 / total as f64
+ correct_n as f64 / total as f64
};
-
let by_type = by_type
.into_iter()
.map(|(k, (t, c))| {
- let s = if t == 0 { 0.0 } else { c as f64 / t as f64 };
(
k,
TypeScore {
total: t,
correct: c,
- score: s,
+ score: if t == 0 { 0.0 } else { c as f64 / t as f64 },
},
)
})
.collect();
- LocomoResult {
- conversation_id: conv.conversation_id.clone(),
+ LocomoConversationResult {
+ sample_id: sample.sample_id.clone(),
total_questions: total,
- correct: total_correct,
+ correct: correct_n,
score,
by_type,
+ questions,
}
}
-/// Aggregate results across all conversations.
-///
-/// Computes overall totals, correct counts, and per-type breakdowns.
-pub fn aggregate_results(results: &[LocomoResult]) -> LocomoAggregateResult {
+pub fn aggregate_results(
+ results: &[LocomoConversationResult],
+ kind: ScorerKind,
+ dataset: &str,
+ judge_label: &str,
+ model: Option,
+ temperature: Option,
+) -> LocomoAggregateResult {
let mut total_questions = 0;
let mut total_correct = 0;
let mut by_type: HashMap = HashMap::new();
-
for r in results {
total_questions += r.total_questions;
total_correct += r.correct;
@@ -172,224 +455,228 @@ pub fn aggregate_results(results: &[LocomoResult]) -> LocomoAggregateResult {
entry.1 += ts.correct;
}
}
-
let overall_score = if total_questions == 0 {
0.0
} else {
total_correct as f64 / total_questions as f64
};
-
let by_type = by_type
.into_iter()
.map(|(k, (t, c))| {
- let s = if t == 0 { 0.0 } else { c as f64 / t as f64 };
(
k,
TypeScore {
total: t,
correct: c,
- score: s,
+ score: if t == 0 { 0.0 } else { c as f64 / t as f64 },
},
)
})
.collect();
+ let mut caveats = vec![
+ "one isolated mock store per conversation (no cross-conversation bleed)".into(),
+ format!(
+ "metric is '{}' — do not quote as a published LOCOMO leaderboard number unless scorer is llm-judge with a pinned model",
+ kind.metric_name()
+ ),
+ "dataset license is CC BY-NC 4.0; verify LICENSE.txt before commercial use".into(),
+ ];
+ if kind == ScorerKind::Mock {
+ caveats.push(
+ "mock scorer is substring context_hit_rate over token-overlap retrieval; \
+ not comparable to Mem0/MemMachine LLM-judge numbers"
+ .into(),
+ );
+ }
+
LocomoAggregateResult {
+ metric: kind.metric_name().to_string(),
+ judge: judge_label.to_string(),
+ temperature,
+ model,
+ dataset: dataset.to_string(),
+ isolation: "per-conversation temp store".into(),
conversations: results.len(),
total_questions,
overall_score,
by_type,
per_conversation: results.to_vec(),
+ caveats,
+ }
+}
+
+/// CLI backend helper: ingest via `memory add --timestamp --session-id`.
+/// Kept so a live daemon run can exercise real timestamps.
+pub fn ingest_sample_cli(sample: &LocomoSample, config: &RunConfig) -> Result {
+ let mut n = 0usize;
+ for session in &sample.sessions {
+ let sid = format!("{}-session-{}", sample.sample_id, session.index);
+ let ts = session.timestamp.to_rfc3339();
+ for turn in &session.turns {
+ let content = format!("{}: {}", turn.speaker, turn.text);
+ let kind = "episodic";
+ let output = std::process::Command::new(&config.memory_bin)
+ .args([
+ "add",
+ "--content",
+ &content,
+ "--kind",
+ kind,
+ "--session-id",
+ &sid,
+ "--timestamp",
+ &ts,
+ "--role",
+ "user",
+ "--endpoint",
+ &config.endpoint,
+ ])
+ .output()
+ .context("spawning memory add for LOCOMO turn")?;
+ if !output.status.success() {
+ bail!(
+ "memory add failed for {} {}: {}",
+ sid,
+ turn.dia_id,
+ String::from_utf8_lossy(&output.stderr)
+ );
+ }
+ n += 1;
+ }
}
+ Ok(n)
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::judge::MockJudge;
- #[test]
- fn test_locomo_conversation_parses() {
- let json = r#"{
- "conversation_id": "conv-001",
- "turns": [
- {"role": "user", "content": "Hello"},
- {"role": "assistant", "content": "Hi there"}
+ const REAL_SHAPE: &str = r#"{
+ "sample_id": "conv-41",
+ "qa": [
+ {
+ "question": "When did Caroline go to the LGBTQ support group?",
+ "answer": "7 May 2023",
+ "evidence": ["D1:3"],
+ "category": 2
+ },
+ {
+ "question": "What activities has Maria done with her church friends?",
+ "answer": "Hiking, picnic, volunteer work",
+ "evidence": ["D25:2", "D24:6"],
+ "category": 1
+ },
+ {
+ "question": "Would John be open to moving?",
+ "answer": "No",
+ "evidence": ["D7:2"],
+ "category": 3
+ },
+ {
+ "question": "When did Melanie paint a sunrise?",
+ "answer": 2022,
+ "evidence": ["D1:12"],
+ "category": 2
+ }
+ ],
+ "conversation": {
+ "speaker_a": "Caroline",
+ "speaker_b": "Melanie",
+ "session_1_date_time": "1:56 pm on 8 May, 2023",
+ "session_1": [
+ {"speaker": "Caroline", "dia_id": "D1:1", "text": "Hey Mel! How have you been?"},
+ {"speaker": "Melanie", "dia_id": "D1:2", "text": "Swamped with the kids."},
+ {"speaker": "Caroline", "dia_id": "D1:3", "text": "I went to a LGBTQ support group yesterday and it was so powerful."}
],
- "questions": [
- {"question": "What did the user say?", "answer": "Hello", "type": "single_hop"}
+ "session_2_date_time": "4:04 pm on 20 January, 2023",
+ "session_2": [
+ {"speaker": "Melanie", "dia_id": "D2:1", "text": "I painted a sunrise back in 2022."}
]
- }"#;
- let conv: LocomoConversation = serde_json::from_str(json).unwrap();
- assert_eq!(conv.conversation_id, "conv-001");
- assert_eq!(conv.turns.len(), 2);
- assert_eq!(conv.questions.len(), 1);
- assert_eq!(conv.questions[0].question_type, "single_hop");
+ }
+ }"#;
+
+ #[test]
+ fn real_schema_parses_including_numeric_answer() {
+ let v: Value = serde_json::from_str(REAL_SHAPE).unwrap();
+ let s = parse_sample(&v, "x".into()).unwrap();
+ assert_eq!(s.sample_id, "conv-41");
+ assert_eq!(s.speaker_a, "Caroline");
+ assert_eq!(s.sessions.len(), 2);
+ assert_eq!(s.sessions[0].turns.len(), 3);
+ assert_eq!(s.qa.len(), 4);
+ assert_eq!(s.qa[3].answer, "2022");
+ assert_eq!(category_name(s.qa[0].category), "temporal");
+ assert_eq!(category_name(s.qa[1].category), "multi_hop");
}
#[test]
- fn test_locomo_conversation_multiple_types() {
- let json = r#"{
- "conversation_id": "conv-002",
- "turns": [{"role": "user", "content": "test"}],
- "questions": [
- {"question": "q1", "answer": "a1", "type": "single_hop"},
- {"question": "q2", "answer": "a2", "type": "multi_hop"},
- {"question": "q3", "answer": "a3", "type": "temporal"},
- {"question": "q4", "answer": "a4", "type": "open_domain"}
- ]
- }"#;
- let conv: LocomoConversation = serde_json::from_str(json).unwrap();
- assert_eq!(conv.questions.len(), 4);
- let types: Vec<&str> = conv
- .questions
- .iter()
- .map(|q| q.question_type.as_str())
- .collect();
- assert!(types.contains(&"single_hop"));
- assert!(types.contains(&"multi_hop"));
- assert!(types.contains(&"temporal"));
- assert!(types.contains(&"open_domain"));
+ fn invented_v1_schema_is_rejected() {
+ let json = r#"{"conversation_id":"conv-001","turns":[{"role":"user","content":"hi"}],"questions":[{"question":"q","answer":"a","type":"single_hop"}]}"#;
+ let v: Value = serde_json::from_str(json).unwrap();
+ let err = parse_sample(&v, "x".into()).unwrap_err().to_string();
+ assert!(err.contains("invented v1"), "{err}");
}
#[test]
- fn test_score_conversation_all_correct() {
- let conv = LocomoConversation {
- conversation_id: "test".to_string(),
- turns: vec![],
- questions: vec![
- Question {
- question: "q1".into(),
- answer: "alpha".into(),
- question_type: "single_hop".into(),
- },
- Question {
- question: "q2".into(),
- answer: "beta".into(),
- question_type: "multi_hop".into(),
- },
- ],
- };
- let answers = vec![
- "The answer is Alpha obviously".to_string(),
- "It was beta all along".to_string(),
- ];
- let result = score_conversation(&conv, &answers);
- assert!((result.score - 1.0).abs() < f64::EPSILON);
- assert_eq!(result.correct, 2);
- assert_eq!(result.total_questions, 2);
+ fn parse_locomo_datetime_pm() {
+ let dt = parse_locomo_datetime("1:56 pm on 8 May, 2023").unwrap();
+ assert_eq!(dt.format("%Y-%m-%dT%H:%M").to_string(), "2023-05-08T13:56");
}
#[test]
- fn test_score_conversation_partial() {
- let conv = LocomoConversation {
- conversation_id: "test".to_string(),
- turns: vec![],
- questions: vec![
- Question {
- question: "q1".into(),
- answer: "alpha".into(),
- question_type: "single_hop".into(),
- },
- Question {
- question: "q2".into(),
- answer: "beta".into(),
- question_type: "single_hop".into(),
- },
- Question {
- question: "q3".into(),
- answer: "gamma".into(),
- question_type: "temporal".into(),
- },
- Question {
- question: "q4".into(),
- answer: "delta".into(),
- question_type: "temporal".into(),
- },
- ],
- };
- let answers = vec![
- "alpha is here".to_string(),
- "no match".to_string(),
- "gamma found".to_string(),
- "wrong answer".to_string(),
- ];
- let result = score_conversation(&conv, &answers);
- assert!((result.score - 0.5).abs() < f64::EPSILON);
- assert_eq!(result.correct, 2);
+ fn isolation_no_cross_conversation_bleed() {
+ let v: Value = serde_json::from_str(REAL_SHAPE).unwrap();
+ let a = parse_sample(&v, "a".into()).unwrap();
+ let store_a = ingest_sample_mock(&a);
+
+ let mut b = a.clone();
+ b.sample_id = "other".into();
+ b.sessions[0].turns[2].text = "I adopted a UNIQUE_ZEBRA_TOKEN last week".into();
+ let store_b = ingest_sample_mock(&b);
+
+ let hits_b = store_b.search("UNIQUE_ZEBRA_TOKEN", 5);
+ assert!(hits_b
+ .ranked
+ .iter()
+ .any(|h| h.text.contains("UNIQUE_ZEBRA_TOKEN")));
+ let hits_a = store_a.search("UNIQUE_ZEBRA_TOKEN", 5);
+ assert!(
+ hits_a
+ .ranked
+ .iter()
+ .all(|h| !h.text.contains("UNIQUE_ZEBRA_TOKEN")),
+ "conversation A must not see B's unique token"
+ );
}
#[test]
- fn test_aggregate_results() {
- let r1 = LocomoResult {
- conversation_id: "c1".into(),
- total_questions: 4,
- correct: 3,
- score: 0.75,
- by_type: HashMap::from([
- (
- "single_hop".into(),
- TypeScore {
- total: 2,
- correct: 2,
- score: 1.0,
- },
- ),
- (
- "temporal".into(),
- TypeScore {
- total: 2,
- correct: 1,
- score: 0.5,
- },
- ),
- ]),
- };
- let r2 = LocomoResult {
- conversation_id: "c2".into(),
- total_questions: 2,
- correct: 1,
- score: 0.5,
- by_type: HashMap::from([
- (
- "single_hop".into(),
- TypeScore {
- total: 1,
- correct: 0,
- score: 0.0,
- },
- ),
- (
- "temporal".into(),
- TypeScore {
- total: 1,
- correct: 1,
- score: 1.0,
- },
- ),
- ]),
- };
- let agg = aggregate_results(&[r1, r2]);
- assert_eq!(agg.conversations, 2);
- assert_eq!(agg.total_questions, 6);
- assert_eq!(agg.by_type["single_hop"].total, 3);
- assert_eq!(agg.by_type["single_hop"].correct, 2);
- assert_eq!(agg.by_type["temporal"].total, 3);
- assert_eq!(agg.by_type["temporal"].correct, 2);
- // overall: 4/6
- assert!((agg.overall_score - 4.0 / 6.0).abs() < 0.001);
+ fn mock_pipeline_parse_ingest_retrieve_score() {
+ let v: Value = serde_json::from_str(REAL_SHAPE).unwrap();
+ let sample = parse_sample(&v, "x".into()).unwrap();
+ let store = ingest_sample_mock(&sample);
+ assert!(store.event_count() > 0);
+ let result = evaluate_sample(&sample, &store, &MockJudge, 5);
+ assert_eq!(result.total_questions, 4);
+ // Gold "7 May 2023" is in session_1 D1:3 context via "yesterday" date? The
+ // text says "yesterday" not "7 May 2023". Date is on the session stamp
+ // "8 May, 2023" — substring "7 May 2023" may miss. "2022" is in session_2.
+ assert!(
+ result.by_type.contains_key("temporal") || result.by_type.contains_key("multi_hop")
+ );
+ let agg = aggregate_results(&[result], ScorerKind::Mock, "fixture", "mock", None, None);
+ assert_eq!(agg.metric, "context_hit_rate");
+ assert!(agg.caveats.iter().any(|c| c.contains("not comparable")));
}
#[test]
- fn test_load_dataset_from_dir() {
+ fn load_dataset_array_file() {
let dir = tempfile::tempdir().unwrap();
- let conv1 = r#"{"conversation_id":"c1","turns":[{"role":"user","content":"hi"}],"questions":[{"question":"q","answer":"a","type":"single_hop"}]}"#;
- let conv2 = r#"{"conversation_id":"c2","turns":[{"role":"user","content":"bye"}],"questions":[{"question":"q2","answer":"a2","type":"temporal"}]}"#;
- std::fs::write(dir.path().join("conv1.json"), conv1).unwrap();
- std::fs::write(dir.path().join("conv2.json"), conv2).unwrap();
- // Non-json file should be ignored
- std::fs::write(dir.path().join("readme.txt"), "ignore me").unwrap();
-
- let convs = load_dataset(dir.path()).unwrap();
- assert_eq!(convs.len(), 2);
+ let path = dir.path().join("locomo10.json");
+ std::fs::write(&path, format!("[{REAL_SHAPE}]")).unwrap();
+ let samples = load_dataset(&path).unwrap();
+ assert_eq!(samples.len(), 1);
+ assert_eq!(samples[0].sample_id, "conv-41");
}
}
diff --git a/crates/memory-bench/src/main.rs b/crates/memory-bench/src/main.rs
index 860f525..6c43098 100644
--- a/crates/memory-bench/src/main.rs
+++ b/crates/memory-bench/src/main.rs
@@ -1,14 +1,20 @@
use clap::Parser;
+use std::path::Path;
mod cli;
+use memory_bench::judge::{ApiJudge, Judge, MockJudge, ScorerKind};
+use memory_bench::runner::{BackendKind, MockStore, RunConfig};
use memory_bench::{baseline, fixture, locomo, report, runner, scorer};
use scorer::BenchmarkReport;
fn main() -> anyhow::Result<()> {
let cli = cli::Cli::parse();
- let config = runner::RunConfig {
+ let backend = BackendKind::parse(&cli.backend)?;
+ let config = RunConfig {
memory_bin: cli.memory_bin.clone(),
+ endpoint: cli.endpoint.clone(),
+ backend,
};
match cli.command {
@@ -32,7 +38,7 @@ fn main() -> anyhow::Result<()> {
} => {
let bench_report = run_all(&fixtures, &config)?;
let baselines_data = if compare {
- Some(baseline::Baselines::load(std::path::Path::new(&baselines))?)
+ Some(baseline::Baselines::load(Path::new(&baselines))?)
} else {
None
};
@@ -47,50 +53,57 @@ fn main() -> anyhow::Result<()> {
cli::Commands::Locomo {
dataset,
output,
+ scorer,
+ top,
compare,
- baselines,
+ baselines: _baselines,
} => {
- let conversations = locomo::load_dataset(std::path::Path::new(&dataset))?;
- eprintln!(
- "Loaded {} conversations from {}",
- conversations.len(),
- dataset
- );
-
- let mut results = Vec::new();
- for conv in &conversations {
- // Convert turns to JSONL and ingest via runner
- let temp_dir = tempfile::tempdir()?;
- let session_path = temp_dir.path().join("session.jsonl");
- let mut lines = Vec::new();
- for turn in &conv.turns {
- lines.push(format!(
- "{{\"role\":\"{}\",\"content\":\"{}\"}}",
- turn.role,
- turn.content.replace('\\', "\\\\").replace('"', "\\\"")
- ));
- }
- std::fs::write(&session_path, lines.join("\n"))?;
- let _ = runner::ingest_session(session_path.to_str().unwrap_or_default(), &config);
-
- // Run each question through runner and collect answers
- let mut answers = Vec::new();
- for q in &conv.questions {
- let result = runner::run_query(&q.question, &config);
- answers.push(result.raw_output);
- }
-
- let result = locomo::score_conversation(conv, &answers);
- results.push(result);
+ let kind = ScorerKind::parse(&scorer)?;
+ if compare && kind == ScorerKind::Mock {
+ anyhow::bail!(
+ "--compare refused for mock scorer: context_hit_rate is not a LOCOMO score \
+ and must not share a table with published LLM-judge numbers"
+ );
}
-
- let aggregate = locomo::aggregate_results(&results);
-
- if compare {
- let _baselines_data = baseline::Baselines::load(std::path::Path::new(&baselines))?;
- eprintln!("Loaded baselines for comparison");
+ let judge: Box = match kind {
+ ScorerKind::Mock => Box::new(MockJudge),
+ ScorerKind::LlmJudge => Box::new(ApiJudge::from_env()?),
+ };
+ let aggregate = run_locomo(&dataset, judge.as_ref(), kind, top, &config)?;
+ let json = serde_json::to_string_pretty(&aggregate)?;
+ println!("{json}");
+ if let Some(path) = output {
+ std::fs::write(&path, &json)?;
+ eprintln!("Results written to {path}");
}
-
+ }
+ cli::Commands::Smoke { dataset, output } => {
+ let judge = MockJudge;
+ let aggregate = run_locomo(
+ &dataset,
+ &judge,
+ ScorerKind::Mock,
+ 5,
+ &RunConfig {
+ backend: BackendKind::Mock,
+ ..config
+ },
+ )?;
+ if aggregate.conversations != 1 {
+ anyhow::bail!(
+ "smoke fixture must contain exactly 1 conversation, got {}",
+ aggregate.conversations
+ );
+ }
+ if aggregate.total_questions == 0 {
+ anyhow::bail!(
+ "smoke fixture produced 0 questions — parse/ingest/score pipeline did not run"
+ );
+ }
+ eprintln!(
+ "smoke ok: 1 conversation, {} questions, metric={}",
+ aggregate.total_questions, aggregate.metric
+ );
let json = serde_json::to_string_pretty(&aggregate)?;
println!("{json}");
if let Some(path) = output {
@@ -102,55 +115,215 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
+fn run_locomo(
+ dataset: &str,
+ judge: &dyn Judge,
+ kind: ScorerKind,
+ top: usize,
+ config: &RunConfig,
+) -> anyhow::Result {
+ let conversations = locomo::load_dataset(Path::new(dataset))?;
+ eprintln!(
+ "Loaded {} conversations from {} (backend={} scorer={})",
+ conversations.len(),
+ dataset,
+ config.backend.as_str(),
+ kind.metric_name()
+ );
+
+ let mut results = Vec::new();
+ for conv in &conversations {
+ // Fresh store per conversation — no shared-store bleed.
+ match config.backend {
+ BackendKind::Mock => {
+ let store = locomo::ingest_sample_mock(conv);
+ results.push(locomo::evaluate_sample(conv, &store, judge, top));
+ }
+ BackendKind::Cli => {
+ locomo::ingest_sample_cli(conv, config)?;
+ results.push(evaluate_sample_cli(conv, config, judge, top)?);
+ }
+ }
+ }
+
+ let (judge_label, model, temperature) = match kind {
+ ScorerKind::Mock => ("mock".to_string(), None, None),
+ ScorerKind::LlmJudge => {
+ let model =
+ std::env::var("MEMORY_BENCH_JUDGE_MODEL").unwrap_or_else(|_| "gpt-4o-mini".into());
+ ("api".to_string(), Some(model), Some(0.0))
+ }
+ };
+
+ Ok(locomo::aggregate_results(
+ &results,
+ kind,
+ dataset,
+ &judge_label,
+ model,
+ temperature,
+ ))
+}
+
+fn evaluate_sample_cli(
+ sample: &locomo::LocomoSample,
+ config: &RunConfig,
+ judge: &dyn Judge,
+ top: usize,
+) -> anyhow::Result {
+ // Build a one-shot mock store from CLI retrieval *per question* by
+ // stuffing CLI hits into evaluate_sample would mix questions. Do it
+ // question-by-question and reuse locomo types.
+ use locomo::{QuestionResult, TypeScore};
+ use std::collections::HashMap;
+
+ let mut questions = Vec::new();
+ let mut by_type: HashMap = HashMap::new();
+ let mut correct_n = 0usize;
+ for qa in &sample.qa {
+ let retrieved = runner::run_query_cli(&qa.question, config, top)?;
+ let context = retrieved
+ .ranked
+ .iter()
+ .map(|h| h.text.as_str())
+ .collect::>()
+ .join("\n");
+ let predicted = judge.generate_answer(&qa.question, &context)?;
+ let verdict = judge.judge(&qa.question, &qa.answer, &predicted, &context)?;
+ if verdict.correct {
+ correct_n += 1;
+ }
+ let cat = locomo::category_name(qa.category).to_string();
+ let entry = by_type.entry(cat.clone()).or_insert((0, 0));
+ entry.0 += 1;
+ if verdict.correct {
+ entry.1 += 1;
+ }
+ questions.push(QuestionResult {
+ question: qa.question.clone(),
+ gold: qa.answer.clone(),
+ category: cat,
+ category_id: qa.category,
+ predicted: predicted.chars().take(500).collect(),
+ correct: verdict.correct,
+ rationale: verdict.rationale,
+ latency_ms: retrieved.latency_ms,
+ context_tokens: retrieved.tokens_estimated,
+ });
+ }
+ let total = sample.qa.len();
+ let by_type = by_type
+ .into_iter()
+ .map(|(k, (t, c))| {
+ (
+ k,
+ TypeScore {
+ total: t,
+ correct: c,
+ score: if t == 0 { 0.0 } else { c as f64 / t as f64 },
+ },
+ )
+ })
+ .collect();
+ Ok(locomo::LocomoConversationResult {
+ sample_id: sample.sample_id.clone(),
+ total_questions: total,
+ correct: correct_n,
+ score: if total == 0 {
+ 0.0
+ } else {
+ correct_n as f64 / total as f64
+ },
+ by_type,
+ questions,
+ })
+}
+
/// Run benchmarks for a single category by filtering test case IDs by prefix.
fn run_category(
category_prefix: &str,
fixtures_dir: &str,
- config: &runner::RunConfig,
+ config: &RunConfig,
) -> anyhow::Result {
- let all_tests = fixture::Fixture::load_dir(std::path::Path::new(fixtures_dir))?;
+ let all_tests = fixture::Fixture::load_dir(Path::new(fixtures_dir))?;
let tests: Vec<_> = all_tests
.into_iter()
- .filter(|t| t.id.starts_with(category_prefix))
+ .filter(|t| {
+ t.id.starts_with(category_prefix)
+ || t.category
+ .as_deref()
+ .is_some_and(|c| c.starts_with(category_prefix))
+ })
.collect();
- run_tests(&tests, config)
+ run_tests(&tests, Path::new(fixtures_dir), config)
}
/// Run all benchmark categories and aggregate into one report.
-fn run_all(fixtures_dir: &str, config: &runner::RunConfig) -> anyhow::Result {
- let tests = fixture::Fixture::load_dir(std::path::Path::new(fixtures_dir))?;
- run_tests(&tests, config)
+fn run_all(fixtures_dir: &str, config: &RunConfig) -> anyhow::Result {
+ let tests = fixture::Fixture::load_dir(Path::new(fixtures_dir))?;
+ run_tests(&tests, Path::new(fixtures_dir), config)
}
-/// Execute a set of test cases and produce a benchmark report.
+/// Execute a set of test cases. Each test gets a fresh mock store.
fn run_tests(
tests: &[fixture::TestCase],
- config: &runner::RunConfig,
+ fixtures_dir: &Path,
+ config: &RunConfig,
) -> anyhow::Result {
let mut hits = Vec::new();
+ let mut failed_ids = Vec::new();
let mut latencies = Vec::new();
let mut total_tokens = 0usize;
let mut compression_ratios = Vec::new();
+ let mut recalls = Vec::new();
+ let mut k = 5usize;
for test in tests {
- // Ingest setup session files
- for setup_path in &test.setup {
- let _ = runner::ingest_session(setup_path, config);
- }
+ k = test.k;
+ let result = match config.backend {
+ BackendKind::Mock => {
+ let mut store = MockStore::new();
+ for setup_path in &test.setup {
+ let path = runner::resolve_setup(fixtures_dir, setup_path);
+ store.ingest_file(&path)?;
+ }
+ store.search(&test.query, test.k.max(5))
+ }
+ BackendKind::Cli => {
+ for setup_path in &test.setup {
+ let path = runner::resolve_setup(fixtures_dir, setup_path);
+ runner::ingest_session_cli(path.to_str().unwrap_or_default(), config)?;
+ }
+ runner::run_query_cli(&test.query, config, test.k.max(5))?
+ }
+ };
- // Run the query
- let result = runner::run_query(&test.query, config);
- let hit = scorer::score_result(&result.raw_output, &test.expected_contains);
+ let ranked_texts: Vec = result.ranked.iter().map(|h| h.text.clone()).collect();
+ let blob = ranked_texts.join("\n");
+ let hit = scorer::score_result(&blob, &test.expected_contains);
hits.push(hit);
+ if !hit {
+ failed_ids.push(test.id.clone());
+ }
latencies.push(result.latency_ms);
total_tokens += result.tokens_estimated;
- // Compute compression ratio
- let raw_tokens = scorer::estimate_raw_tokens(&test.setup);
+ if let Some(r) = scorer::compute_recall_at_k(&ranked_texts, &test.relevant, test.k) {
+ recalls.push(r);
+ }
+
+ let paths: Vec<_> = test
+ .setup
+ .iter()
+ .map(|s| runner::resolve_setup(fixtures_dir, s))
+ .collect();
+ let raw_tokens = scorer::estimate_raw_tokens_from_files(&paths)?;
if raw_tokens > 0 {
- let ratio = scorer::compute_compression_ratio(result.tokens_estimated, raw_tokens);
- compression_ratios.push(ratio);
+ compression_ratios.push(scorer::compute_compression_ratio(
+ result.tokens_estimated,
+ raw_tokens,
+ ));
}
}
@@ -159,7 +332,11 @@ fn run_tests(
let test_count = tests.len();
let pass_count = hits.iter().filter(|&&h| h).count();
let accuracy = scorer::compute_accuracy(&hits);
- let recall_at_5 = scorer::compute_recall_at_k(&hits, test_count);
+ let recall_at_k = if recalls.is_empty() {
+ None
+ } else {
+ Some(recalls.iter().sum::() / recalls.len() as f64)
+ };
let token_usage_avg = total_tokens.checked_div(test_count).unwrap_or(0);
let latency_p50_ms = scorer::percentile(&latencies, 50.0);
let latency_p95_ms = scorer::percentile(&latencies, 95.0);
@@ -169,15 +346,34 @@ fn run_tests(
compression_ratios.iter().sum::() / compression_ratios.len() as f64
};
+ let mut caveats = vec![
+ "accuracy is expected_contains over this fixture suite, not LOCOMO".into(),
+ "recall@k uses labeled relevant items in top-k, not accuracy under another name".into(),
+ "compression_ratio compares retrieved tokens to setup file contents (not path strings)"
+ .into(),
+ "each test ran against a fresh store (mock isolation; cli shares the daemon unless you restart it)"
+ .into(),
+ ];
+ if config.backend == BackendKind::Mock {
+ caveats.push(
+ "backend=mock uses in-process token-overlap retrieval; not a production quality number"
+ .into(),
+ );
+ }
+
Ok(BenchmarkReport {
+ backend: config.backend.as_str().to_string(),
accuracy,
- recall_at_5,
+ recall_at_k,
+ k,
token_usage_avg,
latency_p50_ms,
latency_p95_ms,
compression_ratio,
test_count,
pass_count,
+ failed_ids,
+ caveats,
})
}
diff --git a/crates/memory-bench/src/report.rs b/crates/memory-bench/src/report.rs
index 130960c..63f4139 100644
--- a/crates/memory-bench/src/report.rs
+++ b/crates/memory-bench/src/report.rs
@@ -6,75 +6,97 @@ pub fn to_json(report: &BenchmarkReport) -> String {
serde_json::to_string_pretty(report).unwrap_or_default()
}
-/// Generate a markdown table from a benchmark report, optionally including competitor baselines.
+/// Markdown report. Competitor rows never share a unified "Accuracy" column
+/// across incommensurable measurement regimes.
pub fn to_markdown(report: &BenchmarkReport, baselines: Option<&Baselines>) -> String {
let mut out = String::new();
- out.push_str("# Benchmark Results\n\n");
+ out.push_str("# Custom harness results\n\n");
+ out.push_str(&format!("**Backend:** {}\n\n", report.backend));
- if let Some(bl) = baselines {
- out.push_str("| Metric | Agent-Memory | MemMachine | Mem0 |\n");
- out.push_str("|--------|-------------|------------|------|\n");
-
- let mm = bl.memmachine.as_ref();
- let m0 = bl.mem0.as_ref();
-
- out.push_str(&format!(
- "| Accuracy | {:.1}% | {} | {} |\n",
- report.accuracy * 100.0,
- mm.and_then(|m| m.locomo_score)
- .map_or("-".to_string(), |v| format!("{:.1}%", v * 100.0)),
- m0.and_then(|m| m.accuracy_vs_openai_memory)
- .map_or("-".to_string(), |v| format!("+{:.0}%", v * 100.0)),
- ));
- out.push_str(&format!(
- "| Recall@5 | {:.1}% | - | - |\n",
- report.recall_at_5 * 100.0
- ));
- out.push_str(&format!(
- "| Avg Tokens | {} | - | - |\n",
- report.token_usage_avg
- ));
- out.push_str(&format!(
- "| Latency p50 | {}ms | {} | {} |\n",
- report.latency_p50_ms,
- mm.and_then(|m| m.latency_improvement)
- .map_or("-".to_string(), |v| format!("{:.0}% faster", v * 100.0)),
- m0.and_then(|m| m.latency_reduction)
- .map_or("-".to_string(), |v| format!("{:.0}% reduction", v * 100.0)),
- ));
- out.push_str(&format!(
- "| Latency p95 | {}ms | - | - |\n",
- report.latency_p95_ms
- ));
- out.push_str(&format!(
- "| Compression | {:.1}% | {} | {} |\n",
- report.compression_ratio * 100.0,
- mm.and_then(|m| m.token_reduction)
- .map_or("-".to_string(), |v| format!("{:.0}%", v * 100.0)),
- m0.and_then(|m| m.token_reduction)
- .map_or("-".to_string(), |v| format!("{:.0}%", v * 100.0)),
- ));
- } else {
- out.push_str("| Metric | Value |\n");
- out.push_str("|--------|-------|\n");
- out.push_str(&format!("| Accuracy | {:.1}% |\n", report.accuracy * 100.0));
- out.push_str(&format!(
- "| Recall@5 | {:.1}% |\n",
- report.recall_at_5 * 100.0
- ));
- out.push_str(&format!("| Avg Tokens | {} |\n", report.token_usage_avg));
- out.push_str(&format!("| Latency p50 | {}ms |\n", report.latency_p50_ms));
- out.push_str(&format!("| Latency p95 | {}ms |\n", report.latency_p95_ms));
- out.push_str(&format!(
- "| Compression | {:.1}% |\n",
- report.compression_ratio * 100.0
- ));
+ out.push_str("| Metric | Value | Notes |\n");
+ out.push_str("|--------|-------|-------|\n");
+ out.push_str(&format!(
+ "| expected_contains accuracy | {:.1}% | this fixture suite only |\n",
+ report.accuracy * 100.0
+ ));
+ match report.recall_at_k {
+ Some(r) => out.push_str(&format!(
+ "| recall@{} | {:.1}% | labeled relevant items in top-k |\n",
+ report.k,
+ r * 100.0
+ )),
+ None => out.push_str(&format!(
+ "| recall@{} | n/a | no test supplied a relevant set |\n",
+ report.k
+ )),
}
+ out.push_str(&format!(
+ "| avg retrieved tokens | {} | envelope / mock estimate |\n",
+ report.token_usage_avg
+ ));
+ out.push_str(&format!(
+ "| latency p50 | {}ms | |\n",
+ report.latency_p50_ms
+ ));
+ out.push_str(&format!(
+ "| latency p95 | {}ms | |\n",
+ report.latency_p95_ms
+ ));
+ out.push_str(&format!(
+ "| compression (1 - ctx/raw) | {:.1}% | raw = setup file *contents* |\n",
+ report.compression_ratio * 100.0
+ ));
out.push_str(&format!(
"\n**Tests:** {}/{} passed\n",
report.pass_count, report.test_count
));
+ if !report.failed_ids.is_empty() {
+ out.push_str(&format!("**Failed:** {}\n", report.failed_ids.join(", ")));
+ }
+
+ if !report.caveats.is_empty() {
+ out.push_str("\n## Caveats\n\n");
+ for c in &report.caveats {
+ out.push_str(&format!("- {c}\n"));
+ }
+ }
+
+ if let Some(bl) = baselines {
+ out.push_str("\n## Published competitor numbers (NOT the same metric)\n\n");
+ out.push_str("| System | Metric | Value | Commensurable with this row? |\n");
+ out.push_str("|--------|--------|-------|------------------------------|\n");
+ out.push_str(&format!(
+ "| Agent-Memory custom harness | expected_contains accuracy | {:.1}% | this suite only |\n",
+ report.accuracy * 100.0
+ ));
+ if let Some(mm) = bl.memmachine.as_ref() {
+ if let Some(v) = mm.locomo_score {
+ out.push_str(&format!(
+ "| MemMachine | {} | {:.1}% | no — different dataset, judge, and protocol |\n",
+ mm.metric
+ .as_deref()
+ .unwrap_or("LOCOMO LLM-judge (their paper)"),
+ v * 100.0
+ ));
+ }
+ }
+ if let Some(m0) = bl.mem0.as_ref() {
+ if let Some(v) = m0.accuracy_vs_openai_memory {
+ out.push_str(&format!(
+ "| Mem0 | {} | +{:.0}% | no — relative delta vs OpenAI memory, not LOCOMO |\n",
+ m0.metric
+ .as_deref()
+ .unwrap_or("relative delta vs OpenAI memory"),
+ v * 100.0
+ ));
+ }
+ }
+ out.push_str(
+ "\nIncommensurable metrics never share a unified Accuracy column. \
+ A LOCOMO LLM-judge number belongs next to other LOCOMO LLM-judge numbers only.\n",
+ );
+ }
out
}
@@ -82,17 +104,22 @@ pub fn to_markdown(report: &BenchmarkReport, baselines: Option<&Baselines>) -> S
#[cfg(test)]
mod tests {
use super::*;
+ use crate::baseline::CompetitorScore;
fn sample_report() -> BenchmarkReport {
BenchmarkReport {
+ backend: "mock".into(),
accuracy: 0.85,
- recall_at_5: 0.70,
+ recall_at_k: Some(0.70),
+ k: 5,
token_usage_avg: 300,
latency_p50_ms: 45,
latency_p95_ms: 120,
compression_ratio: 0.75,
test_count: 10,
pass_count: 8,
+ failed_ids: vec![],
+ caveats: vec!["mock retrieval".into()],
}
}
@@ -108,7 +135,39 @@ mod tests {
fn test_to_markdown_contains_headers() {
let report = sample_report();
let md = to_markdown(&report, None);
- assert!(md.contains("Accuracy"), "Should contain Accuracy header");
- assert!(md.contains("Recall@5"), "Should contain Recall@5 header");
+ assert!(md.contains("expected_contains accuracy"));
+ assert!(md.contains("recall@5"));
+ assert!(!md.contains("Recall@5 | 0.70"), "old unified column gone");
+ }
+
+ #[test]
+ fn compare_table_does_not_unify_accuracy() {
+ let report = sample_report();
+ let bl = Baselines {
+ memmachine: Some(CompetitorScore {
+ locomo_score: Some(0.91),
+ token_reduction: None,
+ latency_improvement: None,
+ accuracy_vs_openai_memory: None,
+ latency_reduction: None,
+ metric: Some("LOCOMO LLM-judge".into()),
+ }),
+ mem0: Some(CompetitorScore {
+ locomo_score: None,
+ token_reduction: None,
+ latency_improvement: None,
+ accuracy_vs_openai_memory: Some(0.26),
+ latency_reduction: None,
+ metric: Some("relative delta vs OpenAI memory".into()),
+ }),
+ };
+ let md = to_markdown(&report, Some(&bl));
+ assert!(md.contains("NOT the same metric"));
+ assert!(md.contains("LOCOMO LLM-judge"));
+ assert!(md.contains("relative delta vs OpenAI memory"));
+ assert!(
+ !md.contains("| Accuracy |"),
+ "must not emit a unified Accuracy column: {md}"
+ );
}
}
diff --git a/crates/memory-bench/src/runner.rs b/crates/memory-bench/src/runner.rs
index 6b2a5ed..d41194b 100644
--- a/crates/memory-bench/src/runner.rs
+++ b/crates/memory-bench/src/runner.rs
@@ -1,86 +1,458 @@
+//! Run queries against either a mock isolated store or the `memory` CLI.
+//!
+//! Errors from `memory add` / `memory search` fail the run. A dead daemon
+//! is not scored as accuracy 0.0.
+
+use anyhow::{bail, Context, Result};
+use serde_json::Value;
use std::io::BufRead;
+use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
+/// Which retrieval backend to drive.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BackendKind {
+ /// In-process token-overlap store. Always isolated per test.
+ Mock,
+ /// Shell out to the `memory` CLI against a running daemon.
+ Cli,
+}
+
+impl BackendKind {
+ pub fn parse(s: &str) -> Result {
+ match s {
+ "mock" => Ok(Self::Mock),
+ "cli" => Ok(Self::Cli),
+ other => bail!("unknown backend '{other}' (expected mock|cli)"),
+ }
+ }
+
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Mock => "mock",
+ Self::Cli => "cli",
+ }
+ }
+}
+
/// Configuration for the benchmark runner.
pub struct RunConfig {
/// Path to the memory binary (default: "memory").
pub memory_bin: String,
+ /// gRPC endpoint for CLI backend.
+ pub endpoint: String,
+ pub backend: BackendKind,
}
impl Default for RunConfig {
fn default() -> Self {
Self {
memory_bin: "memory".to_string(),
+ endpoint: "http://127.0.0.1:50051".to_string(),
+ backend: BackendKind::Mock,
}
}
}
-/// Result of running a single query against the memory binary.
+/// One ranked retrieval hit.
+#[derive(Debug, Clone)]
+pub struct RankedHit {
+ pub text: String,
+ pub score: f64,
+}
+
+/// Result of running a single query.
+#[derive(Debug, Clone)]
pub struct QueryResult {
- /// Raw stdout output from the binary.
+ /// Raw stdout (CLI) or synthesized JSON (mock).
pub raw_output: String,
- /// Elapsed time in milliseconds.
pub latency_ms: u64,
- /// Token count from meta.tokens_estimated in JSON envelope.
pub tokens_estimated: usize,
- /// Whether the command exited successfully.
- pub success: bool,
+ pub ranked: Vec,
}
-/// Run a search query against the memory binary and capture JSON output + latency.
-pub fn run_query(query: &str, config: &RunConfig) -> QueryResult {
- let start = Instant::now();
- let output = Command::new(&config.memory_bin)
- .args(["search", query, "--format=json"])
- .output();
+/// A single ingested event sitting in a mock store.
+#[derive(Debug, Clone)]
+struct StoredEvent {
+ text: String,
+}
- let elapsed = start.elapsed().as_millis() as u64;
+/// Isolated in-memory store. One of these per test / per LOCOMO conversation.
+#[derive(Debug, Default)]
+pub struct MockStore {
+ events: Vec,
+}
+
+impl MockStore {
+ pub fn new() -> Self {
+ Self { events: Vec::new() }
+ }
+
+ pub fn ingest_text(&mut self, _session_id: impl Into, text: impl Into) {
+ self.events.push(StoredEvent { text: text.into() });
+ }
+
+ /// Ingest a JSONL session file. Each line is either a JSON object with
+ /// `content` (and optional `role`) or raw text.
+ pub fn ingest_file(&mut self, path: &Path) -> Result {
+ let file = std::fs::File::open(path)
+ .with_context(|| format!("opening session file {}", path.display()))?;
+ let reader = std::io::BufReader::new(file);
+ let session_id = path
+ .file_stem()
+ .map(|s| s.to_string_lossy().into_owned())
+ .unwrap_or_else(|| "session".to_string());
+ let mut n = 0usize;
+ for line in reader.lines() {
+ let line = line?;
+ let trimmed = line.trim();
+ if trimmed.is_empty() {
+ continue;
+ }
+ let text = parse_jsonl_line(trimmed);
+ if text.is_empty() {
+ continue;
+ }
+ self.ingest_text(&session_id, text);
+ n += 1;
+ }
+ Ok(n)
+ }
+
+ /// Rank events by query-term overlap. Isolated: only this store's events.
+ pub fn search(&self, query: &str, top_k: usize) -> QueryResult {
+ let start = Instant::now();
+ let terms = tokenize(query);
+ let mut scored: Vec<(f64, &StoredEvent)> = self
+ .events
+ .iter()
+ .map(|e| {
+ let hay = e.text.to_lowercase();
+ let score = terms.iter().filter(|t| hay.contains(t.as_str())).count() as f64;
+ (score, e)
+ })
+ .collect();
+ scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
+ scored.truncate(top_k);
+
+ let ranked: Vec = scored
+ .into_iter()
+ .map(|(score, e)| RankedHit {
+ text: e.text.clone(),
+ score,
+ })
+ .collect();
+
+ let joined = ranked
+ .iter()
+ .map(|h| h.text.as_str())
+ .collect::>()
+ .join("\n");
+ let tokens_estimated = crate::scorer::estimate_tokens_from_text(&joined);
+ let results: Vec = ranked
+ .iter()
+ .map(|h| {
+ serde_json::json!({
+ "text_preview": h.text,
+ "score": h.score,
+ })
+ })
+ .collect();
+ let envelope = serde_json::json!({
+ "status": "ok",
+ "query": query,
+ "results": results,
+ "meta": {
+ "retrieval_ms": start.elapsed().as_millis() as u64,
+ "tokens_estimated": tokens_estimated,
+ "confidence": ranked.first().map(|h| h.score).unwrap_or(0.0),
+ "backend": "mock",
+ }
+ });
+
+ QueryResult {
+ raw_output: envelope.to_string(),
+ latency_ms: start.elapsed().as_millis() as u64,
+ tokens_estimated,
+ ranked,
+ }
+ }
+
+ pub fn event_count(&self) -> usize {
+ self.events.len()
+ }
+}
+
+fn tokenize(query: &str) -> Vec {
+ query
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|s| s.len() > 1)
+ .map(|s| s.to_lowercase())
+ .collect()
+}
- match output {
- Ok(out) => {
- let raw_output = String::from_utf8_lossy(&out.stdout).to_string();
- let tokens_estimated = extract_tokens_estimated(&raw_output);
- QueryResult {
- raw_output,
- latency_ms: elapsed,
- tokens_estimated,
- success: out.status.success(),
+fn parse_jsonl_line(line: &str) -> String {
+ if let Ok(v) = serde_json::from_str::(line) {
+ if let Some(c) = v.get("content").and_then(|x| x.as_str()) {
+ let role = v.get("role").and_then(|x| x.as_str()).unwrap_or("");
+ if role.is_empty() {
+ return c.to_string();
}
+ return format!("{role}: {c}");
+ }
+ if let Some(t) = v.get("text").and_then(|x| x.as_str()) {
+ return t.to_string();
}
- Err(_) => QueryResult {
- raw_output: String::new(),
- latency_ms: elapsed,
- tokens_estimated: 0,
- success: false,
- },
}
+ line.to_string()
+}
+
+/// Parse ranked hits out of a `memory search --format=json` envelope.
+pub fn parse_ranked_hits(json_output: &str) -> Vec {
+ let Ok(v) = serde_json::from_str::(json_output) else {
+ return Vec::new();
+ };
+ let Some(results) = v.get("results").and_then(|r| r.as_array()) else {
+ return Vec::new();
+ };
+ results
+ .iter()
+ .map(|r| RankedHit {
+ text: r
+ .get("text_preview")
+ .and_then(|t| t.as_str())
+ .unwrap_or("")
+ .to_string(),
+ score: r.get("score").and_then(|s| s.as_f64()).unwrap_or(0.0),
+ })
+ .collect()
}
/// Extract meta.tokens_estimated from JSON envelope output.
-fn extract_tokens_estimated(json_output: &str) -> usize {
- serde_json::from_str::(json_output)
+pub fn extract_tokens_estimated(json_output: &str) -> usize {
+ serde_json::from_str::(json_output)
.ok()
.and_then(|v| v.get("meta")?.get("tokens_estimated")?.as_u64())
.unwrap_or(0) as usize
}
+/// Resolve a fixture setup path against the fixtures directory.
+pub fn resolve_setup(fixtures_dir: &Path, setup: &str) -> PathBuf {
+ let p = Path::new(setup);
+ if p.is_absolute() {
+ return p.to_path_buf();
+ }
+ let from_fixtures = fixtures_dir.join(setup);
+ if from_fixtures.exists() {
+ return from_fixtures;
+ }
+ p.to_path_buf()
+}
+
/// Ingest a JSONL session file by calling `memory add` for each line.
-pub fn ingest_session(session_path: &str, config: &RunConfig) -> anyhow::Result<()> {
- let file = std::fs::File::open(session_path)?;
+/// Failures (missing binary, non-zero exit, daemon down) abort the run.
+pub fn ingest_session_cli(session_path: &str, config: &RunConfig) -> Result {
+ let file = std::fs::File::open(session_path)
+ .with_context(|| format!("opening session file {session_path}"))?;
let reader = std::io::BufReader::new(file);
+ let mut n = 0usize;
- for line in reader.lines() {
+ for (idx, line) in reader.lines().enumerate() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
+ let content = parse_jsonl_line(trimmed);
+ let output = Command::new(&config.memory_bin)
+ .args([
+ "add",
+ "--content",
+ &content,
+ "--kind",
+ "episodic",
+ "--endpoint",
+ &config.endpoint,
+ ])
+ .output()
+ .with_context(|| {
+ format!(
+ "spawning `{} add` (is the memory binary on PATH?)",
+ config.memory_bin
+ )
+ })?;
+ if !output.status.success() {
+ let stderr = String::from_utf8_lossy(&output.stderr);
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ bail!(
+ "memory add failed on {session_path}:{idx}: status={} stderr={stderr} stdout={stdout}",
+ output.status
+ );
+ }
+ n += 1;
+ }
+ Ok(n)
+}
- let _ = Command::new(&config.memory_bin)
- .args(["add", "--content", trimmed, "--kind", "episodic"])
- .output();
+/// Run a search query against the memory CLI. Non-zero exit fails the run.
+pub fn run_query_cli(query: &str, config: &RunConfig, top: usize) -> Result {
+ let start = Instant::now();
+ let output = Command::new(&config.memory_bin)
+ .args([
+ "search",
+ query,
+ "--format",
+ "json",
+ "--top",
+ &top.to_string(),
+ "--endpoint",
+ &config.endpoint,
+ ])
+ .output()
+ .with_context(|| {
+ format!(
+ "spawning `{} search` (is the memory binary on PATH?)",
+ config.memory_bin
+ )
+ })?;
+ let elapsed = start.elapsed().as_millis() as u64;
+ let stdout = String::from_utf8_lossy(&output.stdout).to_string();
+ let stderr = String::from_utf8_lossy(&output.stderr);
+ if !output.status.success() {
+ bail!(
+ "memory search failed for query {query:?}: status={} stderr={stderr} stdout={stdout}",
+ output.status
+ );
}
+ if stdout.trim().is_empty() {
+ bail!("memory search returned empty stdout for query {query:?} (daemon down?)");
+ }
+ let ranked = parse_ranked_hits(&stdout);
+ let tokens_estimated = extract_tokens_estimated(&stdout);
+ Ok(QueryResult {
+ raw_output: stdout,
+ latency_ms: elapsed,
+ tokens_estimated,
+ ranked,
+ })
+}
- Ok(())
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn mock_stores_do_not_bleed() {
+ let mut a = MockStore::new();
+ a.ingest_text("s1", "UNIQUE_ALPHA_TOKEN jwt rotation");
+ let mut b = MockStore::new();
+ b.ingest_text("s2", "UNIQUE_BETA_TOKEN redis cache");
+
+ let hits = b.search("UNIQUE_ALPHA_TOKEN", 5);
+ assert!(
+ hits.ranked
+ .iter()
+ .all(|h| !h.text.contains("UNIQUE_ALPHA_TOKEN")),
+ "store B must not see store A's events: {:?}",
+ hits.ranked
+ );
+ let hits_a = a.search("UNIQUE_ALPHA_TOKEN", 5);
+ assert!(hits_a
+ .ranked
+ .iter()
+ .any(|h| h.text.contains("UNIQUE_ALPHA_TOKEN")));
+ }
+
+ #[test]
+ fn mock_search_ranks_term_overlap() {
+ let mut s = MockStore::new();
+ s.ingest_text("s", "the weather is nice today");
+ s.ingest_text("s", "we chose JWT with refresh token rotation for auth");
+ s.ingest_text("s", "redis caching hit rate");
+ let hits = s.search("JWT refresh rotation", 5);
+ assert!(!hits.ranked.is_empty());
+ assert!(hits.ranked[0].text.contains("JWT"));
+ }
+
+ #[test]
+ fn ingest_file_reads_jsonl_content_not_path() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("auth.jsonl");
+ std::fs::write(
+ &path,
+ r#"{"role":"user","content":"pick JWT please"}
+{"role":"assistant","content":"JWT with rotation it is"}
+"#,
+ )
+ .unwrap();
+ let mut s = MockStore::new();
+ let n = s.ingest_file(&path).unwrap();
+ assert_eq!(n, 2);
+ let hits = s.search("JWT", 5);
+ assert!(hits.ranked.iter().any(|h| h.text.contains("JWT")));
+ }
+
+ #[test]
+ fn cli_ingest_fails_loud_when_binary_missing() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("s.jsonl");
+ std::fs::write(&path, "{\"content\":\"hi\"}\n").unwrap();
+ let cfg = RunConfig {
+ memory_bin: "/definitely/not/a/memory/binary".into(),
+ endpoint: "http://127.0.0.1:1".into(),
+ backend: BackendKind::Cli,
+ };
+ let err = ingest_session_cli(path.to_str().unwrap(), &cfg).unwrap_err();
+ let msg = err.to_string();
+ assert!(
+ msg.contains("spawning") || msg.contains("No such file") || msg.contains("failed"),
+ "unexpected error: {msg}"
+ );
+ }
+
+ #[test]
+ fn committed_fixtures_are_at_least_25() {
+ let dir =
+ std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/fixtures");
+ let tests = crate::fixture::Fixture::load_dir(&dir).unwrap();
+ assert!(
+ tests.len() >= 25,
+ "Phase 56 requires ≥25 custom-harness tests, found {}",
+ tests.len()
+ );
+ assert!(tests.iter().any(|t| t.id.starts_with("temporal-")));
+ assert!(tests.iter().any(|t| t.id.starts_with("multi-")));
+ assert!(tests.iter().any(|t| t.id.starts_with("compress-")));
+ assert!(tests.iter().all(|t| !t.relevant.is_empty()));
+ }
+
+ #[test]
+ fn run_committed_fixtures_isolated_mock() {
+ let dir =
+ std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/fixtures");
+ let tests = crate::fixture::Fixture::load_dir(&dir).unwrap();
+ let mut pass = 0usize;
+ let mut recalls = Vec::new();
+ for test in &tests {
+ let mut store = MockStore::new();
+ for setup in &test.setup {
+ let path = resolve_setup(&dir, setup);
+ store.ingest_file(&path).unwrap();
+ }
+ let result = store.search(&test.query, test.k.max(5));
+ let texts: Vec = result.ranked.iter().map(|h| h.text.clone()).collect();
+ if crate::scorer::score_result(&texts.join("\n"), &test.expected_contains) {
+ pass += 1;
+ }
+ if let Some(r) = crate::scorer::compute_recall_at_k(&texts, &test.relevant, test.k) {
+ recalls.push(r);
+ }
+ }
+ assert!(pass > 0, "mock retrieval should hit at least one fixture");
+ assert_eq!(recalls.len(), tests.len());
+ // Isolation: running the suite must not require a daemon and must
+ // not panic. Accuracy is a mock number — asserted only as a pipeline.
+ assert!(tests.len() >= 25);
+ }
}
diff --git a/crates/memory-bench/src/scorer.rs b/crates/memory-bench/src/scorer.rs
index e615742..327e59a 100644
--- a/crates/memory-bench/src/scorer.rs
+++ b/crates/memory-bench/src/scorer.rs
@@ -1,5 +1,16 @@
+//! Scoring for the custom TOML-fixture harness.
+//!
+//! `recall_at_k` is computed against *labeled relevant items* found in the
+//! top-k ranked result texts. It is not `accuracy` under another name.
+
+use anyhow::{Context, Result};
+use std::path::Path;
+
/// Returns true if result text contains at least one expected string (case-insensitive).
pub fn score_result(result: &str, expected_contains: &[String]) -> bool {
+ if expected_contains.is_empty() {
+ return false;
+ }
let lower = result.to_lowercase();
expected_contains
.iter()
@@ -14,12 +25,30 @@ pub fn compute_accuracy(hits: &[bool]) -> f64 {
hits.iter().filter(|&&h| h).count() as f64 / hits.len() as f64
}
-/// Compute recall@k: fraction of relevant items found in top-k results.
-pub fn compute_recall_at_k(hits_in_top_k: &[bool], total_relevant: usize) -> f64 {
- if total_relevant == 0 {
- return 0.0;
+/// Recall@k against a labeled relevant-item set.
+///
+/// For each relevant string, count a hit if it appears (case-insensitive) in
+/// any of the top-`k` ranked result texts. Divide by `|relevant|`.
+///
+/// Returns `None` when `relevant` is empty so the caller can omit the test
+/// from the aggregate rather than reporting a fake 0.0 or 1.0.
+pub fn compute_recall_at_k(ranked_texts: &[String], relevant: &[String], k: usize) -> Option {
+ if relevant.is_empty() {
+ return None;
}
- hits_in_top_k.iter().filter(|&&h| h).count() as f64 / total_relevant as f64
+ let top: Vec = ranked_texts
+ .iter()
+ .take(k)
+ .map(|t| t.to_lowercase())
+ .collect();
+ let found = relevant
+ .iter()
+ .filter(|item| {
+ let needle = item.to_lowercase();
+ top.iter().any(|t| t.contains(&needle))
+ })
+ .count();
+ Some(found as f64 / relevant.len() as f64)
}
/// Return the value at the given percentile from a sorted slice.
@@ -31,15 +60,9 @@ pub fn percentile(sorted_values: &[u64], p: f64) -> u64 {
sorted_values[idx.min(sorted_values.len() - 1)]
}
-/// Compute compression ratio: how much smaller the context_tokens are compared to raw input.
-///
-/// Formula: `1.0 - (context_tokens as f64 / raw_tokens as f64)`
+/// Compression ratio: how much smaller retrieved context is vs corpus file *contents*.
///
-/// - `context_tokens`: tokens_estimated returned by the memory search JSON envelope.
-/// - `raw_tokens`: derived by counting total characters across all JSONL setup lines,
-/// divided by 4.0 as a standard chars-per-token approximation.
-///
-/// Returns 0.0 if raw_tokens is 0 (prevents divide-by-zero).
+/// `1.0 - (context_tokens / raw_tokens)`. Returns 0.0 if `raw_tokens` is 0.
pub fn compute_compression_ratio(context_tokens: usize, raw_tokens: usize) -> f64 {
if raw_tokens == 0 {
return 0.0;
@@ -47,24 +70,45 @@ pub fn compute_compression_ratio(context_tokens: usize, raw_tokens: usize) -> f6
1.0 - (context_tokens as f64 / raw_tokens as f64)
}
-/// Estimate raw token count from JSONL setup strings (TestCase.setup lines).
-/// Sums character lengths of all setup strings and divides by 4 (chars-per-token approximation).
-pub fn estimate_raw_tokens(setup_lines: &[String]) -> usize {
- let total_chars: usize = setup_lines.iter().map(|s| s.len()).sum();
- (total_chars as f64 / 4.0).ceil() as usize
+/// Estimate tokens from a blob of text: `ceil(chars / 4)`.
+pub fn estimate_tokens_from_text(text: &str) -> usize {
+ (text.len() as f64 / 4.0).ceil() as usize
+}
+
+/// Estimate raw tokens from **file contents** of the setup corpus.
+///
+/// v3.0 summed the character lengths of the *path strings*. That is not a
+/// token estimate of the corpus. This reads each file.
+pub fn estimate_raw_tokens_from_files(paths: &[impl AsRef]) -> Result {
+ let mut total_chars = 0usize;
+ for p in paths {
+ let path = p.as_ref();
+ let content = std::fs::read_to_string(path)
+ .with_context(|| format!("reading setup corpus {}", path.display()))?;
+ total_chars += content.len();
+ }
+ Ok((total_chars as f64 / 4.0).ceil() as usize)
}
-/// Aggregated benchmark report with all computed metrics.
+/// Aggregated custom-harness report.
+///
+/// `recall_at_k` is `None` when no test supplied a labeled relevant set.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)]
pub struct BenchmarkReport {
+ pub backend: String,
pub accuracy: f64,
- pub recall_at_5: f64,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub recall_at_k: Option,
+ pub k: usize,
pub token_usage_avg: usize,
pub latency_p50_ms: u64,
pub latency_p95_ms: u64,
pub compression_ratio: f64,
pub test_count: usize,
pub pass_count: usize,
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub failed_ids: Vec,
+ pub caveats: Vec,
}
#[cfg(test)]
@@ -134,9 +178,52 @@ mod tests {
}
#[test]
- fn test_estimate_raw_tokens() {
- let lines = vec!["hello world".to_string()];
- // ceil(11/4) = 3
- assert_eq!(estimate_raw_tokens(&lines), 3);
+ fn recall_at_k_is_not_accuracy() {
+ // Accuracy would be 1.0 (the query "hit" because JWT is present).
+ // Recall@5 against three labeled items of which only one appears is 1/3.
+ let ranked = vec![
+ "We chose JWT with refresh rotation".to_string(),
+ "Unrelated deploy notes".to_string(),
+ ];
+ let relevant = vec!["JWT".to_string(), "OAuth2".to_string(), "SAML".to_string()];
+ let recall = compute_recall_at_k(&ranked, &relevant, 5).unwrap();
+ assert!((recall - 1.0 / 3.0).abs() < 0.001);
+ assert!(score_result(&ranked.join("\n"), &["JWT".to_string()]));
+ }
+
+ #[test]
+ fn recall_at_k_empty_relevant_is_none() {
+ assert!(compute_recall_at_k(&["anything".into()], &[], 5).is_none());
+ }
+
+ #[test]
+ fn recall_at_k_respects_k_cutoff() {
+ let ranked = vec![
+ "alpha appears here".to_string(),
+ "beta appears here".to_string(),
+ "gamma appears here".to_string(),
+ ];
+ let relevant = vec!["gamma".to_string()];
+ let at_2 = compute_recall_at_k(&ranked, &relevant, 2).unwrap();
+ let at_3 = compute_recall_at_k(&ranked, &relevant, 3).unwrap();
+ assert!((at_2 - 0.0).abs() < f64::EPSILON);
+ assert!((at_3 - 1.0).abs() < f64::EPSILON);
+ }
+
+ #[test]
+ fn estimate_raw_tokens_reads_file_contents_not_paths() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("corpus.jsonl");
+ // 40 chars of content. Path string is much shorter.
+ let content = "abcdefghijklmnopqrstuvwxyzabcdefghijklmn";
+ assert_eq!(content.len(), 40);
+ std::fs::write(&path, content).unwrap();
+ let from_contents = estimate_raw_tokens_from_files(&[path.as_path()]).unwrap();
+ let from_path_string = estimate_tokens_from_text(path.to_str().unwrap());
+ assert_eq!(from_contents, 10); // ceil(40/4)
+ assert_ne!(
+ from_contents, from_path_string,
+ "path-length estimate must not equal content estimate"
+ );
}
}
diff --git a/crates/memory-cli/src/cli.rs b/crates/memory-cli/src/cli.rs
index 7612eef..1133ca7 100644
--- a/crates/memory-cli/src/cli.rs
+++ b/crates/memory-cli/src/cli.rs
@@ -91,6 +91,18 @@ pub struct AddArgs {
/// Agent identifier.
#[arg(long)]
pub agent: Option,
+
+ /// RFC3339 timestamp for the event (default: now). Required for temporal ingest.
+ #[arg(long)]
+ pub timestamp: Option,
+
+ /// Session identifier (default: generated). Same id groups a conversation session.
+ #[arg(long)]
+ pub session_id: Option,
+
+ /// Role: user, assistant, system, tool (default: user).
+ #[arg(long)]
+ pub role: Option,
}
/// Arguments for the `timeline` subcommand.
diff --git a/crates/memory-cli/src/commands/add.rs b/crates/memory-cli/src/commands/add.rs
index c9b1f21..d3244a8 100644
--- a/crates/memory-cli/src/commands/add.rs
+++ b/crates/memory-cli/src/commands/add.rs
@@ -1,7 +1,5 @@
-//! `memory add` command -- ingest a new memory event via gRPC.
-
use anyhow::Result;
-use chrono::Utc;
+use chrono::{DateTime, Utc};
use serde_json::json;
use ulid::Ulid;
@@ -22,13 +20,38 @@ fn kind_to_event_type(kind: &str) -> EventType {
}
}
+fn parse_role(role: Option<&str>) -> EventRole {
+ match role.map(|s| s.to_ascii_lowercase()) {
+ Some(s) if s == "assistant" => EventRole::Assistant,
+ Some(s) if s == "system" => EventRole::System,
+ Some(s) if s == "tool" => EventRole::Tool,
+ _ => EventRole::User,
+ }
+}
+
+fn parse_timestamp(raw: Option<&str>) -> Result> {
+ match raw {
+ None => Ok(Utc::now()),
+ Some(s) => DateTime::parse_from_rfc3339(s)
+ .map(|dt| dt.with_timezone(&Utc))
+ .map_err(|e| anyhow::anyhow!("invalid --timestamp {s:?}: {e} (want RFC3339)")),
+ }
+}
+
/// Build an `Event` from CLI arguments.
-fn build_event(content: &str, kind: &str, agent: Option<&str>) -> Event {
+fn build_event(
+ content: &str,
+ kind: &str,
+ agent: Option<&str>,
+ timestamp: DateTime,
+ session_id: Option<&str>,
+ role: EventRole,
+) -> Event {
let event_id = Ulid::new().to_string();
- let session_id = format!("cli-{}", Ulid::new());
- let timestamp = Utc::now();
+ let session_id = session_id
+ .map(|s| s.to_string())
+ .unwrap_or_else(|| format!("cli-{}", Ulid::new()));
let event_type = kind_to_event_type(kind);
- let role = EventRole::User;
let event = Event::new(
event_id,
@@ -58,7 +81,15 @@ pub async fn run(args: AddArgs, global: &GlobalArgs) -> Result<()> {
}
};
- let event = build_event(&args.content, &args.kind, args.agent.as_deref());
+ let timestamp = parse_timestamp(args.timestamp.as_deref())?;
+ let event = build_event(
+ &args.content,
+ &args.kind,
+ args.agent.as_deref(),
+ timestamp,
+ args.session_id.as_deref(),
+ parse_role(args.role.as_deref()),
+ );
let event_id = event.event_id.clone();
match client.ingest(event).await {
@@ -157,7 +188,7 @@ mod tests {
#[test]
fn test_build_event_episodic_no_agent() {
- let event = build_event("hello", "episodic", None);
+ let event = build_event("hello", "episodic", None, Utc::now(), None, EventRole::User);
assert!(!event.event_id.is_empty());
assert!(event.session_id.starts_with("cli-"));
assert!(matches!(event.event_type, EventType::UserMessage));
@@ -168,7 +199,14 @@ mod tests {
#[test]
fn test_build_event_tool_result_with_agent() {
- let event = build_event("note", "tool_result", Some("claude"));
+ let event = build_event(
+ "note",
+ "tool_result",
+ Some("claude"),
+ Utc::now(),
+ None,
+ EventRole::User,
+ );
assert!(matches!(event.event_type, EventType::ToolResult));
assert_eq!(event.agent.as_deref(), Some("claude"));
assert_eq!(event.text, "note");
@@ -176,9 +214,33 @@ mod tests {
#[test]
fn test_build_event_generates_unique_ids() {
- let e1 = build_event("a", "episodic", None);
- let e2 = build_event("b", "episodic", None);
+ let e1 = build_event("a", "episodic", None, Utc::now(), None, EventRole::User);
+ let e2 = build_event("b", "episodic", None, Utc::now(), None, EventRole::User);
assert_ne!(e1.event_id, e2.event_id);
assert_ne!(e1.session_id, e2.session_id);
}
+
+ #[test]
+ fn test_build_event_honors_session_and_timestamp() {
+ let ts = DateTime::parse_from_rfc3339("2023-05-08T13:56:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let event = build_event(
+ "hey",
+ "episodic",
+ None,
+ ts,
+ Some("locomo-session-1"),
+ EventRole::Assistant,
+ );
+ assert_eq!(event.session_id, "locomo-session-1");
+ assert_eq!(event.timestamp, ts);
+ assert!(matches!(event.role, EventRole::Assistant));
+ }
+
+ #[test]
+ fn test_parse_timestamp_rejects_garbage() {
+ let err = parse_timestamp(Some("yesterday")).unwrap_err().to_string();
+ assert!(err.contains("RFC3339"), "{err}");
+ }
}
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 64bfa2f..d7fee64 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -148,3 +148,98 @@ debug run is ~4 s because each search embeds the query string with Candle on
CPU; that is a real query-path cost in this profile, not embedder init. Do
not quote it as production HNSW latency — re-run `--release` before
publishing a product number.
+
+
+---
+
+# Retrieval-quality benchmarks (Phase 56)
+
+`perf_bench` (above) measures **latency**. `memory-bench` measures **retrieval
+quality**. They are different tools. This section is the methodology for the
+quality harness. No number here is a published LOCOMO leaderboard score
+unless its `metric` field is exactly `locomo_llm_judge` and a pinned model
+id is recorded.
+
+## Custom TOML-fixture harness
+
+```bash
+cargo run -p memory-bench -- all --backend mock --output benchmarks/results/custom-harness-mock.json
+```
+
+`--backend mock` (default) uses an isolated in-process token-overlap store
+**per test**. That is a pipeline number, not production retrieval quality.
+`--backend cli` shells out to a running `memory` daemon; `memory add` /
+`memory search` failures abort the run (a dead daemon is not accuracy 0.0).
+
+Metrics:
+
+| Metric | What it is |
+|--------|------------|
+| `accuracy` | fraction of tests whose retrieved text contains at least one `expected_contains` string |
+| `recall_at_k` | fraction of **labeled `relevant` items** found in the top-k ranked texts. Not equal to accuracy. Omitted when no test supplies `relevant`. |
+| `compression_ratio` | `1 - context_tokens / raw_tokens` where `raw_tokens` is `ceil(chars/4)` of setup **file contents** (not path-string lengths) |
+| `latency_p50_ms` / `p95` | search latency |
+
+`--compare` prints competitor rows with a **Metric** column. MemMachine's
+0.91 is `LOCOMO LLM-judge (their paper)`. Mem0's +26% is `relative delta vs
+OpenAI memory`. Those rows are not commensurable with fixture `accuracy`.
+
+## LOCOMO adapter v2
+
+Real schema (`data/locomo10.json` from [snap-research/locomo](https://github.com/snap-research/locomo)):
+
+- top-level JSON array
+- each sample: `sample_id`, `conversation` (`speaker_a`/`speaker_b`, `session_N` + `session_N_date_time`), `qa[]` with `question`, `answer` (string **or** number), integer `category`, `evidence[]`
+
+Category map (from `task_eval/evaluation.py` + inspection of `category: 2` "When did…" items): `1=multi_hop`, `2=temporal`, `3=open_domain`, `4=single_hop`, `5=adversarial`.
+
+Download (prints `LICENSE.txt`, CC BY-NC 4.0, before the data file):
+
+```bash
+./benchmarks/scripts/download-locomo.sh locomo-data
+```
+
+`locomo-data/` is gitignored. Do not commit the dataset.
+
+One **isolated store per `sample_id`**. Sessions are ingested with their
+parsed timestamps (`1:56 pm on 8 May, 2023`) and speakers.
+
+### Scorers
+
+| `--scorer` | `metric` in results.json | When to use |
+|------------|--------------------------|-------------|
+| `mock` (default) | `context_hit_rate` | CI smoke. Gold-answer substring in retrieved context. **Not a LOCOMO score.** `--compare` is refused. |
+| `llm-judge` | `locomo_llm_judge` | Retrieve → generate answer → judge at temperature 0. Requires `OPENAI_API_KEY` or `ANTHROPIC_API_KEY`. Model id is recorded. |
+
+```bash
+# CI / local smoke (committed fixture, 1 conversation, mock judge)
+cargo run -p memory-bench -- smoke --output benchmarks/results/locomo-smoke.json
+
+# Full dataset, still not a LOCOMO score
+cargo run -p memory-bench -- locomo --dataset locomo-data --scorer mock
+
+# The only path that may be labeled locomo_llm_judge
+cargo run -p memory-bench -- locomo --dataset locomo-data --scorer llm-judge --output benchmarks/results/locomo-$(date -u +%F).json
+```
+
+`memory add --timestamp RFC3339 --session-id ID --role user|assistant` exists
+so a live-daemon LOCOMO run can preserve session time.
+
+## Decision gate (2026-08-30)
+
+**HOLD comparison marketing.** This phase commits:
+
+1. `benchmarks/results/locomo-smoke.json` — 1-conversation fixture, `metric=context_hit_rate`, mock retrieval. Pipeline evidence, not a leaderboard number.
+2. `benchmarks/results/custom-harness-mock.json` — ≥25 fixtures, `backend=mock`. Pipeline evidence.
+
+No `locomo_llm_judge` artifact is committed because no API key was used for a
+full `locomo10.json` run. Until that artifact exists, README/docs must not
+claim a LOCOMO score, and `--compare` must not imply Agent-Memory "beats"
+Mem0/MemMachine.
+
+## Smoke fixture
+
+`benchmarks/fixtures/locomo-smoke.json` is a 1-conversation file in the real
+schema (including a numeric `answer` in the v2 tests). It is not a subset of
+the official dataset; it exists so CI can execute parse → ingest → retrieve →
+score without downloading CC-BY-NC data.