From a28bc89c977ff359546363f5f61d65d401127ee8 Mon Sep 17 00:00:00 2001
From: Victor <70475442+vsolano9@users.noreply.github.com>
Date: Sun, 23 Aug 2026 16:07:24 +0200
Subject: [PATCH] fix(eval): score unknown principals consistently

Score an unresolvable principal as perfect recall only when no documents were expected, while preserving a miss when expected documents exist. Remove the dead optionality filter and cover both boundaries with real-database regression tests.

Closes AgentPostmortem/VaultRAG#8
---
 tests/test_evaluate.py | 27 +++++++++++++++++++++++++++
 vaultrag/evaluate.py   |  3 ++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py
index 724705b..755430c 100644
--- a/tests/test_evaluate.py
+++ b/tests/test_evaluate.py
@@ -87,6 +87,33 @@ async def test_real_llm_is_held_to_refusal_correctness(corpus, conn, embedder):
     assert report.pass_rate == 0.0
 
 
+@pytest.mark.asyncio
+async def test_unknown_principal_with_no_expected_docs_has_perfect_recall(conn, embedder):
+    cases = [_case("unknown-refusal", "ghost-user", "anything", should_answer=False)]
+
+    report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False)
+
+    assert report.cases[0].recall == 1.0
+    assert report.mean_recall == 1.0
+
+
+@pytest.mark.asyncio
+async def test_unknown_principal_still_misses_expected_docs(conn, embedder):
+    cases = [
+        _case(
+            "unknown-miss",
+            "ghost-user",
+            "anything",
+            expected=["required-document"],
+        )
+    ]
+
+    report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False)
+
+    assert report.cases[0].recall == 0.0
+    assert report.mean_recall == 0.0
+
+
 # --------------------------------------------------------------------------- scoring arithmetic
 
 
diff --git a/vaultrag/evaluate.py b/vaultrag/evaluate.py
index 1b07923..693d067 100644
--- a/vaultrag/evaluate.py
+++ b/vaultrag/evaluate.py
@@ -91,7 +91,7 @@ def leak_rate(self) -> float:
 
     @property
     def mean_recall(self) -> float:
-        scored = [c.recall for c in self.cases if c.recall is not None]
+        scored = [c.recall for c in self.cases]
         return sum(scored) / len(scored) if scored else 0.0
 
     @property
@@ -148,6 +148,7 @@ async def run_case(
         return CaseResult(
             case_id=case.id,
             user_id=case.user_id,
+            recall=1.0 if not case.expected_docs else 0.0,
             correct_refusal=not case.should_answer,
             llm_judged=llm_judged,
         )