fix(eval): score unknown principals consistently

tests/test_evaluate.py

@@ -87,6 +87,33 @@ async def test_real_llm_is_held_to_refusal_correctness(corpus, conn, embedder):
     assert report.pass_rate == 0.0
 
 
+@pytest.mark.asyncio
+async def test_unknown_principal_with_no_expected_docs_has_perfect_recall(conn, embedder):
+    cases = [_case("unknown-refusal", "ghost-user", "anything", should_answer=False)]
+
+    report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False)
+
+    assert report.cases[0].recall == 1.0
+    assert report.mean_recall == 1.0
+
+
+@pytest.mark.asyncio
+async def test_unknown_principal_still_misses_expected_docs(conn, embedder):
+    cases = [
+        _case(
+            "unknown-miss",
+            "ghost-user",
+            "anything",
+            expected=["required-document"],
+        )
+    ]
+
+    report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False)
+
+    assert report.cases[0].recall == 0.0
+    assert report.mean_recall == 0.0
+
+
 # --------------------------------------------------------------------------- scoring arithmetic
 
 

vaultrag/evaluate.py

@@ -91,7 +91,7 @@ def leak_rate(self) -> float:
 
     @property
     def mean_recall(self) -> float:
-        scored = [c.recall for c in self.cases if c.recall is not None]
+        scored = [c.recall for c in self.cases]
         return sum(scored) / len(scored) if scored else 0.0
 
     @property
@@ -148,6 +148,7 @@ async def run_case(
         return CaseResult(
             case_id=case.id,
             user_id=case.user_id,
+            recall=1.0 if not case.expected_docs else 0.0,
             correct_refusal=not case.should_answer,
             llm_judged=llm_judged,
         )