diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py index 724705b..755430c 100644 --- a/tests/test_evaluate.py +++ b/tests/test_evaluate.py @@ -87,6 +87,33 @@ async def test_real_llm_is_held_to_refusal_correctness(corpus, conn, embedder): assert report.pass_rate == 0.0 +@pytest.mark.asyncio +async def test_unknown_principal_with_no_expected_docs_has_perfect_recall(conn, embedder): + cases = [_case("unknown-refusal", "ghost-user", "anything", should_answer=False)] + + report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False) + + assert report.cases[0].recall == 1.0 + assert report.mean_recall == 1.0 + + +@pytest.mark.asyncio +async def test_unknown_principal_still_misses_expected_docs(conn, embedder): + cases = [ + _case( + "unknown-miss", + "ghost-user", + "anything", + expected=["required-document"], + ) + ] + + report = await run_eval(conn, embedder, _ANSWERS, cases, llm_judged=False) + + assert report.cases[0].recall == 0.0 + assert report.mean_recall == 0.0 + + # --------------------------------------------------------------------------- scoring arithmetic diff --git a/vaultrag/evaluate.py b/vaultrag/evaluate.py index 1b07923..693d067 100644 --- a/vaultrag/evaluate.py +++ b/vaultrag/evaluate.py @@ -91,7 +91,7 @@ def leak_rate(self) -> float: @property def mean_recall(self) -> float: - scored = [c.recall for c in self.cases if c.recall is not None] + scored = [c.recall for c in self.cases] return sum(scored) / len(scored) if scored else 0.0 @property @@ -148,6 +148,7 @@ async def run_case( return CaseResult( case_id=case.id, user_id=case.user_id, + recall=1.0 if not case.expected_docs else 0.0, correct_refusal=not case.should_answer, llm_judged=llm_judged, )