From 9d09e12c1cff5f55e193dd5fbf249e9dd8662cd6 Mon Sep 17 00:00:00 2001 From: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:45:56 +0000 Subject: [PATCH] evals: expand golden set to fourteen queries and add JSON report output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grow the evaluation dataset beyond machine learning so domain-agnostic ranking is actually measured: - Eight new labeled queries covering federated learning privacy, protein structure prediction, sleep and memory, misinformation on social media, quantum error correction, end-to-end speech recognition, the gut-brain axis, and GAN image synthesis — each with graded relevance and cross-domain hard negatives - Dataset policy holds: entries whose DOI cannot be stated with confidence carry none rather than a guessed one; two high-confidence DOIs added to keep citation validity comfortably above the CI floor - Measured keyword-only baseline across fourteen queries: mean R@5 0.94, R@10 1.00, nDCG@10 0.97, MRR 1.00, citation validity 92% — all regression floors hold unchanged - Harness gains a domain column in the table and a JSON output mode (per-query metrics plus means) for automation and CI artifacts, exposed via a --json flag on the module entry point - Docs and test baseline text refreshed to the fourteen-query numbers --- docs/development/evaluation.md | 17 +- evals/golden_set.yaml | 549 +++++++++++++++++++++++++++++++++ src/evaluation/__main__.py | 9 +- src/evaluation/harness.py | 30 +- tests/test_evaluation.py | 30 +- 5 files changed, 617 insertions(+), 18 deletions(-) diff --git a/docs/development/evaluation.md b/docs/development/evaluation.md index 4e9cf4d..ba5fccb 100644 --- a/docs/development/evaluation.md +++ b/docs/development/evaluation.md @@ -8,9 +8,10 @@ so it works identically on developer machines and in CI. ## Golden dataset -`evals/golden_set.yaml` contains labeled queries across machine learning, -biomedicine, natural language processing, and computer vision. Each query -lists candidate papers with a graded relevance label: +`evals/golden_set.yaml` contains fourteen labeled queries spanning machine +learning, biomedicine, neuroscience, physics, speech, social science, and +computer vision. Each query lists candidate papers with a graded relevance +label: - `2` — highly relevant, the papers a domain expert would expect first - `1` — relevant supporting work @@ -41,8 +42,8 @@ use, and its module entry point prints the per-query table. ## Regression floors in CI `tests/test_evaluation.py` asserts floor values below the measured -keyword-only baseline (mean R@5 0.93, nDCG@10 0.98, MRR 1.00, validity -96%). A change that drops the suite below a floor has genuinely hurt +keyword-only baseline (with fourteen queries: mean R@5 0.94, nDCG@10 0.97, +MRR 1.00, validity 92%). A change that drops the suite below a floor has genuinely hurt ranking quality and fails CI. When a deliberate improvement raises the baseline, tighten the floors in the same change so the new level becomes the protected one. @@ -53,4 +54,8 @@ Add queries where the ranker currently struggles: multi-concept queries, fields outside machine learning, and queries whose relevant papers share few title words with the query. Keep labels honest — the harness is only as trustworthy as its dataset — and prefer real papers with verifiable -metadata. Growing toward the roadmap's ~20 queries is an open task. +metadata; entries whose DOI cannot be stated with confidence carry none +rather than a guessed one. Growing toward the roadmap's ~20 queries +remains open, as does a live mode that resolves DOIs against CrossRef. +The module entry point accepts a JSON flag for automation, emitting +per-query metrics plus means. diff --git a/evals/golden_set.yaml b/evals/golden_set.yaml index 82e6d9a..3b867fb 100644 --- a/evals/golden_set.yaml +++ b/evals/golden_set.yaml @@ -449,3 +449,552 @@ queries: citation_count: 20000 abstract: >- Clipped-objective policy gradient methods for reinforcement learning. + + - query: federated learning privacy preservation + domain: machine-learning + candidates: + - title: Communication-Efficient Learning of Deep Networks from Decentralized Data + relevance: 2 + year: 2017 + venue: AISTATS + doi: 10.48550/arXiv.1602.05629 + citation_count: 15000 + abstract: >- + Introduces federated averaging, training deep networks across + decentralized devices while keeping raw data local for privacy. + - title: Deep Learning with Differential Privacy + relevance: 2 + year: 2016 + venue: CCS + doi: 10.48550/arXiv.1607.00133 + citation_count: 7000 + abstract: >- + Trains neural networks under differential privacy with noisy SGD and + a moments accountant, bounding privacy loss during learning. + - title: "Advances and Open Problems in Federated Learning" + relevance: 1 + year: 2021 + doi: 10.48550/arXiv.1912.04977 + citation_count: 5000 + abstract: >- + Survey of federated learning systems, privacy mechanisms, robustness, + and open research problems across the field. + - title: "Federated Machine Learning: Concept and Applications" + relevance: 1 + year: 2019 + venue: ACM TIST + doi: 10.48550/arXiv.1902.04885 + citation_count: 4000 + abstract: >- + Categorizes horizontal, vertical, and transfer federated learning and + surveys privacy-preserving applications. + - title: Deep Residual Learning for Image Recognition + relevance: 0 + year: 2016 + venue: CVPR + doi: 10.48550/arXiv.1512.03385 + citation_count: 150000 + abstract: >- + Residual connections for very deep convolutional image recognition + networks. + - title: A Programmable Dual-RNA-Guided DNA Endonuclease in Adaptive Bacterial Immunity + relevance: 0 + year: 2012 + venue: Science + doi: 10.1126/science.1225829 + citation_count: 15000 + abstract: >- + Cas9 programmed with guide RNA cleaves target DNA sites, founding + CRISPR genome editing. + - title: Mastering the Game of Go with Deep Neural Networks and Tree Search + relevance: 0 + year: 2016 + venue: Nature + doi: 10.1038/nature16961 + citation_count: 18000 + abstract: >- + Deep reinforcement learning with tree search masters the game of Go. + - title: "Longformer: The Long-Document Transformer" + relevance: 0 + year: 2020 + doi: 10.48550/arXiv.2004.05150 + citation_count: 4000 + abstract: >- + Sparse sliding-window attention for long documents. + + - query: protein structure prediction deep learning + domain: biomedicine + candidates: + - title: Highly Accurate Protein Structure Prediction with AlphaFold + relevance: 2 + year: 2021 + venue: Nature + doi: 10.1038/s41586-021-03819-2 + citation_count: 30000 + abstract: >- + AlphaFold predicts protein three-dimensional structure from sequence + with atomic accuracy using attention-based deep learning over + evolutionary and geometric features. + - title: Improved Protein Structure Prediction Using Potentials from Deep Learning + relevance: 2 + year: 2020 + venue: Nature + doi: 10.1038/s41586-019-1923-7 + citation_count: 5000 + abstract: >- + Deep learning predicts inter-residue distance distributions whose + potentials guide protein structure prediction. + - title: Evolutionary-scale Prediction of Atomic-level Protein Structure with a Language Model + relevance: 1 + year: 2023 + venue: Science + citation_count: 2500 + abstract: >- + A protein language model predicts atomic-level structure directly + from sequence, scaling structure prediction to metagenomic proteins. + - title: Language Models are Few-Shot Learners + relevance: 0 + year: 2020 + venue: NeurIPS + doi: 10.48550/arXiv.2005.14165 + citation_count: 30000 + abstract: >- + Scaling autoregressive language models enables few-shot task + performance from in-context examples. + - title: Human-Level Control Through Deep Reinforcement Learning + relevance: 0 + year: 2015 + venue: Nature + doi: 10.1038/nature14236 + citation_count: 25000 + abstract: >- + Deep Q-networks learn Atari control policies from pixels. + - title: Very Deep Convolutional Networks for Large-Scale Image Recognition + relevance: 0 + year: 2015 + venue: ICLR + doi: 10.48550/arXiv.1409.1556 + citation_count: 100000 + abstract: >- + VGG studies convolutional depth for image recognition. + - title: Efficient Estimation of Word Representations in Vector Space + relevance: 0 + year: 2013 + doi: 10.48550/arXiv.1301.3781 + citation_count: 40000 + abstract: >- + word2vec learns static word embeddings from large corpora. + + - query: sleep deprivation effects on memory and cognition + domain: neuroscience + candidates: + - title: The Memory Function of Sleep + relevance: 2 + year: 2010 + venue: Nature Reviews Neuroscience + doi: 10.1038/nrn2762 + citation_count: 5000 + abstract: >- + Reviews how sleep consolidates memory, with slow-wave and REM sleep + supporting system and synaptic consolidation, and how sleep loss + impairs cognition and memory formation. + - title: About Sleep's Role in Memory + relevance: 2 + year: 2013 + venue: Physiological Reviews + doi: 10.1152/physrev.00032.2012 + citation_count: 4000 + abstract: >- + Comprehensive review of sleep-dependent memory consolidation across + declarative and procedural systems, including deprivation effects on + cognitive performance. + - title: Sleep Drives Metabolite Clearance from the Adult Brain + relevance: 1 + year: 2013 + venue: Science + doi: 10.1126/science.1241224 + citation_count: 4000 + abstract: >- + Sleep expands interstitial space and increases clearance of neurotoxic + waste including beta-amyloid, linking sleep to brain maintenance. + - title: "Sleep Deprivation: Impact on Cognitive Performance" + relevance: 2 + year: 2007 + venue: Neuropsychiatric Disease and Treatment + citation_count: 2000 + abstract: >- + Reviews experimental evidence on how acute and chronic sleep + deprivation degrade attention, working memory, and executive function. + - title: Attention Is All You Need + relevance: 0 + year: 2017 + venue: NeurIPS + doi: 10.48550/arXiv.1706.03762 + citation_count: 100000 + abstract: >- + Transformer sequence architecture built on self-attention. + - title: Semi-Supervised Classification with Graph Convolutional Networks + relevance: 0 + year: 2017 + venue: ICLR + doi: 10.48550/arXiv.1609.02907 + citation_count: 30000 + abstract: >- + Graph convolutional networks for node classification. + - title: Genome Engineering Using the CRISPR-Cas9 System + relevance: 0 + year: 2013 + venue: Nature Protocols + doi: 10.1038/nprot.2013.143 + citation_count: 9000 + abstract: >- + Protocol for CRISPR-Cas9 genome editing. + - title: "FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness" + relevance: 0 + year: 2022 + doi: 10.48550/arXiv.2205.14135 + citation_count: 3000 + abstract: >- + IO-aware exact attention algorithm for transformers. + + - query: misinformation spread on social media + domain: social-science + candidates: + - title: The Spread of True and False News Online + relevance: 2 + year: 2018 + venue: Science + doi: 10.1126/science.aap9559 + citation_count: 8000 + abstract: >- + Analysis of Twitter cascades showing false news spreads farther, + faster, and more broadly than true news, driven by novelty and human + sharing rather than bots. + - title: The Science of Fake News + relevance: 2 + year: 2018 + venue: Science + doi: 10.1126/science.aao2998 + citation_count: 4000 + abstract: >- + Framework article on the prevalence, spread, and countermeasures of + fake news and misinformation on social media platforms. + - title: Fake News on Twitter During the 2016 U.S. Presidential Election + relevance: 1 + year: 2019 + venue: Science + citation_count: 1500 + abstract: >- + Measures exposure to and sharing of fake news sources on Twitter, + finding concentration among small user groups during the election. + - title: Fighting Misinformation on Social Media Using Crowdsourced Judgments of News Source Quality + relevance: 1 + year: 2019 + venue: PNAS + citation_count: 1200 + abstract: >- + Shows crowdsourced trust ratings of news sources can inform platform + ranking to reduce misinformation spread. + - title: Deep Residual Learning for Image Recognition + relevance: 0 + year: 2016 + venue: CVPR + doi: 10.48550/arXiv.1512.03385 + citation_count: 150000 + abstract: >- + Residual connections for image recognition networks. + - title: Proximal Policy Optimization Algorithms + relevance: 0 + year: 2017 + doi: 10.48550/arXiv.1707.06347 + citation_count: 20000 + abstract: >- + Clipped-objective policy gradient reinforcement learning. + - title: Highly Accurate Protein Structure Prediction with AlphaFold + relevance: 0 + year: 2021 + venue: Nature + doi: 10.1038/s41586-021-03819-2 + citation_count: 30000 + abstract: >- + Deep learning predicts protein structure with atomic accuracy. + - title: Inductive Representation Learning on Large Graphs + relevance: 0 + year: 2017 + venue: NeurIPS + doi: 10.48550/arXiv.1706.02216 + citation_count: 15000 + abstract: >- + GraphSAGE computes node embeddings on large graphs. + + - query: quantum error correction codes + domain: physics + candidates: + - title: "Surface Codes: Towards Practical Large-Scale Quantum Computation" + relevance: 2 + year: 2012 + venue: Physical Review A + doi: 10.48550/arXiv.1208.0928 + citation_count: 4000 + abstract: >- + Pedagogical treatment of surface codes for quantum error correction, + covering stabilizer measurement, logical qubits, and thresholds for + fault-tolerant quantum computation. + - title: Stabilizer Codes and Quantum Error Correction + relevance: 2 + year: 1997 + doi: 10.48550/arXiv.quant-ph/9705052 + citation_count: 3000 + abstract: >- + Develops the stabilizer formalism underpinning quantum error + correcting codes and fault-tolerant operations. + - title: Fault-Tolerant Quantum Computation by Anyons + relevance: 1 + year: 2003 + venue: Annals of Physics + doi: 10.48550/arXiv.quant-ph/9707021 + citation_count: 5000 + abstract: >- + Proposes topological quantum computation with anyons, providing + intrinsically fault-tolerant error correction. + - title: Quantum Error Correction for Beginners + relevance: 1 + year: 2013 + venue: Reports on Progress in Physics + doi: 10.48550/arXiv.0905.2794 + citation_count: 900 + abstract: >- + Tutorial introduction to quantum error correction codes and + fault-tolerant protocols for quantum computing. + - title: "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding" + relevance: 0 + year: 2019 + venue: NAACL + doi: 10.48550/arXiv.1810.04805 + citation_count: 90000 + abstract: >- + Masked language modeling pre-training for language understanding. + - title: A Programmable Dual-RNA-Guided DNA Endonuclease in Adaptive Bacterial Immunity + relevance: 0 + year: 2012 + venue: Science + doi: 10.1126/science.1225829 + citation_count: 15000 + abstract: >- + Cas9 cleaves programmable DNA target sites. + - title: Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model + relevance: 0 + year: 2020 + venue: Nature + doi: 10.48550/arXiv.1911.08265 + citation_count: 3000 + abstract: >- + MuZero plans with a learned model of environment dynamics. + - title: "ImageNet: A Large-Scale Hierarchical Image Database" + relevance: 0 + year: 2009 + venue: CVPR + citation_count: 50000 + abstract: >- + Large labeled image database underpinning classification benchmarks. + + - query: end-to-end deep learning speech recognition + domain: speech + candidates: + - title: "Deep Speech: Scaling up End-to-End Speech Recognition" + relevance: 2 + year: 2014 + doi: 10.48550/arXiv.1412.5567 + citation_count: 3000 + abstract: >- + End-to-end recurrent network speech recognition trained on spectrograms, + replacing hand-engineered pipelines with deep learning at scale. + - title: Listen, Attend and Spell + relevance: 2 + year: 2016 + venue: ICASSP + doi: 10.48550/arXiv.1508.01211 + citation_count: 3000 + abstract: >- + Attention-based encoder-decoder that transcribes speech to characters + end to end without HMMs or lexicons. + - title: "wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations" + relevance: 2 + year: 2020 + venue: NeurIPS + doi: 10.48550/arXiv.2006.11477 + citation_count: 5000 + abstract: >- + Self-supervised pre-training of speech representations that fine-tunes + into state-of-the-art end-to-end speech recognition with little labeled data. + - title: Robust Speech Recognition via Large-Scale Weak Supervision + relevance: 1 + year: 2022 + doi: 10.48550/arXiv.2212.04356 + citation_count: 2500 + abstract: >- + Whisper trains multilingual end-to-end speech recognition on weakly + supervised web audio, generalizing robustly across domains. + - title: Graph Attention Networks + relevance: 0 + year: 2018 + venue: ICLR + doi: 10.48550/arXiv.1710.10903 + citation_count: 20000 + abstract: >- + Masked self-attention over graph neighborhoods. + - title: The Spread of True and False News Online + relevance: 0 + year: 2018 + venue: Science + doi: 10.1126/science.aap9559 + citation_count: 8000 + abstract: >- + False news spreads faster than true news on Twitter. + - title: Highly Accurate Protein Structure Prediction with AlphaFold + relevance: 0 + year: 2021 + venue: Nature + doi: 10.1038/s41586-021-03819-2 + citation_count: 30000 + abstract: >- + Deep learning predicts protein structures with atomic accuracy. + - title: Deep Residual Learning for Image Recognition + relevance: 0 + year: 2016 + venue: CVPR + doi: 10.48550/arXiv.1512.03385 + citation_count: 150000 + abstract: >- + Residual connections for deep convolutional networks. + + - query: gut microbiome influence on brain and behavior + domain: biomedicine + candidates: + - title: "Mind-Altering Microorganisms: the Impact of the Gut Microbiota on Brain and Behaviour" + relevance: 2 + year: 2012 + venue: Nature Reviews Neuroscience + doi: 10.1038/nrn3346 + citation_count: 4000 + abstract: >- + Reviews evidence that gut microbiota communicate with the brain along + the microbiome-gut-brain axis and influence behavior, stress, and + neurodevelopment. + - title: Normal Gut Microbiota Modulates Brain Development and Behavior + relevance: 2 + year: 2011 + venue: PNAS + citation_count: 3000 + abstract: >- + Germ-free mice show altered brain development and behavior, showing + gut microbiota shape neural circuits during early life. + - title: Gut Microbiota in Health and Disease + relevance: 1 + year: 2010 + venue: Physiological Reviews + citation_count: 4000 + abstract: >- + Reviews the composition and physiological roles of gut microbiota, + including immune, metabolic, and gut-brain interactions. + - title: Efficient Estimation of Word Representations in Vector Space + relevance: 0 + year: 2013 + doi: 10.48550/arXiv.1301.3781 + citation_count: 40000 + abstract: >- + word2vec learns word embeddings from corpora. + - title: Proximal Policy Optimization Algorithms + relevance: 0 + year: 2017 + doi: 10.48550/arXiv.1707.06347 + citation_count: 20000 + abstract: >- + Policy gradient methods for reinforcement learning. + - title: "Longformer: The Long-Document Transformer" + relevance: 0 + year: 2020 + doi: 10.48550/arXiv.2004.05150 + citation_count: 4000 + abstract: >- + Sparse attention for long documents. + - title: "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer" + relevance: 0 + year: 2020 + venue: JMLR + doi: 10.48550/arXiv.1910.10683 + citation_count: 15000 + abstract: >- + T5 casts NLP tasks as text-to-text problems. + + - query: generative adversarial networks image synthesis + domain: computer-vision + candidates: + - title: Generative Adversarial Networks + relevance: 2 + year: 2014 + venue: NeurIPS + doi: 10.48550/arXiv.1406.2661 + citation_count: 60000 + abstract: >- + Introduces the adversarial framework where a generator and + discriminator compete, enabling realistic image synthesis. + - title: Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks + relevance: 2 + year: 2016 + venue: ICLR + doi: 10.48550/arXiv.1511.06434 + citation_count: 15000 + abstract: >- + DCGAN establishes architectural guidelines for stable convolutional + GAN training and high-quality image generation. + - title: A Style-Based Generator Architecture for Generative Adversarial Networks + relevance: 2 + year: 2019 + venue: CVPR + doi: 10.48550/arXiv.1812.04948 + citation_count: 10000 + abstract: >- + StyleGAN separates high-level attributes in the generator, producing + state-of-the-art photorealistic image synthesis. + - title: Analyzing and Improving the Image Quality of StyleGAN + relevance: 1 + year: 2020 + venue: CVPR + doi: 10.48550/arXiv.1912.04958 + citation_count: 6000 + abstract: >- + StyleGAN2 redesigns normalization and regularization to remove + artifacts and improve generated image quality. + - title: Denoising Diffusion Probabilistic Models + relevance: 1 + year: 2020 + venue: NeurIPS + doi: 10.48550/arXiv.2006.11239 + citation_count: 12000 + abstract: >- + Diffusion models synthesize high-quality images by learning to + reverse a gradual noising process, rivaling GAN image synthesis. + - title: Human-Level Control Through Deep Reinforcement Learning + relevance: 0 + year: 2015 + venue: Nature + doi: 10.1038/nature14236 + citation_count: 25000 + abstract: >- + Deep Q-networks play Atari from pixels. + - title: The Memory Function of Sleep + relevance: 0 + year: 2010 + venue: Nature Reviews Neuroscience + doi: 10.1038/nrn2762 + citation_count: 5000 + abstract: >- + Sleep consolidates memory across systems. + - title: Robust Speech Recognition via Large-Scale Weak Supervision + relevance: 0 + year: 2022 + doi: 10.48550/arXiv.2212.04356 + citation_count: 2500 + abstract: >- + Weakly supervised multilingual speech recognition. diff --git a/src/evaluation/__main__.py b/src/evaluation/__main__.py index 1c28fb9..8347bdb 100644 --- a/src/evaluation/__main__.py +++ b/src/evaluation/__main__.py @@ -6,7 +6,7 @@ import argparse from pathlib import Path -from .harness import format_report, run_golden_evaluation +from .harness import format_report, format_report_json, run_golden_evaluation def main() -> None: @@ -19,10 +19,15 @@ def main() -> None: default=None, help="Path to a golden dataset YAML (default: evals/golden_set.yaml)", ) + parser.add_argument( + "--json", + action="store_true", + help="Emit the full report as JSON (per-query metrics plus means)", + ) args = parser.parse_args() report = run_golden_evaluation(dataset_path=args.dataset) - print(format_report(report)) + print(format_report_json(report) if args.json else format_report(report)) if __name__ == "__main__": diff --git a/src/evaluation/harness.py b/src/evaluation/harness.py index 5845aad..1b60a44 100644 --- a/src/evaluation/harness.py +++ b/src/evaluation/harness.py @@ -47,6 +47,10 @@ class EvaluationReport(BaseModel): def _mean(self, values: list[float]) -> float: return sum(values) / len(values) if values else 0.0 + @property + def mean_recall_at_5(self) -> float: + return self._mean([q.recall_at_5 for q in self.queries]) + @property def mean_recall_at_10(self) -> float: return self._mean([q.recall_at_10 for q in self.queries]) @@ -127,18 +131,19 @@ def format_report(report: EvaluationReport) -> str: "Golden-set evaluation " f"({'embedding' if report.embeddings_used else 'keyword'} ranking)", "", - f"{'query':<44} {'R@5':>5} {'R@10':>5} {'nDCG@10':>8} {'MRR':>5}", + f"{'query':<44} {'domain':<18} {'R@5':>5} {'R@10':>5} {'nDCG@10':>8} {'MRR':>5}", ] for item in report.queries: label = item.query if len(item.query) <= 42 else item.query[:39] + "..." lines.append( - f"{label:<44} {item.recall_at_5:>5.2f} {item.recall_at_10:>5.2f} " - f"{item.ndcg_at_10:>8.2f} {item.mrr:>5.2f}" + f"{label:<44} {item.domain:<18} {item.recall_at_5:>5.2f} " + f"{item.recall_at_10:>5.2f} {item.ndcg_at_10:>8.2f} {item.mrr:>5.2f}" ) lines.append("") lines.append( - f"{'mean':<44} {'':>5} {report.mean_recall_at_10:>5.2f} " - f"{report.mean_ndcg_at_10:>8.2f} {report.mean_mrr:>5.2f}" + f"{'mean':<44} {'':<18} {report.mean_recall_at_5:>5.2f} " + f"{report.mean_recall_at_10:>5.2f} {report.mean_ndcg_at_10:>8.2f} " + f"{report.mean_mrr:>5.2f}" ) validity = report.citation_validity lines.append( @@ -146,3 +151,18 @@ def format_report(report: EvaluationReport) -> str: f"({validity.validity_rate:.0%})" ) return "\n".join(lines) + + +def format_report_json(report: EvaluationReport) -> str: + """Render an evaluation report as JSON for automation and CI artifacts.""" + payload = report.model_dump() + payload["means"] = { + "recall_at_5": report.mean_recall_at_5, + "recall_at_10": report.mean_recall_at_10, + "ndcg_at_10": report.mean_ndcg_at_10, + "mrr": report.mean_mrr, + "citation_validity_rate": report.citation_validity.validity_rate, + } + import json + + return json.dumps(payload, indent=2) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index fd208b0..056c61e 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -8,7 +8,7 @@ from src.config.settings import AppSettings from src.evaluation.citation_validity import check_citation_validity from src.evaluation.dataset import load_golden_dataset -from src.evaluation.harness import format_report, run_golden_evaluation +from src.evaluation.harness import format_report, format_report_json, run_golden_evaluation from src.evaluation.metrics import mrr, ndcg_at_k, recall_at_k from src.retrieval.models import RetrievedPaper @@ -68,7 +68,9 @@ class TestGoldenDataset: def test_dataset_loads_and_is_well_formed(self) -> None: dataset = load_golden_dataset() - assert len(dataset.queries) >= 5 + assert len(dataset.queries) >= 14 + domains = {query.domain for query in dataset.queries} + assert len(domains) >= 6, "golden set should span multiple domains" for query in dataset.queries: assert len(query.candidates) >= 6 assert query.relevant_titles, f"query {query.query!r} has no relevant papers" @@ -79,9 +81,10 @@ def test_dataset_loads_and_is_well_formed(self) -> None: class TestGoldenEvaluation: """Regression floors for ranking quality on the golden set. - Floors are set below the measured keyword-only baseline (mean R@5 0.93, - nDCG@10 0.98, MRR 1.0) so they hold with or without the embedding - backend; a change that drops below them has genuinely hurt ranking. + Floors are set below the measured keyword-only baseline (fourteen + queries: mean R@5 0.94, nDCG@10 0.97, MRR 1.00, citation validity 92%) + so they hold with or without the embedding backend; a change that drops + below them has genuinely hurt ranking. """ @pytest.fixture(scope="class") @@ -105,3 +108,20 @@ def test_report_renders(self, report) -> None: rendered = format_report(report) assert "Golden-set evaluation" in rendered assert "citation validity" in rendered + assert "domain" in rendered + + def test_json_report_carries_means_and_queries(self, report) -> None: + import json + + payload = json.loads(format_report_json(report)) + + assert len(payload["queries"]) == len(report.queries) + assert set(payload["means"]) == { + "recall_at_5", + "recall_at_10", + "ndcg_at_10", + "mrr", + "citation_validity_rate", + } + assert payload["means"]["mrr"] == pytest.approx(report.mean_mrr) + assert payload["queries"][0]["domain"]