From 33ab3124572608457f67619c5161ae601eae4837 Mon Sep 17 00:00:00 2001 From: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:10:22 +0000 Subject: [PATCH] feat: iterative research loop and claim verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the one-shot pipeline into an assess-refine-stop cycle and check report claims against their sources before publication. Research loop (new stage between relevance scoring and fulltext): - Assesses whether the kept papers answer the query — a structured LLM verdict when synthesis LLM mode is on, and a concept-coverage heuristic otherwise (every core query concept must be matched by at least one paper, with a minimum paper floor) - When coverage is thin, generates targeted follow-up queries, retrieves for them, and merges candidates through the existing deduplication, ranking, and relevance machinery — no special-case scoring path — repeating up to research_loop.max_iterations - The final coverage assessment lands in the coverage_assessment artifact for debug dumps Verification (new stage between gap analysis and citation export): - Checks each paper's key points against that paper's own sources (abstract plus grounded full-text passages); a structured LLM verdict per paper when available, and a conservative term-coverage heuristic otherwise - Unsupported points are flagged, never rewritten: paper analyses gain unverified_points, reports mark them '(unverified)', and the report carries an aggregate VerificationSummary (claims checked/unverified, method) in markdown and JSON output Both stages are config-gated (research_loop / verification sections), degrade gracefully on any failure, and are documented in the stage pages, environment reference, and .env.example. Author: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue Sep 1 23:10:22 2026 +0000 --- .env.example | 7 + config/default.yaml | 12 + docs/architecture/stages/research-loop.md | 48 ++++ docs/architecture/stages/verification.md | 45 ++++ docs/configuration/environment-variables.md | 8 +- mkdocs.yml | 2 + src/analysis/verification.py | 182 +++++++++++++ src/config/settings.py | 20 ++ src/models/base.py | 15 ++ src/models/structured.py | 9 + src/reporting/markdown.py | 17 +- src/reporting/report_generation.py | 7 + src/research/research_loop.py | 230 +++++++++++++++++ src/retrieval/models.py | 14 + src/retrieval/orchestrator.py | 4 + src/utils/progress_reporter.py | 2 + tests/test_research_loop_verification.py | 272 ++++++++++++++++++++ 17 files changed, 891 insertions(+), 3 deletions(-) create mode 100644 docs/architecture/stages/research-loop.md create mode 100644 docs/architecture/stages/verification.md create mode 100644 src/analysis/verification.py create mode 100644 src/research/research_loop.py create mode 100644 tests/test_research_loop_verification.py diff --git a/.env.example b/.env.example index 03fae70..66f49d8 100644 --- a/.env.example +++ b/.env.example @@ -113,5 +113,12 @@ RA_PIPELINE__STREAM_PROGRESS=true # RA_FULLTEXT__ENABLED=false # RA_FULLTEXT__MAX_PAPERS=5 +# Coverage-driven retrieval refinement (follow-up queries when coverage is thin) +# RA_RESEARCH_LOOP__ENABLED=false +# RA_RESEARCH_LOOP__MAX_ITERATIONS=1 + +# Claim verification before report generation +# RA_VERIFICATION__ENABLED=false + # Optional: override config directory # RA_CONFIG_DIR=/path/to/config diff --git a/config/default.yaml b/config/default.yaml index 00d9182..d9fe940 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -95,13 +95,25 @@ pipeline: snowball: true rerank: true relevance_scoring: true + research_loop: true fulltext: true clustering: true synthesis: true gap_analysis: true + verification: true citation_export: true report_generation: true +research_loop: + enabled: true + max_iterations: 1 + min_sufficient_papers: 6 + max_follow_up_queries: 3 + +verification: + enabled: true + min_term_coverage: 0.5 + fulltext: enabled: true max_papers: 5 diff --git a/docs/architecture/stages/research-loop.md b/docs/architecture/stages/research-loop.md new file mode 100644 index 0000000..04cf2e8 --- /dev/null +++ b/docs/architecture/stages/research-loop.md @@ -0,0 +1,48 @@ +# Research Loop (Coverage-Driven Refinement) + +The research loop runs between [relevance scoring](relevance-scoring.md) +and [fulltext](fulltext.md). It turns the one-shot pipeline into the +assess–refine–stop cycle deep-research agents use: after filtering, the +stage judges whether the kept papers actually answer the query; when +coverage is thin it generates targeted follow-up queries, retrieves for +them, and merges the new candidates through the same deduplication, +ranking, and relevance machinery — repeating up to a configured budget. + +## Coverage assessment + +Two assessors, chosen by the LLM feature resolution: + +- **LLM** (when synthesis LLM mode is on and structured outputs are + enabled) — a structured verdict: `sufficient`, `missing_aspects`, and + `follow_up_queries` proposed as concise academic search phrases. +- **Heuristic** (always available) — coverage requires at least + `min_sufficient_papers` kept papers *and* every core query concept + matched by at least one paper; follow-up queries combine each missing + concept with the covered ones. + +The final assessment is stored in the `coverage_assessment` artifact, so +debug dumps show why the loop did or did not fire. + +## Refinement round + +Follow-up queries go through the standard retrieval fan-out, and the new +candidates merge with the current set through deduplication (with +cross-provider reconciliation), ranking, and the relevance filter — no +special-case scoring path. Embedding-backend failures fall back to +keyword ranking exactly as the ranking stage does. + +## Configuration + +| Key | Default | Meaning | +|-----|---------|---------| +| `research_loop.enabled` | `true` | Toggle the stage | +| `research_loop.max_iterations` | `1` | Refinement rounds per run | +| `research_loop.min_sufficient_papers` | `6` | Heuristic floor for sufficiency | +| `research_loop.max_follow_up_queries` | `3` | Queries retrieved per round | + +## Failure behavior + +Retrieval failures during a refinement round pass the current ranking +through with a warning; an assessment that never reaches sufficiency +stops at the iteration budget. Stage metrics record iterations run, the +follow-up queries used, and whether coverage ended sufficient. diff --git a/docs/architecture/stages/verification.md b/docs/architecture/stages/verification.md new file mode 100644 index 0000000..2ba288d --- /dev/null +++ b/docs/architecture/stages/verification.md @@ -0,0 +1,45 @@ +# Verification (Claim Checking) + +The verification stage runs between [gap analysis](gap-analysis.md) and +[citation export](citation-export.md). Before a report is generated, +each paper's key points are checked against that paper's own sources — +its abstract plus any grounded full-text passages from the +[fulltext stage](fulltext.md). Unsupported claims are flagged rather +than presented as fact. + +## What readers see + +- Flagged key points render with an _(unverified)_ marker in the + thematic findings. +- The report's summary area carries an aggregate line, e.g. + *Claim verification (heuristic): 11/12 key points supported by paper + sources* — the `verification` field of the JSON report holds the same + numbers (`claims_checked`, `claims_unverified`, `method`). + +## How claims are checked + +- **LLM** (when synthesis LLM mode is on and structured outputs are + enabled) — a structured verdict per paper naming the claim numbers the + sources do not support, judged strictly from the provided material. +- **Heuristic** (always available) — a claim is flagged when fewer than + `min_term_coverage` of its content terms appear in the paper's + sources. Flagging is deliberately conservative: with no sources at + all, nothing is flagged, and abstract-derived heuristic findings pass + by construction. + +Identity note: verification never rewrites claims — it only annotates +`unverified_points` on the paper analyses and records the +`verification_summary` artifact consumed by report generation. + +## Configuration + +| Key | Default | Meaning | +|-----|---------|---------| +| `verification.enabled` | `true` | Toggle the stage | +| `verification.min_term_coverage` | `0.5` | Heuristic support threshold | + +## Failure behavior + +A failed LLM verdict falls back to the heuristic for that paper. The +stage passes gap-analysis output through unchanged, so downstream +stages are unaffected by verification results. diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 4533b52..e80eda6 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -122,7 +122,7 @@ Full weight list: [YAML reference](yaml-reference.md#ranking). | `RA_PIPELINE__SYNTHESIS_TIMEOUT_SECONDS` | `600` | Synthesis stage timeout | | `RA_PIPELINE__ENABLED_STAGES__` | `true` | Disable individual pipeline stages | -Stage names: `query_understanding`, `query_expansion`, `retrieval`, `deduplication`, `ranking`, `snowball`, `rerank`, `relevance_scoring`, `fulltext`, `clustering`, `synthesis`, `gap_analysis`, `citation_export`, `report_generation`. See [Stage toggles](stage-toggles.md). +Stage names: `query_understanding`, `query_expansion`, `retrieval`, `deduplication`, `ranking`, `snowball`, `rerank`, `relevance_scoring`, `research_loop`, `fulltext`, `clustering`, `synthesis`, `gap_analysis`, `verification`, `citation_export`, `report_generation`. See [Stage toggles](stage-toggles.md). --- @@ -141,8 +141,12 @@ Stage names: `query_understanding`, `query_expansion`, `retrieval`, `deduplicati | `RA_FULLTEXT__MAX_PAPERS` | `5` | Top papers attempted per run | | `RA_FULLTEXT__MAX_PDF_MB` | `15` | Per-PDF download size cap | | `RA_FULLTEXT__TOP_CHUNKS_PER_PAPER` | `3` | Passages retrieved per paper | +| `RA_RESEARCH_LOOP__ENABLED` | `true` | Coverage-driven retrieval refinement | +| `RA_RESEARCH_LOOP__MAX_ITERATIONS` | `1` | Refinement rounds per run | +| `RA_VERIFICATION__ENABLED` | `true` | Claim verification before reporting | +| `RA_VERIFICATION__MIN_TERM_COVERAGE` | `0.5` | Heuristic support threshold | -Full key lists live on the stage pages: [Snowball](../architecture/stages/snowball.md), [Rerank](../architecture/stages/rerank.md), [Fulltext](../architecture/stages/fulltext.md). +Full key lists live on the stage pages: [Snowball](../architecture/stages/snowball.md), [Rerank](../architecture/stages/rerank.md), [Fulltext](../architecture/stages/fulltext.md), [Research loop](../architecture/stages/research-loop.md), [Verification](../architecture/stages/verification.md). --- diff --git a/mkdocs.yml b/mkdocs.yml index 6a870e6..adb0d3e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,10 +79,12 @@ nav: - architecture/stages/snowball.md - architecture/stages/rerank.md - architecture/stages/relevance-scoring.md + - architecture/stages/research-loop.md - architecture/stages/fulltext.md - architecture/stages/clustering.md - architecture/stages/synthesis.md - architecture/stages/gap-analysis.md + - architecture/stages/verification.md - architecture/stages/citation-export.md - architecture/stages/report-generation.md - Configuration: diff --git a/src/analysis/verification.py b/src/analysis/verification.py new file mode 100644 index 0000000..fdc5eb6 --- /dev/null +++ b/src/analysis/verification.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +"""Claim verification against paper sources before report generation. + +Each paper's key points are checked against that paper's own sources — +abstract plus any grounded full-text passages. Unsupported claims are +flagged in the report rather than presented as fact, and the report +carries an aggregate verification summary. + +Verification is LLM-backed (structured output) when synthesis LLM mode is +on, with a term-coverage heuristic otherwise. Flagging is deliberately +conservative: a claim is marked unverified only when its content terms +are largely absent from the paper's sources. +""" + +from __future__ import annotations + +import time + +from pydantic import BaseModel, Field + +from ..core.context import PipelineContext, StageResult +from ..models.base import AgentRole +from ..research.text_utils import extract_query_terms, term_matches_text +from ..retrieval.models import ( + GapAnalysisResult, + PaperAnalysis, + RankedPaper, + VerificationSummary, +) + +VERIFICATION_ARTIFACT = "verification_summary" + + +class _ClaimVerdicts(BaseModel): + """Structured output: which numbered claims lack support in the sources.""" + + unsupported_claim_numbers: list[int] = Field(default_factory=list) + + +def _paper_sources( + analysis: PaperAnalysis, + abstracts: dict[str, str], + passages: dict[str, list[str]], +) -> str: + parts: list[str] = [] + if analysis.paper_id: + if analysis.paper_id in abstracts: + parts.append(abstracts[analysis.paper_id]) + parts.extend(passages.get(analysis.paper_id, [])) + return "\n".join(parts).lower() + + +def heuristic_unsupported( + claims: list[str], + sources: str, + *, + min_term_coverage: float, +) -> list[str]: + """Claims whose content terms are largely absent from the sources.""" + if not sources: + return [] + + unsupported: list[str] = [] + for claim in claims: + terms = extract_query_terms(claim) + if not terms: + continue + matched = sum(1 for term in terms if term_matches_text(term, sources)) + if matched / len(terms) < min_term_coverage: + unsupported.append(claim) + return unsupported + + +def _verification_prompt(analysis: PaperAnalysis, sources: str) -> str: + numbered = "\n".join( + f"{index}. {claim}" for index, claim in enumerate(analysis.key_points, start=1) + ) + return ( + f"Paper: {analysis.title}\n\n" + f"Source material (abstract and full-text passages):\n{sources[:4000]}\n\n" + f"Claims made about this paper:\n{numbered}\n\n" + "Which claim numbers are NOT supported by the source material?" + ) + + +async def _llm_unsupported( + analysis: PaperAnalysis, + sources: str, + ctx: PipelineContext, +) -> list[str] | None: + from ..models.structured import try_run_structured + + verdicts = await try_run_structured( + AgentRole.VERIFICATION, + _verification_prompt(analysis, sources), + _ClaimVerdicts, + ctx.config.llm, + ) + if verdicts is None: + return None + return [ + analysis.key_points[number - 1] + for number in verdicts.unsupported_claim_numbers + if 1 <= number <= len(analysis.key_points) + ] + + +class VerificationStage: + """Pipeline stage that flags unsupported claims before reporting.""" + + name = "verification" + + async def run( + self, + ctx: PipelineContext, + data: GapAnalysisResult, + ) -> StageResult[GapAnalysisResult]: + started = time.perf_counter() + config = ctx.config.verification + + def _result(summary: VerificationSummary | None) -> StageResult[GapAnalysisResult]: + metrics: dict[str, object] = {} + if summary is not None: + ctx.set_artifact(VERIFICATION_ARTIFACT, summary.model_dump()) + metrics = { + "claims_checked": summary.claims_checked, + "claims_unverified": summary.claims_unverified, + "method": summary.method, + } + return StageResult( + output=data, + duration_ms=(time.perf_counter() - started) * 1000, + metrics=metrics, + ) + + analyses: list[PaperAnalysis] = ctx.get_artifact("paper_analyses") or [] + if not config.enabled or not analyses: + return _result(None) + + ranked: list[RankedPaper] = ctx.get_artifact("ranked_papers") or [] + abstracts = { + item.paper.paper_id: item.paper.abstract or "" for item in ranked + } + passages: dict[str, list[str]] = ctx.get_artifact("fulltext_passages") or {} + + use_llm = ctx.config.synthesis.llm_enabled and ctx.config.llm.structured_outputs + method = "llm" if use_llm else "heuristic" + + checked = 0 + unverified_total = 0 + verified_analyses: list[PaperAnalysis] = [] + + for analysis in analyses: + sources = _paper_sources(analysis, abstracts, passages) + claims = analysis.key_points + checked += len(claims) + + unsupported: list[str] | None = None + if use_llm and claims and sources: + unsupported = await _llm_unsupported(analysis, sources, ctx) + if unsupported is None: + method = "heuristic" if not use_llm else method + unsupported = heuristic_unsupported( + claims, + sources, + min_term_coverage=config.min_term_coverage, + ) + + unverified_total += len(unsupported) + verified_analyses.append( + analysis.model_copy(update={"unverified_points": unsupported}) + if unsupported + else analysis + ) + + ctx.set_artifact("paper_analyses", verified_analyses) + summary = VerificationSummary( + claims_checked=checked, + claims_unverified=unverified_total, + method=method, + ) + return _result(summary) diff --git a/src/config/settings.py b/src/config/settings.py index 2f67efc..2a8b0da 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -151,6 +151,22 @@ class RetrievalConfig(BaseModel): ) +class ResearchLoopConfig(BaseModel): + """Iterative coverage-driven retrieval refinement.""" + + enabled: bool = True + max_iterations: int = 1 + min_sufficient_papers: int = 6 + max_follow_up_queries: int = 3 + + +class VerificationConfig(BaseModel): + """Claim verification against paper sources before report generation.""" + + enabled: bool = True + min_term_coverage: float = 0.5 + + class FulltextConfig(BaseModel): """Open-access full-text grounding for top-ranked papers.""" @@ -200,10 +216,12 @@ class PipelineConfig(BaseModel): "snowball": True, "rerank": True, "relevance_scoring": True, + "research_loop": True, "fulltext": True, "clustering": True, "synthesis": True, "gap_analysis": True, + "verification": True, "citation_export": True, "report_generation": True, } @@ -280,6 +298,8 @@ class AppSettings(BaseSettings): snowball: SnowballConfig = Field(default_factory=SnowballConfig) rerank: RerankConfig = Field(default_factory=RerankConfig) fulltext: FulltextConfig = Field(default_factory=FulltextConfig) + research_loop: ResearchLoopConfig = Field(default_factory=ResearchLoopConfig) + verification: VerificationConfig = Field(default_factory=VerificationConfig) pipeline: PipelineConfig = Field(default_factory=PipelineConfig) memory: MemoryConfig = Field(default_factory=MemoryConfig) synthesis: SynthesisConfig = Field(default_factory=SynthesisConfig) diff --git a/src/models/base.py b/src/models/base.py index d9437d0..fb77483 100644 --- a/src/models/base.py +++ b/src/models/base.py @@ -25,6 +25,8 @@ class AgentRole(str, Enum): GAP_ANALYSIS = "gap_analysis" INTERACTIVE = "interactive" ANALYSIS = "analysis" + COVERAGE = "coverage" + VERIFICATION = "verification" ROLE_SYSTEM_PROMPTS: dict[AgentRole, str] = { @@ -59,6 +61,19 @@ class AgentRole(str, Enum): '"underexplored_areas" (list of strings). ' "Do not include conversational filler." ), + AgentRole.COVERAGE: ( + "You assess literature-search coverage. Given a research query and the " + "papers found so far, respond with ONLY a JSON object with keys: " + '"sufficient" (boolean), "missing_aspects" (list of strings), ' + '"follow_up_queries" (list of strings — concise academic search phrases ' + "targeting the missing aspects). Do not include conversational filler." + ), + AgentRole.VERIFICATION: ( + "You verify claims against source material. Respond with ONLY a JSON " + 'object with key "unsupported_claim_numbers" (list of integers naming ' + "the claims the sources do not support). Judge strictly from the given " + "sources. Do not include conversational filler." + ), AgentRole.INTERACTIVE: ( "You are a research assistant helping refine literature review follow-up questions. " "Respond concisely and stay focused on the user's research topic." diff --git a/src/models/structured.py b/src/models/structured.py index ebf9dcf..fa5cffd 100644 --- a/src/models/structured.py +++ b/src/models/structured.py @@ -47,6 +47,15 @@ "prioritized research gaps, actionable opportunities, and " "underexplored areas." ), + AgentRole.COVERAGE: ( + "You assess literature-search coverage. Judge whether the papers found " + "so far answer the research query; when they do not, name the missing " + "aspects and propose concise follow-up search queries targeting them." + ), + AgentRole.VERIFICATION: ( + "You verify claims against source material. Name the claims the given " + "sources do not support, judging strictly from those sources." + ), } diff --git a/src/reporting/markdown.py b/src/reporting/markdown.py index b0cfaa8..80e9c49 100644 --- a/src/reporting/markdown.py +++ b/src/reporting/markdown.py @@ -78,7 +78,12 @@ def _render_thematic_findings( lines.append(meta) lines.append(f"Source: {_format_paper_source(paper)}") if paper.key_points: - lines.extend(["", "Key points:", *_bullet_lines(paper.key_points)]) + unverified = set(paper.unverified_points) + point_lines = [ + f"- {point} _(unverified)_" if point in unverified else f"- {point}" + for point in paper.key_points + ] + lines.extend(["", "Key points:", *point_lines]) if paper.why_relevant: lines.extend(["", "Why relevant:", *_bullet_lines(paper.why_relevant)]) if paper.evidence: @@ -150,6 +155,16 @@ def render_enhanced_markdown( lines.append(f"Literature review for: **{report.query}**") lines.append("") + if report.verification and report.verification.claims_checked: + verification = report.verification + lines.append( + f"_Claim verification ({verification.method}): " + f"{verification.claims_supported}/{verification.claims_checked} " + "key points supported by paper sources; unsupported points are " + "marked (unverified) below._" + ) + lines.append("") + lines.extend(["# Research Query & Scope", "", f"**Query:** {report.query}", ""]) lines.extend(["# Thematic Findings", ""]) diff --git a/src/reporting/report_generation.py b/src/reporting/report_generation.py index adbf4d8..fd6f922 100644 --- a/src/reporting/report_generation.py +++ b/src/reporting/report_generation.py @@ -17,6 +17,7 @@ PaperCluster, RankedPaper, SynthesisResult, + VerificationSummary, ) @@ -140,6 +141,11 @@ def assemble_report(ctx: PipelineContext, exports: dict[str, str]) -> EnhancedRe elif synthesis: gaps.extend(synthesis.gaps) + verification_raw = ctx.get_artifact("verification_summary") + verification = ( + VerificationSummary.model_validate(verification_raw) if verification_raw else None + ) + report = EnhancedResearchReport( query=ctx.query, executive_summary=_build_executive_summary( @@ -150,6 +156,7 @@ def assemble_report(ctx: PipelineContext, exports: dict[str, str]) -> EnhancedRe ranked_papers=ranked_papers, relevance_config=ctx.config.relevance_scoring, ), + verification=verification, papers=analyses, clusters=clusters, synthesis=synthesis, diff --git a/src/research/research_loop.py b/src/research/research_loop.py new file mode 100644 index 0000000..4b10b9e --- /dev/null +++ b/src/research/research_loop.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +"""Iterative coverage-driven retrieval refinement. + +After relevance filtering, the loop asks whether the kept papers actually +answer the query. When coverage is thin it generates targeted follow-up +queries, retrieves for them, and merges the new candidates through the +same deduplication, ranking, and relevance machinery — repeating up to a +configured budget. This turns the one-shot pipeline into the +assess-refine-stop cycle deep-research agents use. + +Coverage assessment is LLM-backed (structured output) when synthesis LLM +mode is on, with a concept-coverage heuristic otherwise, so the loop +functions in fully local heuristic runs too. +""" + +from __future__ import annotations + +import time + +from pydantic import BaseModel, Field + +from ..core.context import PipelineContext, StageResult +from ..models.base import AgentRole +from ..retrieval.models import ExpandedQuerySet, RankedPaper +from ..utils.logging_system import logger +from .ranking import rank_papers +from .text_utils import extract_core_concepts, term_matches_text + +COVERAGE_ARTIFACT = "coverage_assessment" + + +class CoverageAssessment(BaseModel): + """Structured verdict on whether retrieved papers answer the query.""" + + sufficient: bool = True + missing_aspects: list[str] = Field(default_factory=list) + follow_up_queries: list[str] = Field(default_factory=list) + + +def heuristic_coverage( + query: str, + ranked_papers: list[RankedPaper], + *, + min_sufficient_papers: int, +) -> CoverageAssessment: + """Concept-coverage heuristic: every core concept needs a matching paper.""" + concepts = extract_core_concepts(query) + texts = [ + f"{item.paper.title} {item.paper.abstract or ''}".lower() + for item in ranked_papers + ] + + missing = [ + concept + for concept in concepts + if not any(term_matches_text(concept.lower(), text) for text in texts) + ] + + covered_terms = [c for c in concepts if c not in missing] + follow_ups = [ + " ".join([concept, *covered_terms[:2]]).strip() for concept in missing + ] + + sufficient = len(ranked_papers) >= min_sufficient_papers and not missing + return CoverageAssessment( + sufficient=sufficient, + missing_aspects=missing, + follow_up_queries=follow_ups, + ) + + +def _coverage_prompt(query: str, ranked_papers: list[RankedPaper]) -> str: + lines = [f"Research query: {query}", "", "Papers found so far:"] + for item in ranked_papers[:15]: + abstract = (item.paper.abstract or "")[:200] + lines.append(f"- {item.paper.title} ({item.paper.year or 'n.d.'}): {abstract}") + lines.append("") + lines.append( + "Is this set sufficient to answer the query? If not, list the missing " + "aspects and propose follow-up search queries." + ) + return "\n".join(lines) + + +async def assess_coverage( + query: str, + ranked_papers: list[RankedPaper], + ctx: PipelineContext, +) -> CoverageAssessment: + """LLM coverage assessment with heuristic fallback.""" + config = ctx.config.research_loop + heuristic = heuristic_coverage( + query, + ranked_papers, + min_sufficient_papers=config.min_sufficient_papers, + ) + + if not (ctx.config.synthesis.llm_enabled and ctx.config.llm.structured_outputs): + return heuristic + + from ..models.structured import try_run_structured + + assessment = await try_run_structured( + AgentRole.COVERAGE, + _coverage_prompt(query, ranked_papers), + CoverageAssessment, + ctx.config.llm, + ) + return assessment if assessment is not None else heuristic + + +class ResearchLoopStage: + """Pipeline stage that refines retrieval until coverage is sufficient.""" + + name = "research_loop" + + async def run( + self, + ctx: PipelineContext, + data: list[RankedPaper], + ) -> StageResult[list[RankedPaper]]: + started = time.perf_counter() + config = ctx.config.research_loop + warnings: list[str] = [] + iterations = 0 + follow_ups_used: list[str] = [] + papers = data + + def _result(assessment: CoverageAssessment | None) -> StageResult[list[RankedPaper]]: + if assessment is not None: + ctx.set_artifact(COVERAGE_ARTIFACT, assessment.model_dump()) + return StageResult( + output=papers, + duration_ms=(time.perf_counter() - started) * 1000, + metrics={ + "iterations_run": iterations, + "follow_up_queries": follow_ups_used, + "papers_after_loop": len(papers), + "coverage_sufficient": bool(assessment.sufficient) + if assessment + else None, + }, + warnings=warnings, + ) + + if not config.enabled or not papers: + return _result(None) + + assessment = await assess_coverage(ctx.query, papers, ctx) + + while ( + not assessment.sufficient + and assessment.follow_up_queries + and iterations < config.max_iterations + ): + iterations += 1 + follow_ups = [ + q.strip() + for q in assessment.follow_up_queries[: config.max_follow_up_queries] + if q.strip() + ] + if not follow_ups: + break + follow_ups_used.extend(follow_ups) + logger.info( + "Research loop iteration %d: retrieving %d follow-up quer(ies)", + iterations, + len(follow_ups), + ) + + try: + papers = await self._refine(ctx, papers, follow_ups) + except Exception as exc: + warnings.append(f"Research loop retrieval failed: {exc}") + break + + assessment = await assess_coverage(ctx.query, papers, ctx) + + ctx.set_artifact("ranked_papers", papers) + return _result(assessment) + + async def _refine( + self, + ctx: PipelineContext, + current: list[RankedPaper], + follow_ups: list[str], + ) -> list[RankedPaper]: + """Retrieve follow-up queries and re-run dedup, ranking, and filtering.""" + from ..embeddings import try_create_embedding_provider + from ..retrieval.deduplication import deduplicate_papers + from ..retrieval.retrieval_stage import retrieve_papers + from .embedding_context import store_ranking_embedding_result + from .relevance_scoring import RelevanceScoringStage + + expanded = ExpandedQuerySet(original=follow_ups[0], variants=follow_ups[1:]) + new_papers, warnings, _failed = await retrieve_papers(expanded, ctx.config) + if not new_papers: + return current + + embedder = try_create_embedding_provider(ctx.config.embedding) + base = [item.paper for item in current] + try: + combined, _stats = deduplicate_papers( + base + new_papers, + config=ctx.config.deduplication, + embedder=embedder, + ) + ranking_result = rank_papers( + combined, ctx.query, config=ctx.config.ranking, embedder=embedder + ) + except ImportError: + combined, _stats = deduplicate_papers( + base + new_papers, + config=ctx.config.deduplication.model_copy( + update={"enable_embedding_dedup": False} + ), + embedder=None, + ) + ranking_result = rank_papers( + combined, ctx.query, config=ctx.config.ranking, embedder=None + ) + + store_ranking_embedding_result( + ctx, + ranking_result.query_embedding, + ranking_result.paper_embeddings, + ) + + filtered = await RelevanceScoringStage().run(ctx, ranking_result.ranked) + return filtered.output diff --git a/src/retrieval/models.py b/src/retrieval/models.py index 49b67f5..075afd0 100644 --- a/src/retrieval/models.py +++ b/src/retrieval/models.py @@ -53,6 +53,7 @@ class PaperAnalysis(BaseModel): key_points: list[str] = Field(default_factory=list) why_relevant: list[str] = Field(default_factory=list) evidence: list[str] = Field(default_factory=list) + unverified_points: list[str] = Field(default_factory=list) class ResearchReport(BaseModel): @@ -126,11 +127,24 @@ class GapAnalysisResult(BaseModel): underexplored_areas: list[str] = Field(default_factory=list) +class VerificationSummary(BaseModel): + """Outcome of the claim-verification pass.""" + + claims_checked: int = 0 + claims_unverified: int = 0 + method: str = "heuristic" + + @property + def claims_supported(self) -> int: + return self.claims_checked - self.claims_unverified + + class EnhancedResearchReport(BaseModel): """Extended research report with synthesis, clusters, and exports.""" query: str executive_summary: str = "" + verification: Optional[VerificationSummary] = None papers: list[PaperAnalysis] = Field(default_factory=list) clusters: list[PaperCluster] = Field(default_factory=list) synthesis: Optional[SynthesisResult] = None diff --git a/src/retrieval/orchestrator.py b/src/retrieval/orchestrator.py index bb1346f..f4b1222 100644 --- a/src/retrieval/orchestrator.py +++ b/src/retrieval/orchestrator.py @@ -7,6 +7,7 @@ from ..analysis.gap_analysis import GapAnalysisStage from ..analysis.synthesis import SynthesisStage +from ..analysis.verification import VerificationStage from ..config.settings import AppSettings from ..core.context import ResearchSession from ..core.events import get_event_bus @@ -21,6 +22,7 @@ from ..research.query_understanding import QueryUnderstandingStage from ..research.ranking import RankingStage from ..research.relevance_scoring import RelevanceScoringStage +from ..research.research_loop import ResearchLoopStage from ..research.reranker import RerankStage from ..fulltext.stage import FulltextStage from ..retrieval.deduplication import DeduplicationStage @@ -52,10 +54,12 @@ def build_pipeline(settings: AppSettings) -> ResearchPipeline: SnowballStage(), RerankStage(), RelevanceScoringStage(), + ResearchLoopStage(), FulltextStage(), ClusteringStage(), SynthesisStage(), GapAnalysisStage(), + VerificationStage(), CitationExportStage(), ReportGenerationStage(), ], diff --git a/src/utils/progress_reporter.py b/src/utils/progress_reporter.py index a53e792..4fc0c63 100644 --- a/src/utils/progress_reporter.py +++ b/src/utils/progress_reporter.py @@ -40,10 +40,12 @@ "snowball": "Following citation trails", "rerank": "Reranking top papers", "relevance_scoring": "Scoring semantic relevance", + "research_loop": "Checking coverage and refining search", "fulltext": "Reading full papers", "clustering": "Grouping papers by theme", "synthesis": "Synthesizing cross-paper insights", "gap_analysis": "Identifying research gaps", + "verification": "Verifying claims against sources", "citation_export": "Formatting citations", "report_generation": "Organizing final report", } diff --git a/tests/test_research_loop_verification.py b/tests/test_research_loop_verification.py new file mode 100644 index 0000000..850c5d1 --- /dev/null +++ b/tests/test_research_loop_verification.py @@ -0,0 +1,272 @@ +# -*- coding: utf-8 -*- +"""Tests for the iterative research loop and claim verification.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from src.analysis.verification import ( + VERIFICATION_ARTIFACT, + VerificationStage, + heuristic_unsupported, +) +from src.config.settings import AppSettings +from src.core.context import PipelineContext +from src.reporting.markdown import render_enhanced_markdown +from src.research.research_loop import ( + COVERAGE_ARTIFACT, + CoverageAssessment, + ResearchLoopStage, + heuristic_coverage, +) +from src.retrieval.models import ( + EnhancedResearchReport, + GapAnalysisResult, + PaperAnalysis, + RankedPaper, + RetrievedPaper, + VerificationSummary, +) + + +def _paper(title: str, abstract: str = "") -> RetrievedPaper: + return RetrievedPaper(title=title, abstract=abstract or None, provider="test") + + +def _ranked(title: str, abstract: str = "", score: float = 0.8) -> RankedPaper: + return RankedPaper(paper=_paper(title, abstract), rank_score=score, score_breakdown={}) + + +class TestHeuristicCoverage: + def test_sufficient_when_concepts_covered_and_enough_papers(self) -> None: + papers = [ + _ranked(f"Transformer attention study {i}", "Attention mechanisms in transformers.") + for i in range(6) + ] + assessment = heuristic_coverage( + "transformer attention", papers, min_sufficient_papers=6 + ) + + assert assessment.sufficient is True + assert assessment.missing_aspects == [] + + def test_missing_concept_generates_follow_up(self) -> None: + papers = [_ranked("Transformer models overview", "Transformers for NLP.")] * 6 + assessment = heuristic_coverage( + "transformer interpretability methods", papers, min_sufficient_papers=3 + ) + + assert assessment.sufficient is False + assert "interpretability" in assessment.missing_aspects + assert any("interpretability" in q for q in assessment.follow_up_queries) + + def test_too_few_papers_is_insufficient(self) -> None: + papers = [_ranked("Attention study", "attention transformers")] + assessment = heuristic_coverage( + "transformer attention", papers, min_sufficient_papers=6 + ) + assert assessment.sufficient is False + + +class TestResearchLoopStage: + async def test_passthrough_when_disabled(self) -> None: + settings = AppSettings(research_loop={"enabled": False}) + ctx = PipelineContext.create("query", settings) + ranked = [_ranked("Paper")] + + result = await ResearchLoopStage().run(ctx, ranked) + + assert result.output == ranked + assert result.metrics["iterations_run"] == 0 + + async def test_no_iteration_when_coverage_sufficient(self) -> None: + settings = AppSettings(research_loop={"enabled": True, "min_sufficient_papers": 1}) + ctx = PipelineContext.create("attention", settings) + ranked = [_ranked("Attention study", "attention in models")] + + result = await ResearchLoopStage().run(ctx, ranked) + + assert result.metrics["iterations_run"] == 0 + assert result.metrics["coverage_sufficient"] is True + assert ctx.get_artifact(COVERAGE_ARTIFACT)["sufficient"] is True + + async def test_insufficient_coverage_triggers_refinement(self) -> None: + settings = AppSettings(research_loop={"enabled": True, "min_sufficient_papers": 1}) + ctx = PipelineContext.create("transformer interpretability", settings) + seed = [_ranked("Transformer overview", "transformer architectures")] + refined = seed + [ + _ranked( + "Interpretability of transformers", + "Probing and interpretability methods for transformer attention.", + ) + ] + + async def fake_refine(self, ctx, current, follow_ups): + assert any("interpretability" in q for q in follow_ups) + return refined + + with patch.object(ResearchLoopStage, "_refine", fake_refine): + result = await ResearchLoopStage().run(ctx, seed) + + assert result.metrics["iterations_run"] == 1 + assert result.metrics["papers_after_loop"] == 2 + assert result.metrics["coverage_sufficient"] is True + assert ctx.get_artifact("ranked_papers") == refined + + async def test_iteration_budget_is_respected(self) -> None: + settings = AppSettings( + research_loop={"enabled": True, "max_iterations": 1, "min_sufficient_papers": 99} + ) + ctx = PipelineContext.create("transformer interpretability", settings) + seed = [_ranked("Transformer overview", "transformers")] + + calls = {"count": 0} + + async def fake_refine(self, ctx, current, follow_ups): + calls["count"] += 1 + return current # never improves + + with patch.object(ResearchLoopStage, "_refine", fake_refine): + result = await ResearchLoopStage().run(ctx, seed) + + assert calls["count"] == 1 + assert result.metrics["iterations_run"] == 1 + assert result.metrics["coverage_sufficient"] is False + + async def test_refinement_failure_degrades_gracefully(self) -> None: + settings = AppSettings( + research_loop={"enabled": True, "min_sufficient_papers": 99} + ) + ctx = PipelineContext.create("transformer interpretability", settings) + seed = [_ranked("Transformer overview", "transformers")] + + async def failing_refine(self, ctx, current, follow_ups): + raise RuntimeError("network down") + + with patch.object(ResearchLoopStage, "_refine", failing_refine): + result = await ResearchLoopStage().run(ctx, seed) + + assert result.output == seed + assert any("failed" in warning.lower() for warning in result.warnings) + + +class TestHeuristicVerification: + def test_supported_claim_passes(self) -> None: + sources = "attention mechanisms improve translation quality in transformers" + unsupported = heuristic_unsupported( + ["Attention improves translation quality"], + sources, + min_term_coverage=0.5, + ) + assert unsupported == [] + + def test_unrelated_claim_is_flagged(self) -> None: + sources = "attention mechanisms improve translation quality" + unsupported = heuristic_unsupported( + ["Quantum error correction reduces decoherence"], + sources, + min_term_coverage=0.5, + ) + assert unsupported == ["Quantum error correction reduces decoherence"] + + def test_no_sources_flags_nothing(self) -> None: + assert heuristic_unsupported(["Any claim"], "", min_term_coverage=0.5) == [] + + +class TestVerificationStage: + def _ctx(self, analyses, ranked, passages=None, enabled=True): + settings = AppSettings(verification={"enabled": enabled}) + ctx = PipelineContext.create("query", settings) + ctx.set_artifact("paper_analyses", analyses) + ctx.set_artifact("ranked_papers", ranked) + if passages: + ctx.set_artifact("fulltext_passages", passages) + return ctx + + async def test_flags_unsupported_points_and_sets_summary(self) -> None: + ranked = [_ranked("Attention Paper", "Attention improves translation quality.")] + analysis = PaperAnalysis( + paper_id=ranked[0].paper.paper_id, + title="Attention Paper", + key_points=[ + "Attention improves translation quality", + "Quantum decoherence entangles qubits catastrophically", + ], + ) + ctx = self._ctx([analysis], ranked) + + result = await VerificationStage().run(ctx, GapAnalysisResult()) + + updated = ctx.get_artifact("paper_analyses")[0] + assert updated.unverified_points == [ + "Quantum decoherence entangles qubits catastrophically" + ] + summary = ctx.get_artifact(VERIFICATION_ARTIFACT) + assert summary["claims_checked"] == 2 + assert summary["claims_unverified"] == 1 + assert result.metrics["method"] == "heuristic" + + async def test_passthrough_when_disabled(self) -> None: + ctx = self._ctx([PaperAnalysis(title="T", key_points=["p"])], [], enabled=False) + gap = GapAnalysisResult(gaps=["g"]) + + result = await VerificationStage().run(ctx, gap) + + assert result.output is gap + assert ctx.get_artifact(VERIFICATION_ARTIFACT) is None + + async def test_passages_extend_sources(self) -> None: + ranked = [_ranked("Grounded Paper", "Short abstract.")] + paper_id = ranked[0].paper.paper_id + analysis = PaperAnalysis( + paper_id=paper_id, + title="Grounded Paper", + key_points=["Sparse retrieval outperforms dense baselines on long tails"], + ) + passages = { + paper_id: [ + "Our experiments show sparse retrieval outperforms dense baselines " + "on long-tail queries across benchmarks." + ] + } + ctx = self._ctx([analysis], ranked, passages=passages) + + await VerificationStage().run(ctx, GapAnalysisResult()) + + assert ctx.get_artifact("paper_analyses")[0].unverified_points == [] + + +class TestReportRendering: + def test_unverified_points_and_summary_render(self) -> None: + report = EnhancedResearchReport( + query="q", + verification=VerificationSummary( + claims_checked=3, claims_unverified=1, method="heuristic" + ), + papers=[ + PaperAnalysis( + paper_id="p1", + title="Paper", + key_points=["Solid claim", "Shaky claim"], + unverified_points=["Shaky claim"], + ) + ], + clusters=[], + ) + report.clusters = [ + __import__("src.retrieval.models", fromlist=["PaperCluster"]).PaperCluster( + theme="Theme", paper_ids=["p1"] + ) + ] + + rendered = render_enhanced_markdown(report) + + assert "Claim verification (heuristic): 2/3" in rendered + assert "- Shaky claim _(unverified)_" in rendered + assert "- Solid claim\n" in rendered + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])