diff --git a/.env.example b/.env.example index 9836e6e..0f1c43c 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,20 @@ # RA_CROSSREF_MAILTO=you@example.com # CROSSREF_MAILTO=you@example.com +# PubMed (NCBI E-utilities) — optional, raises rate limits +# NCBI_API_KEY=your_ncbi_key_here + +# CORE — required when the core provider is enabled +# (free keys: https://core.ac.uk/services/api) +# CORE_API_KEY=your_core_key_here + +# Enable additional providers (openalex + semantic_scholar are on by default) +# RA_RETRIEVAL__PROVIDERS__ARXIV__ENABLED=true +# RA_RETRIEVAL__PROVIDERS__CROSSREF__ENABLED=true +# RA_RETRIEVAL__PROVIDERS__PUBMED__ENABLED=true +# RA_RETRIEVAL__PROVIDERS__DBLP__ENABLED=true +# RA_RETRIEVAL__PROVIDERS__CORE__ENABLED=true + # ============================================================================= # LLM — Ollama (default, local-first) # ============================================================================= @@ -55,7 +69,7 @@ RA_LLM__API_KEY=ollama # ============================================================================= # RA_LLM__PROVIDER=anthropic -# RA_LLM__MODEL=claude-3-5-haiku-latest +# RA_LLM__MODEL=claude-sonnet-5 # ANTHROPIC_API_KEY=sk-ant-... # Or set the unified key: # RA_LLM__API_KEY=sk-ant-... @@ -70,7 +84,29 @@ RA_RANKING__TOP_K=25 RA_PIPELINE__DEBUG=false # Live stage + LLM progress on stderr (auto-disabled when stderr is not a TTY) RA_PIPELINE__STREAM_PROGRESS=true -RA_DEBUG=1 +# Verbose debug mode (JSON debug dumps, DEBUG console logs) +# RA_DEBUG=1 + +# Console log level (files under logs/ always get full detail) +# RA_CONSOLE_LOG_LEVEL=INFO + +# Skip the Ollama setup/health check on startup (CI, containers) +# RA_SKIP_SETUP_CHECK=1 + +# ============================================================================= +# Accuracy stages (snowball / rerank / fulltext) — on by default +# ============================================================================= + +# Citation-graph expansion of top-ranked papers +# RA_SNOWBALL__ENABLED=false + +# Cross-encoder reranking of the top papers +# RA_RERANK__ENABLED=false +# RA_RERANK__MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2 + +# Open-access full-text grounding (PDF download, passage evidence) +# RA_FULLTEXT__ENABLED=false +# RA_FULLTEXT__MAX_PAPERS=5 # Optional: override config directory # RA_CONFIG_DIR=/path/to/config diff --git a/Pipfile b/Pipfile index 8ff707a..b0f11a8 100644 --- a/Pipfile +++ b/Pipfile @@ -10,6 +10,7 @@ name = "pytorch-cpu" [packages] pydantic-ai = "*" +pymupdf = {version = "*", index = "pypi"} aiohttp = "*" pydantic = "*" pytest = "*" diff --git a/Pipfile.lock b/Pipfile.lock index e2a0d0b..ef5c717 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "368c72cfc2325fe573783ad05d073cd094a11284f8013fe340b3c2338567b0dd" + "sha256": "1a80a55ad055f6ee126e6c46dccec0f021db0d7e7c3208f158370b60faabf13d" }, "pipfile-spec": 6, "requires": { @@ -2310,6 +2310,24 @@ "markers": "python_version >= '3.9'", "version": "==2.13.0" }, + "pymupdf": { + "hashes": [ + "sha256:2e1b574c0fd2cb238021033fd3c0f9c4388816638df064e4bfb56d9d81736dc8", + "sha256:3050a233dde1211efe89ada74e2add6238436434159f46097a1423aad2842545", + "sha256:397d6715c1f0df7548a92d0afd8ce370fc48fa47aeefac16be2bc04a16a8227f", + "sha256:530ef543a3885b3b81cb72a854e7c5a625a9233201221132bb6c31698c6a2bdb", + "sha256:5e0be7908a715aa20333caddd73f1d6f01e4cd0c26e869fa2dd0b7f344da2249", + "sha256:5fc315b425ff1f7afdd1ea2f348205cb19b806767daae7ce4d64115799c2bae1", + "sha256:7113846b35dbf0a033f088e4f4fb543dabeb4b0b12c112966a1ca1ee2d5eacae", + "sha256:ebd244918798502d7b4504c90410d1711a4d7675a32584ca30f1bab419ecbffe", + "sha256:f89fb2d86d07d643a269f17a093105057e20c79c1d06c103b53600067b6d2b01", + "sha256:fd481ed48bef56305c41fb7e05a055c03345c899c7b101dad086258b438f8168", + "sha256:ffe91a24edc75c80da2a4b62f50fc0f54632d34fc8fe4cbc48e5c7ff07cf8fb4" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==1.28.2" + }, "pyperclip": { "hashes": [ "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", diff --git a/config/default.yaml b/config/default.yaml index 072eeb0..329f4f0 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -94,12 +94,23 @@ pipeline: snowball: true rerank: true relevance_scoring: true + fulltext: true clustering: true synthesis: true gap_analysis: true citation_export: true report_generation: true +fulltext: + enabled: true + max_papers: 5 + max_pdf_mb: 15 + request_timeout_seconds: 30 + cache_dir: data/fulltext + chunk_chars: 1400 + chunk_overlap: 200 + top_chunks_per_paper: 3 + rerank: enabled: true model: cross-encoder/ms-marco-MiniLM-L-6-v2 diff --git a/docs/architecture/stages/fulltext.md b/docs/architecture/stages/fulltext.md new file mode 100644 index 0000000..9680ff7 --- /dev/null +++ b/docs/architecture/stages/fulltext.md @@ -0,0 +1,51 @@ +# Fulltext (Grounded Passages) + +The fulltext stage runs between [relevance scoring](relevance-scoring.md) +and [clustering](clustering.md). For the top relevance-filtered papers it +resolves open-access PDFs, downloads and parses them, chunks the text +section-aware, and retrieves the passages most relevant to the query. The +passages ground synthesis: extractions quote verbatim evidence, and +reports show it under each paper as **Evidence (from full text)** — the +difference between a summary generator and a research assistant whose +claims can be checked. + +## Flow + +1. **Resolve** — metadata-first, no extra API calls: arXiv IDs map + directly to PDF URLs, OpenAlex work records already carry open-access + locations in raw metadata, and CORE results expose download URLs. + Closed-access papers simply resolve to nothing. +2. **Download** — cached under `data/fulltext/` keyed by paper ID, with + size and content-type guards (`CachingPDFDownloader`). +3. **Parse** — PyMuPDF text extraction with heuristic section detection; + the references section is truncated to keep bibliography noise out of + retrieval (`src/fulltext/parser.py`). +4. **Chunk** — paragraphs packed into overlapping windows that carry + their section name (`SectionAwareChunker`). +5. **Retrieve** — chunks are embedded through the existing embedding + provider and searched by cosine similarity, falling back to BM25 when + the embedding backend is unavailable (`InMemoryFulltextIndex`). + +The per-paper passages land in the `fulltext_passages` artifact. +Synthesis includes them in extraction prompts (LLM mode asks for short +verbatim quotes; heuristic mode attaches trimmed passages directly), and +the markdown renderer prints the quotes under each paper. + +## Configuration + +| Key | Default | Meaning | +|-----|---------|---------| +| `fulltext.enabled` | `true` | Toggle the stage | +| `fulltext.max_papers` | `5` | Top papers attempted per run | +| `fulltext.max_pdf_mb` | `15` | Per-PDF download size cap | +| `fulltext.request_timeout_seconds` | `30` | Per-download timeout | +| `fulltext.cache_dir` | `data/fulltext` | PDF cache location | +| `fulltext.chunk_chars` / `chunk_overlap` | `1400` / `200` | Chunk window sizing | +| `fulltext.top_chunks_per_paper` | `3` | Passages retrieved per paper | + +## Failure behavior + +Grounding is best-effort by design: closed-access papers, failed +downloads, malformed PDFs, or a missing `pymupdf` backend reduce coverage +and never break the run. Stage metrics report PDFs resolved, downloaded, +chunks indexed, and papers that ended up with passages. diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index c0d5aef..23e6a89 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -20,6 +20,8 @@ Boolean env values accept standard truthy strings (`true`, `1`, `yes`). |----------|---------|-------------| | `RA_CONFIG_DIR` | `config/` (project root) | Override directory for YAML files | | `RA_DEBUG` | unset | Alias for debug mode (`1`, `true`, `yes`) — OR-combined with `RA_PIPELINE__DEBUG` | +| `RA_SKIP_SETUP_CHECK` | unset | Skip the Ollama setup/health check on startup (CI, containers) | +| `RA_CONSOLE_LOG_LEVEL` | `WARNING` | Log level shown on the console; files always get full detail | --- @@ -103,6 +105,8 @@ Full weight list: [YAML reference](yaml-reference.md#ranking). | `S2_API_KEY` | No | Semantic Scholar — higher rate limits when set | | `RA_CROSSREF_MAILTO` | Recommended | CrossRef polite pool (User-Agent mailto) | | `CROSSREF_MAILTO` | Recommended | Alias for CrossRef mailto | +| `NCBI_API_KEY` | No | PubMed E-utilities — higher rate limits when set | +| `CORE_API_KEY` | For `core` | Required when the CORE provider is enabled | --- @@ -117,7 +121,27 @@ Full weight list: [YAML reference](yaml-reference.md#ranking). | `RA_PIPELINE__SYNTHESIS_TIMEOUT_SECONDS` | `600` | Synthesis stage timeout | | `RA_PIPELINE__ENABLED_STAGES__` | `true` | Disable individual pipeline stages | -Stage names: `query_understanding`, `query_expansion`, `retrieval`, `deduplication`, `ranking`, `relevance_scoring`, `clustering`, `synthesis`, `gap_analysis`, `citation_export`, `report_generation`. See [Stage toggles](stage-toggles.md). +Stage names: `query_understanding`, `query_expansion`, `retrieval`, `deduplication`, `ranking`, `snowball`, `rerank`, `relevance_scoring`, `fulltext`, `clustering`, `synthesis`, `gap_analysis`, `citation_export`, `report_generation`. See [Stage toggles](stage-toggles.md). + +--- + +## Snowball, rerank, fulltext + +| Variable | Default | Description | +|----------|---------|-------------| +| `RA_SNOWBALL__ENABLED` | `true` | Citation-graph expansion of top-ranked papers | +| `RA_SNOWBALL__MAX_SEED_PAPERS` | `5` | Seeds for the one-hop expansion | +| `RA_SNOWBALL__MAX_NEW_PAPERS` | `30` | Cap on candidates entering re-ranking | +| `RA_RERANK__ENABLED` | `true` | Cross-encoder reranking of the top papers | +| `RA_RERANK__MODEL` | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Any sentence-transformers cross-encoder | +| `RA_RERANK__TOP_N` | `25` | Papers scored by the cross-encoder | +| `RA_RERANK__BLEND_WEIGHT` | `0.5` | Cross-encoder share of the final score | +| `RA_FULLTEXT__ENABLED` | `true` | Open-access full-text grounding | +| `RA_FULLTEXT__MAX_PAPERS` | `5` | Top papers attempted per run | +| `RA_FULLTEXT__MAX_PDF_MB` | `15` | Per-PDF download size cap | +| `RA_FULLTEXT__TOP_CHUNKS_PER_PAPER` | `3` | Passages retrieved per paper | + +Full key lists live on the stage pages: [Snowball](../architecture/stages/snowball.md), [Rerank](../architecture/stages/rerank.md), [Fulltext](../architecture/stages/fulltext.md). --- diff --git a/mkdocs.yml b/mkdocs.yml index 6ae8154..6a870e6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,7 @@ nav: - architecture/stages/snowball.md - architecture/stages/rerank.md - architecture/stages/relevance-scoring.md + - architecture/stages/fulltext.md - architecture/stages/clustering.md - architecture/stages/synthesis.md - architecture/stages/gap-analysis.md diff --git a/src/analysis/synthesis.py b/src/analysis/synthesis.py index dcba385..e0aa140 100644 --- a/src/analysis/synthesis.py +++ b/src/analysis/synthesis.py @@ -114,7 +114,21 @@ def _detect_conflicting_findings(extractions: list[PaperExtraction]) -> list[str return list(dict.fromkeys(conflicts))[:3] -def _heuristic_extraction(paper: RankedPaper) -> PaperExtraction: +def _evidence_from_passages(passages: list[str] | None, limit: int = 2) -> list[str]: + """Trim grounded passages into short verbatim evidence quotes.""" + if not passages: + return [] + quotes: list[str] = [] + for passage in passages[:limit]: + cleaned = " ".join(passage.split()) + quotes.append(cleaned[:300] + ("…" if len(cleaned) > 300 else "")) + return quotes + + +def _heuristic_extraction( + paper: RankedPaper, + passages: list[str] | None = None, +) -> PaperExtraction: """Build a fallback extraction when the LLM pass is unavailable.""" abstract = paper.paper.abstract or "" sentences = [part.strip() for part in abstract.split(".") if part.strip()] @@ -126,14 +140,21 @@ def _heuristic_extraction(paper: RankedPaper) -> PaperExtraction: if not datasets and abstract: datasets = ["Not explicitly stated in abstract"] + source_note = ( + ["Details inferred from abstract and full-text passages"] + if passages + else ["Details inferred from abstract only"] + ) + return PaperExtraction( paper_id=paper.paper.paper_id, title=paper.paper.title, - methodology=["Details inferred from abstract only"], + methodology=source_note, datasets=datasets, benchmarks=[], limitations=limitations, findings=findings, + evidence=_evidence_from_passages(passages), ) @@ -183,17 +204,33 @@ def _heuristic_synthesis( ) -def _build_extraction_prompt(paper: RankedPaper, query: str) -> str: +def _build_extraction_prompt( + paper: RankedPaper, + query: str, + passages: list[str] | None = None, +) -> str: abstract = paper.paper.abstract or "No abstract available." - return ( + prompt = ( f"Research query: {query}\n\n" f"Paper ID: {paper.paper.paper_id}\n" f"Title: {paper.paper.title}\n" f"Year: {paper.paper.year or 'unknown'}\n" f"Venue: {paper.paper.venue or 'unknown'}\n" - f"Abstract: {abstract[:1200]}\n\n" - "Extract methodology, datasets, benchmarks, limitations, and findings." + f"Abstract: {abstract[:1200]}\n" ) + if passages: + numbered = "\n\n".join( + f"[Passage {index}] {passage[:1500]}" + for index, passage in enumerate(passages, start=1) + ) + prompt += ( + f"\nFull-text passages retrieved for this query:\n{numbered}\n\n" + "Ground the extraction in these passages and quote short verbatim " + "evidence snippets from them." + ) + else: + prompt += "\nExtract methodology, datasets, benchmarks, limitations, and findings." + return prompt def _build_synthesis_prompt( @@ -244,6 +281,7 @@ def _extractions_to_paper_analyses( doi=source.doi if source else None, key_points=extraction.findings, why_relevant=extraction.methodology, + evidence=extraction.evidence, ) ) return analyses @@ -285,7 +323,11 @@ def recover_synthesis_output( ) resolved_clusters = clusters or ctx.get_artifact("paper_clusters") or [] - extractions = [_heuristic_extraction(paper) for paper in ranked_papers] + recovery_passages: dict[str, list[str]] = ctx.get_artifact("fulltext_passages") or {} + extractions = [ + _heuristic_extraction(paper, recovery_passages.get(paper.paper.paper_id)) + for paper in ranked_papers + ] synthesis = _heuristic_synthesis( ctx.query, extractions, @@ -307,8 +349,9 @@ async def _extract_single_paper( agent: Agent, handler: EnhancedResponseHandler, context: RequestContext, + passages: list[str] | None = None, ) -> tuple[PaperExtraction, bool]: - prompt = _build_extraction_prompt(paper, query) + prompt = _build_extraction_prompt(paper, query, passages) result = await handler.process_structured_response( agent, prompt, @@ -317,12 +360,17 @@ async def _extract_single_paper( schema_description=( '{"paper_id": "...", "title": "...", "methodology": ["..."], ' '"datasets": ["..."], "benchmarks": ["..."], ' - '"limitations": ["..."], "findings": ["..."]}' + '"limitations": ["..."], "findings": ["..."], "evidence": ["..."]}' ), ) if result.success and isinstance(result.data, PaperExtraction): - return result.data, True - return _heuristic_extraction(paper), False + extraction = result.data + if not extraction.evidence and passages: + extraction = extraction.model_copy( + update={"evidence": _evidence_from_passages(passages)} + ) + return extraction, True + return _heuristic_extraction(paper, passages), False async def extract_papers( @@ -334,10 +382,16 @@ async def extract_papers( concurrency: int = 4, session_id: str = "", synthesis_config: SynthesisConfig | None = None, + passages: dict[str, list[str]] | None = None, ) -> list[PaperExtraction]: - """Pass A — structured extraction for each ranked paper.""" + """Pass A — structured extraction for each ranked paper. + + ``passages`` maps paper IDs to query-relevant full-text passages from the + fulltext stage; when present they ground both LLM and heuristic paths. + """ if not ranked_papers: return [] + passages = passages or {} if synthesis_config is None: from ..config.settings import get_settings @@ -349,7 +403,10 @@ async def extract_papers( "Using heuristic paper extraction for %d paper(s) (synthesis.llm_enabled=false)", len(ranked_papers), ) - return [_heuristic_extraction(paper) for paper in ranked_papers] + return [ + _heuristic_extraction(paper, passages.get(paper.paper.paper_id)) + for paper in ranked_papers + ] max_llm_papers = max(1, synthesis_config.max_llm_papers) llm_targets = ranked_papers[:max_llm_papers] @@ -372,8 +429,9 @@ async def extract_papers( async def _extract_with_circuit(index: int, paper: RankedPaper) -> PaperExtraction: async with semaphore: + paper_passages = passages.get(paper.paper.paper_id) if breaker["open"]: - return _heuristic_extraction(paper) + return _heuristic_extraction(paper, paper_passages) title_preview = paper.paper.title[:72] logger.info( @@ -395,6 +453,7 @@ async def _extract_with_circuit(index: int, paper: RankedPaper) -> PaperExtracti agent, response_handler, context, + passages=paper_passages, ) if llm_success: @@ -424,7 +483,10 @@ async def _extract_with_circuit(index: int, paper: RankedPaper) -> PaperExtracti ) ) - heuristic_extractions = [_heuristic_extraction(paper) for paper in heuristic_targets] + heuristic_extractions = [ + _heuristic_extraction(paper, passages.get(paper.paper.paper_id)) + for paper in heuristic_targets + ] return llm_extractions + heuristic_extractions @@ -503,6 +565,7 @@ async def run_synthesis( concurrency: int = 4, session_id: str = "", synthesis_config: SynthesisConfig | None = None, + passages: dict[str, list[str]] | None = None, ) -> tuple[SynthesisResult, list[PaperExtraction], list[PaperAnalysis]]: """Run the full two-pass synthesis workflow.""" if synthesis_config is None: @@ -518,6 +581,7 @@ async def run_synthesis( concurrency=synthesis_config.concurrency or concurrency, session_id=session_id, synthesis_config=synthesis_config, + passages=passages, ) synthesis = await synthesize_collective( query, @@ -592,6 +656,7 @@ async def run( ctx.query, ranked_papers, data, + passages=ctx.get_artifact("fulltext_passages") or {}, llm_config=ctx.config.llm, handler=self.handler, concurrency=ctx.config.synthesis.concurrency, diff --git a/src/config/settings.py b/src/config/settings.py index 72c9226..cc212f7 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -150,6 +150,19 @@ class RetrievalConfig(BaseModel): ) +class FulltextConfig(BaseModel): + """Open-access full-text grounding for top-ranked papers.""" + + enabled: bool = True + max_papers: int = 5 + max_pdf_mb: int = 15 + request_timeout_seconds: int = 30 + cache_dir: str = "data/fulltext" + chunk_chars: int = 1400 + chunk_overlap: int = 200 + top_chunks_per_paper: int = 3 + + class RerankConfig(BaseModel): """Cross-encoder reranking of the top-ranked papers.""" @@ -186,6 +199,7 @@ class PipelineConfig(BaseModel): "snowball": True, "rerank": True, "relevance_scoring": True, + "fulltext": True, "clustering": True, "synthesis": True, "gap_analysis": True, @@ -264,6 +278,7 @@ class AppSettings(BaseSettings): retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig) snowball: SnowballConfig = Field(default_factory=SnowballConfig) rerank: RerankConfig = Field(default_factory=RerankConfig) + fulltext: FulltextConfig = Field(default_factory=FulltextConfig) pipeline: PipelineConfig = Field(default_factory=PipelineConfig) memory: MemoryConfig = Field(default_factory=MemoryConfig) synthesis: SynthesisConfig = Field(default_factory=SynthesisConfig) diff --git a/src/core/registry.py b/src/core/registry.py index 5438a48..63ced91 100644 --- a/src/core/registry.py +++ b/src/core/registry.py @@ -149,8 +149,8 @@ def bootstrap_default_plugins() -> None: for name, factory in stage_factories.items(): _global_registry.register_stage(name, factory) - from ..fulltext.downloader import StubPDFDownloader - from ..fulltext.rag import StubRAGIndex + from ..fulltext.downloader import CachingPDFDownloader + from ..fulltext.rag import InMemoryFulltextIndex - _global_registry.register_fulltext_downloader("stub", StubPDFDownloader) - _global_registry.register_fulltext_index("stub", StubRAGIndex) + _global_registry.register_fulltext_downloader("default", CachingPDFDownloader) + _global_registry.register_fulltext_index("default", InMemoryFulltextIndex) diff --git a/src/fulltext/__init__.py b/src/fulltext/__init__.py index b1142c0..4060e15 100644 --- a/src/fulltext/__init__.py +++ b/src/fulltext/__init__.py @@ -1,18 +1,22 @@ # -*- coding: utf-8 -*- -"""Full-text ingestion architecture (Phase 3 interfaces only).""" +"""Full-text ingestion: PDF resolution, parsing, chunking, and retrieval.""" from .base import FullTextChunk, FullTextDocument, PDFDownloader, RAGIndex, TextChunker -from .chunker import StubTextChunker -from .downloader import StubPDFDownloader -from .rag import StubRAGIndex +from .chunker import SectionAwareChunker +from .downloader import CachingPDFDownloader +from .rag import InMemoryFulltextIndex +from .resolver import resolve_pdf_url +from .stage import FulltextStage __all__ = [ + "CachingPDFDownloader", "FullTextChunk", "FullTextDocument", + "FulltextStage", + "InMemoryFulltextIndex", "PDFDownloader", "RAGIndex", - "StubPDFDownloader", - "StubRAGIndex", - "StubTextChunker", + "SectionAwareChunker", "TextChunker", + "resolve_pdf_url", ] diff --git a/src/fulltext/chunker.py b/src/fulltext/chunker.py index 52d901e..9447570 100644 --- a/src/fulltext/chunker.py +++ b/src/fulltext/chunker.py @@ -1,17 +1,78 @@ # -*- coding: utf-8 -*- -"""Text chunking stubs for future full-text ingestion.""" +"""Section-aware chunking of extracted full text.""" from __future__ import annotations from .base import FullTextChunk, FullTextDocument, TextChunker -class StubTextChunker(TextChunker): - """Placeholder chunker that documents the intended chunking flow.""" +class SectionAwareChunker(TextChunker): + """Packs paragraphs into overlapping chunks that respect section marks. + + Section markers (``## Heading`` lines emitted by the parser) start a new + chunk and tag every chunk that follows until the next marker, so a + retrieved passage can say which part of the paper it came from. + """ + + def __init__(self, max_chars: int = 1400, overlap_chars: int = 200) -> None: + self.max_chars = max_chars + self.overlap_chars = overlap_chars def chunk(self, document: FullTextDocument) -> list[FullTextChunk]: - raise NotImplementedError( - "Text chunking is not yet implemented. " - "Future versions will split extracted PDF text into overlapping " - "windows for embedding and RAG retrieval." - ) + if not document.text: + return [] + + chunks: list[FullTextChunk] = [] + current_parts: list[str] = [] + current_length = 0 + section = "body" + + def flush() -> None: + nonlocal current_parts, current_length + text = "\n".join(current_parts).strip() + if len(text) >= 200: + chunks.append( + FullTextChunk( + paper_id=document.paper_id, + chunk_index=len(chunks), + text=text, + metadata={"section": section}, + ) + ) + if text and self.overlap_chars > 0: + tail = text[-self.overlap_chars :] + current_parts = [tail] + current_length = len(tail) + else: + current_parts = [] + current_length = 0 + + for paragraph in document.text.split("\n\n"): + stripped = paragraph.strip() + if not stripped: + continue + + if stripped.startswith("## "): + flush() + current_parts = [] + current_length = 0 + section = stripped[3:].strip().lower() + continue + + if current_length + len(stripped) > self.max_chars and current_parts: + flush() + + current_parts.append(stripped) + current_length += len(stripped) + + text = "\n".join(current_parts).strip() + if len(text) >= 200: + chunks.append( + FullTextChunk( + paper_id=document.paper_id, + chunk_index=len(chunks), + text=text, + metadata={"section": section}, + ) + ) + return chunks diff --git a/src/fulltext/downloader.py b/src/fulltext/downloader.py index a82909e..3b23b8e 100644 --- a/src/fulltext/downloader.py +++ b/src/fulltext/downloader.py @@ -1,17 +1,66 @@ # -*- coding: utf-8 -*- -"""PDF download stubs for future full-text ingestion.""" +"""Open-access PDF download with an on-disk cache.""" from __future__ import annotations +import hashlib +from pathlib import Path + +import aiohttp + +from ..utils.logging_system import logger from .base import FullTextDocument, PDFDownloader -class StubPDFDownloader(PDFDownloader): - """Placeholder downloader that documents the intended download flow.""" +class CachingPDFDownloader(PDFDownloader): + """Downloads PDFs once and caches them under a local directory.""" + + def __init__( + self, + cache_dir: str = "data/fulltext", + *, + max_bytes: int = 15 * 1024 * 1024, + timeout_seconds: int = 30, + ) -> None: + self.cache_dir = Path(cache_dir) + self.max_bytes = max_bytes + self.timeout_seconds = timeout_seconds + + def _cache_path(self, paper_id: str) -> Path: + digest = hashlib.sha256(paper_id.encode("utf-8")).hexdigest()[:24] + return self.cache_dir / f"{digest}.pdf" + + async def download( + self, + paper_id: str, + url: str, + session: aiohttp.ClientSession | None = None, + ) -> FullTextDocument: + """Download a PDF (or reuse the cached copy) and return its metadata.""" + path = self._cache_path(paper_id) + if path.is_file() and path.stat().st_size > 0: + return FullTextDocument(paper_id=paper_id, source_url=url, local_path=path) + + if session is None: + async with aiohttp.ClientSession() as owned_session: + return await self.download(paper_id, url, session=owned_session) + + async with session.get( + url, + timeout=self.timeout_seconds, + allow_redirects=True, + headers={"User-Agent": "ResearchAssistant/1.0 (fulltext)"}, + ) as response: + response.raise_for_status() + content_type = (response.content_type or "").lower() + body = await response.content.read(self.max_bytes + 1) + + if len(body) > self.max_bytes: + raise ValueError(f"PDF exceeds size limit ({self.max_bytes} bytes): {url}") + if not body.startswith(b"%PDF") and "pdf" not in content_type: + raise ValueError(f"URL did not return a PDF (content-type {content_type}): {url}") - async def download(self, paper_id: str, url: str) -> FullTextDocument: - raise NotImplementedError( - "PDF download is not yet implemented. " - "Future versions will fetch open-access PDFs, cache them under " - "data/fulltext/, and return a FullTextDocument with local_path set." - ) + self.cache_dir.mkdir(parents=True, exist_ok=True) + path.write_bytes(body) + logger.info("Cached full text for %s (%d KB)", paper_id, len(body) // 1024) + return FullTextDocument(paper_id=paper_id, source_url=url, local_path=path) diff --git a/src/fulltext/parser.py b/src/fulltext/parser.py new file mode 100644 index 0000000..1950a25 --- /dev/null +++ b/src/fulltext/parser.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +"""PDF text extraction with lightweight section detection. + +Uses PyMuPDF for dependable local-first extraction. Section headings are +detected heuristically (numbered headings and common scholarly section +names) and recorded per text block so chunking can stay section-aware. +The references section is truncated — bibliography lines add noise to +retrieval without adding evidence. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from .base import FullTextDocument + +_HEADING_RE = re.compile( + r"^(?:\d+(?:\.\d+)*\.?\s+)?" + r"(abstract|introduction|background|related work|methods?|methodology|" + r"materials and methods|approach|experiments?|results?|evaluation|" + r"discussion|limitations|conclusions?|future work|acknowledg\w+|references)" + r"\s*$", + re.IGNORECASE, +) + +_STOP_SECTIONS = {"references", "acknowledgment", "acknowledgments", "acknowledgements"} + + +def _detect_heading(line: str) -> str | None: + stripped = line.strip() + if not stripped or len(stripped) > 60: + return None + match = _HEADING_RE.match(stripped) + if match: + return match.group(1).lower() + return None + + +def extract_document(paper_id: str, path: Path) -> FullTextDocument: + """Extract text from a PDF, annotated with per-page section markers. + + The returned document's ``text`` holds the body up to the references + section; ``metadata['sections']`` maps section names to their first page. + """ + import pymupdf + + sections: dict[str, int] = {} + pages: list[str] = [] + current_section = "body" + stopped = False + + with pymupdf.open(path) as pdf: + page_count = pdf.page_count + for page_number, page in enumerate(pdf, start=1): + if stopped: + break + lines = page.get_text("text").splitlines() + kept: list[str] = [] + for line in lines: + heading = _detect_heading(line) + if heading: + if heading in _STOP_SECTIONS: + stopped = True + break + current_section = heading + sections.setdefault(heading, page_number) + kept.append(f"\n## {heading.title()}\n") + continue + kept.append(line) + if kept: + pages.append("\n".join(kept)) + + text = "\n".join(pages) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text).strip() + + return FullTextDocument( + paper_id=paper_id, + local_path=path, + text=text or None, + page_count=page_count, + metadata={"sections": sections, "current_section": current_section}, + ) diff --git a/src/fulltext/rag.py b/src/fulltext/rag.py index b0dc66a..c2f160d 100644 --- a/src/fulltext/rag.py +++ b/src/fulltext/rag.py @@ -1,23 +1,87 @@ # -*- coding: utf-8 -*- -"""RAG index stubs for future full-text grounded retrieval.""" +"""Session-scoped retrieval over full-text chunks. + +Chunks are embedded through the existing embedding provider (reusing its +disk cache) and searched by cosine similarity. When the embedding backend +is unavailable, retrieval falls back to BM25 over the chunk texts, so +grounding works in every environment. +""" from __future__ import annotations +import numpy as np + +from ..embeddings.base import EmbeddingProvider, cosine_similarity +from ..research.bm25 import normalized_bm25_scores from .base import FullTextChunk, RAGIndex -class StubRAGIndex(RAGIndex): - """Placeholder RAG index that documents the intended indexing flow.""" +class InMemoryFulltextIndex(RAGIndex): + """In-memory chunk index for one pipeline run.""" + + def __init__(self, embedder: EmbeddingProvider | None = None) -> None: + self.embedder = embedder + self._chunks: list[FullTextChunk] = [] + self._vectors: np.ndarray | None = None async def index_chunks(self, chunks: list[FullTextChunk]) -> int: - raise NotImplementedError( - "Full-text RAG indexing is not yet implemented. " - "Future versions will embed chunks and persist vectors for " - "session-scoped similarity search." - ) - - async def search(self, query: str, *, top_k: int = 5) -> list[FullTextChunk]: - raise NotImplementedError( - "Full-text RAG search is not yet implemented. " - "Future versions will retrieve the top-k chunks by embedding similarity." - ) + self._chunks.extend(chunks) + self._vectors = None # rebuilt lazily on next search + return len(chunks) + + def _ensure_vectors(self) -> None: + if self.embedder is None or self._vectors is not None or not self._chunks: + return + try: + self._vectors = np.asarray( + self.embedder.embed_texts([chunk.text for chunk in self._chunks]) + ) + except Exception: + # Embedding backend unavailable at runtime; BM25 fallback takes over. + self.embedder = None + self._vectors = None + + def _rank(self, query: str, indices: list[int]) -> list[tuple[int, float]]: + """Score the given chunk indices against the query, best first.""" + self._ensure_vectors() + + if self._vectors is not None and self.embedder is not None: + try: + query_vector = self.embedder.embed_texts([query])[0] + except Exception: + self.embedder = None + return self._rank(query, indices) + scored = [ + (index, cosine_similarity(query_vector, self._vectors[index])) + for index in indices + ] + else: + scores = normalized_bm25_scores( + query, [self._chunks[index].text for index in indices] + ) + scored = list(zip(indices, scores, strict=True)) + + scored.sort(key=lambda item: item[1], reverse=True) + return scored + + async def search( + self, + query: str, + *, + top_k: int = 5, + paper_id: str | None = None, + ) -> list[FullTextChunk]: + """Return the most relevant chunks, optionally scoped to one paper.""" + if not self._chunks: + return [] + + indices = [ + index + for index, chunk in enumerate(self._chunks) + if paper_id is None or chunk.paper_id == paper_id + ] + if not indices: + return [] + + ranked = self._rank(query, indices) + return [self._chunks[index] for index, _ in ranked[:top_k]] diff --git a/src/fulltext/resolver.py b/src/fulltext/resolver.py new file mode 100644 index 0000000..6eae664 --- /dev/null +++ b/src/fulltext/resolver.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +"""Resolve open-access PDF URLs from retrieved-paper metadata. + +Resolution is metadata-first and free of extra network calls: arXiv IDs +map directly to PDF URLs, OpenAlex work records already carry open-access +locations in their raw metadata, and CORE results expose direct download +URLs. Papers without any of these simply resolve to ``None`` — full-text +grounding is best-effort by design. +""" + +from __future__ import annotations + +import re + +from ..retrieval.models import RetrievedPaper + +_ARXIV_ID_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/([0-9]{4}\.[0-9]{4,5})(v\d+)?", re.IGNORECASE) +_ARXIV_DOI_RE = re.compile(r"10\.48550/arxiv\.([0-9]{4}\.[0-9]{4,5})", re.IGNORECASE) + + +def _arxiv_pdf_url(paper: RetrievedPaper) -> str | None: + for candidate in (paper.url or "", paper.doi or ""): + match = _ARXIV_ID_RE.search(candidate) or _ARXIV_DOI_RE.search(candidate) + if match: + return f"https://arxiv.org/pdf/{match.group(1)}" + return None + + +def _openalex_pdf_url(paper: RetrievedPaper) -> str | None: + raw = paper.raw_metadata + open_access = raw.get("open_access") or {} + best_location = raw.get("best_oa_location") or {} + primary = raw.get("primary_location") or {} + + for candidate in ( + best_location.get("pdf_url"), + primary.get("pdf_url"), + open_access.get("oa_url"), + best_location.get("landing_page_url") if best_location.get("is_oa") else None, + ): + if candidate: + return str(candidate) + return None + + +def _core_pdf_url(paper: RetrievedPaper) -> str | None: + raw = paper.raw_metadata + candidate = raw.get("downloadUrl") or (raw.get("sourceFulltextUrls") or [None])[0] + return str(candidate) if candidate else None + + +def resolve_pdf_url(paper: RetrievedPaper) -> str | None: + """Return the most direct open-access PDF URL for a paper, if any.""" + for resolver in (_arxiv_pdf_url, _openalex_pdf_url, _core_pdf_url): + url = resolver(paper) + if url: + return url + return None diff --git a/src/fulltext/stage.py b/src/fulltext/stage.py new file mode 100644 index 0000000..5365e58 --- /dev/null +++ b/src/fulltext/stage.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +"""Pipeline stage that grounds top papers in their full text. + +For the relevance-filtered papers, the stage resolves open-access PDF +URLs, downloads and parses the PDFs, chunks the text section-aware, and +retrieves the query-relevant passages per paper. The passages land in the +``fulltext_passages`` artifact, which synthesis uses to ground extractions +and reports use to show verbatim evidence. + +Everything is best-effort and budgeted: closed-access papers, failed +downloads, or a missing PDF backend reduce coverage, never break the run. +""" + +from __future__ import annotations + +import time + +import aiohttp + +from ..core.context import PipelineContext, StageResult +from ..retrieval.models import RankedPaper +from ..utils.logging_system import logger +from .base import FullTextChunk +from .chunker import SectionAwareChunker +from .downloader import CachingPDFDownloader +from .rag import InMemoryFulltextIndex +from .resolver import resolve_pdf_url + +PASSAGES_ARTIFACT = "fulltext_passages" +SECTIONS_ARTIFACT = "fulltext_sections" + + +class FulltextStage: + """Download, parse, chunk, and retrieve grounded passages.""" + + name = "fulltext" + + def __init__( + self, + downloader: CachingPDFDownloader | None = None, + chunker: SectionAwareChunker | None = None, + index: InMemoryFulltextIndex | None = None, + ) -> None: + self.downloader = downloader + self.chunker = chunker + self.index = index + + async def run( + self, + ctx: PipelineContext, + data: list[RankedPaper], + ) -> StageResult[list[RankedPaper]]: + started = time.perf_counter() + config = ctx.config.fulltext + warnings: list[str] = [] + + def _passthrough(**metrics: object) -> StageResult[list[RankedPaper]]: + return StageResult( + output=data, + duration_ms=(time.perf_counter() - started) * 1000, + metrics={"papers_with_passages": 0, **metrics}, + warnings=warnings, + ) + + if not config.enabled or not data: + return _passthrough(skipped=True) + + try: + import pymupdf # noqa: F401 + except ImportError: + warnings.append( + "Full-text grounding skipped: pymupdf is not installed. Run: pipenv install" + ) + return _passthrough() + + from .parser import extract_document + + candidates = [ + (ranked.paper, resolve_pdf_url(ranked.paper)) + for ranked in data[: config.max_papers] + ] + resolvable = [(paper, url) for paper, url in candidates if url] + if not resolvable: + return _passthrough(pdfs_resolved=0) + + downloader = self.downloader or CachingPDFDownloader( + cache_dir=config.cache_dir, + max_bytes=config.max_pdf_mb * 1024 * 1024, + timeout_seconds=config.request_timeout_seconds, + ) + chunker = self.chunker or SectionAwareChunker( + max_chars=config.chunk_chars, + overlap_chars=config.chunk_overlap, + ) + + chunks_by_paper: dict[str, list[FullTextChunk]] = {} + sections_by_paper: dict[str, dict[str, int]] = {} + downloaded = 0 + + async with aiohttp.ClientSession() as session: + for paper, url in resolvable: + try: + document = await downloader.download(paper.paper_id, url, session=session) + parsed = extract_document(paper.paper_id, document.local_path) + chunks = chunker.chunk(parsed) + except Exception as exc: + logger.info("Full text unavailable for %s: %s", paper.title[:60], exc) + continue + downloaded += 1 + if chunks: + chunks_by_paper[paper.paper_id] = chunks + sections_by_paper[paper.paper_id] = dict( + parsed.metadata.get("sections") or {} + ) + + if not chunks_by_paper: + return _passthrough(pdfs_resolved=len(resolvable), pdfs_downloaded=downloaded) + + index = self.index + if index is None: + from ..embeddings import try_create_embedding_provider + + index = InMemoryFulltextIndex( + embedder=try_create_embedding_provider(ctx.config.embedding) + ) + + total_chunks = 0 + for chunks in chunks_by_paper.values(): + total_chunks += await index.index_chunks(chunks) + + passages: dict[str, list[str]] = {} + for paper_id in chunks_by_paper: + top = await index.search( + ctx.query, + top_k=config.top_chunks_per_paper, + paper_id=paper_id, + ) + if top: + passages[paper_id] = [chunk.text for chunk in top] + + ctx.set_artifact(PASSAGES_ARTIFACT, passages) + ctx.set_artifact(SECTIONS_ARTIFACT, sections_by_paper) + + return StageResult( + output=data, + duration_ms=(time.perf_counter() - started) * 1000, + metrics={ + "pdfs_resolved": len(resolvable), + "pdfs_downloaded": downloaded, + "chunks_indexed": total_chunks, + "papers_with_passages": len(passages), + }, + warnings=warnings, + ) diff --git a/src/models/base.py b/src/models/base.py index c8292d2..d9437d0 100644 --- a/src/models/base.py +++ b/src/models/base.py @@ -38,7 +38,10 @@ class AgentRole(str, Enum): "Extract structured information from a single paper and respond with ONLY a JSON object " 'with keys: "paper_id" (string), "title" (string), "methodology" (list of strings), ' '"datasets" (list of strings), "benchmarks" (list of strings), ' - '"limitations" (list of strings), "findings" (list of strings). ' + '"limitations" (list of strings), "findings" (list of strings), ' + '"evidence" (list of strings). ' + "When full-text passages are provided, evidence must contain short verbatim quotes " + "from those passages that support the findings; otherwise leave evidence empty. " "Do not include conversational filler or markdown." ), AgentRole.SYNTHESIS: ( diff --git a/src/reporting/markdown.py b/src/reporting/markdown.py index 350e37e..b0cfaa8 100644 --- a/src/reporting/markdown.py +++ b/src/reporting/markdown.py @@ -81,6 +81,14 @@ def _render_thematic_findings( lines.extend(["", "Key points:", *_bullet_lines(paper.key_points)]) if paper.why_relevant: lines.extend(["", "Why relevant:", *_bullet_lines(paper.why_relevant)]) + if paper.evidence: + lines.extend( + [ + "", + "Evidence (from full text):", + *[f"> {quote}" for quote in paper.evidence], + ] + ) lines.append("") return lines diff --git a/src/retrieval/models.py b/src/retrieval/models.py index ce90fce..49b67f5 100644 --- a/src/retrieval/models.py +++ b/src/retrieval/models.py @@ -52,6 +52,7 @@ class PaperAnalysis(BaseModel): doi: Optional[str] = None key_points: list[str] = Field(default_factory=list) why_relevant: list[str] = Field(default_factory=list) + evidence: list[str] = Field(default_factory=list) class ResearchReport(BaseModel): @@ -103,6 +104,7 @@ class PaperExtraction(BaseModel): benchmarks: list[str] = Field(default_factory=list) limitations: list[str] = Field(default_factory=list) findings: list[str] = Field(default_factory=list) + evidence: list[str] = Field(default_factory=list) class SynthesisResult(BaseModel): diff --git a/src/retrieval/orchestrator.py b/src/retrieval/orchestrator.py index e051cd6..bb1346f 100644 --- a/src/retrieval/orchestrator.py +++ b/src/retrieval/orchestrator.py @@ -22,6 +22,7 @@ from ..research.ranking import RankingStage from ..research.relevance_scoring import RelevanceScoringStage from ..research.reranker import RerankStage +from ..fulltext.stage import FulltextStage from ..retrieval.deduplication import DeduplicationStage from ..retrieval.snowball import SnowballStage from ..retrieval.models import ( @@ -51,6 +52,7 @@ def build_pipeline(settings: AppSettings) -> ResearchPipeline: SnowballStage(), RerankStage(), RelevanceScoringStage(), + FulltextStage(), ClusteringStage(), SynthesisStage(), GapAnalysisStage(), diff --git a/src/utils/progress_reporter.py b/src/utils/progress_reporter.py index 0e05469..a53e792 100644 --- a/src/utils/progress_reporter.py +++ b/src/utils/progress_reporter.py @@ -40,6 +40,7 @@ "snowball": "Following citation trails", "rerank": "Reranking top papers", "relevance_scoring": "Scoring semantic relevance", + "fulltext": "Reading full papers", "clustering": "Grouping papers by theme", "synthesis": "Synthesizing cross-paper insights", "gap_analysis": "Identifying research gaps", diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py new file mode 100644 index 0000000..7c8db0e --- /dev/null +++ b/tests/test_fulltext.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +"""Tests for full-text grounding: resolver, parser, chunker, index, stage.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from src.config.settings import AppSettings +from src.core.context import PipelineContext +from src.fulltext.base import FullTextChunk, FullTextDocument +from src.fulltext.chunker import SectionAwareChunker +from src.fulltext.rag import InMemoryFulltextIndex +from src.fulltext.resolver import resolve_pdf_url +from src.fulltext.stage import PASSAGES_ARTIFACT, FulltextStage +from src.retrieval.models import RankedPaper, RetrievedPaper + + +def _paper(title: str = "Paper", **overrides) -> RetrievedPaper: + defaults = {"title": title, "provider": "test"} + defaults.update(overrides) + return RetrievedPaper(**defaults) + + +def _ranked(paper: RetrievedPaper, score: float = 0.8) -> RankedPaper: + return RankedPaper(paper=paper, rank_score=score, score_breakdown={}) + + +class TestResolver: + def test_arxiv_url_from_abs_link(self) -> None: + paper = _paper(url="https://arxiv.org/abs/1706.03762v5") + assert resolve_pdf_url(paper) == "https://arxiv.org/pdf/1706.03762" + + def test_arxiv_url_from_datacite_doi(self) -> None: + paper = _paper(doi="https://doi.org/10.48550/arXiv.2205.14135") + assert resolve_pdf_url(paper) == "https://arxiv.org/pdf/2205.14135" + + def test_openalex_best_oa_location(self) -> None: + paper = _paper( + raw_metadata={ + "best_oa_location": {"pdf_url": "https://repo.org/paper.pdf", "is_oa": True} + } + ) + assert resolve_pdf_url(paper) == "https://repo.org/paper.pdf" + + def test_openalex_oa_url_fallback(self) -> None: + paper = _paper(raw_metadata={"open_access": {"oa_url": "https://oa.org/x.pdf"}}) + assert resolve_pdf_url(paper) == "https://oa.org/x.pdf" + + def test_core_download_url(self) -> None: + paper = _paper(raw_metadata={"downloadUrl": "https://core.ac.uk/download/9.pdf"}) + assert resolve_pdf_url(paper) == "https://core.ac.uk/download/9.pdf" + + def test_closed_access_resolves_to_none(self) -> None: + assert resolve_pdf_url(_paper(url="https://publisher.example/paywall")) is None + + +class TestParser: + def test_extracts_sections_and_truncates_references(self, tmp_path: Path) -> None: + pymupdf = pytest.importorskip("pymupdf") + + pdf_path = tmp_path / "sample.pdf" + document = pymupdf.open() + page = document.new_page() + body = ( + "A Study of Testing\n" + "Abstract\n" + "This paper studies automated testing of research pipelines in depth.\n" + "1 Introduction\n" + "Testing pipelines end to end catches integration regressions early.\n" + "References\n" + "[1] Some Citation That Should Not Appear.\n" + ) + page.insert_text((72, 72), body) + document.save(pdf_path) + document.close() + + from src.fulltext.parser import extract_document + + parsed = extract_document("paper-1", pdf_path) + + assert parsed.page_count == 1 + assert parsed.text is not None + assert "studies automated testing" in parsed.text + assert "## Abstract" in parsed.text + assert "## Introduction" in parsed.text + assert "Some Citation" not in parsed.text + assert parsed.metadata["sections"]["abstract"] == 1 + + +class TestChunker: + def test_section_markers_tag_chunks(self) -> None: + text = ( + "## Abstract\n\n" + + "Sentence about attention mechanisms. " * 10 + + "\n\n## Methods\n\n" + + "Details of the experimental setup and datasets. " * 10 + ) + document = FullTextDocument(paper_id="p1", text=text) + + chunks = SectionAwareChunker(max_chars=600, overlap_chars=50).chunk(document) + + assert chunks + sections = {chunk.metadata["section"] for chunk in chunks} + assert {"abstract", "methods"} <= sections + assert all(chunk.paper_id == "p1" for chunk in chunks) + + def test_long_text_is_split_with_overlap(self) -> None: + text = "\n\n".join(f"Paragraph {i} " + "content words here. " * 20 for i in range(6)) + document = FullTextDocument(paper_id="p1", text=text) + + chunks = SectionAwareChunker(max_chars=800, overlap_chars=100).chunk(document) + + assert len(chunks) >= 2 + assert all(len(chunk.text) >= 200 for chunk in chunks) + + def test_empty_document_yields_no_chunks(self) -> None: + assert SectionAwareChunker().chunk(FullTextDocument(paper_id="p1")) == [] + + +class TestFulltextIndex: + async def test_bm25_fallback_search_scoped_to_paper(self) -> None: + index = InMemoryFulltextIndex(embedder=None) + await index.index_chunks( + [ + FullTextChunk( + paper_id="p1", + chunk_index=0, + text="Scaled dot-product attention drives the transformer architecture.", + ), + FullTextChunk( + paper_id="p1", + chunk_index=1, + text="The optimizer used warmup steps during training.", + ), + FullTextChunk( + paper_id="p2", + chunk_index=0, + text="Attention in convolutional models differs entirely.", + ), + ] + ) + + results = await index.search("transformer attention", top_k=1, paper_id="p1") + + assert len(results) == 1 + assert results[0].paper_id == "p1" + assert "transformer" in results[0].text + + async def test_empty_index_returns_nothing(self) -> None: + index = InMemoryFulltextIndex(embedder=None) + assert await index.search("anything") == [] + + +class FakeDownloader: + def __init__(self, path: Path) -> None: + self.path = path + self.calls: list[str] = [] + + async def download(self, paper_id: str, url: str, session=None) -> FullTextDocument: + self.calls.append(url) + return FullTextDocument(paper_id=paper_id, source_url=url, local_path=self.path) + + +class TestFulltextStage: + async def test_passthrough_when_disabled(self) -> None: + settings = AppSettings(fulltext={"enabled": False}) + ctx = PipelineContext.create("query", settings) + ranked = [_ranked(_paper())] + + result = await FulltextStage().run(ctx, ranked) + + assert result.output == ranked + assert result.metrics["papers_with_passages"] == 0 + + async def test_passthrough_when_no_open_access_urls(self) -> None: + settings = AppSettings(fulltext={"enabled": True}) + ctx = PipelineContext.create("query", settings) + ranked = [_ranked(_paper(url="https://publisher.example/paywalled"))] + + result = await FulltextStage().run(ctx, ranked) + + assert result.output == ranked + assert result.metrics.get("pdfs_resolved") == 0 + + async def test_stage_produces_passages_artifact(self, tmp_path: Path) -> None: + pytest.importorskip("pymupdf") + from unittest.mock import patch + + settings = AppSettings(fulltext={"enabled": True, "top_chunks_per_paper": 2}) + ctx = PipelineContext.create("transformer attention", settings) + paper = _paper( + "Attention Paper", + url="https://arxiv.org/abs/1706.03762", + abstract="Attention mechanisms.", + ) + ranked = [_ranked(paper)] + + parsed = FullTextDocument( + paper_id=paper.paper_id, + text=( + "## Abstract\n\n" + "Scaled dot-product attention is the core of the transformer model " + "and enables parallel sequence processing at scale for translation. " + "It replaces recurrence entirely across encoder and decoder stacks, " + "and multi-head attention lets the model attend to information from " + "different representation subspaces jointly." + ), + metadata={"sections": {"abstract": 1}}, + ) + + with patch( + "src.fulltext.parser.extract_document", return_value=parsed + ) as mock_extract: + stage = FulltextStage( + downloader=FakeDownloader(tmp_path / "cached.pdf"), + index=InMemoryFulltextIndex(embedder=None), + ) + result = await stage.run(ctx, ranked) + + assert mock_extract.called + assert result.metrics["papers_with_passages"] == 1 + passages = ctx.get_artifact(PASSAGES_ARTIFACT) + assert paper.paper_id in passages + assert "dot-product attention" in passages[paper.paper_id][0] + + async def test_download_failure_degrades_gracefully(self) -> None: + settings = AppSettings(fulltext={"enabled": True}) + ctx = PipelineContext.create("query", settings) + ranked = [_ranked(_paper(url="https://arxiv.org/abs/1706.03762"))] + + class FailingDownloader: + async def download(self, paper_id, url, session=None): + raise RuntimeError("download failed") + + result = await FulltextStage(downloader=FailingDownloader()).run(ctx, ranked) + + assert result.output == ranked + assert result.metrics["papers_with_passages"] == 0 + + +class TestEvidenceInSynthesis: + def test_heuristic_extraction_includes_evidence_from_passages(self) -> None: + from src.analysis.synthesis import _heuristic_extraction + + ranked = _ranked(_paper("Grounded Paper", abstract="First point. Second point.")) + passages = ["A verbatim passage from the paper full text " * 12] + + extraction = _heuristic_extraction(ranked, passages) + + assert extraction.evidence + assert extraction.evidence[0].endswith("…") + assert len(extraction.evidence[0]) <= 301 + assert "full-text passages" in extraction.methodology[0] + + def test_extraction_prompt_embeds_passages(self) -> None: + from src.analysis.synthesis import _build_extraction_prompt + + ranked = _ranked(_paper("Grounded Paper", abstract="Abstract text.")) + prompt = _build_extraction_prompt( + ranked, "test query", ["Passage one text.", "Passage two text."] + ) + + assert "[Passage 1] Passage one text." in prompt + assert "[Passage 2] Passage two text." in prompt + assert "verbatim" in prompt + + def test_markdown_renders_evidence_quotes(self) -> None: + from src.reporting.markdown import render_enhanced_markdown + from src.retrieval.models import EnhancedResearchReport, PaperAnalysis, PaperCluster + + report = EnhancedResearchReport( + query="q", + papers=[ + PaperAnalysis( + paper_id="p1", + title="Grounded Paper", + key_points=["Finding one"], + evidence=["Attention replaces recurrence entirely."], + ) + ], + clusters=[PaperCluster(theme="Theme", summary="", paper_ids=["p1"])], + ) + + rendered = render_enhanced_markdown(report) + + assert "Evidence (from full text):" in rendered + assert "> Attention replaces recurrence entirely." in rendered diff --git a/tests/test_phase3_extensibility.py b/tests/test_phase3_extensibility.py index 62eba57..ce3a0b9 100644 --- a/tests/test_phase3_extensibility.py +++ b/tests/test_phase3_extensibility.py @@ -14,8 +14,8 @@ from src.core.events import StageEventCollector from src.core.pipeline import ResearchPipeline from src.core.registry import bootstrap_default_plugins, get_registry -from src.fulltext.downloader import StubPDFDownloader -from src.fulltext.rag import StubRAGIndex +from src.fulltext.downloader import CachingPDFDownloader +from src.fulltext.rag import InMemoryFulltextIndex from src.reporting.html import render_pdf_ready_html from src.reporting.output import render_report_output from src.retrieval.providers import ( @@ -170,19 +170,27 @@ async def failing_request(*args, **kwargs): class TestFullTextScaffold: - @pytest.mark.asyncio - async def test_stub_pdf_downloader_not_implemented(self) -> None: - downloader = StubPDFDownloader() + def test_downloader_uses_stable_cache_paths(self, tmp_path) -> None: + downloader = CachingPDFDownloader(cache_dir=str(tmp_path)) + + first = downloader._cache_path("paper-1") + again = downloader._cache_path("paper-1") + other = downloader._cache_path("paper-2") - with pytest.raises(NotImplementedError, match="PDF download"): - await downloader.download("paper-1", "https://example.com/paper.pdf") + assert first == again + assert first != other + assert first.suffix == ".pdf" @pytest.mark.asyncio - async def test_stub_rag_index_not_implemented(self) -> None: - index = StubRAGIndex() + async def test_index_accepts_and_counts_chunks(self) -> None: + from src.fulltext.base import FullTextChunk + + index = InMemoryFulltextIndex(embedder=None) + count = await index.index_chunks( + [FullTextChunk(paper_id="p", chunk_index=0, text="chunk text")] + ) - with pytest.raises(NotImplementedError, match="RAG indexing"): - await index.index_chunks([]) + assert count == 1 class TestPluginRegistry: @@ -198,8 +206,8 @@ def test_bootstrap_registers_builtin_providers_and_stages(self) -> None: assert "dblp" in providers assert "query_understanding" in stages assert "report_generation" in stages - assert "stub" in global_registry.list_fulltext_downloaders() - assert "stub" in global_registry.list_fulltext_indexes() + assert "default" in global_registry.list_fulltext_downloaders() + assert "default" in global_registry.list_fulltext_indexes() def test_registry_create_retrieval_provider(self) -> None: bootstrap_default_plugins()