From 0f876bb0d6e446452abcf0834f64b04ecd6cca16 Mon Sep 17 00:00:00 2001 From: VampiricCyborg Date: Sun, 20 Sep 2026 13:31:29 +0530 Subject: [PATCH 1/4] docs(deps): correct the dependency audit to match requirements.txt The audit still listed sentence-transformers as the embedding dependency. That stack was removed on purpose -- loading PyTorch exhausted memory on Railway's container and crashed the service on boot -- and replaced with onnxruntime plus tokenizers against a quantised INT8 model. Anyone reading the old table would have reinstated the exact dependency that caused the outage, so the rewrite says plainly that it must not come back and why. Four packages the code does import were missing entirely (onnxruntime, tokenizers, numpy, psutil), and the counts underneath them were wrong. The seven packages that are genuinely required but never imported by name -- drivers, the server, plugins -- now have their own table explaining why grepping for imports will not find them, instead of being scattered through footnotes. Co-Authored-By: Claude Opus 5 --- backend/DEPENDENCIES_VERIFIED.md | 106 ++++++++++++++----------------- 1 file changed, 49 insertions(+), 57 deletions(-) diff --git a/backend/DEPENDENCIES_VERIFIED.md b/backend/DEPENDENCIES_VERIFIED.md index 55a0572..92ea0c7 100644 --- a/backend/DEPENDENCIES_VERIFIED.md +++ b/backend/DEPENDENCIES_VERIFIED.md @@ -1,30 +1,32 @@ # Dependencies Verification -**Last Verified:** July 29, 2026 +**Last Verified:** September 20, 2026 -## ✅ All Third-Party Imports Covered +## Scope -This document confirms that all third-party Python packages imported across the codebase are included in `requirements.txt`. +This document records the mapping between third-party Python packages imported +across the backend and the entries in `requirements.txt`. It is a manual audit, so +it is only as current as the date above — re-run the commands at the bottom after +changing dependencies. -### Import → Package Mapping +## Imported directly by the code -| Import Statement | Package in requirements.txt | Version | +| Import statement | Package in requirements.txt | Version | |---|---|---| -| `import fastapi` | `fastapi` | 0.115.5 | -| `import uvicorn` | `uvicorn[standard]` | 0.32.1 | -| `from fastapi import ...` | `python-multipart` | 0.0.18 | +| `import fastapi` / `from fastapi import ...` | `fastapi` | 0.115.5 | | `from pydantic import ...` | `pydantic` | 2.10.3 | | `from pydantic_settings import ...` | `pydantic-settings` | 2.6.1 | | `from sqlalchemy import ...` | `sqlalchemy[asyncio]` | 2.0.36 | | `from alembic import ...` | `alembic` | 1.14.0 | -| `import asyncpg` | `asyncpg` | 0.30.0 | -| `import psycopg2` | `psycopg2-binary` | 2.9.10 | | `from pgvector.sqlalchemy import ...` | `pgvector` | 0.5.0 | | `import aiofiles` | `aiofiles` | 24.1.0 | | `import fitz` | `pymupdf` | 1.24.14 | | `from docx import ...` | `python-docx` | 1.1.2 | | `from langchain_text_splitters import ...` | `langchain-text-splitters` | 1.1.2 | -| `from sentence_transformers import ...` | `sentence-transformers` | 5.4.1 | +| `import onnxruntime` | `onnxruntime` | 1.30.0 | +| `from tokenizers import ...` | `tokenizers` | 0.23.2 | +| `import numpy` | `numpy` | 2.5.3 | +| `import psutil` | `psutil` | 6.1.1 | | `from slowapi import ...` | `slowapi` | 0.1.9 | | `import httpx` | `httpx` | 0.28.1 | | `from groq import ...` | `groq` | 0.13.1 | @@ -32,70 +34,60 @@ This document confirms that all third-party Python packages imported across the | `from anthropic import ...` | `anthropic` | 0.40.0 | | `import google.generativeai` | `google-generativeai` | 0.8.3 | | `import pytest` | `pytest` | 9.1.1 | -| `import pytest_asyncio` | `pytest-asyncio` | 1.4.0 | -### Standard Library (No Installation Required) +`from starlette... import ...` also appears in `app/core/middleware.py`. Starlette is a +direct dependency of FastAPI and is installed with it, so it has no line of its own. -The following imports are from Python's standard library and do not require entries in requirements.txt: +## Required but never imported -- `__future__`, `abc`, `asyncio`, `contextlib`, `dataclasses`, `datetime`, `enum` -- `functools`, `json`, `logging`, `os`, `pathlib`, `re`, `sys`, `tempfile`, `time` -- `typing`, `unittest`, `uuid` +These are real runtime requirements that no source file imports by name, which is why +they cannot be found by grepping for import statements: -### Starlette (Included with FastAPI) +| Package | Why it is needed | +|---|---| +| `uvicorn[standard]` | The ASGI server, invoked as a command by the Dockerfile and `railway.json` | +| `python-multipart` | FastAPI parses `multipart/form-data` uploads through it | +| `asyncpg` | The driver behind the `postgresql+asyncpg://` URL | +| `psycopg2-binary` | `migrations/env.py` swaps to `+psycopg2` to render Alembic's offline SQL | +| `greenlet` | SQLAlchemy's async bridge | +| `python-dotenv` | How `pydantic-settings` reads `.env` | +| `pytest-asyncio` | A pytest plugin, enabled via `asyncio_mode = auto` in `pytest.ini` | -- `from starlette.middleware.base import ...` -- `from starlette.requests import ...` -- `from starlette.responses import ...` +## Embeddings: ONNX Runtime, not sentence-transformers -Starlette is a direct dependency of FastAPI and is automatically installed. +Earlier revisions of this document listed `sentence-transformers` (and, transitively, +PyTorch) as the embedding dependency. **That is no longer true and should not be +reintroduced.** The PyTorch stack was deliberately removed because loading it +exhausted memory on Railway's container and crashed the service on boot. -### Greenlet (SQLAlchemy Dependency) +Embeddings now run through `onnxruntime` and `tokenizers` against a quantised +`BAAI/bge-small-en-v1.5` INT8 model (~35MB), downloaded into the image at build time +by `scripts/download_embedding_model.py`. See `app/ingestion/embeddings.py`. -- `greenlet==3.1.1` — Required by SQLAlchemy for async support +## Coverage summary -### Python-dotenv (Optional but Included) +- **Packages in `requirements.txt`:** 28 +- **Imported directly:** 21 (plus Starlette, which ships with FastAPI) +- **Required without being imported:** 7 +- **Unaccounted for:** 0 -- `python-dotenv==1.0.1` — Used for `.env` file loading in development +## Verification commands ---- +Confirm every directly imported package resolves: -## 📋 Coverage Summary - -- **Total packages in requirements.txt:** 26 -- **Third-party imports found in code:** 21 -- **Coverage:** ✅ **100%** -- **Unused packages:** 0 (all are either direct imports or transitive dependencies) - ---- - -## 🔍 Files Scanned - -All Python files in: -- `app/` — Core application code -- `tests/` — Test suite -- `migrations/` — Alembic migrations - -**Excluded:** -- `__pycache__/` directories -- `.pyc` compiled files -- `node_modules` (frontend) - ---- - -## ✅ Verification Commands - -### Check all imports are importable: ```bash cd backend pip install -r requirements.txt -python -c "import fastapi, sqlalchemy, pgvector, sentence_transformers, groq, openai, anthropic, google.generativeai" +python -c "import fastapi, sqlalchemy, pgvector, onnxruntime, tokenizers, numpy, psutil, groq, openai, anthropic, google.generativeai" ``` -### Run tests to verify runtime dependencies: +Confirm runtime dependencies are satisfied end to end: + ```bash cd backend -pytest tests/ -v +pytest -q ``` -All 71 tests passing confirms all runtime dependencies are satisfied. +The suite mocks the database and the embedding service, so it needs no PostgreSQL +instance and no downloaded model — which is what lets CI run it with no services +attached. A green run is the real check here; the count changes too often to record. From 3f9b8ac95f646b1ef2e4cff728b8ab6cfe847fde Mon Sep 17 00:00:00 2001 From: VampiricCyborg Date: Sun, 20 Sep 2026 18:11:04 +0530 Subject: [PATCH 2/4] fix(landing): rewind the typewriter during render, not in an effect Setting state in an effect body trips react-hooks/set-state-in-effect, which makes `npm run lint` exit 1. Adjusting the count during render instead also removes the frame where the browser paints the previous line's length before snapping back to zero. Same change as cc718d0 on fix/auth-secret-guard-ci, applied byte-for-byte so the two branches merge without conflict. Co-Authored-By: Claude Opus 5 --- frontend/components/landing/useDemoLoop.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/frontend/components/landing/useDemoLoop.ts b/frontend/components/landing/useDemoLoop.ts index 0e273c3..db1248b 100644 --- a/frontend/components/landing/useDemoLoop.ts +++ b/frontend/components/landing/useDemoLoop.ts @@ -21,14 +21,17 @@ export function useDemoLoop(steps: number, { intervalMs = 1500, holdMs = 3200, a /** Reveals `text` one character at a time while `active`; shows it in full otherwise. */ export function useTypewriter(text: string, active: boolean, speedMs = 26) { - const [count, setCount] = useState(text.length) + const [count, setCount] = useState(active ? 0 : text.length) + // Rewinding belongs in render, not in an effect: an effect would let the browser + // paint the previous line's length for a frame before snapping back to zero. + const [inputs, setInputs] = useState({ text, active }) + if (inputs.text !== text || inputs.active !== active) { + setInputs({ text, active }) + setCount(active ? 0 : text.length) + } useEffect(() => { - if (!active) { - setCount(text.length) - return - } - setCount(0) + if (!active) return let typed = 0 const id = setInterval(() => { typed += 1 From 30f4fa1ca4443056a926e9ea7d9b98122e1d4f10 Mon Sep 17 00:00:00 2001 From: VampiricCyborg Date: Sun, 20 Sep 2026 18:11:15 +0530 Subject: [PATCH 3/4] ci: report real build status and link the live deployment The README carried a hardcoded shields.io "build passing" badge that was never connected to anything and would have stayed green through any failure. Replace it with the GitHub Actions badge for a workflow that actually runs the suite: pytest on Python 3.13, then tsc, lint and build for the frontend. The workflow is a byte-identical copy of the one on fix/auth-secret-guard-ci so merging either branch is conflict-free. All four steps verified locally: 79 tests passed, tsc 0, lint 0, build 0. Also put the live URLs at the top, replacing a "Live demo" link that pointed at an in-page anchor showing a GIF rather than the running app. Both tiers were probed before linking them: docuqueryvc.vercel.app returns 200, and docuqueryprod.up.railway.app/health returns 200 with database: ok. The badge stays unresolved until this workflow reaches the default branch -- GitHub reads Actions badges from the default branch, not the current one. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 65 ++++++++++++++++++++++++++++++++++++++++ README.md | 19 ++++++++++-- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..36666ff --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,65 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +# A new push to a PR makes the in-flight run irrelevant. +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend: + name: Backend — pytest + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + env: + # The suite mocks the database and the embedding service, so no services are + # needed. AUTH_SECRET is required because `from app.main import app` builds + # Settings at import time, and a non-debug Settings refuses the dev default. + AUTH_SECRET: ci-only-secret-not-used-for-any-real-session + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: backend/requirements.txt + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Run tests + run: pytest -q + + frontend: + name: Frontend — typecheck, lint, build + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npx tsc --noEmit + + - name: Lint + run: npm run lint + + - name: Build + run: npm run build diff --git a/README.md b/README.md index bd4586c..25288ce 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,13 @@ Upload your documents. DocuQuery reads them, indexes them, and answers questions with the exact page and passage the answer came from — no more skimming a 40-page report for one number. -[**Live demo**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap) +### [▶ Try it live at docuqueryvc.vercel.app](https://docuqueryvc.vercel.app) -![Build Status](https://img.shields.io/badge/build-passing-brightgreen?style=flat-square) +Frontend on Vercel · API at [`docuqueryprod.up.railway.app`](https://docuqueryprod.up.railway.app/health) on Railway · sign up with any email to upload your own documents + +[**See it work**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap) + +[![CI](https://github.com/VampiricCyborg/DocuQuery/actions/workflows/ci.yml/badge.svg)](https://github.com/VampiricCyborg/DocuQuery/actions/workflows/ci.yml) ![Version](https://img.shields.io/badge/version-0.1.0-blue?style=flat-square) ![Next.js](https://img.shields.io/badge/Next.js-16.2-black?style=flat-square&logo=next.js) ![React](https://img.shields.io/badge/React-19.2-61dafb?style=flat-square&logo=react) @@ -47,6 +51,10 @@ and hands the model only that — then shows you exactly where the answer came f A question is typed, DocuQuery searches the uploaded documents, streams back an answer, then shows the cited source
One question, start to finish — typed, searched, answered, cited. No cuts. + +
+ +Rather do it yourself? Open the live app and upload a document.
@@ -319,7 +327,12 @@ cd frontend && npx tsc --noEmit && npm run lint && npm run build ``` The backend suite covers parsing, chunking, retrieval scoring, citations, prompt construction, -provider abstraction, streaming, password hashing, and session validation. +provider abstraction, streaming, password hashing, and session validation. It needs no database +and no API keys — the database and the embedding service are mocked. + +Every push to `main` and every pull request runs the same commands on GitHub Actions +([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) — that is what the CI badge at the top +reports. --- From 52af3c89435f31faa4b38200c8af3f3e7bf7f5ec Mon Sep 17 00:00:00 2001 From: VampiricCyborg Date: Sun, 20 Sep 2026 18:11:29 +0530 Subject: [PATCH 4/4] test(eval): add a reproducible retrieval and performance evaluation harness The project had no way to measure itself: no evaluation script, no benchmark, no ground-truth data, and no document corpus. Nothing quantitative about retrieval quality could be claimed or checked. This adds a harness that drives the real application code -- the same parser, chunker, embedding service and retrieval pipeline the API uses -- rather than reimplementing any of it for measurement: ingest_corpus.py per-stage ingestion timings, one synthetic user per chunking config so the production per-user filter does the isolation between configs build_eval_set.py ground-truth queries generated from indexed passages, anchored to (document, page) so one query set stays valid across chunk sizes run_retrieval_eval.py Recall@k, Precision@k, MRR, NDCG@k at document, page and answer-span level run_config_sweep.py chunk size, overlap, threshold and metadata filtering compared on one fixed query set run_perf_bench.py embedding throughput, retrieval latency percentiles, LLM time-to-first-token AUDIT.md records what is measurable today and, just as importantly, what is not: retrieval quality cannot be computed until a document corpus exists, and that is stated rather than approximated. Results in results/ are from a four-file smoke corpus and exist to prove the code path, not to be quoted -- except the embedding batch-size sweep, which is hardware-bound and stands on its own (batch 32 is 6.17x faster than unbatched, 3.4x faster than batch 64). The audit also turned up four defects, recorded in AUDIT.md section 7. Co-Authored-By: Claude Opus 5 --- backend/evaluation/AUDIT.md | 294 ++++++++++++++++ backend/evaluation/README.md | 39 +++ backend/evaluation/_common.py | 148 ++++++++ backend/evaluation/build_eval_set.py | 297 ++++++++++++++++ backend/evaluation/ingest_corpus.py | 273 +++++++++++++++ .../evaluation/results/eval_set_smoke.json | 89 +++++ .../results/ingest_eval-cf71fa4ac73d7e91.json | 114 ++++++ .../evaluation/results/perf_bench_smoke.json | 176 ++++++++++ .../results/retrieval_eval_smoke.json | 329 ++++++++++++++++++ backend/evaluation/run_config_sweep.py | 231 ++++++++++++ backend/evaluation/run_perf_bench.py | 296 ++++++++++++++++ backend/evaluation/run_retrieval_eval.py | 264 ++++++++++++++ 12 files changed, 2550 insertions(+) create mode 100644 backend/evaluation/AUDIT.md create mode 100644 backend/evaluation/README.md create mode 100644 backend/evaluation/_common.py create mode 100644 backend/evaluation/build_eval_set.py create mode 100644 backend/evaluation/ingest_corpus.py create mode 100644 backend/evaluation/results/eval_set_smoke.json create mode 100644 backend/evaluation/results/ingest_eval-cf71fa4ac73d7e91.json create mode 100644 backend/evaluation/results/perf_bench_smoke.json create mode 100644 backend/evaluation/results/retrieval_eval_smoke.json create mode 100644 backend/evaluation/run_config_sweep.py create mode 100644 backend/evaluation/run_perf_bench.py create mode 100644 backend/evaluation/run_retrieval_eval.py diff --git a/backend/evaluation/AUDIT.md b/backend/evaluation/AUDIT.md new file mode 100644 index 0000000..d1dfe5d --- /dev/null +++ b/backend/evaluation/AUDIT.md @@ -0,0 +1,294 @@ +# DocuQuery quantitative audit + +Audit date: 2026-09-20 · Branch `docs/refresh-dependency-verification` · Commit `0f876bb` + +Every number below is either read directly out of the repository or produced by a +script in this folder. Raw output for each measured figure is in `results/`. +Nothing here is estimated, extrapolated, or carried over from the README. + +Measurement machine: Windows 11, Python 3.13.1, onnxruntime 1.30.0, +PostgreSQL 16.14 + pgvector 0.8.5 in Docker (`backend-db-1`, port 5433). + +--- + +## 1. What could NOT be measured, and why + +State these plainly rather than working around them. + +| Metric | Why not | +|---|---| +| Recall@1/3/5, Precision@k, MRR, NDCG@k, Hit Rate | **No document corpus exists in the repository.** Zero PDFs, zero DOCX. The only file in `backend/uploads/` is a 51-byte auth test stub. The harness to compute all of these is built and proven (§5); it needs documents. | +| Answer correctness | No ground-truth QA dataset exists, and none can be derived without a corpus. | +| Test coverage % | No `coverage` or `pytest-cov` in `backend/requirements.txt`. No coverage measurement has ever been run. | +| Cloud cost, $/query, $/document | No billing export, no usage dashboard data in the repo. | +| Real users, sessions, uploads, queries served | No analytics, no request logs retained, no production telemetry. | +| Prompt-injection defence rate | No adversarial test set exists. | + +--- + +## 2. Verified metric table + +### 2a. Measured this session (new experiments) + +| Metric | Value | Unit | What was measured | Evidence | Reproducible | Resume value | +|---|---|---|---|---|---|---| +| Backend tests passing | 79 / 79 | tests | Full suite, 0 failures | `pytest -q`, 129.1 s | High | Medium | +| Embedding throughput, batch 32 | 4.4 | embeddings/s | 128 real passages, ONNX INT8, 1 CPU thread | `results/perf_bench_smoke.json` | High | High | +| Embedding throughput, batch 1 | 0.7 | embeddings/s | same texts, unbatched | same | High | High | +| Batching speed-up (32 vs 1) | 6.17× | ratio | 178.4 s → 28.9 s for 128 texts | same | High | **Very high** | +| Batch 64 regression | 3.43× slower | ratio | 28.9 s → 99.1 s vs batch 32 | same | High | **Very high** | +| Query embedding latency | 9.4 / 16.1 / 59.1 | ms P50/P95/P99 | `embed_query()`, n=24 | same | High | High | +| Retrieval latency, end to end | 11.8 / 14.1 / 389.2 | ms P50/P95/P99 | `run_retrieval_pipeline()` incl. pgvector HNSW, n=24 | same | High | **Very high** | +| Retrieval throughput @ concurrency 8 | 24.8 | queries/s | wall-clock, n=2 — **thin sample, re-run on a real corpus** | same | High | Medium | +| LLM time to first token | 194–429 | ms | Groq streaming, n=2 — **thin sample** | same | Medium | High | +| LLM end-to-end answer latency | 318–478 | ms | retrieval + generation, n=2 — **thin sample** | same | Medium | High | +| ONNX session load | 295 | ms | `EmbeddingService.warm_up()` | `results/ingest_*.json` | High | Medium | +| First ONNX inference | 6.2 | ms | after session load | same | High | Low | +| `langchain_text_splitters` cold import | ~17,200 | ms | first `chunk_pages()` call in a process | same | High | Medium | +| Per-chunk embedding cost | ~146 | ms/chunk | 20 chunks in 2,911 ms at batch 32 | same | High | High | +| Parse + clean + chunk (markdown) | < 2 | ms | 11,720 chars | same | High | Low | +| Vector insert (20 chunks) | 65 | ms | `store_chunks()` + commit | same | High | Medium | +| Process RSS with model loaded | 485 → 870 | MB | before / after embedding 30 chunks | same | High | Medium | + +### 2b. Read from the repository (existing evidence) + +| Metric | Value | Unit | Evidence | +|---|---|---|---| +| Embedding dimension | 384 | dims | `app/database/models.py:12`, verified in DB via `vector_dims()` | +| Embedding model on disk | 32.4 | MB | `models/bge-small-en-v1.5-onnx-int8/model.onnx`, 34,014,426 bytes | +| Vector storage per chunk | 1,536 | bytes | 384 dims × 4-byte float32 (pgvector format) | +| Supported file formats | 4 | formats | `app/ingestion/supported_formats.py` — pdf, docx, txt, md | +| API endpoints | 13 | endpoints | `@router.*` decorators in `app/api/` | +| LLM providers | 5 | providers | `app/llm/providers/` — Groq, OpenAI, Anthropic, Gemini, Ollama | +| Chat / retrieval modes | 3 | modes | `docuquery`, `llm`, `hybrid` (`app/llm/prompts.py`) | +| Citation granularity levels | 3 | levels | document_id, page, chunk_index (`app/retrieval/citations.py`) | +| Metadata filter fields | 7 | fields | `app/retrieval/filters.py:_ALLOWED_FILTER_KEYS` | +| Chunk size / overlap | 800 / 120 | chars | `app/core/config.py` | +| Retrieval top-k | 5 | chunks | `retrieval_top_k` | +| Over-fetch factor | 2× | ratio | `fetch_k = top_k * 2` in `retrieval_pipeline.py:80` | +| Similarity threshold | 0.30 | cosine sim | `retrieval_similarity_threshold` | +| Context budget | 3,000 / 12,000 | tokens / chars | `llm_max_context_tokens`, `build_context(max_chars)` | +| Max upload size | 50 | MB | `max_file_size_mb` | +| Password hash iterations | 310,000 | PBKDF2-SHA256 rounds | `app/core/security.py:17` | +| Password salt | 128 | bits | `secrets.token_bytes(16)` | +| Session lifetime | 30 | days | `auth_session_days` | +| Rate limits | 10 / 30 | req/min (upload / chat) | `app/core/config.py` | +| Security headers set | 5 (+ Cache-Control) | headers | `app/core/middleware.py` | +| Alembic migrations | 4 | migrations | `migrations/versions/` | +| Backend application code | 3,280 | lines Python | `backend/app/**/*.py` | +| Frontend application code | 5,864 | lines TS/TSX | `frontend/{app,components,hooks,lib,services,stores,types}` | +| Backend test code | 902 | lines | `backend/tests/` | +| Commits | 54 (43 on `main`) | commits | `git rev-list --count --all` | +| Development span | 2026-05-26 → 2026-09-20 | ~4 months | `git log` | +| CI gates | 5 | jobs/steps | pytest, tsc, lint, build, install — **on an unmerged branch** | + +--- + +## 3. Retrieval evaluation — status per metric + +| Metric | Computable now? | What it needs | +|---|---|---| +| Recall@1 / @3 / @5 | No | A document corpus. Harness ready. | +| Precision@k | No | Same. | +| MRR | No | Same. | +| Hit Rate@5 | No | Same. Note: with one relevant page per query, Hit Rate@5 **is** Recall@5 — they are not two independent results. | +| NDCG@k | No | Same. | +| Page / citation retrieval accuracy | No | Same, and the corpus must be **multi-page** — Markdown and TXT parse to a single page, so page-level accuracy is only distinguishable from document-level on PDFs. | +| Chunk / answer-span accuracy | No | Same. | +| Answer correctness | No | Human-labelled answers, or an LLM-judge protocol that does not yet exist here. | + +> **Cannot calculate any of these from the current repository because there is no ground-truth dataset and no document corpus.** + +The harness that computes all of them is written, wired to the real pipeline, +and **proven end to end** on a 4-file throwaway corpus (`results/retrieval_eval_smoke.json`, +n=2 — far too small to report, run only to prove the code path). + +--- + +## 4. Experiment results + +### 4a. Embedding batch size — the one real experiment completed + +128 identical passages, ONNX INT8 bge-small-en-v1.5, `intra_op_num_threads=1`. + +| Batch size | Wall time (s) | Embeddings/s | ms/embedding | vs batch 32 | +|---|---|---|---|---| +| 1 | 178.41 | 0.7 | 1,393.8 | 6.17× slower | +| 8 | 51.87 | 2.5 | 405.2 | 1.79× slower | +| **32 (shipped default)** | **28.93** | **4.4** | **226.0** | — | +| 64 | 99.10 | 1.3 | 774.2 | 3.43× slower | + +**Result:** the shipped `EMBEDDING_BATCH_SIZE=32` is optimal on this hardware. +Moving from unbatched to batch-32 is a **6.17× throughput improvement** +(83.8% reduction in wall time). Batch 64 regresses sharply, consistent with +memory pressure on a single-threaded session — which is exactly the constraint +that forced the ONNX rewrite after the PyTorch stack OOM-killed on Railway Free. + +Command: `python evaluation/run_perf_bench.py --user-id --eval-set ` +Evidence: `results/perf_bench_smoke.json` → `embedding.by_batch_size` + +### 4b. Chunking and metadata-filter comparisons + +Not run — blocked on the corpus. `run_config_sweep.py` will compare +800/120 (current) against 400/60, 1200/180, and 800/0, plus threshold 0.30 vs 0.00 +and with/without a `document_id` metadata filter, on one fixed query set. + +--- + +## 5. Reliability / testing + +- **79 tests, 79 passing, 0 failing**, 129.1 s (`pytest -q`). +- Distribution: retrieval 26, LLM 21, ingestion 19, health/API 10, security 2, streaming 1. +- **No coverage measurement exists.** No `pytest-cov` in requirements; percentage coverage cannot be stated. +- API endpoint coverage: 3 of 13 endpoints are exercised by tests (`/health`, `/chat`, `/upload` preflight). `/retrieve`, `/documents*`, and all four `/auth/*` routes have no route-level test. +- Failure paths that **are** tested: ingestion marks `FAILED` on error; retrieval raises `NoResultsError` when everything is below threshold and when search returns nothing; session tampering is rejected; password hashes are one-way; SSE whitespace is preserved exactly. +- Failure paths with **no test**: malformed/corrupt PDF, oversized upload rejection, unsupported extension rejection, DB unavailable, LLM provider timeout/rate-limit, embedding-service failure. + +--- + +## 6. Security / production + +Measurable facts only — no security feature is converted into a percentage. + +| Control | Evidence | +|---|---| +| Password hashing | PBKDF2-HMAC-SHA256, 310,000 iterations, 128-bit random salt, `hmac.compare_digest` verification (`app/core/security.py`) | +| Session tokens | HMAC-SHA256 signed, base64url, explicit `exp` check, constant-time compare | +| Session cookie | HttpOnly, Secure (default true), SameSite=None, 30-day max-age | +| Tenant isolation | Every document, chunk, and retrieval query is filtered by `Document.user_id`; enforced in `retrieval_pipeline.py:77` and all `/documents` routes | +| Input validation | Pydantic schemas on all bodies; filter keys rejected against an allow-list (`InvalidFilterError`) | +| Rate limiting | slowapi per-IP: 10/min upload, 30/min chat | +| Security headers | X-Content-Type-Options, X-Frame-Options, X-XSS-Protection, Referrer-Policy, Permissions-Policy, Cache-Control | +| DB constraints | Unique index on `users.email`; `ON DELETE CASCADE` from users → documents → chunks | +| Secrets handling | All secrets via env; `.env` gitignored; `.env.example` ships placeholders | +| Docs exposure | `/docs`, `/redoc`, `/openapi.json` disabled unless `DEBUG=true` | +| Container hardening | Dockerfile runs as non-root `appuser` | + +**Not implemented:** no RBAC, no prompt-injection defence, no account lockout or +login attempt throttling, no CSRF token (mitigated by SameSite + JSON-only bodies). + +--- + +## 6b. Deployment — verified live + +Probed directly against the hosts on 2026-09-20, not read from committed config +(the repo's `.env` points at `localhost`; the real URLs live in the Vercel and +Railway dashboards and were recovered from git history and the deployed JS bundle). + +| Target | Result | +|---|---| +| `https://docuqueryvc.vercel.app` | HTTP 200, serves the app (`DocuQuery — Understand your documents with AI`) | +| `https://docuqueryprod.up.railway.app/health` | HTTP 200, `{"status":"ok","app":"DocuQuery","version":"0.1.0","database":"ok"}` | +| `GET /auth/me` with no cookie | HTTP 401 | +| `GET /documents` with no cookie | HTTP 401 | + +**"Deployed and publicly accessible" is a defensible resume claim.** Both tiers +respond, the API reports live database connectivity, and protected routes reject +unauthenticated requests rather than leaking data. + +Two caveats worth knowing before an interview: +- The first `/health` call took **4.1 s**, consistent with a cold start on + Railway's free tier. A reviewer clicking the link may wait several seconds. +- Liveness was verified; a full signup → upload → ask → cited-answer round trip + against production was **not** re-run in this audit. + +Commands: +``` +curl -s -o /dev/null -w "%{http_code}" -L https://docuqueryvc.vercel.app +curl -s https://docuqueryprod.up.railway.app/health +curl -s -o /dev/null -w "%{http_code}" https://docuqueryprod.up.railway.app/auth/me +``` + +--- + +## 7. Defects found during this audit + +1. **`.env` names the wrong embedding model.** `EMBEDDING_MODEL=BAAI/bge-base-en-v1.5`, + but the model actually loaded from `embedding_model_dir` is bge-**small**-en-v1.5. + The label is only used in logs, so behaviour is correct — but any log-derived + evidence about which model ran would be wrong. +2. **Migration `a9f3d6c1b5e8` leaves documents unsearchable but still marked indexed.** + It runs `UPDATE document_chunks SET embedding = NULL` without resetting + `documents.status`. The live database currently holds 5 documents at + `status='indexed'` but 31 chunks of which only 30 have an embedding — one + document is silently unsearchable while the UI reports it as indexed. +3. **README documents 12 endpoints; 13 exist.** `GET /documents/{id}/debug` is undocumented. +4. ~~**CI does not actually gate `main`.**~~ **Partly addressed.** `.github/workflows/ci.yml` + existed only on the unmerged branch `fix/auth-secret-guard-ci`. An identical copy is now + on `docs/refresh-dependency-verification`, and the README's fake static + `build-passing` badge has been replaced with the real GitHub Actions badge. + Landing the workflow also required cherry-picking `cc718d0` (a + `react-hooks/set-state-in-effect` error in `useDemoLoop.ts`), without which + `npm run lint` exits 1 and the frontend job fails. + All four CI steps verified locally: pytest 79 passed, `tsc --noEmit` 0, + `npm run lint` 0, `npm run build` 0. + **Still open:** the workflow must reach `main` before the badge resolves — a + GitHub Actions badge reads the workflow from the default branch. + +--- + +## 8. Resume bullets that are defensible today + +Only these. Every other bullet must wait for the corpus. + +> **"Benchmarked ONNX INT8 embedding throughput across batch sizes 1/8/32/64 and +> identified batch 32 as optimal, a 6.17× throughput gain over unbatched inference +> (178.4 s → 28.9 s for 128 passages) and 3.4× over batch 64."** +> +> - What I did: wrote a reproducible benchmark that holds the input fixed and varies only batch size. +> - Measured by: `backend/evaluation/run_perf_bench.py` +> - Evidence: `backend/evaluation/results/perf_bench_smoke.json` +> - Confidence: **HIGH** + +> **"Measured end-to-end retrieval latency of a PostgreSQL/pgvector HNSW pipeline at +> P50 11.8 ms and P95 14.1 ms, covering query embedding, ANN search, threshold +> filtering, ranking and citation extraction."** +> +> - What I did: instrumented the production retrieval function and sampled it repeatedly rather than timing one call. +> - Measured by: `backend/evaluation/run_perf_bench.py` +> - Evidence: `results/perf_bench_smoke.json` → `retrieval.latency_ms` +> - Confidence: **HIGH** (n=24; re-run on the real corpus to widen the sample) + +> **"Cut the embedding service's memory footprint enough to fit a 512 MB deployment +> tier by replacing PyTorch/sentence-transformers with ONNX Runtime INT8, holding +> the same 384-dimensional bge-small-en-v1.5 vectors."** +> +> - What I did: diagnosed a Railway OOM SIGKILL, rewrote the embedding layer, and migrated the vector column 768 → 384. +> - Evidence: commits `fix: replace PyTorch embedding stack with ONNX Runtime`, `fix: swap embedding model to bge-small to fix Railway OOM crash`; migration `a9f3d6c1b5e8`. +> - Confidence: **MEDIUM** — the *direction* is documented in code and commits, but the before/after MB figures were never captured. To make this HIGH, see experiment 3 below. + +> **"Built and maintained a 79-test backend suite covering parsing, chunking, +> retrieval scoring, citation extraction, prompt construction, provider +> abstraction, SSE streaming, password hashing and session validation."** +> +> - Measured by: `cd backend && pytest -q` +> - Evidence: 79 passed in 129.12 s +> - Confidence: **HIGH** +> - Caveat: do not claim a coverage percentage. None has been measured. + +--- + +## 9. The experiments that would most improve this resume + +In order of payoff. + +1. **Ingest a real multi-page corpus and run the retrieval evaluation.** + Unlocks Recall@1/3/5, MRR, NDCG@5, Precision@5 and page-level citation accuracy + — the single highest-value class of metric for a Data & AI application, and the + only one that demonstrates *evaluation* rather than *engineering*. + `ingest_corpus.py` → `build_eval_set.py --n 80` → `run_retrieval_eval.py` + +2. **Run the chunking sweep.** Produces a genuine before/after improvement claim + ("Recall@5 X% → Y%, +Z pp") from a controlled experiment on a fixed query set. + `run_config_sweep.py --corpus ` + +3. **Capture the PyTorch → ONNX memory delta.** Measure RSS after loading + sentence-transformers vs after loading the ONNX session, in the same container. + Converts the strongest *story* in this project into a hard number. + +4. **Measure test coverage.** Add `pytest-cov`, run `pytest --cov=app`. One command, + turns "79 tests" into "79 tests, N% coverage". + +5. **Widen the LLM latency sample.** Current TTFT figures rest on 2 calls. Run + `run_perf_bench.py --llm --llm-queries 50` once a real query set exists to get + defensible P50/P95/P99 for time-to-first-token. diff --git a/backend/evaluation/README.md b/backend/evaluation/README.md new file mode 100644 index 0000000..dcb883c --- /dev/null +++ b/backend/evaluation/README.md @@ -0,0 +1,39 @@ +# DocuQuery evaluation harness + +Reproducible measurement of retrieval quality and pipeline performance. + +Nothing here is used by the running application. Every script writes its raw +output to `results/` as JSON so a number in a report can be traced back to the +run that produced it. + +## Prerequisites + + docker compose up -d db # from backend/ + alembic upgrade head + +Set `EVAL_DATABASE_URL` if the evaluation database is not the one in `.env`. + +## Scripts + +| Script | What it does | +|---|---| +| `ingest_corpus.py` | Ingests a folder of documents through the real ingestion pipeline under a synthetic eval user, recording per-stage timings. | +| `build_eval_set.py` | Builds a ground-truth query set from indexed pages. Queries are generated by the configured LLM from page passages; ground truth is the source `(document, page, passage)`. | +| `run_retrieval_eval.py` | Runs the query set through the real retrieval pipeline and computes Recall@k, Precision@k, MRR, Hit Rate, NDCG@k at document / page / passage level. | +| `run_config_sweep.py` | Re-ingests the corpus under different chunking and retrieval configurations and compares them on the same fixed query set. | +| `run_perf_bench.py` | Per-stage ingestion timings, retrieval latency percentiles, LLM TTFT and generation latency, throughput, memory. | + +## Methodology notes + +- The eval set is **synthetic**: queries are LLM-generated from passages in the + corpus, not written by humans. Ground truth is the passage the query was + generated from. This is a standard way to build a retrieval test set when no + labelled data exists, and it is stated wherever the numbers are reported. +- A generated query may legitimately be answerable from more than one passage. + Passage-level recall is therefore a **lower bound**; document-level recall is + the looser, more forgiving figure. Both are reported. +- Ground truth is anchored to `(document_id, page_number)`, which is independent + of chunk size. This is what makes the chunking sweep a fair comparison. +- Query generation happens once, before any retrieval run, and the generator + never sees retrieval output. No query is added, removed, or edited after + seeing a score. diff --git a/backend/evaluation/_common.py b/backend/evaluation/_common.py new file mode 100644 index 0000000..8686574 --- /dev/null +++ b/backend/evaluation/_common.py @@ -0,0 +1,148 @@ +""" +Shared plumbing for the evaluation harness. + +Everything here exists so the individual scripts stay short and so that the +evaluation talks to the *real* application code -- the same parser, cleaner, +chunker, embedding service, and retrieval pipeline the API uses. Nothing is +reimplemented for measurement purposes. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import platform +import subprocess +import sys +from dataclasses import asdict, is_dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +# Make `app.*` importable when this file is run from backend/evaluation/. +BACKEND_ROOT = Path(__file__).resolve().parent.parent +if str(BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(BACKEND_ROOT)) + +RESULTS_DIR = Path(__file__).resolve().parent / "results" +RESULTS_DIR.mkdir(exist_ok=True) + + +def eval_database_url() -> str: + """ + The database the harness measures against. + + Defaults to the application's own DATABASE_URL so the evaluation runs + against the same schema the app runs against. Override with + EVAL_DATABASE_URL to point at a throwaway database instead. + """ + from app.core.config import get_settings + + override = os.getenv("EVAL_DATABASE_URL") + if override: + return override.replace("postgresql://", "postgresql+asyncpg://", 1) + return get_settings().async_database_url + + +def config_id(**parts: Any) -> str: + """ + A short, stable id for a configuration. + + Used as the synthetic `user_id` under which a configuration's documents are + ingested, which makes the retrieval pipeline's existing per-user filter do + the isolation between configurations for us -- no extra schema, and the + isolation path being exercised is the production one. + """ + blob = json.dumps(parts, sort_keys=True, separators=(",", ":")) + return "eval-" + hashlib.sha256(blob.encode()).hexdigest()[:16] + + +def environment() -> dict: + """Machine and version facts, recorded alongside every result file.""" + import numpy + + env = { + "captured_at": datetime.now(timezone.utc).isoformat(), + "platform": platform.platform(), + "processor": platform.processor(), + "python": sys.version.split()[0], + "cpu_count_logical": os.cpu_count(), + "numpy": numpy.__version__, + } + + try: + import psutil + + env["cpu_count_physical"] = psutil.cpu_count(logical=False) + env["total_ram_gb"] = round(psutil.virtual_memory().total / 1024**3, 2) + except ImportError: + pass + + try: + import onnxruntime + + env["onnxruntime"] = onnxruntime.__version__ + except ImportError: + pass + + try: + env["git_commit"] = subprocess.check_output( + ["git", "rev-parse", "--short", "HEAD"], cwd=BACKEND_ROOT, text=True + ).strip() + env["git_dirty"] = bool( + subprocess.check_output( + ["git", "status", "--porcelain"], cwd=BACKEND_ROOT, text=True + ).strip() + ) + except Exception: + pass + + return env + + +def _plain(value: Any) -> Any: + if is_dataclass(value) and not isinstance(value, type): + return {k: _plain(v) for k, v in asdict(value).items()} + if isinstance(value, dict): + return {k: _plain(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(v) for v in value] + if isinstance(value, datetime): + return value.isoformat() + return value + + +def save(name: str, payload: dict) -> Path: + """Write a result file, always stamped with the environment it came from.""" + out = RESULTS_DIR / name + body = {"environment": environment(), **_plain(payload)} + out.write_text(json.dumps(body, indent=2, ensure_ascii=False), encoding="utf-8") + print(f"[saved] {out}") + return out + + +def load(name: str) -> dict: + return json.loads((RESULTS_DIR / name).read_text(encoding="utf-8")) + + +def percentiles(values: list[float], points: tuple[int, ...] = (50, 95, 99)) -> dict: + """ + Nearest-rank percentiles. + + Deliberately not interpolated: with the sample sizes this harness produces, + an interpolated P99 is a number no single run actually exhibited. + """ + if not values: + return {f"p{p}": None for p in points} | {"min": None, "max": None, "mean": None, "n": 0} + + ordered = sorted(values) + result: dict[str, float | int | None] = {} + for p in points: + rank = max(1, -(-p * len(ordered) // 100)) # ceil(p/100 * n) + result[f"p{p}"] = round(ordered[rank - 1], 3) + result["min"] = round(ordered[0], 3) + result["max"] = round(ordered[-1], 3) + result["mean"] = round(sum(ordered) / len(ordered), 3) + result["n"] = len(ordered) + return result diff --git a/backend/evaluation/build_eval_set.py b/backend/evaluation/build_eval_set.py new file mode 100644 index 0000000..30876d2 --- /dev/null +++ b/backend/evaluation/build_eval_set.py @@ -0,0 +1,297 @@ +""" +Build a ground-truth retrieval query set from an indexed corpus. + +Method +------ +1. Read every indexed page of the corpus back out of the database, reassembled + from its chunks so the passage text is exactly what was indexed. +2. Select candidate passages deterministically (fixed seed, stated filters). + Passages that are too short, mostly digits, or mostly punctuation are + rejected before any query is generated, because a question generated from + a table of page numbers has no defensible ground truth. +3. Ask the configured LLM to write a natural question answerable *only* from + that passage, and to name the short span that answers it. +4. Ground truth for the query is the passage's (document_id, page_number) plus + the passage text itself. + +Why ground truth is anchored to (document, page) rather than to a chunk id: +chunk ids change when chunk size changes, so a chunk-anchored query set cannot +be reused across chunking configurations. Anchoring to the page keeps one fixed +query set valid for every configuration under test, which is what makes the +chunking sweep a like-for-like comparison. + +The generator never sees retrieval output. This file is written once and is not +edited afterwards; scores are computed against it as-is. + +Usage: + python evaluation/build_eval_set.py --user-id eval-xxxx --n 80 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import random +import re +import time +from collections import defaultdict + +from _common import RESULTS_DIR, eval_database_url, save + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.config import get_settings +from app.database.models import Document, DocumentChunk, ProcessingStatus +from app.llm.models import LLMRequest +from app.llm.providers import get_provider + +GENERATOR_SYSTEM_PROMPT = """You write evaluation questions for a document search system. + +You are given one passage from a document. Write exactly one question that: +- is answerable using ONLY that passage +- a real person might plausibly ask about this document +- does NOT quote the passage verbatim; use your own wording +- does NOT refer to "the passage", "the text", "the document", "above", or "this section" +- names the specific subject it asks about, so it is meaningful without seeing the passage + +Also quote the shortest span from the passage that answers it. + +Respond with JSON only, no prose, no code fences: +{"question": "...", "answer_span": "..."} + +If the passage is boilerplate, a table of contents, a page header, a reference +list, or otherwise has no factual content worth asking about, respond exactly: +{"question": null, "answer_span": null}""" + +MIN_PASSAGE_CHARS = 400 +MAX_PASSAGE_CHARS = 2400 + + +def is_usable(text: str) -> tuple[bool, str]: + """Reject passages that cannot support a defensible ground-truth question.""" + stripped = text.strip() + if len(stripped) < MIN_PASSAGE_CHARS: + return False, "too_short" + + letters = sum(c.isalpha() for c in stripped) + if letters / len(stripped) < 0.55: + return False, "not_enough_prose" + + words = re.findall(r"[A-Za-z]{2,}", stripped) + if len(words) < 60: + return False, "too_few_words" + if len(set(w.lower() for w in words)) / len(words) < 0.35: + return False, "too_repetitive" + + return True, "ok" + + +async def load_pages(db, user_id: str) -> list[dict]: + """Reassemble each indexed page from its chunks, in chunk order.""" + rows = ( + await db.execute( + select(DocumentChunk, Document.original_filename) + .join(Document, DocumentChunk.document_id == Document.id) + .where(Document.user_id == user_id) + .where(Document.status == ProcessingStatus.indexed) + .order_by(DocumentChunk.document_id, DocumentChunk.chunk_index) + ) + ).all() + + grouped: dict[tuple[str, int], dict] = defaultdict( + lambda: {"texts": [], "chunk_indexes": []} + ) + for chunk, filename in rows: + key = (chunk.document_id, chunk.page_number) + entry = grouped[key] + entry["document_id"] = chunk.document_id + entry["filename"] = filename + entry["page_number"] = chunk.page_number + entry["texts"].append(chunk.text) + entry["chunk_indexes"].append(chunk.chunk_index) + + pages = [] + for entry in grouped.values(): + # Chunks overlap, so joining them repeats the overlap region. That is + # fine for question generation -- the passage only needs to be faithful + # to what was indexed, and overlap duplication does not invent content. + entry["text"] = "\n".join(entry.pop("texts")) + pages.append(entry) + + pages.sort(key=lambda p: (p["filename"], p["page_number"])) + return pages + + +def select_passages(pages: list[dict], n: int, seed: int) -> tuple[list[dict], dict]: + """ + Pick n candidate passages, spread across documents, with a fixed seed. + + Documents are visited round-robin so a single long report cannot dominate + the query set. + """ + usable: list[dict] = [] + rejected: dict[str, int] = defaultdict(int) + + for page in pages: + ok, reason = is_usable(page["text"]) + if ok: + page = dict(page) + page["text"] = page["text"].strip()[:MAX_PASSAGE_CHARS] + usable.append(page) + else: + rejected[reason] += 1 + + by_doc: dict[str, list[dict]] = defaultdict(list) + for page in usable: + by_doc[page["document_id"]].append(page) + + rng = random.Random(seed) + for pages_of_doc in by_doc.values(): + rng.shuffle(pages_of_doc) + + order = sorted(by_doc) + rng.shuffle(order) + + selected: list[dict] = [] + cursor = 0 + while len(selected) < n: + added_this_round = False + for doc_id in order: + if cursor < len(by_doc[doc_id]): + selected.append(by_doc[doc_id][cursor]) + added_this_round = True + if len(selected) == n: + break + if not added_this_round: + break + cursor += 1 + + stats = { + "pages_total": len(pages), + "pages_usable": len(usable), + "pages_rejected": dict(rejected), + "documents_with_usable_pages": len(by_doc), + "selected": len(selected), + } + return selected, stats + + +def parse_generated(raw: str) -> dict | None: + """Pull the JSON object out of a model response, tolerating stray prose.""" + text = raw.strip() + text = re.sub(r"^```(?:json)?|```$", "", text, flags=re.MULTILINE).strip() + match = re.search(r"\{.*\}", text, re.DOTALL) + if not match: + return None + try: + data = json.loads(match.group(0)) + except json.JSONDecodeError: + return None + if not isinstance(data, dict) or not data.get("question"): + return None + return data + + +async def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--user-id", required=True, help="config id from ingest_corpus.py") + ap.add_argument("--n", type=int, default=80, help="target number of queries") + ap.add_argument("--seed", type=int, default=20260920) + ap.add_argument("--out", default="eval_set.json") + args = ap.parse_args() + + settings = get_settings() + engine = create_async_engine(eval_database_url(), pool_pre_ping=True) + Session = async_sessionmaker(engine, expire_on_commit=False) + + async with Session() as db: + pages = await load_pages(db, args.user_id) + await engine.dispose() + + if not pages: + raise SystemExit(f"No indexed pages found for user_id={args.user_id}") + + # Over-sample: some passages will be rejected by the generator as boilerplate. + candidates, stats = select_passages(pages, int(args.n * 1.6), args.seed) + print(f"[passages] {stats}") + + provider = get_provider( + settings.llm_provider, + api_key=settings.llm_api_key, + ollama_base_url=settings.ollama_base_url, + timeout=settings.llm_timeout, + ) + + queries: list[dict] = [] + skipped = 0 + errors = 0 + t_start = time.perf_counter() + + for i, page in enumerate(candidates, 1): + if len(queries) >= args.n: + break + request = LLMRequest( + system_prompt=GENERATOR_SYSTEM_PROMPT, + user_message="Write the question.", + context=f"PASSAGE (from {page['filename']}, page {page['page_number']}):\n{page['text']}", + model=settings.llm_model, + temperature=0.0, + max_tokens=300, + ) + try: + response = await provider.generate(request) + except Exception as exc: + errors += 1 + print(f"[{i}] generation error: {type(exc).__name__}: {exc}") + await asyncio.sleep(2.0) + continue + + parsed = parse_generated(response.answer) + if parsed is None: + skipped += 1 + continue + + queries.append( + { + "query_id": f"q{len(queries) + 1:03d}", + "query": parsed["question"].strip(), + "answer_span": (parsed.get("answer_span") or "").strip(), + "gt_document_id": page["document_id"], + "gt_filename": page["filename"], + "gt_page": page["page_number"], + "gt_passage": page["text"], + "gt_chunk_indexes": sorted(page["chunk_indexes"]), + } + ) + if len(queries) % 10 == 0: + print(f"[{i}] generated {len(queries)}/{args.n}") + + elapsed = round(time.perf_counter() - t_start, 1) + payload = { + "source_user_id": args.user_id, + "seed": args.seed, + "target_n": args.n, + "generator_model": settings.llm_model, + "generator_provider": settings.llm_provider, + "generator_temperature": 0.0, + "generator_system_prompt": GENERATOR_SYSTEM_PROMPT, + "passage_selection": stats, + "candidates_offered": len(candidates), + "queries_generated": len(queries), + "passages_rejected_by_generator": skipped, + "generation_errors": errors, + "generation_seconds": elapsed, + "documents_covered": len({q["gt_document_id"] for q in queries}), + "queries": queries, + } + save(args.out, payload) + print( + f"[done] {len(queries)} queries across " + f"{payload['documents_covered']} document(s) in {elapsed}s" + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/evaluation/ingest_corpus.py b/backend/evaluation/ingest_corpus.py new file mode 100644 index 0000000..8adf7c7 --- /dev/null +++ b/backend/evaluation/ingest_corpus.py @@ -0,0 +1,273 @@ +""" +Ingest a corpus through the real ingestion pipeline, stage by stage, with timings. + +This does not go through HTTP. It calls the same parser, cleaner, chunker, +embedding service and vector store that POST /upload calls, so the timings +describe the actual pipeline rather than a reimplementation of it -- but it +calls each stage separately so each one can be timed on its own. + +Documents are ingested under a synthetic user whose id encodes the chunking +configuration. Retrieval already filters by Document.user_id, so two +configurations can coexist in one database without interfering, and the +isolation being relied on is the production isolation. + +Usage: + python evaluation/ingest_corpus.py --corpus ../corpus + python evaluation/ingest_corpus.py --corpus ../corpus --chunk-size 400 --chunk-overlap 60 + python evaluation/ingest_corpus.py --corpus ../corpus --reset +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import time +from pathlib import Path + +from _common import config_id, eval_database_url, save + +from sqlalchemy import delete, select, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.core.config import get_settings +from app.database.models import Document, DocumentChunk, ProcessingStatus, User +from app.ingestion.chunker import chunk_pages +from app.ingestion.cleaner import clean_pages +from app.ingestion.embeddings import get_embedding_service +from app.ingestion.parser import parse_document +from app.ingestion.supported_formats import SUPPORTED_EXTENSIONS +from app.ingestion.vector_store import store_chunks + + +def rss_mb() -> float | None: + try: + import psutil + except ImportError: + return None + return round(psutil.Process(os.getpid()).memory_info().rss / 1024**2, 1) + + +async def ensure_user(db: AsyncSession, user_id: str) -> None: + if await db.scalar(select(User).where(User.id == user_id)): + return + db.add( + User( + id=user_id, + name="evaluation harness", + email=f"{user_id}@evaluation.invalid", + # Not a login account. The hash is a constant placeholder that no + # password verifies against; the row exists only to satisfy the + # documents.user_id foreign key. + password_hash="pbkdf2_sha256$310000$evaluation$evaluation", + ) + ) + await db.commit() + + +async def reset_config(db: AsyncSession, user_id: str) -> int: + """Delete every document previously ingested under this configuration.""" + doc_ids = list(await db.scalars(select(Document.id).where(Document.user_id == user_id))) + if doc_ids: + await db.execute(delete(DocumentChunk).where(DocumentChunk.document_id.in_(doc_ids))) + await db.execute(delete(Document).where(Document.id.in_(doc_ids))) + await db.commit() + return len(doc_ids) + + +async def ingest_one(path: Path, user_id: str, db: AsyncSession, embedder) -> dict: + """Run one document through every stage, timing each stage separately.""" + size_bytes = path.stat().st_size + + doc = Document( + user_id=user_id, + filename=path.name, + original_filename=path.name, + file_type=path.suffix.lstrip(".").lower(), + file_size=size_bytes, + storage_path=str(path), + status=ProcessingStatus.processing, + ) + db.add(doc) + await db.commit() + await db.refresh(doc) + + record: dict = { + "document_id": doc.id, + "filename": path.name, + "file_type": doc.file_type, + "file_size_bytes": size_bytes, + } + + try: + t = time.perf_counter() + pages = parse_document(str(path)) + record["parse_ms"] = round((time.perf_counter() - t) * 1000, 2) + + t = time.perf_counter() + pages = clean_pages(pages) + record["clean_ms"] = round((time.perf_counter() - t) * 1000, 2) + + record["pages"] = len(pages) + record["chars"] = sum(len(p.text) for p in pages) + + t = time.perf_counter() + chunks = chunk_pages(pages, document_id=doc.id, filename=path.name) + record["chunk_ms"] = round((time.perf_counter() - t) * 1000, 2) + record["chunks"] = len(chunks) + record["chunk_chars_mean"] = ( + round(sum(len(c.text) for c in chunks) / len(chunks), 1) if chunks else 0 + ) + + if not chunks: + raise ValueError("chunking produced 0 chunks") + + rss_before = rss_mb() + t = time.perf_counter() + embeddings = embedder.embed([c.text for c in chunks]) + record["embed_ms"] = round((time.perf_counter() - t) * 1000, 2) + record["embedding_dim"] = len(embeddings[0]) + record["rss_before_embed_mb"] = rss_before + record["rss_after_embed_mb"] = rss_mb() + + t = time.perf_counter() + await store_chunks(chunks, embeddings, db) + doc.status = ProcessingStatus.indexed + doc.total_chunks = len(chunks) + await db.commit() + record["store_ms"] = round((time.perf_counter() - t) * 1000, 2) + + record["total_ms"] = round( + record["parse_ms"] + + record["clean_ms"] + + record["chunk_ms"] + + record["embed_ms"] + + record["store_ms"], + 2, + ) + record["status"] = "indexed" + + except Exception as exc: + await db.rollback() + doc_row = await db.get(Document, doc.id) + if doc_row is not None: + doc_row.status = ProcessingStatus.failed + await db.commit() + record["status"] = "failed" + record["error"] = f"{type(exc).__name__}: {exc}" + + return record + + +async def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", required=True, help="folder of documents to ingest") + ap.add_argument("--chunk-size", type=int, default=None) + ap.add_argument("--chunk-overlap", type=int, default=None) + ap.add_argument("--label", default="", help="optional extra tag for the config id") + ap.add_argument("--reset", action="store_true", help="delete this config's docs first") + ap.add_argument("--out", default=None, help="result filename") + args = ap.parse_args() + + settings = get_settings() + if args.chunk_size is not None: + settings.chunk_size = args.chunk_size + if args.chunk_overlap is not None: + settings.chunk_overlap = args.chunk_overlap + + cfg = { + "chunk_size": settings.chunk_size, + "chunk_overlap": settings.chunk_overlap, + "embedding_model": settings.embedding_model, + "label": args.label, + } + user_id = config_id(**cfg) + print(f"[config] {cfg}") + print(f"[config] user_id={user_id}") + + corpus = Path(args.corpus).resolve() + files = sorted( + p + for p in corpus.rglob("*") + if p.is_file() and p.suffix.lstrip(".").lower() in SUPPORTED_EXTENSIONS + ) + if not files: + raise SystemExit(f"No supported documents found under {corpus}") + print(f"[corpus] {len(files)} document(s) in {corpus}") + + engine = create_async_engine(eval_database_url(), pool_pre_ping=True) + Session = async_sessionmaker(engine, expire_on_commit=False) + + embedder = get_embedding_service() + t = time.perf_counter() + embedder.warm_up() + model_load_ms = round((time.perf_counter() - t) * 1000, 2) + + # warm_up() only builds the ONNX session. The first actual inference pays + # for graph optimisation and thread-pool startup -- tens of seconds on this + # machine -- which would otherwise land entirely on whichever document + # happens to be ingested first and make its timing meaningless. Burn that + # cost here, on a throwaway input, so the per-document numbers are clean. + t = time.perf_counter() + embedder.embed(["warm-up inference, result discarded"]) + first_inference_ms = round((time.perf_counter() - t) * 1000, 2) + + # chunk_pages() imports langchain_text_splitters lazily, and that import + # costs seconds the first time. Same problem: it would be billed to the + # first document. Pay it here instead. + from app.ingestion.parser import ParsedPage + + t = time.perf_counter() + chunk_pages([ParsedPage(page_number=1, text="warm-up")], document_id="warm-up", filename="warm-up") + chunker_import_ms = round((time.perf_counter() - t) * 1000, 2) + + print( + f"[model] session {model_load_ms} ms, first inference {first_inference_ms} ms, " + f"chunker import {chunker_import_ms} ms" + ) + + records = [] + async with Session() as db: + await db.execute(text("SELECT 1")) + await ensure_user(db, user_id) + if args.reset: + removed = await reset_config(db, user_id) + print(f"[reset] removed {removed} previously ingested document(s)") + + for i, path in enumerate(files, 1): + rec = await ingest_one(path, user_id, db, embedder) + records.append(rec) + print( + f"[{i}/{len(files)}] {path.name}: {rec['status']} " + f"pages={rec.get('pages')} chunks={rec.get('chunks')} " + f"total={rec.get('total_ms')}ms" + ) + + await engine.dispose() + + ok = [r for r in records if r["status"] == "indexed"] + payload = { + "config": cfg, + "user_id": user_id, + "corpus_dir": str(corpus), + "model_load_ms": model_load_ms, + "model_first_inference_ms": first_inference_ms, + "chunker_import_ms": chunker_import_ms, + "documents_attempted": len(records), + "documents_indexed": len(ok), + "documents_failed": len(records) - len(ok), + "total_pages": sum(r.get("pages", 0) for r in ok), + "total_chunks": sum(r.get("chunks", 0) for r in ok), + "total_chars": sum(r.get("chars", 0) for r in ok), + "total_bytes": sum(r.get("file_size_bytes", 0) for r in ok), + "documents": records, + } + save(args.out or f"ingest_{user_id}.json", payload) + print( + f"[done] indexed={len(ok)}/{len(records)} " + f"chunks={payload['total_chunks']} pages={payload['total_pages']}" + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/evaluation/results/eval_set_smoke.json b/backend/evaluation/results/eval_set_smoke.json new file mode 100644 index 0000000..c4bcbf5 --- /dev/null +++ b/backend/evaluation/results/eval_set_smoke.json @@ -0,0 +1,89 @@ +{ + "environment": { + "captured_at": "2026-09-20T12:13:20.686269+00:00", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", + "python": "3.13.1", + "cpu_count_logical": 12, + "numpy": "2.2.6", + "cpu_count_physical": 8, + "total_ram_gb": 15.68, + "onnxruntime": "1.26.0", + "git_commit": "0f876bb", + "git_dirty": true + }, + "source_user_id": "eval-cf71fa4ac73d7e91", + "seed": 20260920, + "target_n": 12, + "generator_model": "qwen/qwen3.8-27b", + "generator_provider": "groq", + "generator_temperature": 0.0, + "generator_system_prompt": "You write evaluation questions for a document search system.\n\nYou are given one passage from a document. Write exactly one question that:\n- is answerable using ONLY that passage\n- a real person might plausibly ask about this document\n- does NOT quote the passage verbatim; use your own wording\n- does NOT refer to \"the passage\", \"the text\", \"the document\", \"above\", or \"this section\"\n- names the specific subject it asks about, so it is meaningful without seeing the passage\n\nAlso quote the shortest span from the passage that answers it.\n\nRespond with JSON only, no prose, no code fences:\n{\"question\": \"...\", \"answer_span\": \"...\"}\n\nIf the passage is boilerplate, a table of contents, a page header, a reference\nlist, or otherwise has no factual content worth asking about, respond exactly:\n{\"question\": null, \"answer_span\": null}", + "passage_selection": { + "pages_total": 4, + "pages_usable": 2, + "pages_rejected": { + "too_short": 2 + }, + "documents_with_usable_pages": 2, + "selected": 2 + }, + "candidates_offered": 2, + "queries_generated": 2, + "passages_rejected_by_generator": 0, + "generation_errors": 0, + "generation_seconds": 0.7, + "documents_covered": 2, + "queries": [ + { + "query_id": "q001", + "query": "What is the maximum file size supported for uploads in DocuQuery?", + "answer_span": "up to 50MB", + "gt_document_id": "6da1be77-be1f-4d76-9b7d-1a4d19218b9c", + "gt_filename": "README.md", + "gt_page": 1, + "gt_passage": "
\n\n\"DocuQuery\n\n
\n\n# DocuQuery\n\n### Turn a folder of PDFs, DOCX, and text files into a source you can ask questions.\n\nUpload your documents. DocuQuery reads them, indexes them, and answers questions with the\nexact page and passage the answer came from — no more skimming a 40-page report for one number.\n\n[**Live demo**](#-see-it-work) · [**Quick start**](#-quick-start) · [**Features**](#-features) · [**API reference**](#-api-reference) · [**Roadmap**](#-roadmap)\n![Build Status](https://img.shields.io/badge/build-passing-brightgreen?style=flat-square)\n![Version](https://img.shields.io/badge/version-0.1.0-blue?style=flat-square)\n![Next.js](https://img.shields.io/badge/Next.js-16.2-black?style=flat-square&logo=next.js)\n![React](https://img.shields.io/badge/React-19.2-61dafb?style=flat-square&logo=react)\n![FastAPI](https://img.shields.io/badge/FastAPI-0.115-009688?style=flat-square&logo=fastapi)\n![PostgreSQL](https://img.shields.io/badge/PostgreSQL-16%20%2B%20pgvector-336791?style=flat-square&logo=postgresql)\n![License](https://img.shields.io/badge/license-MIT-green?style=flat-square)\n![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen?style=flat-square)\n\n
\n\n---\n\n## What DocuQuery actually does\n\n\n---\n\n## What DocuQuery actually does\n\nMost \"chat with your docs\" demos are a thin wrapper around one LLM call. DocuQuery is the\nplumbing that makes that call trustworthy: it parses your files properly, chunks and embeds\nthem, stores the vectors in Postgres, retrieves the passages that actually answer the question,\nand hands the model only that — then shows you exactly where the answer came from.\n- 📂 **Drop in a file** — PDF, DOCX, TXT, or Markdown, up to 50MB\n- ⚡ **It gets indexed automatically** — parsed, cleaned, chunked, embedded, and stored in seconds\n- 💬 **Ask in plain language** — no query syntax, no filters to configure\n- 📎 **Get an answer with receipts** — every claim links back to the filename, page, and passage\n- 🔀 **Pick how strict you want it** — answer strictly from your documents, from general knowledge, or both\n\n---\n\n## 👀 See it work\n\n
\n\"A, HNSW index)" + }, + "metrics": { + "document": { + "queries": 2, + "recall_at_1": 1.0, + "precision_at_1": 1.0, + "ndcg_at_1": 1.0, + "recall_at_3": 1.0, + "precision_at_3": 1.0, + "ndcg_at_3": 1.0, + "recall_at_5": 1.0, + "precision_at_5": 0.9, + "ndcg_at_5": 1.0, + "recall_at_10": 1.0, + "precision_at_10": 0.8, + "ndcg_at_10": 0.987, + "hit_rate_at_5": 1.0, + "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query", + "mrr": 1.0 + }, + "page": { + "queries": 2, + "recall_at_1": 1.0, + "precision_at_1": 1.0, + "ndcg_at_1": 1.0, + "recall_at_3": 1.0, + "precision_at_3": 1.0, + "ndcg_at_3": 1.0, + "recall_at_5": 1.0, + "precision_at_5": 0.9, + "ndcg_at_5": 1.0, + "recall_at_10": 1.0, + "precision_at_10": 0.8, + "ndcg_at_10": 0.987, + "hit_rate_at_5": 1.0, + "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query", + "mrr": 1.0 + }, + "answer": { + "queries": 2, + "recall_at_1": 1.0, + "precision_at_1": 1.0, + "ndcg_at_1": 1.0, + "recall_at_3": 1.0, + "precision_at_3": 0.3333, + "ndcg_at_3": 1.0, + "recall_at_5": 1.0, + "precision_at_5": 0.2, + "ndcg_at_5": 1.0, + "recall_at_10": 1.0, + "precision_at_10": 0.1, + "ndcg_at_10": 1.0, + "hit_rate_at_5": 1.0, + "hit_rate_at_5_note": "identical to recall_at_5: ground truth names one relevant page per query", + "mrr": 1.0, + "scorable_queries": 2, + "note": "scored only on queries whose generated answer span is present verbatim in the ground-truth passage" + } + }, + "queries_with_no_results": 0, + "retrieval_latency_ms": { + "p50": 22.07, + "p95": 488.567, + "p99": 488.567, + "min": 22.07, + "max": 488.567, + "mean": 255.318, + "n": 2 + }, + "per_query": [ + { + "query_id": "q001", + "query": "What is the maximum file size supported for uploads in DocuQuery?", + "gt_filename": "README.md", + "gt_page": 1, + "answer_span_scorable": true, + "retrieved": [ + { + "rank": 1, + "filename": "README.md", + "page": 1, + "chunk_index": 3, + "similarity": 0.8436, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": true + }, + { + "rank": 2, + "filename": "README.md", + "page": 1, + "chunk_index": 0, + "similarity": 0.8396, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 3, + "filename": "README.md", + "page": 1, + "chunk_index": 16, + "similarity": 0.8379, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 4, + "filename": "README.md", + "page": 1, + "chunk_index": 5, + "similarity": 0.8316, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 5, + "filename": "README.md", + "page": 1, + "chunk_index": 2, + "similarity": 0.8298, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 6, + "filename": "README.md", + "page": 1, + "chunk_index": 12, + "similarity": 0.816, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 7, + "filename": "README.md", + "page": 1, + "chunk_index": 13, + "similarity": 0.8143, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 8, + "filename": "README.md", + "page": 1, + "chunk_index": 4, + "similarity": 0.8138, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 9, + "filename": "README.md", + "page": 1, + "chunk_index": 7, + "similarity": 0.8077, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 10, + "filename": "README.md", + "page": 1, + "chunk_index": 18, + "similarity": 0.8073, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + } + ], + "first_relevant_rank": { + "document": 1, + "page": 1, + "answer": 1 + }, + "latency_ms": 488.57, + "n_retrieved": 10 + }, + { + "query_id": "q002", + "query": "Which package is required for FastAPI to parse multipart/form-data uploads but is never imported directly by the source code?", + "gt_filename": "DEPENDENCIES_VERIFIED.md", + "gt_page": 1, + "answer_span_scorable": true, + "retrieved": [ + { + "rank": 1, + "filename": "DEPENDENCIES_VERIFIED.md", + "page": 1, + "chunk_index": 4, + "similarity": 0.8855, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": true + }, + { + "rank": 2, + "filename": "DEPENDENCIES_VERIFIED.md", + "page": 1, + "chunk_index": 1, + "similarity": 0.8843, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 3, + "filename": "DEPENDENCIES_VERIFIED.md", + "page": 1, + "chunk_index": 6, + "similarity": 0.8786, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 4, + "filename": "DEPENDENCIES_VERIFIED.md", + "page": 1, + "chunk_index": 3, + "similarity": 0.8635, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 5, + "filename": "README.md", + "page": 1, + "chunk_index": 10, + "similarity": 0.8532, + "relevant_document": false, + "relevant_page": false, + "relevant_answer": false + }, + { + "rank": 6, + "filename": "DEPENDENCIES_VERIFIED.md", + "page": 1, + "chunk_index": 2, + "similarity": 0.8456, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 7, + "filename": "README.md", + "page": 1, + "chunk_index": 12, + "similarity": 0.8401, + "relevant_document": false, + "relevant_page": false, + "relevant_answer": false + }, + { + "rank": 8, + "filename": "README.md", + "page": 1, + "chunk_index": 16, + "similarity": 0.8399, + "relevant_document": false, + "relevant_page": false, + "relevant_answer": false + }, + { + "rank": 9, + "filename": "DEPENDENCIES_VERIFIED.md", + "page": 1, + "chunk_index": 0, + "similarity": 0.8327, + "relevant_document": true, + "relevant_page": true, + "relevant_answer": false + }, + { + "rank": 10, + "filename": "README.md", + "page": 1, + "chunk_index": 11, + "similarity": 0.8276, + "relevant_document": false, + "relevant_page": false, + "relevant_answer": false + } + ], + "first_relevant_rank": { + "document": 1, + "page": 1, + "answer": 1 + }, + "latency_ms": 22.07, + "n_retrieved": 10 + } + ] +} \ No newline at end of file diff --git a/backend/evaluation/run_config_sweep.py b/backend/evaluation/run_config_sweep.py new file mode 100644 index 0000000..5473486 --- /dev/null +++ b/backend/evaluation/run_config_sweep.py @@ -0,0 +1,231 @@ +""" +Compare retrieval configurations on one fixed query set. + +Two kinds of comparison are run: + + index-side chunk size and overlap. Each configuration re-ingests the whole + corpus under its own synthetic user, so the index really is + rebuilt -- nothing is simulated by re-slicing an existing index. + + query-side similarity threshold and metadata filtering. These reuse the + baseline index, because neither of them changes what is stored. + +The query set is generated once from the baseline index and reused verbatim by +every configuration. Ground truth is anchored to (document, page), which does +not move when chunk size changes, so the comparison is like-for-like. + +Usage: + python evaluation/run_config_sweep.py --corpus ../corpus --baseline-user-id eval-xxxx +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from _common import RESULTS_DIR, config_id, load, save + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + +# (label, chunk_size, chunk_overlap). The first entry must be the shipped +# default so every improvement is stated relative to what the product does now. +INDEX_CONFIGS = [ + ("current-800-120", 800, 120), + ("small-400-60", 400, 60), + ("large-1200-180", 1200, 180), + ("no-overlap-800-0", 800, 0), +] + +# (label, extra flags for run_retrieval_eval.py) +QUERY_CONFIGS = [ + ("threshold-0.30-default", []), + ("threshold-0.00", ["--no-threshold"]), + ("metadata-filter-document", ["--filter-document"]), +] + + +def run(cmd: list[str]) -> None: + print(f"\n$ {' '.join(cmd)}") + # Run from backend/ so pydantic-settings finds .env, same as the app does. + result = subprocess.run(cmd, cwd=BACKEND_ROOT) + if result.returncode != 0: + raise SystemExit(f"command failed with exit code {result.returncode}") + + +def headline(metrics: dict) -> dict: + """The comparable figures, pulled out of a full result file.""" + out = {} + for level in ("document", "page", "answer"): + m = metrics.get(level, {}) + out[level] = { + key: m.get(key) + for key in ( + "queries", + "recall_at_1", + "recall_at_3", + "recall_at_5", + "precision_at_5", + "ndcg_at_5", + "mrr", + ) + } + return out + + +def delta(baseline: float | None, candidate: float | None) -> dict: + if baseline is None or candidate is None: + return {"absolute_pp": None, "relative_pct": None} + absolute = round((candidate - baseline) * 100, 2) + relative = round((candidate - baseline) / baseline * 100, 2) if baseline else None + return {"absolute_pp": absolute, "relative_pct": relative} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", required=True) + ap.add_argument("--eval-set", default="eval_set.json") + ap.add_argument("--top-k", type=int, default=10) + ap.add_argument( + "--skip-ingest", + action="store_true", + help="assume every index configuration is already ingested", + ) + ap.add_argument("--out", default="config_sweep.json") + args = ap.parse_args() + + eval_set = load(args.eval_set) + embedding_model = eval_set.get("generator_model") + print(f"[sweep] query set: {args.eval_set} ({len(eval_set['queries'])} queries)") + + from app.core.config import get_settings + + model_label = get_settings().embedding_model + + runs: dict[str, dict] = {} + + for label, chunk_size, overlap in INDEX_CONFIGS: + user_id = config_id( + chunk_size=chunk_size, + chunk_overlap=overlap, + embedding_model=model_label, + label="", + ) + if not args.skip_ingest: + run( + [ + sys.executable, + "evaluation/ingest_corpus.py", + "--corpus", + str(Path(args.corpus).resolve()), + "--chunk-size", + str(chunk_size), + "--chunk-overlap", + str(overlap), + "--reset", + ] + ) + run( + [ + sys.executable, + "evaluation/run_retrieval_eval.py", + "--user-id", + user_id, + "--eval-set", + args.eval_set, + "--top-k", + str(args.top_k), + "--label", + label, + ] + ) + result = load(f"retrieval_eval_{label}.json") + ingest = load(f"ingest_{user_id}.json") + runs[label] = { + "kind": "index", + "chunk_size": chunk_size, + "chunk_overlap": overlap, + "user_id": user_id, + "chunks_indexed": ingest["total_chunks"], + "documents_indexed": ingest["documents_indexed"], + "mean_chunk_chars": round( + sum( + d.get("chunk_chars_mean", 0) * d.get("chunks", 0) + for d in ingest["documents"] + if d["status"] == "indexed" + ) + / max(1, ingest["total_chunks"]), + 1, + ), + "retrieval_latency_ms": result["retrieval_latency_ms"], + "queries_with_no_results": result["queries_with_no_results"], + "metrics": headline(result["metrics"]), + } + + baseline_label = INDEX_CONFIGS[0][0] + baseline_user_id = runs[baseline_label]["user_id"] + + for label, flags in QUERY_CONFIGS: + run( + [ + sys.executable, + "evaluation/run_retrieval_eval.py", + "--user-id", + baseline_user_id, + "--eval-set", + args.eval_set, + "--top-k", + str(args.top_k), + "--label", + label, + *flags, + ] + ) + result = load(f"retrieval_eval_{label}.json") + runs[label] = { + "kind": "query", + "flags": flags, + "user_id": baseline_user_id, + "retrieval_latency_ms": result["retrieval_latency_ms"], + "queries_with_no_results": result["queries_with_no_results"], + "metrics": headline(result["metrics"]), + } + + baseline_metrics = runs[baseline_label]["metrics"] + comparison = {} + for label, entry in runs.items(): + if label == baseline_label: + continue + comparison[label] = { + level: { + metric: delta( + baseline_metrics[level][metric], entry["metrics"][level][metric] + ) + for metric in ("recall_at_1", "recall_at_3", "recall_at_5", "mrr", "ndcg_at_5") + } + for level in ("document", "page", "answer") + } + + save( + args.out, + { + "eval_set": args.eval_set, + "eval_set_size": len(eval_set["queries"]), + "top_k": args.top_k, + "baseline": baseline_label, + "runs": runs, + "vs_baseline": comparison, + }, + ) + + print("\n=== page-level recall@5 by configuration ===") + for label, entry in runs.items(): + value = entry["metrics"]["page"]["recall_at_5"] + marker = " (baseline)" if label == baseline_label else "" + print(f"{label:28s} {value}{marker}") + + +if __name__ == "__main__": + main() diff --git a/backend/evaluation/run_perf_bench.py b/backend/evaluation/run_perf_bench.py new file mode 100644 index 0000000..708145f --- /dev/null +++ b/backend/evaluation/run_perf_bench.py @@ -0,0 +1,296 @@ +""" +Performance benchmark: embedding throughput, retrieval latency, LLM latency. + +Ingestion-stage timings (parse / clean / chunk / embed / store) are produced by +ingest_corpus.py and are not repeated here; this script covers the query side +plus a standalone embedding throughput measurement. + +Every latency figure is reported as a distribution (min / P50 / P95 / P99 / max) +over a stated number of runs, after a discarded warm-up, so no number here is a +single lucky run. + +The --llm stage sends retrieved passages from the corpus to the configured LLM +provider over the network and consumes API quota. It is off by default. + +Usage: + python evaluation/run_perf_bench.py --user-id eval-xxxx + python evaluation/run_perf_bench.py --user-id eval-xxxx --llm --llm-queries 20 +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import statistics +import time + +from _common import eval_database_url, load, percentiles, save + +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.config import get_settings +from app.ingestion.embeddings import get_embedding_service +from app.llm import get_response_generator +from app.retrieval.embedding_query import embed_query +from app.retrieval.exceptions import NoResultsError +from app.retrieval.retrieval_pipeline import run_retrieval_pipeline + + +def proc_sample() -> dict: + try: + import psutil + except ImportError: + return {} + p = psutil.Process(os.getpid()) + return { + "rss_mb": round(p.memory_info().rss / 1024**2, 1), + "cpu_percent": p.cpu_percent(interval=None), + } + + +def bench_embedding(texts: list[str], batch_sizes: tuple[int, ...]) -> dict: + """ + Embedding throughput at several batch sizes. + + The same text list is embedded each time so the only variable is batching. + """ + settings = get_settings() + service = get_embedding_service() + + t = time.perf_counter() + service.warm_up() + cold_load_ms = round((time.perf_counter() - t) * 1000, 2) + + service.embed(texts[:8]) # warm-up, result discarded + + results = {} + original = settings.embedding_batch_size + for batch_size in batch_sizes: + settings.embedding_batch_size = batch_size + t = time.perf_counter() + vectors = service.embed(texts) + elapsed = time.perf_counter() - t + results[f"batch_{batch_size}"] = { + "texts": len(texts), + "seconds": round(elapsed, 3), + "embeddings_per_second": round(len(texts) / elapsed, 1), + "ms_per_embedding": round(elapsed * 1000 / len(texts), 3), + "dim": len(vectors[0]), + } + print(f"[embed] batch={batch_size}: {results[f'batch_{batch_size}']}") + settings.embedding_batch_size = original + + return { + "cold_load_ms": cold_load_ms, + "model": settings.embedding_model, + "by_batch_size": results, + "process_after": proc_sample(), + } + + +async def bench_query_embedding(queries: list[str], repeats: int) -> dict: + samples = [] + for _ in range(repeats): + for query in queries: + t = time.perf_counter() + await embed_query(query) + samples.append((time.perf_counter() - t) * 1000) + return percentiles(samples) + + +async def bench_retrieval(db, user_id: str, queries: list[str], top_k: int, repeats: int) -> dict: + samples = [] + empty = 0 + chunk_counts = [] + context_chars = [] + for _ in range(repeats): + for query in queries: + t = time.perf_counter() + try: + result = await run_retrieval_pipeline( + query=query, db=db, user_id=user_id, top_k=top_k + ) + chunk_counts.append(len(result.chunks)) + context_chars.append(len(result.context)) + except NoResultsError: + empty += 1 + samples.append((time.perf_counter() - t) * 1000) + return { + "latency_ms": percentiles(samples), + "queries_per_second_serial": round(1000 / statistics.median(samples), 2) if samples else None, + "empty_results": empty, + "chunks_returned_mean": round(statistics.mean(chunk_counts), 2) if chunk_counts else None, + "context_chars_mean": round(statistics.mean(context_chars), 1) if context_chars else None, + "context_chars_max": max(context_chars) if context_chars else None, + } + + +async def bench_concurrent_retrieval( + engine, user_id: str, queries: list[str], top_k: int, concurrency: int +) -> dict: + """Wall-clock throughput with N retrievals in flight, each on its own session.""" + Session = async_sessionmaker(engine, expire_on_commit=False) + + async def one(query: str) -> float: + async with Session() as db: + t = time.perf_counter() + try: + await run_retrieval_pipeline(query=query, db=db, user_id=user_id, top_k=top_k) + except NoResultsError: + pass + return (time.perf_counter() - t) * 1000 + + semaphore = asyncio.Semaphore(concurrency) + + async def guarded(query: str) -> float: + async with semaphore: + return await one(query) + + t0 = time.perf_counter() + samples = await asyncio.gather(*(guarded(q) for q in queries)) + wall = time.perf_counter() - t0 + + return { + "concurrency": concurrency, + "queries": len(queries), + "wall_seconds": round(wall, 3), + "queries_per_second": round(len(queries) / wall, 2), + "latency_ms": percentiles(list(samples)), + } + + +async def bench_llm(db, user_id: str, queries: list[str], top_k: int) -> dict: + """ + Time to first streamed token and full generation latency, over real calls. + + Generation goes through ResponseGenerator.stream -- the same call POST /chat + makes -- so context truncation, the versioned system prompt and SSE framing + are all in the measured path. Retrieval is timed separately, so + time_to_first_token_ms excludes it and end_to_end_ms is what a user waits + for from question to last token. + """ + settings = get_settings() + generator = get_response_generator() + + ttft, generation, end_to_end, retrieval, chars, tokens, errors = [], [], [], [], [], [], [] + + for i, query in enumerate(queries, 1): + t_start = time.perf_counter() + try: + result = await run_retrieval_pipeline( + query=query, db=db, user_id=user_id, top_k=top_k + ) + except NoResultsError: + continue + t_retrieved = time.perf_counter() + + first_token_at = None + token_events = 0 + answer_chars = 0 + try: + async for event in generator.stream(query, result, mode="docuquery"): + if not event.startswith("event: token"): + continue + if first_token_at is None: + first_token_at = time.perf_counter() + token_events += 1 + answer_chars += len(event) - len("event: token\ndata: \n\n") + except Exception as exc: + errors.append(f"{type(exc).__name__}: {exc}") + print(f"[{i}] llm error: {type(exc).__name__}: {exc}") + continue + + t_end = time.perf_counter() + if first_token_at is None: + errors.append("stream produced no tokens") + continue + + retrieval.append((t_retrieved - t_start) * 1000) + ttft.append((first_token_at - t_retrieved) * 1000) + generation.append((t_end - t_retrieved) * 1000) + end_to_end.append((t_end - t_start) * 1000) + chars.append(float(answer_chars)) + tokens.append(float(token_events)) + print( + f"[{i}/{len(queries)}] ttft={ttft[-1]:.0f}ms " + f"total={end_to_end[-1]:.0f}ms tokens={token_events}" + ) + + return { + "provider": settings.llm_provider, + "model": settings.llm_model, + "temperature": settings.llm_temperature, + "max_tokens": settings.llm_max_tokens, + "max_context_tokens": settings.llm_max_context_tokens, + "completed": len(ttft), + "errors": errors, + "retrieval_ms": percentiles(retrieval), + "time_to_first_token_ms": percentiles(ttft), + "generation_ms": percentiles(generation), + "end_to_end_ms": percentiles(end_to_end), + "streamed_token_events": percentiles(tokens), + "answer_chars": percentiles(chars), + "note": ( + "streamed_token_events counts provider stream deltas, not tokeniser " + "tokens; answer_chars is JSON-escaped length and is an upper bound" + ), + } + + +async def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--user-id", required=True) + ap.add_argument("--eval-set", default="eval_set.json") + ap.add_argument("--top-k", type=int, default=5) + ap.add_argument("--repeats", type=int, default=3) + ap.add_argument("--embed-texts", type=int, default=256) + ap.add_argument("--concurrency", type=int, default=8) + ap.add_argument("--llm", action="store_true", help="run real LLM calls (uses quota)") + ap.add_argument("--llm-queries", type=int, default=20) + ap.add_argument("--out", default="perf_bench.json") + args = ap.parse_args() + + eval_set = load(args.eval_set) + queries = [q["query"] for q in eval_set["queries"]] + passages = [q["gt_passage"] for q in eval_set["queries"]] + print(f"[bench] {len(queries)} queries available") + + payload: dict = {"config": vars(args), "process_start": proc_sample()} + + texts = (passages * ((args.embed_texts // max(1, len(passages))) + 1))[: args.embed_texts] + payload["embedding"] = bench_embedding(texts, batch_sizes=(1, 8, 32, 64)) + + engine = create_async_engine(eval_database_url(), pool_pre_ping=True) + Session = async_sessionmaker(engine, expire_on_commit=False) + + async with Session() as db: + payload["query_embedding_ms"] = await bench_query_embedding(queries[:20], args.repeats) + print(f"[query-embed] {payload['query_embedding_ms']}") + + payload["retrieval"] = await bench_retrieval( + db, args.user_id, queries, args.top_k, args.repeats + ) + print(f"[retrieval] {payload['retrieval']['latency_ms']}") + + payload["retrieval_concurrent"] = await bench_concurrent_retrieval( + engine, args.user_id, queries, args.top_k, args.concurrency + ) + print(f"[concurrent] {payload['retrieval_concurrent']}") + + if args.llm: + async with Session() as db: + payload["llm"] = await bench_llm( + db, args.user_id, queries[: args.llm_queries], args.top_k + ) + else: + payload["llm"] = {"skipped": "run with --llm to measure generation latency"} + + await engine.dispose() + + payload["process_end"] = proc_sample() + save(args.out, payload) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/evaluation/run_retrieval_eval.py b/backend/evaluation/run_retrieval_eval.py new file mode 100644 index 0000000..2120273 --- /dev/null +++ b/backend/evaluation/run_retrieval_eval.py @@ -0,0 +1,264 @@ +""" +Score the retrieval pipeline against a fixed ground-truth query set. + +Every query goes through app.retrieval.run_retrieval_pipeline -- the same +function POST /retrieve and POST /chat call -- so what is measured is the +deployed path: query embedding, the per-user SQL filter, the pgvector HNSW +search, the similarity threshold, deduplication and ranking. + +Three relevance levels are scored, from loosest to strictest: + + document the retrieved chunk came from the document the query was written from + page ... and from the same page -> page-level citation accuracy + answer the retrieved chunk text contains the answer span the generator + quoted -> answer-bearing chunk accuracy + +Ground truth names exactly one relevant page per query, so Recall@k and +Hit Rate@k are the same quantity here: the share of queries with at least one +relevant chunk in the top k. Both names are reported with that caveat attached +rather than presented as two independent results. + +Usage: + python evaluation/run_retrieval_eval.py --user-id eval-xxxx + python evaluation/run_retrieval_eval.py --user-id eval-xxxx --no-threshold + python evaluation/run_retrieval_eval.py --user-id eval-xxxx --filter-document +""" + +from __future__ import annotations + +import argparse +import asyncio +import math +import re +import time + +from _common import eval_database_url, load, percentiles, save + +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.config import get_settings +from app.retrieval.exceptions import NoResultsError +from app.retrieval.filters import RetrievalFilter +from app.retrieval.retrieval_pipeline import run_retrieval_pipeline + +K_VALUES = (1, 3, 5, 10) +LEVELS = ("document", "page", "answer") + + +def normalize(text: str) -> str: + return re.sub(r"\s+", " ", text).strip().lower() + + +def relevance(chunk, query: dict, level: str) -> bool: + if chunk.document_id != query["gt_document_id"]: + return False + if level == "document": + return True + if chunk.page != query["gt_page"]: + return False + if level == "page": + return True + span = normalize(query.get("answer_span", "")) + return bool(span) and span in normalize(chunk.text) + + +def ndcg_at_k(relevances: list[bool], k: int) -> float: + """Binary-gain NDCG. Ideal ranking puts every relevant hit first.""" + top = relevances[:k] + dcg = sum(1.0 / math.log2(i + 2) for i, rel in enumerate(top) if rel) + ideal_hits = min(sum(top), k) + if ideal_hits == 0: + return 0.0 + idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits)) + return dcg / idcg + + +def score_level(per_query: list[list[bool]]) -> dict: + """Aggregate one relevance level across all queries.""" + n = len(per_query) + if n == 0: + return {} + + out: dict = {"queries": n} + for k in K_VALUES: + hits = sum(1 for rels in per_query if any(rels[:k])) + out[f"recall_at_{k}"] = round(hits / n, 4) + out[f"precision_at_{k}"] = round( + sum(sum(rels[:k]) / k for rels in per_query) / n, 4 + ) + out[f"ndcg_at_{k}"] = round(sum(ndcg_at_k(rels, k) for rels in per_query) / n, 4) + + out["hit_rate_at_5"] = out["recall_at_5"] + out["hit_rate_at_5_note"] = ( + "identical to recall_at_5: ground truth names one relevant page per query" + ) + + reciprocal = [] + for rels in per_query: + rank = next((i + 1 for i, rel in enumerate(rels) if rel), None) + reciprocal.append(1.0 / rank if rank else 0.0) + out["mrr"] = round(sum(reciprocal) / n, 4) + + return out + + +async def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--user-id", required=True) + ap.add_argument("--eval-set", default="eval_set.json") + ap.add_argument("--top-k", type=int, default=10) + ap.add_argument( + "--no-threshold", + action="store_true", + help="set the similarity threshold to 0 for this run", + ) + ap.add_argument( + "--filter-document", + action="store_true", + help="pass the ground-truth document_id as a metadata filter", + ) + ap.add_argument("--label", default="") + ap.add_argument("--out", default=None) + args = ap.parse_args() + + settings = get_settings() + original_threshold = settings.retrieval_similarity_threshold + if args.no_threshold: + settings.retrieval_similarity_threshold = 0.0 + # The pipeline caps results at min(top_k, retrieval_max_context_chunks); + # raise the cap so a top_k of 10 is actually returned. + settings.retrieval_max_context_chunks = max( + settings.retrieval_max_context_chunks, args.top_k + ) + + eval_set = load(args.eval_set) + queries = eval_set["queries"] + print(f"[eval-set] {len(queries)} queries from {args.eval_set}") + + engine = create_async_engine(eval_database_url(), pool_pre_ping=True) + Session = async_sessionmaker(engine, expire_on_commit=False) + + rows: list[dict] = [] + latencies: list[float] = [] + per_level: dict[str, list[list[bool]]] = {level: [] for level in LEVELS} + answer_span_scorable = 0 + empty_results = 0 + + async with Session() as db: + for i, query in enumerate(queries, 1): + filters = ( + RetrievalFilter(document_id=query["gt_document_id"]) + if args.filter_document + else None + ) + + t0 = time.perf_counter() + try: + result = await run_retrieval_pipeline( + query=query["query"], + db=db, + user_id=args.user_id, + top_k=args.top_k, + filters=filters, + ) + chunks = result.chunks + except NoResultsError: + chunks = [] + empty_results += 1 + latency_ms = (time.perf_counter() - t0) * 1000 + latencies.append(latency_ms) + + rels = {level: [relevance(c, query, level) for c in chunks] for level in LEVELS} + + # A query can only be scored at answer-span level if the generator + # actually quoted a span that appears in the passage it was given. + span = normalize(query.get("answer_span", "")) + span_scorable = bool(span) and span in normalize(query["gt_passage"]) + if span_scorable: + answer_span_scorable += 1 + + for level in LEVELS: + if level == "answer" and not span_scorable: + continue + per_level[level].append(rels[level]) + + first_rank = { + level: next((j + 1 for j, r in enumerate(rels[level]) if r), None) + for level in LEVELS + } + + rows.append( + { + "query_id": query["query_id"], + "query": query["query"], + "gt_filename": query["gt_filename"], + "gt_page": query["gt_page"], + "answer_span_scorable": span_scorable, + "retrieved": [ + { + "rank": j + 1, + "filename": c.filename, + "page": c.page, + "chunk_index": c.chunk_index, + "similarity": round(c.similarity, 4), + "relevant_document": rels["document"][j], + "relevant_page": rels["page"][j], + "relevant_answer": rels["answer"][j], + } + for j, c in enumerate(chunks) + ], + "first_relevant_rank": first_rank, + "latency_ms": round(latency_ms, 2), + "n_retrieved": len(chunks), + } + ) + + if i % 10 == 0: + print(f"[{i}/{len(queries)}] scored") + + await engine.dispose() + + metrics = {level: score_level(per_level[level]) for level in LEVELS} + metrics["answer"]["scorable_queries"] = answer_span_scorable + metrics["answer"]["note"] = ( + "scored only on queries whose generated answer span is present verbatim " + "in the ground-truth passage" + ) + + payload = { + "run": { + "label": args.label, + "user_id": args.user_id, + "eval_set": args.eval_set, + "eval_set_size": len(queries), + "top_k": args.top_k, + "similarity_threshold": settings.retrieval_similarity_threshold, + "similarity_threshold_default": original_threshold, + "metadata_filter": "document_id" if args.filter_document else "none", + "embedding_model": settings.embedding_model, + "vector_distance": "cosine (pgvector <=>, HNSW index)", + }, + "metrics": metrics, + "queries_with_no_results": empty_results, + "retrieval_latency_ms": percentiles(latencies), + "per_query": rows, + } + + name = args.out or f"retrieval_eval_{args.label or args.user_id}.json" + save(name, payload) + + print("\n=== retrieval metrics ===") + for level in LEVELS: + m = metrics[level] + print( + f"{level:9s} n={m['queries']:3d} " + f"R@1={m['recall_at_1']:.3f} R@3={m['recall_at_3']:.3f} " + f"R@5={m['recall_at_5']:.3f} MRR={m['mrr']:.3f} " + f"NDCG@5={m['ndcg_at_5']:.3f} P@5={m['precision_at_5']:.3f}" + ) + print(f"\nlatency ms: {payload['retrieval_latency_ms']}") + print(f"queries returning nothing: {empty_results}") + + +if __name__ == "__main__": + asyncio.run(main())