diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a27f7a6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.github +.env +.env.* +!.env.example +!.env.showcase.example +.venv +.venv312 +.pytest_cache +.ruff_cache +.coverage +**/__pycache__ +**/*.py[cod] +frontend/node_modules +frontend/dist +data/ingestion +*.png +*.log diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..df52d7c --- /dev/null +++ b/.env.example @@ -0,0 +1,28 @@ +# Runtime +DOCULENS_ENVIRONMENT=development +DOCULENS_LOG_LEVEL=INFO +DOCULENS_CORS_ORIGINS=["http://localhost:5173"] +DOCULENS_INITIALIZE_DATABASE=true +DOCULENS_SEED_DEMO_USERS=false +DOCULENS_SEED_DEMO_WORKSPACE=false +DOCULENS_SHOWCASE_READ_ONLY=false +DOCULENS_AUTH_SECRET=replace-with-a-long-random-value +DOCULENS_API_KEY= + +# AI providers +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +OPEN_ROUTER_API_KEY= +DOCULENS_PROVIDER_TIMEOUT_SECONDS=30 +DOCULENS_CHUNK_MAX_TOKENS=800 +DOCULENS_EMBEDDING_BATCH_SIZE=64 +DOCULENS_EMBEDDING_CACHE_SIZE=1024 + +# Infrastructure +PROJECT_NAME=doculens +DATABASE_HOST=doculens_database +DATABASE_PORT=5432 +DATABASE_NAME=doculens +DATABASE_USER=postgres +DATABASE_PASSWORD=doculens-local-only +REDIS_URL=redis://doculens_redis:6379/0 diff --git a/.env.showcase.example b/.env.showcase.example new file mode 100644 index 0000000..a541b69 --- /dev/null +++ b/.env.showcase.example @@ -0,0 +1,29 @@ +# Copy to .env.showcase and replace every value marked "replace-me". +PROJECT_NAME=doculens_showcase +CADDY_DOMAIN=demo.example.com + +# Public showcase behavior +DOCULENS_ENVIRONMENT=production +DOCULENS_LOG_LEVEL=INFO +DOCULENS_CORS_ORIGINS=["https://demo.example.com"] +DOCULENS_RUN_MIGRATIONS=true +DOCULENS_INITIALIZE_DATABASE=true +DOCULENS_SEED_DEMO_USERS=false +DOCULENS_SEED_DEMO_WORKSPACE=true +DOCULENS_SHOWCASE_READ_ONLY=true +DOCULENS_AUTH_SECRET=replace-me-with-openssl-rand-hex-32 +DOCULENS_API_KEY= + +# No provider call is reachable while showcase mode is enabled. These values +# only let provider clients initialize during creation of the vector table. +OPENAI_API_KEY=showcase-no-provider-calls +ANTHROPIC_API_KEY=showcase-no-provider-calls +OPEN_ROUTER_API_KEY=showcase-no-provider-calls + +# Private Compose network +DATABASE_HOST=database +DATABASE_PORT=5432 +DATABASE_NAME=doculens +DATABASE_USER=postgres +DATABASE_PASSWORD=replace-me-with-a-long-random-password +REDIS_URL=redis://redis:6379/0 diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index cfcce90..9081b67 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -1,4 +1,4 @@ -name: Backend CI +name: Quality on: push: @@ -23,11 +23,40 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.11" + cache: pip - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r app/requirements.txt + pip install -e '.[dev]' - - name: Run tests - run: pytest + - name: Lint + run: ruff check . + + - name: Type-check + run: pyright + + - name: Run tests with coverage + run: pytest --cov=app --cov-report=term-missing + + frontend: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm --prefix frontend ci + + - name: Lint + run: npm --prefix frontend run lint + + - name: Build and type-check + run: npm --prefix frontend run build diff --git a/.gitignore b/.gitignore index b46498b..d3e8366 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ eggs/ .eggs/ lib/ lib64/ +!frontend/src/lib/ +!frontend/src/lib/** parts/ sdist/ var/ @@ -58,7 +60,7 @@ target/ # DotEnv configuration .env -.env.example +!.env.example # Database *.db @@ -89,9 +91,7 @@ target/ # Virtual environment .venv/ - -# docs -/docs/ +.venv312/ # Local ingestion uploads data/ingestion/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0f2e477 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.3 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-added-large-files + args: [--maxkb=2000] + - id: detect-private-key diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..46b5c42 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +SHELL := /bin/bash +.DEFAULT_GOAL := help + +.PHONY: help install dev up down showcase-up showcase-down showcase-logs test lint format typecheck check sample + +help: ## Show available commands + @awk 'BEGIN {FS = ":.*## "; printf "DocuLens developer commands\n\n"} /^[a-zA-Z_-]+:.*## / {printf " %-12s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +install: ## Install backend and frontend development dependencies + python3 -m pip install -e '.[dev]' + npm --prefix frontend ci + +dev: ## Run API with hot reload + uvicorn app.main:app --reload --port 8080 + +up: ## Start the complete Docker development stack + docker compose --env-file .env -f docker/docker-compose.yml up --build -d + +down: ## Stop the Docker development stack + docker compose --env-file .env -f docker/docker-compose.yml down + +showcase-up: ## Build and start the read-only portfolio showcase + docker compose --env-file .env.showcase -f docker/docker-compose.showcase.yml up --build -d + +showcase-down: ## Stop the read-only portfolio showcase + docker compose --env-file .env.showcase -f docker/docker-compose.showcase.yml down + +showcase-logs: ## Follow portfolio showcase logs + docker compose --env-file .env.showcase -f docker/docker-compose.showcase.yml logs -f --tail=200 + +test: ## Run backend tests + pytest + +lint: ## Run backend and frontend lint checks + ruff check . + npm --prefix frontend run lint + +format: ## Format backend code + ruff format . + ruff check --fix . + +typecheck: ## Type-check backend and build/type-check frontend + pyright + npm --prefix frontend run build + +check: lint typecheck test ## Run the full local quality gate + +sample: ## Submit the sample search event to a running API + python3 requests/send_event.py requests/events/search_query.json diff --git a/README.md b/README.md index bf30fb4..9697c9a 100644 --- a/README.md +++ b/README.md @@ -1,206 +1,235 @@ -# DocuLens AI Console - -DocuLens AI is a full-stack document intelligence workstation. The FastAPI/Celery -backend ingests files, runs summarisation and retrieval pipelines, and exposes a -task-driven event API. The React (Vite + TypeScript) console gives analysts, -operations teams, and integrators a control room for uploads, work queues, QA, -and governance. - -This repository contains the entire stack used in local development and CI: -pipelines, Timescale/pgvector integration, a modern UI, and supporting scripts. - ---- - -## Highlights - -- **Event-driven pipelines** – Queue `document_upload`, `document_summary`, - `qa_query`, and related events. Celery orchestrates extraction, chunking, - embedding, and persistence to Timescale Vector. -- **Operator-first console** – Intake dashboards, role-aware work queues - (Finance, Compliance, Operations, Legal, HR, Integrator), and the dedicated QA - Studio keep reviewers in flow. -- **Retrieval-augmented QA** – QA Studio streams pending questions, polling for - answers without requiring A page reload. Pending states surface inline with - animated typing indicators and citations. -- **Insights & notifications** – Uploads, QA completions, and routing actions - raise toasts and persist in the activity feed. Polling keeps stakeholders - informed when answers or summaries finish processing. -- **Governance-ready settings** – Persona selection, API key management, and - configurable limits (chunk preview, QA `top_k`, search result counts) are - exposed in the console with local persistence. -- **Comprehensive automation** – Docker Compose environments, smoke-test - scripts, pytest coverage, and GitHub Actions ensure the pipelines remain - reliable as the product evolves. - ---- - -## Architecture at a Glance +# DocuLens AI -```text -┌──────────────────┐ ┌──────────────────────┐ ┌──────────────────┐ -│ React Console │ ─────▶ │ FastAPI Gateway │ ─────▶ │ PostgreSQL/ │ -│ (Vite + TS) │ │ /events endpoints │ │ Timescale + pgvec│ -└──────────────────┘ │ └ Celery dispatcher│ └──────────────────┘ - ▲ │ │ ▲ - │ └──────────────┬──────┘ │ - │ │ │ - │ ┌──────────────▼───────────────┐ │ - └──────────────────┤ Celery Worker (pipelines) │ ◀──────┘ - │ • chunking & embedding │ - │ • summarisation & QA │ - │ • semantic search │ - └──────────────────────────────┘ -``` +**Turn unstructured business documents into searchable, cited, operational knowledge.** -**Backend**: FastAPI, SQLAlchemy, Celery, TimescaleDB + pgvector, Redis -**LLM/Retrieval**: OpenAI/Anthropic (configurable), in-house chunker/tokeniser -**Frontend**: React 18, TypeScript, Vite, Tailwind, shadcn/ui components -**Tooling**: Docker Compose, pytest, GitHub Actions, scripts for smoke testing +DocuLens is an open-source document intelligence system for teams that need more than a chat-with-PDF demo. It accepts documents asynchronously, extracts layout-aware content, creates citation-ready embeddings, and exposes classification, summarization, semantic search, and grounded question answering through a versioned API and an operator console. ---- +[![Backend CI](https://github.com/codewithmoin/doculens-ai/actions/workflows/backend-ci.yml/badge.svg)](https://github.com/codewithmoin/doculens-ai/actions/workflows/backend-ci.yml) +[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-3776AB)](https://www.python.org/) +[![FastAPI](https://img.shields.io/badge/FastAPI-API-009688)](https://fastapi.tiangolo.com/) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -## Repository Layout +## Why it is technically interesting -```text -app/ FastAPI application, pipelines, Celery tasks -frontend/ React console (Vite + TypeScript) -docker/ Container definitions and compose files -requests/ Sample payloads and helper scripts -scripts/ Developer utilities and smoke tests -tests/ Pytest suite covering API & pipelines -.github/workflows/ Continuous integration pipelines +- **Layout-aware ingestion:** Docling preserves headings, tables, provenance, and page numbers instead of flattening a document into one string. +- **Asynchronous AI workflows:** FastAPI accepts work quickly; Celery executes extraction and model calls with bounded task time, late acknowledgement, and worker-loss recovery. +- **Citation-first RAG:** every vector carries document, chunk, page, title, and token metadata; QA prompts use stable references and require grounded answers. +- **Measurable retrieval:** dependency-free Recall@K and MRR utilities make retrieval changes testable against labelled fixtures. +- **Operational product surface:** the React console covers intake, work queues, QA history, labels, lifecycle actions, and dashboard insights. + +## Product tour + +![DocuLens AI landing experience](docs/images/landing.jpg) + +The redesigned experience separates the public product story from a focused authenticated workspace. Inside the app, teams get global `⌘K` search, drag-and-drop ingestion, collection-aware document navigation, work queues, processing visibility, and an evidence-first AI chat with source references. + +![DocuLens AI workspace overview](docs/images/workspace-overview.jpg) + +
+View the evidence-first QA studio and work queues + +![DocuLens AI QA studio](docs/images/qa-studio.jpg) + +![DocuLens AI work queues](docs/images/work-queues.jpg) + +
+ +
+View the workspace sign-in experience + +![DocuLens AI sign-in](docs/images/login.jpg) + +
+ +## Architecture + +```mermaid +flowchart LR + UI["React operator console"] -->|REST /api/v1| API["FastAPI gateway"] + Client["API clients"] --> API + API -->|persist event| DB[("TimescaleDB + PostgreSQL")] + API -->|enqueue id| Redis[("Redis broker")] + Redis --> Worker["Celery worker"] + Worker --> Extract["Docling extraction"] + Extract --> Chunk["Layout-aware chunking"] + Chunk --> Embed["Batched embeddings"] + Embed --> Vector[("pgvector / DiskANN")] + Worker --> LLM["Structured LLM calls"] + LLM --> DB + Vector --> Retrieve["Filtered retrieval"] + Retrieve --> LLM ``` ---- +The event record is the durable boundary between HTTP and AI work. A worker validates the stored event, selects a typed pipeline from the registry, runs its nodes, and stores a serializable task context. This design keeps slow or retryable model work away from request threads while retaining an auditable input/output trail. -## Getting Started +## Capabilities -### Prerequisites +- PDF and text ingestion with layout-aware extraction +- document classification and configurable label hierarchies +- structured information extraction +- document summaries with source chunk provenance +- metadata-filtered semantic and keyword search +- retrieval-augmented QA with stable citations and confidence +- archive, restore, and soft-delete lifecycle operations +- JWT personas plus optional API-key protection +- work queues, notifications, dashboards, and request history +- OpenAI, Anthropic, OpenRouter, and local OpenAI-compatible model adapters -- Python 3.12+ -- Node.js 18+ -- Docker Desktop (Postgres/Timescale, Redis, ancillary services) -- OpenAI or Anthropic credentials (configure in `.env`) +## Stack -### 1. Clone & configure environment +| Layer | Technology | Responsibility | +| --- | --- | --- | +| API | FastAPI, Pydantic, SQLAlchemy | contracts, validation, authentication, persistence | +| Jobs | Celery, Redis | resilient long-running document and AI processing | +| Retrieval | Docling, tiktoken, OpenAI embeddings | extraction, bounded chunks, batched embeddings | +| Data | PostgreSQL, Timescale Vector, pgvector | events, metadata, keyword and vector search | +| Models | Instructor, OpenAI, Anthropic | provider-neutral structured outputs | +| Web | React 19, TypeScript, Vite, TanStack Query | operator workflows and evidence review | +| Quality | pytest, Ruff, Pyright, pre-commit, GitHub Actions | repeatable engineering checks | + +## Quick start + +### Prerequisites + +- Docker with Compose v2 +- an OpenAI API key (required for embeddings; model providers are configurable) +- Python 3.11+ and Node 20+ only when running services outside Docker ```bash -git clone -cd "Doculens AI" +git clone https://github.com/codewithmoin/doculens-ai.git +cd doculens-ai cp .env.example .env -# edit .env with provider keys, database overrides, and doculens settings +# Set OPENAI_API_KEY and replace DOCULENS_AUTH_SECRET in .env +make up ``` -Key variables: +Open: -| Variable | Purpose | -| --- | --- | -| `DOCULENS_SUMMARY_CHUNK_LIMIT` | Max chunks fed into the summariser | -| `DOCULENS_QA_TOP_K` | Retrieval breadth for QA Studio | -| `DOCULENS_SEARCH_RESULT_LIMIT` | Default search pagination | -| `DATABASE_*` | Optional overrides when not using Docker Postgres | -| `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | LLM provider credentials | +- console: `http://localhost:5173` +- OpenAPI: `http://localhost:8080/docs` +- liveness: `http://localhost:8080/health/live` -### 2. Start the stack +Schema migrations should be applied with Alembic for deployments. Automatic table initialization is a local-development convenience and can be disabled with `DOCULENS_INITIALIZE_DATABASE=false`. Demo users are opt-in via `DOCULENS_SEED_DEMO_USERS=true` and are explicitly rejected in production. -```bash -docker network create doculens_network # safe to rerun; ensures required network exists -./scripts/dev_stack.sh # boots Postgres, Redis, API, worker, UI -``` +### Publish the portfolio showcase -If Celery exits with a `libpq.5.dylib` import error on macOS, reinstall the bundled driver via: +DocuLens includes a separate single-node deployment for a public, read-only product tour. It serves the frontend and API from one HTTPS domain, seeds synthetic documents idempotently, skips the worker, and blocks every state-changing workspace request in the API. ```bash -.venv/bin/pip install --force-reinstall psycopg2-binary +cp .env.showcase.example .env.showcase +# Set the domain and generated secrets, then: +make showcase-up ``` -Backends start on `http://localhost:8080`, the console on `http://localhost:5173`. +See the [showcase deployment runbook](docs/deploy-showcase.md) for DNS, TLS, verification, backups, and rollback. This mode demonstrates the finished AI workflow without accepting public uploads or spending money on visitor model calls. + +## API examples -### 3. Frontend in development mode +All stable endpoints are under `/api/v1`. Legacy `/events` routes remain available for existing clients. + +Upload a document: ```bash -cd frontend -npm install -npm run dev +curl --fail-with-body http://localhost:8080/api/v1/events/documents/upload \ + -H "X-API-Key: $DOCULENS_API_KEY" \ + -F "file=@data/sample/example.txt" \ + -F "doc_type=invoice" \ + -F 'metadata={"source":"quickstart","department":"finance"}' ``` -### 4. Run tests +Ask a grounded question: ```bash -.venv/bin/python -m pytest # backend + pipeline coverage +curl --fail-with-body http://localhost:8080/api/v1/events/ \ + -H 'Content-Type: application/json' \ + -H "X-API-Key: $DOCULENS_API_KEY" \ + -d '{"event_type":"qa_query","query":"What is the invoice total?","top_k":5}' ``` ---- +The API returns `202 Accepted` with an event id. Poll the event resource until its task context contains the answer and `chunk_references` used to construct it. Ready-made payloads live in [`requests/events`](requests/events). -## Everyday Workflows +## Configuration -1. **Upload documents** – use the Intake page or POST to `/events/documents/upload`. - Uploaded files appear in the Intake dashboard and work queues with automatic - status tags. -2. **Process & route** – Work Queues group documents by inferred or assigned - role (Finance, Compliance, Operations, Legal, HR, Integrator). HR-specific - labels (Resume/CV, Offer Letter, Training Certificate, etc.) now stay within - the HR lane. -3. **Summarise** – Document detail pages expose “Summarise again”. If no - summary exists, placeholders prompt operators to run one (and the processing - timeline remains visible). -4. **Ask questions** – QA Studio stores per-document chat history, keeps pending - messages visible with typing indicators, and pulls in completed answers - automatically—no page refresh needed. -5. **Stay informed** – Notification toasts and the bell keep track of uploads, - QA completions, routing changes, and summary jobs. +Configuration is validated once at startup. See [`.env.example`](.env.example) for the full local template. ---- +| Variable | Default | Purpose | +| --- | ---: | --- | +| `DOCULENS_ENVIRONMENT` | `development` | enables production safety checks | +| `DOCULENS_CORS_ORIGINS` | localhost console | JSON list of allowed browser origins | +| `DOCULENS_MAX_UPLOAD_BYTES` | 25 MiB | streamed upload limit | +| `DOCULENS_CHUNK_MAX_TOKENS` | 800 | retrieval chunk size | +| `DOCULENS_EMBEDDING_BATCH_SIZE` | 64 | provider request batch bound | +| `DOCULENS_EMBEDDING_CACHE_SIZE` | 1024 | process-local repeated-text cache | +| `DOCULENS_PROVIDER_TIMEOUT_SECONDS` | 30 | AI provider network timeout | +| `DOCULENS_QA_TOP_K` | 5 | default QA retrieval breadth | +| `DOCULENS_SHOWCASE_READ_ONLY` | false | blocks workspace mutations and enables the public product-tour UX | -## API Quick Reference +## Development -All routes live under `/events`. Authentication is controlled via the header -defined by `DOCULENS_API_KEY_HEADER` when `DOCULENS_API_KEY` is set. +```bash +make install # backend editable install + npm ci +make check # lint, types, frontend build, tests +pre-commit install # run fast checks before each commit +``` -| Method | Route | Description | -| --- | --- | --- | -| `POST` | `/events/documents/upload` | Multipart upload (`file`, `doc_type`, `metadata`) | -| `POST` | `/events` | Generic event ingestion (summary, QA, search, routing) | -| `GET` | `/events/documents` | Latest documents + metadata | -| `GET` | `/events/qa/history` | Historical QA answers, citations, reasoning | -| `GET` | `/events/search/history` | Semantic search history with snippets | -| `GET` | `/events/insights/dashboard` | Throughput, SLA risk, ROI metrics | +Useful commands are discoverable with `make help`. CI runs the same Ruff, Pyright, pytest, and coverage checks used locally. -Example upload: +### Retrieval evaluation -```bash -curl -X POST "http://localhost:8080/events/documents/upload" \ - -H "X-API-Key: ${DOCULENS_API_KEY:-local-dev}" \ - -F "file=@/path/to/document.pdf" \ - -F 'doc_type=resume' \ - -F 'metadata={"role":"HR","source":"careers-portal"}' +Create labelled `RetrievalExample` cases with known relevant chunk ids and run `evaluate(examples, k=5)`. Track Recall@5 and MRR before changing chunk size, embedding model, filters, or ranking. The utility is intentionally offline and deterministic: it belongs in CI; live provider quality and latency belong in a separate scheduled benchmark. + +## Project structure + +```text +app/ +├── api/ HTTP contracts, auth, dependencies, versioned routers +├── config/ validated runtime and infrastructure settings +├── core/ pipeline primitives and observability +├── database/ SQLAlchemy models, repositories, migrations +├── doc_utils/ extraction, chunking, embedding, retrieval +├── evaluation/ offline RAG quality metrics +├── pipelines/ typed document and QA workflows +├── prompts/ reviewed, versionable Jinja prompt assets +├── services/ domain and external-provider adapters +└── tasks/ durable Celery task entrypoints +frontend/ React/TypeScript operator console +docker/ API, worker, proxy, database, and Redis stack +tests/ API, pipeline, dashboard, and evaluation tests +requests/ executable example payloads ``` ---- +## Engineering decisions and tradeoffs + +- **Modular monolith:** one deployable backend keeps contribution and operations simple. Service boundaries are internal and can be extracted only if scaling evidence demands it. +- **Async jobs, synchronous pipeline nodes:** document work is asynchronous at the system boundary; node code stays easy to reason about because most provider SDKs and extraction libraries are synchronous. Worker concurrency supplies parallelism. +- **Process-local embedding cache:** avoids repeated provider calls without introducing another consistency-sensitive cache. It resets on deploy and is not intended as durable storage. +- **Provider-neutral structured outputs:** improves validation and portability, but provider behavior still differs and must be evaluated per model. +- **Compatibility versioning:** `/api/v1` is canonical while legacy routes remain during migration. Removing aliases is a future breaking release. +- **Product hierarchy over dashboard density:** the public landing page explains the problem and architecture; `/app` is reserved for focused document work. The UI uses route-level code splitting, a paper-and-ink system with cobalt focus and amber evidence, persistent dark mode, and reduced-motion fallbacks instead of a large animation runtime. + +More detail is recorded in [`docs/engineering-notes.md`](docs/engineering-notes.md) and the [`design system specification`](docs/design-system.md). -## Testing & Automation +## Limitations -- **Unit & integration tests:** `pytest` targets FastAPI endpoints, pipeline - orchestration, and dashboard metrics. -- **Smoke tests:** `scripts/backend_smoke.sh` replays sample events. Use - `requests/send_event.py` to post custom payloads. -- **Continuous integration:** `.github/workflows/backend-ci.yml` runs linting - and pytest on every push/PR. -- **Frontend builds:** `npm run build && npm run preview` produce a static Vite - bundle for deployment. +- OCR/model accuracy depends on document quality, language, and provider. +- The current retrieval path is dense-first; hybrid ranking exists but needs a labelled corpus before tuning. +- The included deployment manifest is deliberately scoped to a single-node, read-only portfolio showcase—not a multi-tenant customer environment. +- Authentication is suitable for a single workspace; multi-tenant authorization is not implemented. +- Retrieval metrics are utilities, not a bundled benchmark dataset—the project does not claim quality without domain-labelled examples. ---- +## Roadmap -## Roadmap & Ideas +- persist event lifecycle states and retry diagnostics as first-class columns +- add a small, redistributable labelled retrieval benchmark +- evaluate reciprocal-rank fusion for dense + keyword retrieval +- add OpenTelemetry traces and provider latency/cost dashboards +- split the large legacy event endpoint module by resource as routes stabilize +- publish signed, multi-stage container images and a deployment runbook -- Harden production deployment (container images + helm charts). -- Add role-based access controls inside the console. -- Expand analytics with richer HR/Finance KPIs and Prometheus scraping. -- Extend extraction pipelines with structured field capture. +## Security ---- +Never commit provider keys or production documents. Configure explicit CORS origins, API authentication, a strong `DOCULENS_AUTH_SECRET`, database TLS, and object storage for uploads in deployed environments. Report vulnerabilities privately through GitHub security advisories. ## License -DocuLens AI is released under the CodeWithMoin license. Refer to -[`LICENSE`](LICENSE) for terms. +DocuLens AI is available under the [MIT License](LICENSE). diff --git a/app/alembic.ini b/app/alembic.ini index a2bfc1e..d7c28d7 100644 --- a/app/alembic.ini +++ b/app/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = alembic/ +script_location = %(here)s/alembic/ # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 042f8cd..b04d7bf 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -45,3 +45,19 @@ def require_api_key(request: Request) -> None: status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key.", ) + + +def require_showcase_writable(request: Request) -> None: + """Reject state-changing workspace requests in public showcase mode. + + This guard lives in the API rather than relying on disabled frontend controls, + so direct requests cannot mutate the seeded portfolio workspace. + """ + settings = get_settings() + if not settings.showcase_read_only or request.method in {"GET", "HEAD", "OPTIONS"}: + return + + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="This public showcase is read-only.", + ) diff --git a/app/api/endpoint.py b/app/api/endpoint.py index fab2788..60f7d94 100644 --- a/app/api/endpoint.py +++ b/app/api/endpoint.py @@ -2,17 +2,15 @@ import string from datetime import datetime, timedelta, timezone from http import HTTPStatus -from pathlib import Path import uuid -from datetime import datetime - +from pathlib import Path from typing import Any, Dict, Iterable, List, Literal, Optional, Sequence, Tuple from uuid import UUID, uuid4 from fastapi import APIRouter, Body, Depends, File, Form, HTTPException, Query, Response, UploadFile from sqlalchemy import select, text from sqlalchemy.orm import Session -from pydantic import BaseModel +from pydantic import BaseModel, Field from app.api.dependencies import db_session from app.api.event_schema import EventSchema @@ -121,7 +119,7 @@ class LabelTreeNode(BaseModel): description: Optional[str] = None parent_id: Optional[str] = None workspace_id: Optional[str] = None - children: List["LabelTreeNode"] = [] + children: List["LabelTreeNode"] = Field(default_factory=list) LabelTreeNode.model_rebuild() @@ -295,6 +293,7 @@ def get_runtime_config() -> Dict[str, Any]: "search_preview_limit": settings.search_preview_limit, "chunk_preview_limit": settings.chunk_preview_limit, "auth_required": bool(settings.api_key), + "showcase_read_only": settings.showcase_read_only, "api_key_header": settings.api_key_header, "persona_options": PERSONA_OPTIONS, "role_definitions": ROLE_DEFINITIONS, @@ -403,8 +402,6 @@ def get_dashboard_insights( now = datetime.now(timezone.utc) fourteen_days_ago = now - timedelta(days=13) - thirty_days_ago = now - timedelta(days=30) - uploads_rows = session.execute( text( """ @@ -653,8 +650,7 @@ def list_events( session: Session = Depends(db_session), ): """Return the most recent events and their processing context.""" - repository = GenericRepository(session=session, model=Event) - events = repository.get_latest(limit) + events = session.query(Event).order_by(Event.created_at.desc()).limit(limit).all() return [ { "id": str(event.id), @@ -1060,13 +1056,24 @@ async def upload_document( stored_filename = f"{uuid4().hex}_{original_name}" stored_path = ingestion_dir / stored_filename + bytes_written = 0 + max_upload_bytes = get_settings().max_upload_bytes try: with stored_path.open("wb") as buffer: while True: chunk = await file.read(1 << 20) if not chunk: break + bytes_written += len(chunk) + if bytes_written > max_upload_bytes: + raise HTTPException( + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + detail=f"Upload exceeds the {max_upload_bytes}-byte limit.", + ) buffer.write(chunk) + except Exception: + stored_path.unlink(missing_ok=True) + raise finally: await file.close() @@ -1107,6 +1114,7 @@ async def upload_document( "task_id": task_id, "original_filename": original_name, "stored_path": str(stored_path), + "size_bytes": bytes_written, } diff --git a/app/api/event_schema.py b/app/api/event_schema.py index 2d94eec..e437d3e 100644 --- a/app/api/event_schema.py +++ b/app/api/event_schema.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from typing import List, Optional, Union, Dict, Any, Literal """ @@ -9,45 +9,51 @@ events entering the system through the API endpoints. """ -class DocumentUploadEvent(BaseModel): +class StrictEvent(BaseModel): + """Base contract that rejects misspelled or unsupported event fields.""" + + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + +class DocumentUploadEvent(StrictEvent): event_type: Literal["document_upload"] filename: str file_url: str doc_type: Optional[str] = None - metadata: Dict[str, Any] + metadata: Dict[str, Any] = Field(default_factory=dict) -class DocumentClassificationEvent(BaseModel): +class DocumentClassificationEvent(StrictEvent): event_type: Literal["document_classification"] document_id: str text: str metadata: Dict[str, Any] -class InformationExtractionEvent(BaseModel): +class InformationExtractionEvent(StrictEvent): event_type: Literal["information_extraction"] document_id: str doc_type: str text: str - fields: List[str] + fields: List[str] = Field(min_length=1, max_length=100) -class SearchQueryEvent(BaseModel): +class SearchQueryEvent(StrictEvent): event_type: Literal["search_query"] query: str filters: Optional[Dict[str, Any]] = None - limit: Optional[int] = None + limit: Optional[int] = Field(default=None, ge=1, le=100) -class DocumentRoutingEvent(BaseModel): +class DocumentRoutingEvent(StrictEvent): event_type: Literal["document_routing"] document_id: str target_department: str reason: str -class DocumentSummaryEvent(BaseModel): +class DocumentSummaryEvent(StrictEvent): event_type: Literal["document_summary"] document_id: Optional[str] = None filename: Optional[str] = None doc_type: Optional[str] = None - chunks_limit: Optional[int] = None + chunks_limit: Optional[int] = Field(default=None, ge=1, le=100) @model_validator(mode="after") def validate_identifier(self): @@ -56,10 +62,10 @@ def validate_identifier(self): return self -class QAQueryEvent(BaseModel): +class QAQueryEvent(StrictEvent): event_type: Literal["qa_query"] query: str - top_k: Optional[int] = None + top_k: Optional[int] = Field(default=None, ge=1, le=50) filters: Optional[Dict[str, Any]] = None # Union of all event types for the single endpoint diff --git a/app/api/router.py b/app/api/router.py index 267ab58..ac0cc16 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Depends -from app.api.dependencies import require_api_key +from app.api.dependencies import require_api_key, require_showcase_writable from app.api.endpoint import router as endpoint_router, public_router as public_endpoint_router from app.api.auth_router import router as auth_router @@ -13,7 +13,9 @@ router = APIRouter() -secured_router = APIRouter(dependencies=[Depends(require_api_key)]) +secured_router = APIRouter( + dependencies=[Depends(require_api_key), Depends(require_showcase_writable)] +) secured_router.include_router(endpoint_router, prefix="/events", tags=["events"]) router.include_router(auth_router) diff --git a/app/config/celery_config.py b/app/config/celery_config.py index d3940db..40e3330 100644 --- a/app/config/celery_config.py +++ b/app/config/celery_config.py @@ -46,6 +46,13 @@ def get_celery_config(): "result_serializer": "json", "enable_utc": True, "broker_connection_retry_on_startup": True, + "task_acks_late": True, + "task_reject_on_worker_lost": True, + "task_track_started": True, + "task_time_limit": 600, + "task_soft_time_limit": 540, + "worker_prefetch_multiplier": 1, + "result_expires": 86400, } diff --git a/app/config/database_config.py b/app/config/database_config.py index 73f847e..1bae633 100644 --- a/app/config/database_config.py +++ b/app/config/database_config.py @@ -26,7 +26,7 @@ class DatabaseConfig(BaseSettings): port: str = os.getenv("DATABASE_PORT", "5432") name: str = os.getenv("DATABASE_NAME", "doculens") pg_user: str = os.getenv("DATABASE_USER", "postgres") - password: str = os.getenv("DATABASE_PASSWORD") + password: str = os.getenv("DATABASE_PASSWORD", "") local: bool = False @property @@ -36,4 +36,9 @@ def service_url(self) -> str: return f"postgres://{self.pg_user}:{self.password}@localhost:{self.port}/{self.name}" return f"postgres://{self.pg_user}:{self.password}@{self.host}:{self.port}/{self.name}" + def service_url_for(self, *, local: bool = False) -> str: + """Build a URL without mutating the cached settings singleton.""" + host = "localhost" if local else self.host + return f"postgres://{self.pg_user}:{self.password}@{host}:{self.port}/{self.name}" + vector_store: VectorStoreConfig = VectorStoreConfig() diff --git a/app/config/llm_config.py b/app/config/llm_config.py index 97de355..a7c228c 100644 --- a/app/config/llm_config.py +++ b/app/config/llm_config.py @@ -21,7 +21,7 @@ class LLMProviderSettings(BaseSettings): class OpenAISettings(LLMProviderSettings): """Settings for OpenAI.""" - api_key: str = os.getenv("OPENAI_API_KEY") + api_key: Optional[str] = os.getenv("OPENAI_API_KEY") default_model: str = "gpt-4o" embedding_model: str = "text-embedding-3-small" @@ -29,7 +29,7 @@ class OpenAISettings(LLMProviderSettings): class AnthropicSettings(LLMProviderSettings): """Settings for Anthropic.""" - api_key: str = os.getenv("ANTHROPIC_API_KEY") + api_key: Optional[str] = os.getenv("ANTHROPIC_API_KEY") default_model: str = "claude-3-5-sonnet-20240620" max_tokens: int = 1024 @@ -45,7 +45,7 @@ class LlamaSettings(LLMProviderSettings): class OpenRouterSettings(LLMProviderSettings): """Settings for OpenRouter.""" - api_key: str = os.getenv("OPEN_ROUTER_API_KEY") + api_key: Optional[str] = os.getenv("OPEN_ROUTER_API_KEY") default_model: str = "openai/gpt-4o-mini" base_url: str = "https://openrouter.ai/api/v1" @@ -56,4 +56,4 @@ class LLMConfig(BaseSettings): openai: OpenAISettings = OpenAISettings() anthropic: AnthropicSettings = AnthropicSettings() llama: LlamaSettings = LlamaSettings() - openrouter: OpenRouterSettings = OpenRouterSettings() \ No newline at end of file + openrouter: OpenRouterSettings = OpenRouterSettings() diff --git a/app/config/settings.py b/app/config/settings.py index 29ca4f2..f51fcc6 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -1,44 +1,81 @@ +"""Validated application configuration. + +Environment variables are the public configuration API. Keeping them in one +model makes startup failures explicit and prevents infrastructure clients from +silently inventing their own defaults. +""" + from functools import lru_cache -from typing import Optional +from typing import Literal, Optional -from dotenv import load_dotenv -from pydantic import Field -from pydantic_settings import BaseSettings +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict from app.config.database_config import DatabaseConfig from app.config.llm_config import LLMConfig -load_dotenv() -""" -Main settings for the application using Pydantic Settings. -""" +class Settings(BaseSettings): + """Runtime settings loaded from environment variables and an optional .env.""" + model_config = SettingsConfigDict(env_file=".env", extra="ignore", case_sensitive=False) -class Settings(BaseSettings): - """Main settings for the application.""" + app_name: str = "DocuLens AI" + environment: Literal["development", "test", "staging", "production"] = Field( + default="development", alias="DOCULENS_ENVIRONMENT" + ) + log_level: str = Field(default="INFO", alias="DOCULENS_LOG_LEVEL") + cors_origins: list[str] = Field( + default_factory=lambda: ["http://localhost:5173"], alias="DOCULENS_CORS_ORIGINS" + ) + initialize_database: bool = Field(default=True, alias="DOCULENS_INITIALIZE_DATABASE") + seed_demo_users: bool = Field(default=False, alias="DOCULENS_SEED_DEMO_USERS") + seed_demo_workspace: bool = Field(default=False, alias="DOCULENS_SEED_DEMO_WORKSPACE") + showcase_read_only: bool = Field(default=False, alias="DOCULENS_SHOWCASE_READ_ONLY") - app_name: str = "Doculens AI" - llm: LLMConfig = LLMConfig() - database: DatabaseConfig = DatabaseConfig() + llm: LLMConfig = Field(default_factory=LLMConfig) + database: DatabaseConfig = Field(default_factory=DatabaseConfig) api_key: Optional[str] = Field(default=None, alias="DOCULENS_API_KEY") api_key_header: str = Field(default="X-API-Key", alias="DOCULENS_API_KEY_HEADER") - summary_chunk_limit: int = Field(default=12, ge=1, alias="DOCULENS_SUMMARY_CHUNK_LIMIT") - qa_top_k: int = Field(default=5, ge=1, alias="DOCULENS_QA_TOP_K") - search_result_limit: int = Field(default=10, ge=1, alias="DOCULENS_SEARCH_RESULT_LIMIT") - search_preview_limit: int = Field(default=5, ge=1, alias="DOCULENS_SEARCH_PREVIEW_LIMIT") - chunk_preview_limit: int = Field(default=25, ge=1, alias="DOCULENS_CHUNK_PREVIEW_LIMIT") + summary_chunk_limit: int = Field(default=12, ge=1, le=100, alias="DOCULENS_SUMMARY_CHUNK_LIMIT") + qa_top_k: int = Field(default=5, ge=1, le=50, alias="DOCULENS_QA_TOP_K") + search_result_limit: int = Field(default=10, ge=1, le=100, alias="DOCULENS_SEARCH_RESULT_LIMIT") + search_preview_limit: int = Field(default=5, ge=1, le=50, alias="DOCULENS_SEARCH_PREVIEW_LIMIT") + chunk_preview_limit: int = Field(default=25, ge=1, le=100, alias="DOCULENS_CHUNK_PREVIEW_LIMIT") + chunk_max_tokens: int = Field(default=800, ge=128, le=4000, alias="DOCULENS_CHUNK_MAX_TOKENS") + embedding_batch_size: int = Field(default=64, ge=1, le=256, alias="DOCULENS_EMBEDDING_BATCH_SIZE") + embedding_cache_size: int = Field(default=1024, ge=0, le=10000, alias="DOCULENS_EMBEDDING_CACHE_SIZE") + provider_timeout_seconds: float = Field(default=30.0, ge=1, le=300, alias="DOCULENS_PROVIDER_TIMEOUT_SECONDS") + max_upload_bytes: int = Field(default=25 * 1024 * 1024, ge=1024, alias="DOCULENS_MAX_UPLOAD_BYTES") auth_secret_key: str = Field(default="doculens-dev-secret", alias="DOCULENS_AUTH_SECRET") auth_algorithm: str = Field(default="HS256", alias="DOCULENS_AUTH_ALGORITHM") auth_token_exp_minutes: int = Field(default=120, ge=5, alias="DOCULENS_AUTH_TOKEN_EXP_MINUTES") + @field_validator("log_level") + @classmethod + def validate_log_level(cls, value: str) -> str: + normalized = value.upper() + if normalized not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}: + raise ValueError("DOCULENS_LOG_LEVEL must be a standard Python log level") + return normalized + + def assert_production_safe(self) -> None: + """Reject development credentials that must never reach production.""" + if self.environment == "production" and self.auth_secret_key == "doculens-dev-secret": + raise ValueError("DOCULENS_AUTH_SECRET must be set in production") + if self.environment == "production" and self.seed_demo_users: + raise ValueError("DOCULENS_SEED_DEMO_USERS must be false in production") + if ( + self.environment == "production" + and self.seed_demo_workspace + and not self.showcase_read_only + ): + raise ValueError( + "DOCULENS_SEED_DEMO_WORKSPACE requires DOCULENS_SHOWCASE_READ_ONLY=true in production" + ) + @lru_cache def get_settings() -> Settings: - """ - Get the application settings. - - Returns: - Settings: The application settings. - """ + """Return the immutable-by-convention process configuration singleton.""" return Settings() diff --git a/app/core/observability.py b/app/core/observability.py new file mode 100644 index 0000000..2848109 --- /dev/null +++ b/app/core/observability.py @@ -0,0 +1,33 @@ +"""Small, dependency-free logging and request tracing primitives.""" + +import logging +import time +from uuid import uuid4 + +from fastapi import Request, Response + + +def configure_logging(level: str) -> None: + """Configure consistent process logging for API and worker processes.""" + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + force=True, + ) + + +async def request_context_middleware(request: Request, call_next) -> Response: + """Attach a correlation id and emit one structured request completion log.""" + request_id = request.headers.get("X-Request-ID") or uuid4().hex + started = time.perf_counter() + response = await call_next(request) + response.headers["X-Request-ID"] = request_id + logging.getLogger("doculens.http").info( + "request_complete method=%s path=%s status=%s duration_ms=%.2f request_id=%s", + request.method, + request.url.path, + response.status_code, + (time.perf_counter() - started) * 1000, + request_id, + ) + return response diff --git a/app/doc_utils/chunking.py b/app/doc_utils/chunking.py index 5b21c2e..7eb412e 100644 --- a/app/doc_utils/chunking.py +++ b/app/doc_utils/chunking.py @@ -2,15 +2,17 @@ from docling_core.transforms.chunker.hybrid_chunker import HybridChunker +from app.config.settings import get_settings from app.doc_utils.utils.tokenizer import OpenAITokenizerWrapper -# Maximum context length for text-embedding-3-large -MAX_TOKENS: int = 8191 +# Deliberately much smaller than the embedding model limit. Retrieval quality is +# generally better with focused chunks than with page-sized 8k token passages. +DEFAULT_MAX_TOKENS: int = 800 def chunk_document( docling_document: Any, - max_tokens: int = MAX_TOKENS, + max_tokens: int | None = None, merge_peers: bool = True, ) -> List[Any]: """Chunk a Docling document using the HybridChunker. @@ -29,10 +31,11 @@ def chunk_document( if docling_document is None: raise ValueError("Cannot chunk a null document. Ensure extraction succeeded.") + effective_max_tokens = max_tokens or get_settings().chunk_max_tokens tokenizer = OpenAITokenizerWrapper() chunker = HybridChunker( tokenizer=tokenizer, merge_peers=merge_peers, ) - chunk_iter = chunker.chunk(docling_document, max_length=max_tokens) + chunk_iter = chunker.chunk(docling_document, max_length=effective_max_tokens) return list(chunk_iter) diff --git a/app/doc_utils/embedding.py b/app/doc_utils/embedding.py index dc0d782..f2cb837 100644 --- a/app/doc_utils/embedding.py +++ b/app/doc_utils/embedding.py @@ -1,143 +1,110 @@ +"""Chunk preparation and batched embedding persistence.""" + from datetime import datetime, timedelta, timezone +from functools import lru_cache from typing import Any, Dict, List, Optional + import pandas as pd -from openai import OpenAI +from timescale_vector.client import uuid_from_time from app.doc_utils.utils.tokenizer import OpenAITokenizerWrapper from app.services.vector_store import VectorStore -from timescale_vector.client import uuid_from_time - -# Constants -MAX_TOKENS = 8191 # text-embedding-3-large's maximum context length -# Initialize OpenAI client (make sure you have OPENAI_API_KEY in your environment variables) -client = OpenAI() -tokenizer = OpenAITokenizerWrapper() +MAX_EMBEDDING_TOKENS = 8191 -def _normalize_text(text: Optional[str]) -> str: - if not text: - return "" - return text.strip() +@lru_cache(maxsize=1) +def _get_tokenizer() -> OpenAITokenizerWrapper: + """Load tokenizer assets only when an embedding job needs them.""" + return OpenAITokenizerWrapper() -def _prepare_text_for_embedding(text: str) -> str: - token_ids = tokenizer.tokenizer.encode(text) - if len(token_ids) <= MAX_TOKENS: - return text - trimmed = token_ids[:MAX_TOKENS] - return tokenizer.tokenizer.decode(trimmed) +def _prepare_text(text: str) -> str: + normalized = " ".join(text.split()) + tokenizer = _get_tokenizer() + token_ids = tokenizer.tokenizer.encode(normalized) + if len(token_ids) > MAX_EMBEDDING_TOKENS: + return tokenizer.tokenizer.decode(token_ids[:MAX_EMBEDDING_TOKENS]) + return normalized -def _extract_chunk_text(chunk: Any) -> str: - for attr in ("text", "content", "value"): - value = getattr(chunk, attr, None) +def _chunk_text(chunk: Any) -> str: + for attribute in ("text", "content", "value"): + value = getattr(chunk, attribute, None) if value: return str(value) return str(chunk) -def _extract_metadata( +def _chunk_metadata( chunk: Any, document_id: Optional[str], chunk_index: int, token_count: int, - document_metadata: Optional[Dict[str, Any]] = None, + document_metadata: Optional[Dict[str, Any]], ) -> Dict[str, Any]: meta = getattr(chunk, "meta", None) origin = getattr(meta, "origin", None) - filename = getattr(origin, "filename", None) doc_items = getattr(meta, "doc_items", []) or [] - page_numbers = sorted([ - getattr(prov, "page_no") - for item in doc_items - for prov in getattr(item, "prov", []) or [] - if getattr(prov, "page_no", None) is not None - ]) + pages = sorted( + { + page + for item in doc_items + for provenance in (getattr(item, "prov", []) or []) + if (page := getattr(provenance, "page_no", None)) is not None + } + ) headings = getattr(meta, "headings", []) or [] - metadata: Dict[str, Any] = { - "filename": filename, - "page_numbers": page_numbers or None, + "document_id": document_id, + "filename": getattr(origin, "filename", None), + "page_numbers": pages or None, "title": headings[0] if headings else None, "chunk_index": chunk_index, "token_count": token_count, } - if document_id: - metadata["document_id"] = document_id - if document_metadata: - for key, value in document_metadata.items(): - if value is not None: - metadata[key] = value - return metadata + metadata.update({key: value for key, value in (document_metadata or {}).items() if value is not None}) + return {key: value for key, value in metadata.items() if value is not None} def embed_and_upsert_chunks( chunks: List[Any], - embedding_model: str = "text-embedding-3-small", + embedding_model: str | None = None, document_id: Optional[str] = None, document_metadata: Optional[Dict[str, Any]] = None, + vector_store: VectorStore | None = None, ) -> List[Dict[str, Any]]: - """Generate embeddings for Docling chunks and upsert them into the vector store. - - Args: - chunks: List of Docling chunk objects. - embedding_model: Name of the embedding model to use. - document_id: Optional identifier associated with the source document. + """Embed chunks in bounded batches and persist them with citation metadata. - Returns: - List of dictionaries summarising the inserted chunks (safe for serialization). + The optional store is an intentional injection seam: tests and alternative + vector backends do not need to patch module-level SDK clients. """ - if not chunks: + prepared = [_prepare_text(_chunk_text(chunk)) for chunk in chunks] + non_empty = [(chunk, text) for chunk, text in zip(chunks, prepared) if text] + if not non_empty: return [] - raw_texts = [_extract_chunk_text(chunk) for chunk in chunks] - prepared_texts = [_prepare_text_for_embedding(_normalize_text(text)) for text in raw_texts] - token_counts = [tokenizer.count_tokens(text) for text in prepared_texts] - - response = client.embeddings.create(input=prepared_texts, model=embedding_model) - embeddings = [item.embedding for item in response.data] - + store = vector_store or VectorStore() + tokenizer = _get_tokenizer() + texts = [text for _, text in non_empty] + embeddings = store.embed_texts(texts, model=embedding_model) base_time = datetime.now(timezone.utc) - record_ids: List[str] = [ - str(uuid_from_time(base_time + timedelta(microseconds=index))) - for index in range(len(chunks)) - ] - metadata_list = [ - _extract_metadata( - chunk, - document_id, - index, - token_counts[index], - document_metadata=document_metadata, - ) - for index, chunk in enumerate(chunks) + ids = [str(uuid_from_time(base_time + timedelta(microseconds=index))) for index in range(len(texts))] + metadata = [ + _chunk_metadata(chunk, document_id, index, tokenizer.count_tokens(text), document_metadata) + for index, ((chunk, text)) in enumerate(non_empty) ] + store.upsert(pd.DataFrame({"id": ids, "metadata": metadata, "contents": texts, "embedding": embeddings})) - df = pd.DataFrame( + return [ { - "id": record_ids, - "metadata": metadata_list, - "contents": prepared_texts, - "embedding": embeddings, + "id": record_id, + "preview": text[:160], + "token_count": meta["token_count"], + "page_numbers": meta.get("page_numbers"), + "document_id": meta.get("document_id"), + "doc_type": meta.get("doc_type"), + "filename": meta.get("filename"), } - ) - - vector_store = VectorStore() - vector_store.upsert(df) - - chunk_summaries: List[Dict[str, Any]] = [] - for record_id, text, metadata in zip(record_ids, prepared_texts, metadata_list): - chunk_summaries.append( - { - "id": record_id, - "preview": text[:160], - "token_count": metadata.get("token_count"), - "page_numbers": metadata.get("page_numbers"), - "document_id": metadata.get("document_id"), - "doc_type": metadata.get("doc_type"), - "filename": metadata.get("filename"), - } - ) - - return chunk_summaries + for record_id, text, meta in zip(ids, texts, metadata) + ] diff --git a/app/evaluation/__init__.py b/app/evaluation/__init__.py new file mode 100644 index 0000000..dda9657 --- /dev/null +++ b/app/evaluation/__init__.py @@ -0,0 +1 @@ +"""Offline evaluation helpers for retrieval and grounded generation.""" diff --git a/app/evaluation/retrieval.py b/app/evaluation/retrieval.py new file mode 100644 index 0000000..e0d5086 --- /dev/null +++ b/app/evaluation/retrieval.py @@ -0,0 +1,42 @@ +"""Dependency-free retrieval metrics suitable for CI regression tests.""" + +from dataclasses import dataclass +from statistics import mean +from typing import Iterable, Sequence + + +@dataclass(frozen=True) +class RetrievalExample: + """A ranked retrieval result paired with known relevant chunk identifiers.""" + + query: str + retrieved_ids: Sequence[str] + relevant_ids: frozenset[str] + + +def recall_at_k(example: RetrievalExample, k: int) -> float: + """Return the fraction of known relevant chunks retrieved in the top k.""" + if not example.relevant_ids: + return 1.0 + found = set(example.retrieved_ids[:k]) & example.relevant_ids + return len(found) / len(example.relevant_ids) + + +def reciprocal_rank(example: RetrievalExample) -> float: + """Return inverse rank of the first relevant result, or zero.""" + for rank, chunk_id in enumerate(example.retrieved_ids, start=1): + if chunk_id in example.relevant_ids: + return 1.0 / rank + return 0.0 + + +def evaluate(examples: Iterable[RetrievalExample], k: int = 5) -> dict[str, float]: + """Aggregate metrics for a versioned, human-labelled retrieval fixture.""" + cases = list(examples) + if not cases: + return {f"recall@{k}": 0.0, "mrr": 0.0, "examples": 0.0} + return { + f"recall@{k}": mean(recall_at_k(case, k) for case in cases), + "mrr": mean(reciprocal_rank(case) for case in cases), + "examples": float(len(cases)), + } diff --git a/app/main.py b/app/main.py index 3b207f9..f7096b6 100644 --- a/app/main.py +++ b/app/main.py @@ -1,80 +1,33 @@ +"""FastAPI application factory and lifecycle management.""" + import logging -from contextlib import contextmanager +from contextlib import asynccontextmanager, contextmanager +from typing import Iterator from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy.orm import Session -from app.api.router import router as process_router +from app.api.router import router as api_router +from app.config.settings import Settings, get_settings +from app.core.observability import configure_logging, request_context_middleware +from app.database import event as _event_model # noqa: F401 +from app.database import user as _user_model # noqa: F401 from app.database.session import Base, SessionLocal, engine from app.services.auth_service import AuthService +from app.services.demo_workspace import seed_demo_workspace from app.services.vector_store import VectorStore -# ensure model metadata is imported before creating tables -from app.database import event as _event_model # noqa: F401 -from app.database import user as _user_model # noqa: F401 +logger = logging.getLogger(__name__) -app = FastAPI() - -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -app.include_router(process_router) - -DEFAULT_USER_SEED = [ - { - "email": "admin@doculens.ai", - "password": "Admin!234", - "full_name": "Avery Quinn", - "persona": "executive", - "role": "admin", - "access_level": "full", - }, - { - "email": "analyst@doculens.ai", - "password": "Analyst!234", - "full_name": "Jordan Patel", - "persona": "analyst", - "role": "analyst", - }, - { - "email": "reviewer@doculens.ai", - "password": "Reviewer!234", - "full_name": "Morgan Lee", - "persona": "reviewer", - "role": "reviewer", - }, - { - "email": "manager@doculens.ai", - "password": "Manager!234", - "full_name": "Taylor Brooks", - "persona": "manager", - "role": "manager", - }, - { - "email": "developer@doculens.ai", - "password": "Developer!234", - "full_name": "Riley Chen", - "persona": "developer", - "role": "developer", - }, - { - "email": "viewer@doculens.ai", - "password": "Viewer!234", - "full_name": "Skyler James", - "persona": "executive", - "role": "viewer", - "access_level": "minimal", - }, +DEMO_USERS = [ + {"email": "admin@doculens.ai", "password": "Admin!234", "full_name": "Demo Admin", "persona": "executive", "role": "admin"}, + {"email": "analyst@doculens.ai", "password": "Analyst!234", "full_name": "Demo Analyst", "persona": "analyst", "role": "analyst"}, ] @contextmanager -def _session_scope(): +def session_scope() -> Iterator[Session]: session = SessionLocal() try: yield session @@ -86,17 +39,63 @@ def _session_scope(): session.close() -@app.on_event("startup") -def ensure_vector_store_ready() -> None: - """Initialize database requirements and ensure supporting services are warm.""" +def initialize_dependencies(settings: Settings) -> None: + """Initialize local-development resources; migrations own production schema.""" + if not settings.initialize_database: + return Base.metadata.create_all(bind=engine) + if settings.seed_demo_users: + with session_scope() as session: + AuthService(session).ensure_seed_users(DEMO_USERS) + store = VectorStore() + store.create_tables() + store.create_keyword_search_index() + if settings.seed_demo_workspace: + with session_scope() as session: + inserted = seed_demo_workspace( + session, + embedding_dimensions=settings.database.vector_store.embedding_dimensions, + ) + if inserted: + logger.info("Seeded the synthetic local demo workspace") - with _session_scope() as session: - AuthService(session).ensure_seed_users(DEFAULT_USER_SEED) - try: - vector_store = VectorStore() - vector_store.create_tables() - vector_store.create_keyword_search_index() - except Exception as exc: - logging.getLogger(__name__).warning("Unable to initialize vector store: %s", exc) +def create_app(settings: Settings | None = None) -> FastAPI: + """Build an independently configurable application for runtime and tests.""" + runtime = settings or get_settings() + runtime.assert_production_safe() + configure_logging(runtime.log_level) + + @asynccontextmanager + async def lifespan(_: FastAPI): + try: + initialize_dependencies(runtime) + except Exception: + logger.exception("Dependency initialization failed; readiness will remain unhealthy") + yield + + application = FastAPI( + title=runtime.app_name, + version="1.0.0", + description="Asynchronous document intelligence and retrieval API.", + lifespan=lifespan, + ) + application.middleware("http")(request_context_middleware) + application.add_middleware( + CORSMiddleware, + allow_origins=runtime.cors_origins, + allow_credentials=True, + allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"], + allow_headers=["Authorization", "Content-Type", runtime.api_key_header, "X-Request-ID"], + ) + application.include_router(api_router) + application.include_router(api_router, prefix="/api/v1") + + @application.get("/health/live", tags=["health"], summary="Process liveness") + def liveness() -> dict[str, str]: + return {"status": "ok"} + + return application + + +app = create_app() diff --git a/app/pyproject.toml b/app/pyproject.toml deleted file mode 100644 index bcb067e..0000000 --- a/app/pyproject.toml +++ /dev/null @@ -1,20 +0,0 @@ -[tool.black] -line-length = 135 -target-version = ['py39'] -skip-string-normalization = false -skip-magic-trailing-comma = false -exclude = ''' -/( - \.venv - | \.mypy_cache - | \.pytest_cache - | \.tox - | \.hg - | \.git - | \.svn - | \.ipynb_checkpoints - | dist - | build - | __pycache__ -)/ -''' \ No newline at end of file diff --git a/app/requirements.txt b/app/requirements.txt deleted file mode 100644 index 2c7264c..0000000 --- a/app/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -alembic==1.13.1 -anthropic==0.31.2 -celery==5.4.0 -fastapi==0.111.1 -graphviz==0.20.3 -instructor==1.4.0 -ipython==8.31.0 -pandas==2.2.3 -psycopg2-binary==2.9.9 -pydantic==2.10.4 -pydantic-settings==2.7.0 -python-frontmatter==1.1.0 -redis==5.0.3 -timescale-vector==0.0.7 -docling -tiktoken -requests -python-dotenv -openai -pytest==8.3.3 -passlib[bcrypt]==1.7.4 -bcrypt==3.1.7 -python-jose[cryptography]==3.3.0 diff --git a/app/services/demo_workspace.py b/app/services/demo_workspace.py new file mode 100644 index 0000000..bc31e65 --- /dev/null +++ b/app/services/demo_workspace.py @@ -0,0 +1,437 @@ +"""Idempotent synthetic workspace data for local demos and screenshots. + +The seed is deliberately opt-in and contains no customer or maintainer data. It +exercises the same event, classification, and vector tables used by the product +so the local experience stays representative of the real application. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Sequence +from uuid import NAMESPACE_URL, UUID, uuid1, uuid5 + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.database.event import Event +from app.database.models import DocumentClassificationHistory, DocumentLabel + +DEMO_SEED_VERSION = "workspace-v1" + + +@dataclass(frozen=True) +class DemoDocument: + slug: str + filename: str + doc_type: str + role: str + label: str + status: str + uploaded_hours_ago: int + due_hours_from_now: int | None + summary: str | None + bullet_points: tuple[str, ...] + chunks: tuple[str, ...] + + +DEMO_DOCUMENTS: tuple[DemoDocument, ...] = ( + DemoDocument( + slug="acme-msa", + filename="Acme Cloud — Master Services Agreement.pdf", + doc_type="Master Services Agreement", + role="Legal", + label="Commercial Agreement", + status="Ready", + uploaded_hours_ago=2, + due_hours_from_now=120, + summary="A two-year cloud services agreement with a 60-day renewal notice, annual price protection, and a negotiated liability cap.", + bullet_points=( + "Initial term ends September 30, 2026 and renews automatically for one year.", + "Written non-renewal notice is required at least 60 days before the term ends.", + "Aggregate liability is capped at fees paid during the preceding 12 months.", + ), + chunks=( + "The initial service term continues through September 30, 2026. The agreement renews automatically for successive one-year periods unless either party gives written notice at least 60 days before the current term ends.", + "Subscription fees may increase only once per renewal term and any increase may not exceed three percent. Acme must provide written notice of a proposed increase with the renewal notice.", + "Except for confidentiality, data protection, and indemnification obligations, each party's aggregate liability is limited to fees paid or payable during the twelve months preceding the event giving rise to the claim.", + "Customer data remains the exclusive property of Northstar Labs. Acme must return or securely delete customer data within thirty days after termination and provide written confirmation on request.", + ), + ), + DemoDocument( + slug="vendor-risk", + filename="Q3 Vendor Risk Review — Atlas Payments.pdf", + doc_type="Compliance Review", + role="Compliance", + label="Vendor Risk", + status="Needs review", + uploaded_hours_ago=5, + due_hours_from_now=-1, + summary=None, + bullet_points=(), + chunks=( + "Atlas Payments completed its SOC 2 Type II audit in May 2026. Two low-severity exceptions concerned delayed access recertification for contractor accounts.", + "The vendor processes payment metadata in the United States and Ireland. No primary account numbers are stored in the analytics environment.", + "Open action: obtain evidence that quarterly contractor access reviews were completed by July 10. Compliance owner: Maya Chen.", + "Residual risk is rated medium until the access-review evidence is accepted. No critical availability, privacy, or financial-control gaps were identified.", + ), + ), + DemoDocument( + slug="northstar-invoice", + filename="Northstar Cloud Invoice — June 2026.pdf", + doc_type="Invoice", + role="Finance", + label="Accounts Payable", + status="Ready", + uploaded_hours_ago=27, + due_hours_from_now=72, + summary="June cloud infrastructure invoice totaling $18,420, including a material increase in GPU compute and a one-time reserved-capacity credit.", + bullet_points=( + "Amount due is $18,420 by July 21, 2026.", + "GPU compute increased 24% month over month.", + "A $1,250 reserved-capacity credit was applied.", + ), + chunks=( + "Invoice NS-2026-0617 covers the billing period June 1 through June 30, 2026. Total charges are $19,670 before credits and $18,420 after credits.", + "GPU compute usage was 8,240 accelerator-hours at a blended cost of $1.42 per hour, an increase of twenty-four percent compared with May.", + "Reserved-capacity commitment credit: $1,250. Payment terms are net 15 and the payment due date is July 21, 2026.", + "Cost center allocation: AI Platform 62%, Document Operations 23%, Shared Infrastructure 15%.", + ), + ), + DemoDocument( + slug="security-handbook", + filename="Employee Security Handbook — 2026.pdf", + doc_type="Employee Policy", + role="HR", + label="Security Policy", + status="Ready", + uploaded_hours_ago=52, + due_hours_from_now=None, + summary="Company-wide security expectations covering device management, data handling, access reviews, and incident reporting.", + bullet_points=( + "Managed devices and phishing-resistant MFA are required for production access.", + "Suspected incidents must be reported within 30 minutes.", + "Restricted data may not be copied to personal storage or consumer AI tools.", + ), + chunks=( + "All employees and contractors must use a company-managed device and phishing-resistant multi-factor authentication before accessing production systems.", + "Suspected security incidents, including accidental disclosure or lost devices, must be reported to the security channel within thirty minutes of discovery.", + "Restricted customer data must not be copied to personal storage, public paste services, or consumer AI tools. Approved enterprise services must enforce retention controls.", + "Managers review privileged access quarterly. Access that is no longer required must be removed within one business day of the review decision.", + ), + ), + DemoDocument( + slug="incident-runbook", + filename="Production Incident Response Runbook.pdf", + doc_type="Operations Runbook", + role="Operations", + label="Incident Response", + status="Ready", + uploaded_hours_ago=126, + due_hours_from_now=None, + summary="Operational playbook for severity assessment, incident command, customer communications, mitigation, and post-incident review.", + bullet_points=( + "SEV-1 incidents require an incident commander and communications lead within 10 minutes.", + "Customer updates are published at least every 30 minutes.", + "A blameless review is due within five business days.", + ), + chunks=( + "For a SEV-1 incident, assign an incident commander, operations lead, and communications lead within ten minutes of declaration.", + "Publish the first customer-facing status update within fifteen minutes. Continue updates at least every thirty minutes until service is restored.", + "Prefer reversible mitigations such as traffic shaping, feature rollback, or queue isolation. Preserve logs and a timestamped decision record throughout the incident.", + "The incident commander owns a blameless post-incident review within five business days, including contributing factors, detection gaps, and tracked corrective actions.", + ), + ), + DemoDocument( + slug="ai-governance", + filename="AI Governance Research Brief.pdf", + doc_type="Research Brief", + role="Compliance", + label="AI Governance", + status="Ready", + uploaded_hours_ago=242, + due_hours_from_now=None, + summary="A practical control framework for evaluating model risk, grounding quality, human review, and auditability in document AI systems.", + bullet_points=( + "High-impact workflows require human approval and traceable source evidence.", + "Retrieval quality should be evaluated independently from answer quality.", + "Model, prompt, and index versions belong in the audit record.", + ), + chunks=( + "Document AI controls should be proportional to impact. High-impact decisions require human approval, visible source evidence, and a documented appeal path.", + "Teams should evaluate retrieval recall independently from answer faithfulness. A fluent answer cannot compensate for missing or irrelevant evidence.", + "Audit records should include model version, prompt version, index version, retrieved chunk identifiers, reviewer action, and final disposition.", + "Monitoring should cover citation validity, abstention behavior, latency, cost, drift in document mix, and the rate of human overrides.", + ), + ), +) + + +def _stable_uuid(*parts: object) -> UUID: + return uuid5(NAMESPACE_URL, "/".join(("doculens", DEMO_SEED_VERSION, *(str(part) for part in parts)))) + + +def _utc_now_naive() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0) + + +def seed_demo_workspace(session: Session, *, embedding_dimensions: int = 1536) -> bool: + """Seed the synthetic workspace once and return whether rows were inserted.""" + marker_id = _stable_uuid("event", "upload", DEMO_DOCUMENTS[0].slug) + if session.get(Event, marker_id) is not None: + return False + + now = _utc_now_naive() + _seed_labels(session, now) + + chunk_ids_by_slug: dict[str, list[UUID]] = {} + for document in DEMO_DOCUMENTS: + chunk_ids_by_slug[document.slug] = _seed_document(session, document, now, embedding_dimensions) + + _seed_qa_history(session, now) + _seed_search_history(session, now, chunk_ids_by_slug) + session.flush() + return True + + +def _seed_labels(session: Session, now: datetime) -> None: + parents: dict[str, UUID] = {} + for role in sorted({document.role for document in DEMO_DOCUMENTS}): + role_id = _stable_uuid("label", "domain", role) + parents[role] = role_id + session.add( + DocumentLabel( + id=role_id, + label_name=role, + label_type="domain", + description=f"Demo {role.lower()} workspace", + created_at=now, + updated_at=now, + ) + ) + + for document in DEMO_DOCUMENTS: + session.add( + DocumentLabel( + id=_stable_uuid("label", document.label), + label_name=document.label, + label_type="label", + parent_label_id=parents[document.role], + description=f"Synthetic demo label for {document.doc_type.lower()} documents", + created_at=now, + updated_at=now, + ) + ) + + +def _seed_document( + session: Session, + document: DemoDocument, + now: datetime, + embedding_dimensions: int, +) -> list[UUID]: + document_id = f"demo-{document.slug}" + uploaded_at = now - timedelta(hours=document.uploaded_hours_ago) + due_at = now + timedelta(hours=document.due_hours_from_now) if document.due_hours_from_now is not None else None + # Timescale's time-partitioned vector table extracts timestamps from UUIDv1 + # keys, so vector records intentionally use uuid1 rather than stable ids. + chunk_ids = [uuid1() for _ in document.chunks] + metadata = { + "assigned_role": document.role, + "status": document.status, + "due_at": due_at.isoformat() if due_at else None, + "owner": _owner_for_role(document.role), + "page_count": max(2, len(document.chunks)), + "demo_content": True, + "demo_seed": DEMO_SEED_VERSION, + } + upload_data = { + "event_type": "document_upload", + "filename": f"data/demo/{document.filename}", + "doc_type": document.doc_type, + "metadata": { + "uploaded_filename": document.filename, + "status": document.status, + "demo_seed": DEMO_SEED_VERSION, + }, + } + document_context = { + "id": document_id, + "filename": document.filename, + "original_filename": document.filename, + "doc_type": document.doc_type, + "chunk_count": len(document.chunks), + "embedded_chunk_count": len(document.chunks), + "vector_ids": [str(chunk_id) for chunk_id in chunk_ids], + "status": document.status, + "metadata": metadata, + } + session.add( + Event( + id=_stable_uuid("event", "upload", document.slug), + data=upload_data, + task_context={ + "event": upload_data, + "metadata": {"document": document_context}, + "nodes": {"DocumentIngestionNode": {"status": "completed", "chunk_count": len(document.chunks)}}, + }, + created_at=uploaded_at, + updated_at=uploaded_at + timedelta(minutes=6), + ) + ) + + if document.summary: + summary_time = uploaded_at + timedelta(minutes=8) + summary_payload = { + "summary": document.summary, + "bullet_points": list(document.bullet_points), + "next_steps": _next_steps(document), + "source_chunk_count": len(document.chunks), + "doc_type": document.doc_type, + "filename": document.filename, + "document_id": document_id, + "generated_at": summary_time.isoformat(), + } + session.add( + Event( + id=_stable_uuid("event", "summary", document.slug), + data={"event_type": "document_summary", "document_ids": [document_id]}, + task_context={"metadata": {"document_summaries": {document_id: summary_payload}}}, + created_at=summary_time, + updated_at=summary_time, + ) + ) + + session.add( + DocumentClassificationHistory( + id=_stable_uuid("classification", document.slug), + document_id=document_id, + label_name=document.label, + confidence=0.91, + source="ai", + classifier_version="demo-fixture-v1", + notes="Synthetic classification included with the local demo workspace.", + classification_metadata={"demo_seed": DEMO_SEED_VERSION, "role": document.role}, + created_at=uploaded_at + timedelta(minutes=7), + ) + ) + + for index, (chunk_id, contents) in enumerate(zip(chunk_ids, document.chunks), start=1): + chunk_metadata = { + "document_id": document_id, + "filename": document.filename, + "original_filename": document.filename, + "doc_type": document.doc_type, + "assigned_role": document.role, + "chunk_index": index, + "page_number": index, + "demo_seed": DEMO_SEED_VERSION, + } + session.execute( + text( + f""" + INSERT INTO embeddings (id, metadata, contents, embedding) + VALUES ( + :id, + CAST(:metadata AS jsonb), + :contents, + array_fill(0.0::real, ARRAY[{embedding_dimensions}])::vector + ) + ON CONFLICT (id) DO NOTHING + """ + ), + {"id": str(chunk_id), "metadata": json.dumps(chunk_metadata), "contents": contents}, + ) + return chunk_ids + + +def _seed_qa_history(session: Session, now: datetime) -> None: + document = DEMO_DOCUMENTS[0] + document_id = f"demo-{document.slug}" + query = "When do we need to give notice if we do not want the Acme agreement to renew?" + created_at = now - timedelta(hours=1, minutes=18) + reference = { + "reference": f"{document.filename} · page 1 · chunk 1", + "document_id": document_id, + "filename": document.filename, + "chunk_index": 1, + } + session.add( + Event( + id=_stable_uuid("event", "qa", "acme-renewal"), + data={"event_type": "qa_query", "query": query, "filters": {"document_id": document_id}}, + task_context={ + "event": {"event_type": "qa_query", "query": query}, + "metadata": { + "qa": { + "answer": "Send written non-renewal notice no later than August 1, 2026. The agreement requires at least 60 days' notice before the September 30 term end.", + "reasoning": "The renewal clause states both the term end date and the 60-day notice requirement.", + "confidence": 0.96, + "citations": [reference["reference"]], + } + }, + "nodes": {"QAQueryNode": {"chunk_references": [reference]}}, + }, + created_at=created_at, + updated_at=created_at + timedelta(seconds=3), + ) + ) + + +def _seed_search_history(session: Session, now: datetime, chunk_ids_by_slug: dict[str, list[UUID]]) -> None: + query = "documents with upcoming renewal or payment deadlines" + results = [] + for document in (DEMO_DOCUMENTS[0], DEMO_DOCUMENTS[2]): + results.append( + { + "id": str(chunk_ids_by_slug[document.slug][0]), + "contents": document.chunks[0], + "distance": 0.12 if document.slug == "acme-msa" else 0.19, + "metadata": { + "document_id": f"demo-{document.slug}", + "filename": document.filename, + "doc_type": document.doc_type, + "chunk_index": 1, + }, + } + ) + created_at = now - timedelta(minutes=42) + session.add( + Event( + id=_stable_uuid("event", "search", "upcoming-deadlines"), + data={"event_type": "search_query", "query": query, "filters": {}}, + task_context={ + "metadata": {"search": {"query": query, "filters": {}, "limit": 5, "results": results}}, + "nodes": { + "SemanticSearchNode": { + "limit": 5, + "result_count": len(results), + "preview": results, + "results_truncated": False, + } + }, + }, + created_at=created_at, + updated_at=created_at + timedelta(seconds=1), + ) + ) + + +def _owner_for_role(role: str) -> str: + return { + "Compliance": "Maya Chen", + "Finance": "Jordan Ellis", + "HR": "Priya Shah", + "Legal": "Elena Torres", + "Operations": "Noah Williams", + }.get(role, "Demo Analyst") + + +def _next_steps(document: DemoDocument) -> Sequence[str]: + if document.role == "Legal": + return ("Confirm renewal intent with the service owner.", "Calendar the August 1 notice deadline.") + if document.role == "Finance": + return ("Validate the GPU usage increase.", "Route invoice NS-2026-0617 for approval.") + return ("Share the summary with the document owner.",) diff --git a/app/services/vector_store.py b/app/services/vector_store.py index 81eaf05..c32e10c 100644 --- a/app/services/vector_store.py +++ b/app/services/vector_store.py @@ -1,6 +1,8 @@ import logging +from collections import OrderedDict from datetime import datetime from pathlib import Path +from threading import Lock from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd @@ -25,6 +27,9 @@ class VectorStore: """A class for managing vector operations and database interactions.""" + _embedding_cache: "OrderedDict[tuple[str, str], List[float]]" = OrderedDict() + _cache_lock = Lock() + def __init__(self, local: bool = False): """ Initialize the VectorStore with settings, OpenAI client, and Timescale Vector client. @@ -33,12 +38,17 @@ def __init__(self, local: bool = False): local (bool): If True, overrides .env to use localhost DB for running outside Docker. """ self.settings = get_settings() - self.openai_client = OpenAI(api_key=self.settings.llm.openai.api_key) + self.openai_client = OpenAI( + api_key=self.settings.llm.openai.api_key, + timeout=self.settings.provider_timeout_seconds, + max_retries=self.settings.llm.openai.max_retries, + ) self.embedding_model = self.settings.llm.openai.embedding_model self.vector_settings = self.settings.database.vector_store - self.settings.database.local = local + database_url = self.settings.database.service_url_for(local=local) + self.database_url = database_url self.vec_client = client.Sync( - self.settings.database.service_url, + database_url, self.vector_settings.table_name, self.vector_settings.embedding_dimensions, time_partition_interval=self.vector_settings.time_partition_interval, @@ -52,7 +62,7 @@ def create_keyword_search_index(self): ON {self.vector_settings.table_name} USING gin(to_tsvector('english', contents)); """ try: - with psycopg2.connect(self.settings.database.service_url) as conn: + with psycopg2.connect(self.database_url) as conn: with conn.cursor() as cur: cur.execute(create_index_sql) conn.commit() @@ -70,16 +80,39 @@ def get_embedding(self, text: str) -> List[float]: Returns: A list of floats representing the embedding. """ - text = text.replace("\n", " ") - embedding = ( - self.openai_client.embeddings.create( - input=[text], - model=self.embedding_model, + return self.embed_texts([text])[0] + + def embed_texts(self, texts: List[str], model: Optional[str] = None) -> List[List[float]]: + """Embed texts in bounded batches with a small process-local LRU cache.""" + selected_model = model or self.embedding_model + results: List[Optional[List[float]]] = [None] * len(texts) + missing: List[tuple[int, str, tuple[str, str]]] = [] + with self._cache_lock: + for index, text in enumerate(texts): + key = (selected_model, text.replace("\n", " ")) + cached = self._embedding_cache.get(key) + if cached is None: + missing.append((index, key[1], key)) + else: + self._embedding_cache.move_to_end(key) + results[index] = cached + + batch_size = self.settings.embedding_batch_size + for offset in range(0, len(missing), batch_size): + batch = missing[offset : offset + batch_size] + response = self.openai_client.embeddings.create( + input=[text for _, text, _ in batch], model=selected_model ) - .data[0] - .embedding - ) - return embedding + for (index, _, key), item in zip(batch, response.data): + vector = item.embedding + results[index] = vector + if self.settings.embedding_cache_size: + with self._cache_lock: + self._embedding_cache[key] = vector + self._embedding_cache.move_to_end(key) + while len(self._embedding_cache) > self.settings.embedding_cache_size: + self._embedding_cache.popitem(last=False) + return [vector for vector in results if vector is not None] def create_tables(self) -> None: """Create the necessary tablesin the database""" @@ -252,7 +285,7 @@ def fetch_document_chunks( """ params.append(limit) - with psycopg2.connect(self.settings.database.service_url) as conn: + with psycopg2.connect(self.database_url) as conn: with conn.cursor(cursor_factory=RealDictCursor) as cur: cur.execute(query, params) rows = cur.fetchall() @@ -341,7 +374,7 @@ def keyword_search( LIMIT %s """ - with psycopg2.connect(self.settings.database.service_url) as conn: + with psycopg2.connect(self.database_url) as conn: with conn.cursor(cursor_factory=RealDictCursor) as cur: cur.execute(search_sql, (query, limit)) results = cur.fetchall() diff --git a/app/start.sh b/app/start.sh index 78ed7b1..80970e9 100755 --- a/app/start.sh +++ b/app/start.sh @@ -1,9 +1,7 @@ #!/bin/sh -# Ensure Python can resolve the project package when running inside the container -export PYTHONPATH=${PYTHONPATH:-/} +if [ "${DOCULENS_RUN_MIGRATIONS:-false}" = "true" ]; then + alembic -c /workspace/app/alembic.ini upgrade head +fi -# Uncomment next line to automatically apply Alembic database migrations -# alembic upgrade head - -exec uvicorn app.main:app --host 0.0.0.0 --port 8080 --reload +exec uvicorn app.main:app --host 0.0.0.0 --port 8080 --proxy-headers diff --git a/docker/Caddyfile.showcase b/docker/Caddyfile.showcase new file mode 100644 index 0000000..67db64c --- /dev/null +++ b/docker/Caddyfile.showcase @@ -0,0 +1,26 @@ +{$CADDY_DOMAIN} { + encode zstd gzip + + log { + output stdout + format console + level info + } + + header { + -Server + Referrer-Policy "strict-origin-when-cross-origin" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Permissions-Policy "camera=(), microphone=(), geolocation=()" + } + + @api path /events* /auth* /api* /health* /docs* /redoc* /openapi.json + handle @api { + reverse_proxy api:8080 + } + + handle { + reverse_proxy frontend:80 + } +} diff --git a/docker/Dockerfile.api b/docker/Dockerfile.api index 3131c5b..e1f93b6 100644 --- a/docker/Dockerfile.api +++ b/docker/Dockerfile.api @@ -6,21 +6,21 @@ RUN apt-get update \ python3-dev \ libpq-dev \ build-essential \ + curl \ && rm -rf /var/lib/apt/lists/* ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 -WORKDIR /app +WORKDIR /workspace -ADD app/requirements.txt . -RUN pip install -r requirements.txt +COPY pyproject.toml README.md ./ +COPY app ./app +RUN pip install --no-cache-dir . -ADD app/ /app - -RUN chmod +x /app/start.sh +RUN chmod +x /workspace/app/start.sh EXPOSE 8080 -CMD ["/app/start.sh"] \ No newline at end of file +CMD ["/workspace/app/start.sh"] diff --git a/docker/Dockerfile.celery b/docker/Dockerfile.celery index acf7acb..85ce31a 100644 --- a/docker/Dockerfile.celery +++ b/docker/Dockerfile.celery @@ -19,16 +19,14 @@ ENV DOCLING_CACHE_DIR=${HOME}/.cache/docling RUN mkdir -p "${EASYOCR_HOME}" "${DOCLING_CACHE_DIR}" \ && chown -R celery:celery /home/celery -WORKDIR /app +WORKDIR /workspace -ADD app/requirements.txt . +COPY pyproject.toml README.md ./ +COPY app ./app +RUN pip install --no-cache-dir . -RUN pip install -r requirements.txt watchdog - -ADD app/ /app - -RUN chown -R celery:celery /app +RUN chown -R celery:celery /workspace USER celery -CMD ["sh", "-c", "watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A config.celery_config worker --loglevel=info --concurrency=1"] +CMD ["celery", "-A", "app.config.celery_config", "worker", "--loglevel=info", "--concurrency=2"] diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend new file mode 100644 index 0000000..f3ad07f --- /dev/null +++ b/docker/Dockerfile.frontend @@ -0,0 +1,12 @@ +FROM node:20-alpine AS build + +WORKDIR /workspace +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm ci +COPY frontend/ ./ +RUN npm run build + +FROM nginx:1.27-alpine +COPY --from=build /workspace/dist /usr/share/nginx/html +COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/docker/docker-compose.showcase.yml b/docker/docker-compose.showcase.yml new file mode 100644 index 0000000..d20ed26 --- /dev/null +++ b/docker/docker-compose.showcase.yml @@ -0,0 +1,72 @@ +name: ${PROJECT_NAME:-doculens_showcase} + +services: + frontend: + build: + context: .. + dockerfile: docker/Dockerfile.frontend + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/health"] + interval: 10s + timeout: 3s + retries: 5 + + api: + build: + context: .. + dockerfile: docker/Dockerfile.api + env_file: + - ../${DOCULENS_ENV_FILE:-.env.showcase} + depends_on: + database: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/health/live"] + interval: 10s + timeout: 3s + retries: 10 + + caddy: + image: caddy:2.10-alpine + depends_on: + api: + condition: service_healthy + frontend: + condition: service_healthy + environment: + CADDY_DOMAIN: ${CADDY_DOMAIN:?Set CADDY_DOMAIN in .env.showcase} + ports: + - "80:80" + - "443:443" + - "443:443/udp" + restart: unless-stopped + volumes: + - ./Caddyfile.showcase:/etc/caddy/Caddyfile:ro + - caddy_config:/config + - caddy_data:/data + + database: + image: timescale/timescaledb-ha:pg16 + environment: + POSTGRES_DB: ${DATABASE_NAME:?Set DATABASE_NAME in .env.showcase} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env.showcase} + POSTGRES_USER: ${DATABASE_USER:?Set DATABASE_USER in .env.showcase} + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - postgres_data:/home/postgres/pgdata + +volumes: + caddy_config: + caddy_data: + postgres_data: + +networks: + default: + name: "${PROJECT_NAME:-doculens_showcase}_network" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 45fc900..1063918 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,29 +1,47 @@ services: + frontend: + build: + context: .. + dockerfile: docker/Dockerfile.frontend + container_name: "${PROJECT_NAME}_frontend" + ports: + - "127.0.0.1:5173:80" + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/health"] + interval: 10s + timeout: 3s + retries: 5 api: build: context: .. dockerfile: docker/Dockerfile.api container_name: "${PROJECT_NAME}_api" - environment: - - PYTHONPATH=/ depends_on: - - database - - redis + database: + condition: service_healthy + redis: + condition: service_healthy ports: - "127.0.0.1:8080:8080" - restart: always + restart: unless-stopped + env_file: + - ../.env + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/health/live"] + interval: 10s + timeout: 3s + retries: 5 volumes: - - ./../app/:/app + - ingestion_data:/workspace/data/ingestion caddy: container_name: "${PROJECT_NAME}_caddy" - env_file: - - ./.env image: caddy:latest ports: - "80:80" - "127.0.0.1:2019:2019" - "443:443" - restart: always + restart: unless-stopped volumes: - ./Caddyfile:/etc/caddy/Caddyfile - /var/log/caddy:/var/log/caddy @@ -35,14 +53,18 @@ services: dockerfile: docker/Dockerfile.celery container_name: "${PROJECT_NAME}_celery_worker" depends_on: - - database - - redis - restart: always + database: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + env_file: + - ../.env environment: - EASYOCR_HOME=/home/celery/.EasyOCR - DOCLING_CACHE_DIR=/home/celery/.cache/docling volumes: - - ./../app:/app + - ingestion_data:/workspace/data/ingestion - easyocr_cache:/home/celery/.EasyOCR - docling_cache:/home/celery/.cache/docling database: @@ -53,8 +75,13 @@ services: POSTGRES_PASSWORD: ${DATABASE_PASSWORD} POSTGRES_USER: ${DATABASE_USER} ports: - - "127.0.0.1:5432:5432" - restart: always + - "127.0.0.1:${DATABASE_EXPOSE_PORT:-5432}:5432" + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 volumes: - postgres_data:/home/postgres/pgdata @@ -70,8 +97,8 @@ services: timeout: 10s image: redis:latest ports: - - "127.0.0.1:6379:6379" - restart: always + - "127.0.0.1:${REDIS_EXPOSE_PORT:-6379}:6379" + restart: unless-stopped volumes: - redis_data:/data @@ -83,9 +110,9 @@ volumes: redis_data: easyocr_cache: docling_cache: + ingestion_data: networks: default: driver: bridge - external: true name: "${PROJECT_NAME}_network" diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..e3527ca --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,16 @@ +server { + listen 80; + server_name _; + root /usr/share/nginx/html; + index index.html; + + location / { + try_files $uri $uri/ /index.html; + } + + location = /health { + access_log off; + return 200 "ok\n"; + add_header Content-Type text/plain; + } +} diff --git a/docs/deploy-showcase.md b/docs/deploy-showcase.md new file mode 100644 index 0000000..d8a347a --- /dev/null +++ b/docs/deploy-showcase.md @@ -0,0 +1,82 @@ +# Deploy the portfolio showcase + +This deployment is intentionally a **read-only product tour**, not a public SaaS trial. Visitors can inspect the synthetic workspace, navigate documents, use global search, and review grounded answers with citations. Uploads, live model calls, classification changes, and destructive actions are removed in the UI and rejected by the API. + +## Recommended shape + +Use one Ubuntu VPS with at least 2 vCPUs and 4 GB RAM. The showcase runs four containers on a private Compose network: + +```text +Internet → Caddy (HTTPS) → React frontend + → FastAPI read API → PostgreSQL + pgvector +``` + +There is no Celery worker in showcase mode because visitors cannot enqueue work. This materially reduces memory use and removes surprise AI-provider spend. The tradeoff is deliberate: the hosted site demonstrates completed pipeline outputs rather than processing arbitrary visitor files. + +## 1. Prepare DNS and the server + +1. Create an Ubuntu VPS and add an `A` record such as `demo.example.com` pointing to its public IPv4 address. +2. Allow inbound TCP 22, 80, and 443 and UDP 443. Keep PostgreSQL private. +3. Install Docker Engine and the Compose plugin using Docker's official Ubuntu instructions. +4. Clone the repository onto the server. + +## 2. Configure the showcase + +```bash +cp .env.showcase.example .env.showcase +openssl rand -hex 32 +openssl rand -base64 36 +``` + +Edit `.env.showcase`: + +- set `CADDY_DOMAIN` to the DNS name; +- put the same HTTPS origin in `DOCULENS_CORS_ORIGINS`; +- replace the auth secret and database password with the generated values. + +Do not add real provider keys. The API blocks all state-changing event routes before a pipeline or provider can run. + +## 3. Launch + +```bash +make showcase-up +docker compose --env-file .env.showcase -f docker/docker-compose.showcase.yml ps +curl --fail https://demo.example.com/health/live +``` + +Caddy obtains and renews the TLS certificate automatically after DNS resolves and ports 80/443 are reachable. The first API startup applies Alembic migrations, creates the vector table/index if absent, and inserts the versioned synthetic workspace. Seeding is idempotent, so restarts do not duplicate data. + +## 4. Update and roll back + +Before an update, record the deployed commit and take a database dump: + +```bash +git rev-parse HEAD +docker compose --env-file .env.showcase -f docker/docker-compose.showcase.yml exec -T database \ + pg_dump -U postgres -d doculens -Fc > doculens-showcase.dump +git pull --ff-only +make showcase-up +``` + +To roll back application code, check out the previously recorded commit and run `make showcase-up` again. Restore the database dump only when a migration is not backward-compatible; routine UI/API rollbacks should not require it. + +## 5. Verify the public boundary + +```bash +# Read succeeds. +curl --fail https://demo.example.com/api/v1/events/config + +# Mutation must return HTTP 403. +curl -i https://demo.example.com/api/v1/events \ + -H 'Content-Type: application/json' \ + -d '{"event_type":"qa_query","query":"test"}' +``` + +Also verify `/`, `/app`, `/app/qa`, `/app/pipeline`, and `/docs` in a private browser window. `/app` should open without credentials and show a visible **Read-only showcase** label. + +## Operational notes + +- Back up the `postgres_data` volume or schedule `pg_dump`; Caddy certificates live in `caddy_data`. +- Use provider/VPS uptime monitoring against `/health/live` and the homepage. +- Keep Docker and the host patched. Rebuild monthly even when application code has not changed. +- This manifest is for a single-node portfolio demo. A real customer deployment would need user-scoped authorization, object storage, managed secrets, a worker, durable provider budgets, and tenant isolation. diff --git a/docs/design-system.md b/docs/design-system.md new file mode 100644 index 0000000..6fafb6e --- /dev/null +++ b/docs/design-system.md @@ -0,0 +1,37 @@ +# DocuLens design system + +DocuLens uses a quiet, evidence-first visual language designed for long document-review sessions. The system intentionally favors hierarchy, legibility, and provenance over decorative dashboard density. + +## Foundations + +- **Typography:** Manrope carries interface and editorial copy; DM Mono is reserved for metadata, shortcuts, identifiers, and technical labels. +- **Color:** porcelain surfaces and ink text form the neutral foundation. Cobalt communicates focus and action; amber marks evidence and citations. Semantic success, warning, and danger colors are limited to real system state. +- **Spacing:** an eight-pixel rhythm governs layout, with four-pixel increments used inside compact controls. +- **Shape:** controls use restrained 10–14 px radii; large marketing surfaces use broader radii to create hierarchy without turning every element into a card. +- **Elevation:** borders establish most grouping. Shadows are reserved for floating navigation, dialogs, menus, and the primary product mockup. + +All values are exposed as semantic CSS variables in `frontend/src/styles/globals.css`. Components consume roles such as `background`, `foreground`, `muted`, `border`, and `primary` instead of raw palette values. + +## Interaction principles + +1. Primary actions are visually unambiguous; secondary actions remain quiet until hover or focus. +2. Every interactive control has a keyboard-visible focus ring and a descriptive accessible label. +3. Motion communicates entrance, progress, or state change. It never blocks input and is disabled when the operating system requests reduced motion. +4. Dark mode is a first-class theme, persisted locally and initialized from the operating-system preference. +5. Product pages preserve working context through a stable sidebar and command search, while public pages use a simpler editorial frame. + +## Component boundaries + +```text +components/ +├── brand/ product identity primitives +├── layout/ application shell and navigation +├── ui/ reusable, domain-neutral controls +└── dashboard/ data visualization and workspace composition +``` + +Domain pages compose primitives but do not redefine their visual states. This keeps accessibility and interaction behavior consistent while allowing document workflows to evolve independently. + +## Performance tradeoffs + +DocuLens uses CSS transitions and keyframes instead of an animation library. This limits orchestration features but avoids adding a runtime dependency for effects that are short, local, and deterministic. Authenticated pages are split at route boundaries so marketing visitors download only the public experience. diff --git a/docs/engineering-notes.md b/docs/engineering-notes.md new file mode 100644 index 0000000..17a3162 --- /dev/null +++ b/docs/engineering-notes.md @@ -0,0 +1,43 @@ +# Engineering notes + +## Scope of the production hardening pass + +This project deliberately remains a modular monolith. The main risks were inconsistent configuration, implicit global clients, unsafe startup defaults, coarse retrieval chunks, unbounded embedding requests, permissive API inputs, and a developer workflow that differed from CI. + +## Major changes + +### Application lifecycle and API versioning + +FastAPI now uses an application factory and lifespan hook. This makes isolated testing possible and gives startup policy one owner. `/api/v1` is the canonical namespace; legacy routes are aliases so the React client is not broken. Database initialization is configurable, demo accounts are opt-in, and production rejects demo credentials. + +Tradeoff: aliases duplicate paths in OpenAPI during the compatibility window. Functionality is otherwise unchanged. + +### Configuration and security + +Operational limits, CORS, environment, logging, provider timeout, chunk size, embedding batches, cache size, and upload size now live in the validated settings model. Vector-store construction no longer mutates the cached database configuration. Uploads are streamed with a hard byte limit and partial files are removed after failure. + +Tradeoff: invalid configuration now fails early instead of being tolerated. This is intentional and does not affect correctly configured environments. + +### Retrieval and embeddings + +The default chunk target changed from the embedding model's 8,191-token ceiling to 800 tokens. Embeddings are requested in bounded batches through an injectable vector-store seam, reuse a bounded in-memory LRU, and retain page/chunk provenance for citations. Recall@K and MRR helpers make future retrieval changes measurable. + +Tokenizer construction is lazy because tiktoken may fetch its vocabulary on first use. API imports, health checks, and unit-test collection therefore remain network-independent; the worker pays initialization cost only when it processes an embedding job. Docling is pinned to the Python 3.12-tested 2.95.0 release to prevent expensive resolver backtracking and surprise parser changes. + +Tradeoff: smaller chunks create more rows and can increase embedding cost. They reduce irrelevant context per hit and make citations more precise. The exact value should be tuned against a labelled corpus. + +### Reliability and observability + +Requests receive correlation ids and emit duration/status logs. Celery uses late acknowledgement, worker-loss rejection, bounded execution time, single-message prefetch, and expiring results. Docker services have readiness ordering and restart policies appropriate for development. + +Tradeoff: late-acknowledged tasks can execute more than once after a crash. Pipeline side effects therefore need to stay idempotent; vector upserts and event-based checks provide a foundation, but full idempotency deserves explicit integration coverage. + +### Developer workflow + +The repository root now owns packaging, Ruff, Pyright, pytest, coverage, Make targets, pre-commit, and a documented environment template. CI installs the same editable development package and executes the same quality gates. + +Tradeoff: contributors using the former `app/requirements.txt` workflow must move to the root editable install. One dependency definition is worth the small migration cost because CI, local development, and containers can no longer drift. + +## Intentionally deferred + +The 1,200-line event endpoint module should be split by documents, labels, QA, search, and insights. Doing that safely requires route-level characterization tests first; moving it mechanically would create diff volume without improving behavior. Durable event status/error columns also require a migration and UI contract, so they are recorded on the roadmap instead of hidden inside JSON. diff --git a/docs/images/landing.jpg b/docs/images/landing.jpg new file mode 100644 index 0000000..77f2bfc Binary files /dev/null and b/docs/images/landing.jpg differ diff --git a/docs/images/login.jpg b/docs/images/login.jpg new file mode 100644 index 0000000..f6f3c0b Binary files /dev/null and b/docs/images/login.jpg differ diff --git a/docs/images/qa-studio.jpg b/docs/images/qa-studio.jpg new file mode 100644 index 0000000..c59f307 Binary files /dev/null and b/docs/images/qa-studio.jpg differ diff --git a/docs/images/work-queues.jpg b/docs/images/work-queues.jpg new file mode 100644 index 0000000..54f4f17 Binary files /dev/null and b/docs/images/work-queues.jpg differ diff --git a/docs/images/workspace-overview.jpg b/docs/images/workspace-overview.jpg new file mode 100644 index 0000000..1bbfde3 Binary files /dev/null and b/docs/images/workspace-overview.jpg differ diff --git a/frontend/README.md b/frontend/README.md index d2e7761..1d54c1f 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,73 +1,38 @@ -# React + TypeScript + Vite +# DocuLens web -This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. +The DocuLens web application combines a public product experience with an authenticated document-intelligence workspace. It is built with React 19, TypeScript, Vite, TanStack Query, and a small token-driven component system. -Currently, two official plugins are available: +## Routes -- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh -- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh +| Route | Experience | +| --- | --- | +| `/` | Public product landing page | +| `/login` | Workspace authentication | +| `/app` | Document intake and workspace overview | +| `/app/qa` | Citation-first document chat | +| `/app/work-queues` | Review and processing queues | +| `/app/pipeline` | Pipeline activity and diagnostics | +| `/app/settings` | Workspace preferences | -## React Compiler +Protected routes redirect unauthenticated users to `/login`. Product routes are lazy-loaded so the public landing page does not pay the cost of the authenticated workspace bundle. -The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). +## Local development -## Expanding the ESLint configuration - -If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: - -```js -export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{ts,tsx}'], - extends: [ - // Other configs... +```bash +npm ci +npm run dev +``` - // Remove tseslint.configs.recommended and replace with this - tseslint.configs.recommendedTypeChecked, - // Alternatively, use this for stricter rules - tseslint.configs.strictTypeChecked, - // Optionally, add this for stylistic rules - tseslint.configs.stylisticTypeChecked, +Vite proxies `/api` to `http://localhost:8080` by default. Run the backend stack from the repository root with `make up`, or configure the target in `vite.config.ts`. - // Other configs... - ], - languageOptions: { - parserOptions: { - project: ['./tsconfig.node.json', './tsconfig.app.json'], - tsconfigRootDir: import.meta.dirname, - }, - // other options... - }, - }, -]) +```bash +npm run lint +npm run build +npm run preview ``` -You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: +## Design system -```js -// eslint.config.js -import reactX from 'eslint-plugin-react-x' -import reactDom from 'eslint-plugin-react-dom' +Global semantic tokens, motion primitives, focus treatment, and dark-mode behavior live in `src/styles/globals.css`. Reusable primitives live in `src/components/ui`; product identity is isolated in `src/components/brand`. The interface uses native CSS motion with a `prefers-reduced-motion` fallback to keep the initial bundle small. -export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{ts,tsx}'], - extends: [ - // Other configs... - // Enable lint rules for React - reactX.configs['recommended-typescript'], - // Enable lint rules for React DOM - reactDom.configs.recommended, - ], - languageOptions: { - parserOptions: { - project: ['./tsconfig.node.json', './tsconfig.app.json'], - tsconfigRootDir: import.meta.dirname, - }, - // other options... - }, - }, -]) -``` +The complete system rationale is documented in [`../docs/design-system.md`](../docs/design-system.md). diff --git a/frontend/index.html b/frontend/index.html index 072a57e..776bce4 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -2,9 +2,11 @@ - + - frontend + + + DocuLens AI — Answers that show their work
diff --git a/frontend/public/vite.svg b/frontend/public/vite.svg deleted file mode 100644 index e7b8dfb..0000000 --- a/frontend/public/vite.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b280c8b..ee70771 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,17 +4,18 @@ import { Upload, X } from 'lucide-react'; import { AppShell } from './components/layout/AppShell'; import { Button } from './components/ui/button'; -import { useSettings } from './settings/useSettings'; import { useEventNotifications } from './hooks/useEventNotifications'; import { DocumentUploadForm } from './components/DocumentUploadForm'; import type { UploadResponse } from './api/types'; +import { Badge } from './components/ui/badge'; +import { useSettings } from './settings/useSettings'; export function AppLayout() { - const { serverConfig } = useSettings(); const { pathname } = useLocation(); const navigate = useNavigate(); - const appName = serverConfig?.app_name ?? 'DocuLens AI'; - const isIntake = pathname === '/'; + const isIntake = pathname === '/app'; + const { serverConfig } = useSettings(); + const isShowcaseReadOnly = Boolean(serverConfig?.showcase_read_only); const [isUploadOpen, setIsUploadOpen] = useState(false); useEventNotifications(); @@ -30,7 +31,7 @@ export function AppLayout() { const handleUploadComplete = useCallback( (response: UploadResponse) => { setIsUploadOpen(false); - navigate('/pipeline', { replace: false, state: { recentUpload: response.event_id } }); + navigate('/app/pipeline', { replace: false, state: { recentUpload: response.event_id } }); }, [navigate], ); @@ -49,29 +50,34 @@ export function AppLayout() { return ( <> - - New Upload - + isShowcaseReadOnly ? ( + + Read-only showcase + + ) : ( + + ) } > - {isUploadOpen ? ( -
+ {isUploadOpen && !isShowcaseReadOnly ? ( +

- Upload & ingest documents + Add documents

-

Drop in PDFs or scans to fast-track routing and review.

+

Upload PDFs, scans, or text files to your workspace.

+ {!isShowcaseReadOnly ? <> - {isArchived ? ( + : Curated result}
@@ -557,7 +566,7 @@ export function DocumentDetail({ document, onEventQueued }: DocumentDetailProps) {pendingAction && confirmConfig ? (
-
+
@@ -633,7 +642,7 @@ export function DocumentDetail({ document, onEventQueued }: DocumentDetailProps) Document classification - {displaySource ? `Latest label generated by ${displaySource}.` : 'Predicted label for this document.'} + {displaySource ? `Latest label generated by ${displaySource.toLowerCase() === 'ai' ? 'AI' : displaySource}.` : 'Predicted label for this document.'} @@ -674,7 +683,7 @@ export function DocumentDetail({ document, onEventQueued }: DocumentDetailProps)
) : null} -
+ {!isShowcaseReadOnly ?

Override classification

{candidateLabels.length ? (
@@ -729,7 +738,7 @@ export function DocumentDetail({ document, onEventQueued }: DocumentDetailProps) {isOverriding ? 'Saving…' : 'Save override'}
-
+
: null}

Classification history

@@ -785,10 +794,10 @@ export function DocumentDetail({ document, onEventQueued }: DocumentDetailProps) Summaries help reviewers get context before diving into stage progress. - + : Curated result} @@ -803,15 +812,15 @@ function ProcessingTimeline({ stages }: { stages: ReturnType Processing status - Where this document is in the digitisation journey. + Extraction, indexing, summary, and assignment progress. {stages.map((stage) => ( -
+
{stage.state === 'complete' ? ( - + ) : stage.state === 'active' ? ( - + ) : ( )} diff --git a/frontend/src/components/DocumentList.tsx b/frontend/src/components/DocumentList.tsx index 4d4e4a7..70732d2 100644 --- a/frontend/src/components/DocumentList.tsx +++ b/frontend/src/components/DocumentList.tsx @@ -1,11 +1,10 @@ import { useMemo, useState } from 'react'; -import { Clock, FileText } from 'lucide-react'; +import { Clock, FileText, Search } from 'lucide-react'; import type { DocumentEntry } from '../api/types'; import { cn } from '../lib/utils'; import { inferDueDate, inferRole, inferStatus, formatDateTime } from '../lib/routing'; import { Badge } from './ui/badge'; -import { Card, CardContent, CardHeader, CardTitle } from './ui/card'; import { Input } from './ui/input'; export interface DocumentListProps { @@ -30,23 +29,21 @@ export function DocumentList({ documents, onSelect, selectedId }: DocumentListPr }, [documents, filter]); return ( - - -
- Documents - - {documents.length} - -
+
+
+ setFilter(event.target.value)} - className="h-10 rounded-lg border-platinum-600" + className="h-10 rounded-xl border-border/70 bg-surface-subtle pl-9" /> - - + + {filtered.length} + +
+
{filtered.map((doc) => { const isSelected = doc.document_id === selectedId; @@ -60,12 +57,12 @@ export function DocumentList({ documents, onSelect, selectedId }: DocumentListPr key={doc.document_id} onClick={() => onSelect(doc)} className={cn( - 'w-full rounded-lg border border-platinum-600 bg-white px-4 py-3 text-left transition-colors hover:border-lapis-500/40 hover:bg-sky-blue-900 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-lapis-500/40', - isSelected ? 'border-lapis-500 bg-sky-blue-900 shadow-sm' : '', + 'w-full rounded-xl border border-border/70 bg-background px-4 py-3 text-left transition-colors hover:border-primary/30 hover:bg-surface-subtle focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-primary/30', + isSelected ? 'border-primary/40 bg-primary/[0.06] shadow-sm ring-1 ring-primary/15' : '', )} >
-
+
@@ -103,7 +100,7 @@ export function DocumentList({ documents, onSelect, selectedId }: DocumentListPr
) : null}
- - +
+
); } diff --git a/frontend/src/components/DocumentUploadForm.tsx b/frontend/src/components/DocumentUploadForm.tsx index b23bae0..8659972 100644 --- a/frontend/src/components/DocumentUploadForm.tsx +++ b/frontend/src/components/DocumentUploadForm.tsx @@ -266,7 +266,7 @@ export function DocumentUploadForm({ onUploaded }: DocumentUploadFormProps) { value={docType} onChange={(event) => setDocType(event.target.value)} /> -

Optional label used by downstream pipelines.

+

Optional type used to organize the document and guide review.

@@ -280,7 +280,7 @@ export function DocumentUploadForm({ onUploaded }: DocumentUploadFormProps) { {metadataError ? (

{metadataError}

) : ( -

Attach structured hints for routing or extraction.

+

Attach optional structured context for processing and assignment.

)}
@@ -297,7 +297,7 @@ export function DocumentUploadForm({ onUploaded }: DocumentUploadFormProps) { {queue.map((item) => (
  • diff --git a/frontend/src/components/brand/Logo.tsx b/frontend/src/components/brand/Logo.tsx new file mode 100644 index 0000000..ac79777 --- /dev/null +++ b/frontend/src/components/brand/Logo.tsx @@ -0,0 +1,19 @@ +import { Link } from 'react-router-dom'; +import { ScanText } from 'lucide-react'; + +import { cn } from '../../lib/utils'; + +export function Logo({ compact = false, className }: { compact?: boolean; className?: string }) { + return ( + + + + + {!compact ? ( + + DocuLens AI + + ) : null} + + ); +} diff --git a/frontend/src/components/dashboard/PersonaQuickstart.tsx b/frontend/src/components/dashboard/PersonaQuickstart.tsx index a6a46fc..9592616 100644 --- a/frontend/src/components/dashboard/PersonaQuickstart.tsx +++ b/frontend/src/components/dashboard/PersonaQuickstart.tsx @@ -53,7 +53,7 @@ interface QuickstartActionCardProps { function QuickstartActionCard({ title, description, href }: QuickstartActionCardProps) { return ( -
    +

    {title}

    {description}

    diff --git a/frontend/src/components/dashboard/personaConfig.ts b/frontend/src/components/dashboard/personaConfig.ts index e3def19..39fe813 100644 --- a/frontend/src/components/dashboard/personaConfig.ts +++ b/frontend/src/components/dashboard/personaConfig.ts @@ -24,7 +24,7 @@ export const PERSONA_CONFIG: Record = { { title: 'QA the latest batch', description: 'Run targeted QA across priority documents with citations.', - href: '/qa', + href: '/app/qa', }, { title: 'Approve summaries', @@ -41,17 +41,17 @@ export const PERSONA_CONFIG: Record = { { title: 'Scan today’s KPIs', description: 'Check dashboards for SLA risks and backlog growth.', - href: '/work-queues', + href: '/app/work-queues', }, { title: 'Coach reviewers', description: 'Spot coaching opportunities by filtering documents awaiting approval.', - href: '/pipeline', + href: '/app/pipeline', }, { title: 'Broadcast status', description: 'Share quality and savings snapshots upstream.', - href: '/settings', + href: '/app/settings', }, ], }, @@ -63,7 +63,7 @@ export const PERSONA_CONFIG: Record = { { title: 'Triage review queue', description: 'Filter for documents awaiting human approval and add feedback.', - href: '/work-queues', + href: '/app/work-queues', }, { title: 'Escalate edge cases', @@ -85,17 +85,17 @@ export const PERSONA_CONFIG: Record = { { title: 'Sync API credentials', description: 'Confirm API key headers in Settings match environment config.', - href: '/settings', + href: '/app/settings', }, { title: 'Monitor job feed', description: 'Review QA/search results for anomalies or retriable failures.', - href: '/qa', + href: '/app/qa', }, { title: 'Inspect embedding health', description: 'Check pipeline diagnostics for drift or processing gaps.', - href: '/settings', + href: '/app/settings', }, ], }, @@ -117,7 +117,7 @@ export const PERSONA_CONFIG: Record = { { title: 'Align roadmap', description: 'Spot gaps in coverage and align teams on next investments.', - href: '/settings', + href: '/app/settings', }, ], }, diff --git a/frontend/src/components/layout/AppShell.tsx b/frontend/src/components/layout/AppShell.tsx index 6fd3ef2..1851faa 100644 --- a/frontend/src/components/layout/AppShell.tsx +++ b/frontend/src/components/layout/AppShell.tsx @@ -1,16 +1,34 @@ import { type FormEvent, type ReactNode, useEffect, useState } from 'react'; -import { NavLink, useNavigate } from 'react-router-dom'; -import { LayoutDashboard, UsersRound, GitBranch, MessageSquare, Settings, Search, Plus, Loader2 } from 'lucide-react'; +import { NavLink, useLocation, useNavigate } from 'react-router-dom'; +import { + Activity, + ChevronsUpDown, + Command, + FileStack, + FolderKanban, + LayoutDashboard, + Loader2, + Menu, + MessageSquareText, + PanelLeftClose, + PanelLeftOpen, + Plus, + Search, + Settings, + Sparkles, + X, +} from 'lucide-react'; -import { Button } from '../ui/button'; -import { Input } from '../ui/input'; +import { fetchDocuments } from '../../api/client'; +import type { DocumentEntry } from '../../api/types'; import { cn } from '../../lib/utils'; -import { Badge } from '../ui/badge'; +import { useUIStore } from '../../stores/uiStore'; +import { Logo } from '../brand/Logo'; import { NotificationBell } from '../notifications/NotificationBell'; import { NotificationToaster } from '../notifications/NotificationToaster'; import { ProfileMenu } from '../profile/ProfileMenu'; -import { fetchDocuments } from '../../api/client'; -import type { DocumentEntry } from '../../api/types'; +import { Button } from '../ui/button'; +import { ThemeToggle } from '../ui/theme-toggle'; interface AppShellProps { headerAction?: ReactNode; @@ -20,249 +38,196 @@ interface AppShellProps { onLaunchUpload?: () => void; } -const NAV_ITEMS = [ - { to: '/', label: 'Intake', icon: LayoutDashboard }, - { to: '/work-queues', label: 'Work Queues', icon: UsersRound }, - { to: '/pipeline', label: 'Pipeline', icon: GitBranch }, - { to: '/qa', label: 'QA Studio', icon: MessageSquare }, - { to: '/settings', label: 'Settings', icon: Settings }, +const PRIMARY_NAV = [ + { to: '/app', label: 'Overview', icon: LayoutDashboard, end: true }, + { to: '/app/qa', label: 'Ask DocuLens', icon: MessageSquareText }, + { to: '/app/pipeline', label: 'Documents', icon: FileStack }, + { to: '/app/work-queues', label: 'Work queues', icon: FolderKanban }, ]; -const PLAYBOOK_STEPS = [ - { - title: 'Intake', - description: 'Upload the paperwork, confirm detected fields, and tag urgency so processing kicks off instantly.', - }, - { - title: 'Routing', - description: 'DocuLens routes items by type, risk, and region—manual overrides stay tracked for audit.', - }, - { - title: 'Resolution', - description: 'Review the AI summary, add notes, and complete checklist items before exporting decisions.', - }, +const SECONDARY_NAV = [ + { to: '/app/pipeline', label: 'Activity', icon: Activity }, + { to: '/app/settings', label: 'Settings', icon: Settings }, ]; -export function AppShell({ - headerAction, - children, - productName = 'DocuLens AI', - productTagline = 'Digitize every document. Route every task.', - onLaunchUpload, -}: AppShellProps) { +const PAGE_META: Record = { + '/app': { title: 'Workspace', description: 'Your documents, decisions, and active work in one place.' }, + '/app/qa': { title: 'Ask DocuLens', description: 'Explore your documents with answers grounded in source evidence.' }, + '/app/pipeline': { title: 'Documents', description: 'Review summaries, classifications, source content, and processing status.' }, + '/app/work-queues': { title: 'Work queues', description: 'Route documents to the right team and keep decisions moving.' }, + '/app/settings': { title: 'Workspace settings', description: 'Configure models, retrieval, access, and preferences.' }, +}; + +export function AppShell({ headerAction, children, onLaunchUpload }: AppShellProps) { + const { pathname } = useLocation(); const navigate = useNavigate(); + const { isSidebarOpen, isSidebarCollapsed, toggleSidebar, toggleSidebarCollapsed } = useUIStore(); + const [isCommandOpen, setIsCommandOpen] = useState(false); const [searchValue, setSearchValue] = useState(''); - const [isSearchActive, setIsSearchActive] = useState(false); const [searchResults, setSearchResults] = useState([]); const [isSearching, setIsSearching] = useState(false); const [searchError, setSearchError] = useState(null); + const page = PAGE_META[pathname] ?? PAGE_META['/app']; useEffect(() => { - const trimmed = searchValue.trim().toLowerCase(); - if (!trimmed) { + const handleKey = (event: KeyboardEvent) => { + if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === 'k') { + event.preventDefault(); + setIsCommandOpen((open) => !open); + } + if (event.key === 'Escape') { + setIsCommandOpen(false); + toggleSidebar(false); + } + }; + window.addEventListener('keydown', handleKey); + return () => window.removeEventListener('keydown', handleKey); + }, [toggleSidebar]); + + useEffect(() => { + const query = searchValue.trim().toLowerCase(); + if (!query) { setSearchResults([]); setSearchError(null); - setIsSearching(false); return; } - let cancelled = false; setIsSearching(true); - - const timer = setTimeout(async () => { - try { - const docs = await fetchDocuments(150); - if (cancelled) return; - - const filtered = docs.filter((doc) => { - const status = (doc.status ?? '').toLowerCase(); - if (status === 'deleted') { - return false; - } - const haystack = [ - doc.filename, - doc.doc_type, - doc.summary?.summary, - doc.document_id, - doc.metadata ? JSON.stringify(doc.metadata) : '', - ] - .filter(Boolean) - .join(' ') - .toLowerCase(); - return haystack.includes(trimmed); - }); - - if (cancelled) return; - setSearchResults(filtered.slice(0, 5)); - setSearchError(filtered.length ? null : `No results for “${searchValue.trim()}”.`); - } catch (error) { + const timer = window.setTimeout(() => { + fetchDocuments(150) + .then((documents) => { if (cancelled) return; - const message = error instanceof Error ? error.message : 'Unable to search documents.'; - setSearchError(message); - setSearchResults([]); - } finally { - if (!cancelled) { - setIsSearching(false); - } - } - }, 200); - - return () => { - cancelled = true; - clearTimeout(timer); - }; + const matches = documents.filter((document) => [document.filename, document.doc_type, document.summary?.summary, document.document_id].filter(Boolean).join(' ').toLowerCase().includes(query)); + setSearchResults(matches.slice(0, 6)); + setSearchError(matches.length ? null : `No documents match “${searchValue.trim()}”`); + }) + .catch((error: unknown) => !cancelled && setSearchError(error instanceof Error ? error.message : 'Search is unavailable.')) + .finally(() => !cancelled && setIsSearching(false)); + }, 160); + return () => { cancelled = true; window.clearTimeout(timer); }; }, [searchValue]); - const handleSearchSubmit = (event: FormEvent) => { - event.preventDefault(); - if (searchValue.trim()) { - setIsSearchActive(true); - } + const selectDocument = (id: string) => { + navigate(`/app/pipeline?document=${id}`); + setIsCommandOpen(false); + setSearchValue(''); }; - const handleResultSelect = (documentId: string) => { - navigate(`/pipeline?document=${documentId}`); + const openDestination = (path: string) => { + navigate(path); + setIsCommandOpen(false); setSearchValue(''); - setSearchResults([]); - setSearchError(null); - setIsSearchActive(false); }; return ( -
    - -
    -
    -
    -
    -
    -
    - - setSearchValue(event.target.value)} - onFocus={() => setIsSearchActive(true)} - onBlur={() => setTimeout(() => setIsSearchActive(false), 150)} - placeholder="Search documents, teams, or tags…" - aria-label="Search workspace" - className="h-10 rounded-full border border-platinum-600 bg-white pl-10 pr-4 shadow" - /> - {(isSearchActive && searchValue.trim()) || (searchError && searchValue.trim()) ? ( -
    - {isSearching ? ( -
    - - Searching… -
    - ) : searchResults.length ? ( -
      - {searchResults.map((doc) => ( -
    • - -
      • - ))} -
    - ) : ( -
    - {searchError ?? `No results for “${searchValue.trim()}”.`} -
    - )} -
    - ) : null} - -
    -
    - - {headerAction} - -
    -
    - +
    +
    +
    + + +
    {headerAction}
    -
    {children}
    +
    +
    +
    +

    {page.title}

    {page.description}

    + All systems operational +
    + {children} +
    +
    + {isCommandOpen ? ( +
    setIsCommandOpen(false)}> +
    event.stopPropagation()}> +
    +
    event.preventDefault()} className="flex h-12 items-center gap-3 rounded-xl border border-border/80 bg-surface-subtle px-3 shadow-sm transition focus-within:border-primary/50 focus-within:ring-4 focus-within:ring-primary/10"> + + setSearchValue(event.target.value)} placeholder="Search documents and summaries…" className="h-full min-w-0 flex-1 appearance-none border-0 bg-transparent p-0 text-sm text-foreground shadow-none outline-none placeholder:text-muted-foreground focus:border-0 focus:outline-none focus:ring-0 focus-visible:outline-none focus-visible:ring-0" /> + + +
    +
    + {!searchValue.trim() ? : isSearching ?
    Searching your workspace…
    : searchResults.length ? searchResults.map((document) => ) :

    {searchError}

    } +
    +
    +
    + ) : null}
    ); } -function navLinkPillClass(isActive: boolean) { - return cn( - 'rounded-full border px-3 py-1 text-sm transition-colors', - isActive ? 'border-border bg-card text-foreground' : 'border-transparent text-muted-foreground hover:border-border hover:bg-card', - ); +function SidebarLink({ to, label, icon: Icon, end, collapsed, onClick }: { to: string; label: string; icon: typeof LayoutDashboard; end?: boolean; collapsed: boolean; onClick: () => void }) { + return cn('group flex items-center gap-2.5 rounded-xl px-3 py-2.5 text-xs font-medium transition-all', collapsed && 'lg:justify-center lg:px-2', isActive ? 'bg-background text-foreground shadow-sm ring-1 ring-border/70' : 'text-muted-foreground hover:bg-background/70 hover:text-foreground')}>{label}; } - -function sidebarLinkClass(isActive: boolean) { - return cn( - 'group flex items-center gap-3 rounded-xl px-4 py-2 text-sm font-medium transition-all', - isActive - ? 'bg-gradient-to-r from-lapis-500/15 via-lapis-500/10 to-transparent text-lapis-600 shadow-[0_12px_32px_rgba(47,102,144,0.18)] ring-1 ring-inset ring-lapis-500/40 backdrop-blur-sm' - : 'text-muted-foreground hover:bg-lapis-500/5 hover:text-foreground', +function ChevronHint() { return ↵; } +function CommandEmpty({ onNavigate }: { onNavigate: (path: string) => void }) { + return ( +
    +

    Quick actions

    + + +
    ); } diff --git a/frontend/src/components/notifications/ActivityFeed.tsx b/frontend/src/components/notifications/ActivityFeed.tsx index e55e675..22acc3e 100644 --- a/frontend/src/components/notifications/ActivityFeed.tsx +++ b/frontend/src/components/notifications/ActivityFeed.tsx @@ -1,6 +1,9 @@ -import { type ComponentType } from 'react'; -import { Activity, Brain, FileText, MessageSquare, RefreshCw } from 'lucide-react'; +import { type ComponentType, useMemo } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { Activity, Brain, FileText, MessageSquare, RefreshCw, Search } from 'lucide-react'; +import { fetchEvents } from '../../api/client'; +import type { EventEntry } from '../../api/types'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../ui/card'; import { Button } from '../ui/button'; import { useNotificationStore } from '../../stores/notificationStore'; @@ -9,31 +12,49 @@ const iconMap: Record> = { upload: FileText, summary: Brain, qa: MessageSquare, + search: Search, default: Activity, }; export function ActivityFeed({ limit = 2 }: { limit?: number }) { - const notifications = useNotificationStore((state) => state.notifications).slice(0, limit); + const notifications = useNotificationStore((state) => state.notifications); + const { data: events = [], isFetching, refetch } = useQuery({ + queryKey: ['workspace-activity', limit], + queryFn: () => fetchEvents(Math.max(limit * 3, 10)), + }); + const activity = useMemo(() => { + const localItems: ActivityItem[] = notifications.map((item) => ({ + id: item.id, + title: item.title, + description: item.description, + timestamp: item.timestamp, + variant: 'default', + })); + const eventItems = events.map(toActivityItem).filter((item): item is ActivityItem => item !== null); + return [...localItems, ...eventItems] + .sort((first, second) => second.timestamp - first.timestamp) + .slice(0, limit); + }, [events, limit, notifications]); return (
    Recent activity - Key pipeline events from the past few minutes. + Recent searches, answers, summaries, and uploads.
    -     - {notifications.length === 0 ? ( + {activity.length === 0 ? (

    Actions taken in the console will appear here.

    ) : ( - notifications.map((item) => { + activity.map((item) => { const Icon = iconMap[item.variant] ?? iconMap.default; return ( -
    +

    {item.title}

    @@ -50,3 +71,73 @@ export function ActivityFeed({ limit = 2 }: { limit?: number }) { ); } + +interface ActivityItem { + id: string; + title: string; + description?: string; + timestamp: number; + variant: string; +} + +function toActivityItem(event: EventEntry): ActivityItem | null { + const eventType = stringValue(event.data.event_type); + const timestamp = Date.parse(event.created_at); + if (!eventType || Number.isNaN(timestamp)) return null; + + if (eventType === 'document_upload') { + const metadata = objectValue(event.data.metadata); + const filename = stringValue(metadata.uploaded_filename) ?? stringValue(event.data.filename) ?? 'Document'; + return { + id: event.id, + title: 'Document indexed', + description: filename.split('/').pop(), + timestamp, + variant: 'upload', + }; + } + + if (eventType === 'document_summary') { + const context = objectValue(event.task_context); + const metadata = objectValue(context.metadata); + const summaries = objectValue(metadata.document_summaries); + const firstSummary = objectValue(Object.values(summaries)[0]); + return { + id: event.id, + title: 'Summary completed', + description: stringValue(firstSummary.filename) ?? 'Document summary is ready', + timestamp, + variant: 'summary', + }; + } + + if (eventType === 'qa_query') { + return { + id: event.id, + title: 'Grounded answer generated', + description: stringValue(event.data.query), + timestamp, + variant: 'qa', + }; + } + + if (eventType === 'search_query') { + return { + id: event.id, + title: 'Semantic search completed', + description: stringValue(event.data.query), + timestamp, + variant: 'search', + }; + } + + return null; +} + +function objectValue(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value : undefined; +} diff --git a/frontend/src/components/ui/button.tsx b/frontend/src/components/ui/button.tsx index 52830c7..c99e8ec 100644 --- a/frontend/src/components/ui/button.tsx +++ b/frontend/src/components/ui/button.tsx @@ -5,22 +5,22 @@ import { cva, type VariantProps } from 'class-variance-authority'; import { cn } from '../../lib/utils'; const buttonVariants = cva( - 'inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 ring-offset-background', + 'inline-flex items-center justify-center whitespace-nowrap rounded-xl text-sm font-semibold transition-all duration-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 active:scale-[0.98] ring-offset-background', { variants: { variant: { - default: 'bg-primary text-primary-foreground shadow-subtle hover:bg-primary/90', + default: 'bg-foreground text-background shadow-sm hover:bg-foreground/90 dark:bg-primary dark:text-primary-foreground dark:hover:bg-primary/90', secondary: 'bg-secondary text-secondary-foreground hover:bg-secondary/80', - accent: 'bg-accent text-accent-foreground shadow-subtle hover:bg-accent/85', - outline: 'border border-border bg-transparent hover:bg-muted', - ghost: 'hover:bg-muted hover:text-foreground', + accent: 'bg-primary text-primary-foreground shadow-sm hover:bg-primary/90', + outline: 'border border-border bg-background hover:border-muted-foreground/30 hover:bg-muted/60', + ghost: 'hover:bg-muted/70 hover:text-foreground', destructive: 'bg-destructive text-destructive-foreground shadow-subtle hover:bg-destructive/80', muted: 'bg-muted text-muted-foreground hover:bg-muted/80', }, size: { default: 'h-10 px-4 py-2', - sm: 'h-9 rounded-md px-3', - lg: 'h-11 rounded-md px-5 text-base', + sm: 'h-9 rounded-lg px-3', + lg: 'h-11 rounded-xl px-5 text-base', icon: 'h-10 w-10', }, }, diff --git a/frontend/src/components/ui/card.tsx b/frontend/src/components/ui/card.tsx index cefddb1..0d717e0 100644 --- a/frontend/src/components/ui/card.tsx +++ b/frontend/src/components/ui/card.tsx @@ -7,7 +7,7 @@ export const Card = React.forwardRef(({ className ref={ref} type={type} className={cn( - 'flex h-10 w-full rounded-md border border-input bg-transparent px-3 py-2 text-sm shadow-none transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 disabled:cursor-not-allowed disabled:opacity-50', + 'flex h-10 w-full rounded-xl border border-input bg-background px-3 py-2 text-sm shadow-sm transition-all placeholder:text-muted-foreground focus:border-primary/50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/15 focus-visible:ring-offset-0 disabled:cursor-not-allowed disabled:opacity-50', className, )} {...props} diff --git a/frontend/src/components/ui/theme-toggle.tsx b/frontend/src/components/ui/theme-toggle.tsx new file mode 100644 index 0000000..d59e7aa --- /dev/null +++ b/frontend/src/components/ui/theme-toggle.tsx @@ -0,0 +1,25 @@ +import { useEffect, useState } from 'react'; +import { Moon, Sun } from 'lucide-react'; + +import { Button } from './button'; + +function readTheme() { + if (typeof window === 'undefined') return false; + const stored = window.localStorage.getItem('doculens.theme'); + return stored ? stored === 'dark' : window.matchMedia('(prefers-color-scheme: dark)').matches; +} + +export function ThemeToggle() { + const [dark, setDark] = useState(readTheme); + + useEffect(() => { + document.documentElement.classList.toggle('dark', dark); + window.localStorage.setItem('doculens.theme', dark ? 'dark' : 'light'); + }, [dark]); + + return ( + + ); +} diff --git a/frontend/src/hooks/useEventNotifications.ts b/frontend/src/hooks/useEventNotifications.ts index 5070c60..c31dad7 100644 --- a/frontend/src/hooks/useEventNotifications.ts +++ b/frontend/src/hooks/useEventNotifications.ts @@ -94,7 +94,7 @@ export function useEventNotifications() { title: 'QA answer ready', description: entry.answer ? `${entry.answer.slice(0, 80)}…` : entry.query, variant: 'success', - href: '/qa', + href: '/app/qa', timestamp: Number.isNaN(occurredAt) ? undefined : occurredAt, }); } @@ -115,7 +115,7 @@ export function useEventNotifications() { title: 'Search results updated', description: `${entry.query} (${entry.result_count} results)`, variant: 'info', - href: '/qa', + href: '/app/qa', timestamp: Number.isNaN(occurredAt) ? undefined : occurredAt, }); } diff --git a/frontend/src/lib/pipeline.ts b/frontend/src/lib/pipeline.ts new file mode 100644 index 0000000..dff5a55 --- /dev/null +++ b/frontend/src/lib/pipeline.ts @@ -0,0 +1,47 @@ +import type { DocumentEntry } from '../api/types'; +import { inferRole, inferStatus } from './routing'; + +export interface PipelineStage { + id: 'upload' | 'extraction' | 'embedding' | 'summary' | 'routing'; + label: string; + description: string; + state: 'pending' | 'active' | 'complete'; +} + +export function derivePipelineStages(document: DocumentEntry): PipelineStage[] { + const terminal = ['completed', 'archived'].includes(inferStatus(document).toLowerCase()); + const hasChunks = (document.chunk_count ?? 0) > 0; + const hasEmbeddings = (document.embedded_chunk_count ?? 0) > 0; + const hasSummary = Boolean(document.summary?.summary); + const hasRoute = Boolean(document.assigned_role) || inferRole(document) !== 'Operations'; + const state = (complete: boolean, active: boolean): PipelineStage['state'] => + complete ? 'complete' : active ? 'active' : 'pending'; + + return [ + { id: 'upload', label: 'Received', description: 'Document accepted and stored.', state: 'complete' }, + { + id: 'extraction', + label: 'Extracted', + description: 'Layout and text converted into chunks.', + state: state(hasChunks, !hasChunks), + }, + { + id: 'embedding', + label: 'Indexed', + description: 'Citation-ready chunks embedded for retrieval.', + state: state(hasEmbeddings, hasChunks), + }, + { + id: 'summary', + label: 'Summarised', + description: 'Key facts and next actions generated.', + state: state(hasSummary, hasEmbeddings), + }, + { + id: 'routing', + label: 'Routed', + description: 'Assigned to an operational work queue.', + state: state(hasRoute || terminal, hasSummary), + }, + ]; +} diff --git a/frontend/src/lib/queryClient.ts b/frontend/src/lib/queryClient.ts new file mode 100644 index 0000000..314af02 --- /dev/null +++ b/frontend/src/lib/queryClient.ts @@ -0,0 +1,12 @@ +import { QueryClient } from '@tanstack/react-query'; + +export const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 15_000, + retry: 2, + refetchOnWindowFocus: false, + }, + mutations: { retry: 0 }, + }, +}); diff --git a/frontend/src/lib/routing.ts b/frontend/src/lib/routing.ts new file mode 100644 index 0000000..1758d62 --- /dev/null +++ b/frontend/src/lib/routing.ts @@ -0,0 +1,50 @@ +import type { DocumentEntry } from '../api/types'; + +export const ROLE_ORDER = ['Finance', 'Compliance', 'Operations', 'Legal', 'HR', 'Integrator'] as const; +export type RoleKey = (typeof ROLE_ORDER)[number]; + +function metadataValue(document: DocumentEntry, key: string): unknown { + const metadata = document.metadata; + return metadata && typeof metadata === 'object' ? metadata[key] : undefined; +} + +export function inferRole(document: DocumentEntry): RoleKey { + const candidate = document.assigned_role ?? metadataValue(document, 'assigned_role') ?? metadataValue(document, 'role'); + if (typeof candidate === 'string') { + const match = ROLE_ORDER.find((role) => role.toLowerCase() === candidate.toLowerCase()); + if (match) return match; + } + + const type = (document.doc_type ?? '').toLowerCase(); + if (/resume|cv|offer|employee|training/.test(type)) return 'HR'; + if (/contract|agreement|legal/.test(type)) return 'Legal'; + if (/policy|audit|compliance|tax/.test(type)) return 'Compliance'; + if (/invoice|receipt|expense|financial/.test(type)) return 'Finance'; + return 'Operations'; +} + +export function inferStatus(document: DocumentEntry): string { + if (document.deleted_at) return 'Deleted'; + if (document.archived_at) return 'Archived'; + const explicit = document.status ?? metadataValue(document, 'status'); + if (typeof explicit === 'string' && explicit.trim()) return explicit; + if (document.summary && (document.embedded_chunk_count ?? 0) > 0) return 'Completed'; + if ((document.embedded_chunk_count ?? 0) > 0) return 'Enriching'; + if ((document.chunk_count ?? 0) > 0) return 'Embedding'; + return 'Processing'; +} + +export function inferDueDate(document: DocumentEntry): string | null { + const candidate = document.due_at ?? metadataValue(document, 'due_at') ?? metadataValue(document, 'due_date'); + return typeof candidate === 'string' && candidate ? candidate : null; +} + +export function formatDateTime(value?: string | null): string { + if (!value) return 'Not set'; + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) return 'Invalid date'; + return new Intl.DateTimeFormat(undefined, { + dateStyle: 'medium', + timeStyle: 'short', + }).format(parsed); +} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts new file mode 100644 index 0000000..2819a83 --- /dev/null +++ b/frontend/src/lib/utils.ts @@ -0,0 +1,6 @@ +import { clsx, type ClassValue } from 'clsx'; +import { twMerge } from 'tailwind-merge'; + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 033bc97..7332e9d 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -1,20 +1,33 @@ -import React from 'react'; +import React, { Suspense, lazy } from 'react'; import ReactDOM from 'react-dom/client'; import { QueryClientProvider } from '@tanstack/react-query'; -import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; import { BrowserRouter, Route, Routes } from 'react-router-dom'; -import { AppLayout } from './App'; -import './styles/globals.css'; -import { IntakePage } from './pages/IntakePage'; -import { WorkQueuesPage } from './pages/WorkQueuesPage'; -import { PipelinePage } from './pages/PipelinePage'; -import { QaPage } from './pages/QaPage'; -import { SettingsPage } from './pages/SettingsPage'; -import { SettingsProvider } from './settings/SettingsProvider'; -import { queryClient } from './lib/queryClient'; + import { AuthProvider } from './auth/AuthProvider'; import { ProtectedRoute } from './auth/ProtectedRoute'; -import { LoginPage } from './pages/LoginPage'; +import { queryClient } from './lib/queryClient'; +import { SettingsProvider } from './settings/SettingsProvider'; +import './styles/globals.css'; + +const AppLayout = lazy(() => import('./App')); +const LandingPage = lazy(() => import('./pages/LandingPage')); +const LoginPage = lazy(() => import('./pages/LoginPage').then((module) => ({ default: module.LoginPage }))); +const IntakePage = lazy(() => import('./pages/IntakePage').then((module) => ({ default: module.IntakePage }))); +const WorkQueuesPage = lazy(() => import('./pages/WorkQueuesPage').then((module) => ({ default: module.WorkQueuesPage }))); +const PipelinePage = lazy(() => import('./pages/PipelinePage').then((module) => ({ default: module.PipelinePage }))); +const QaPage = lazy(() => import('./pages/QaPage').then((module) => ({ default: module.QaPage }))); +const SettingsPage = lazy(() => import('./pages/SettingsPage').then((module) => ({ default: module.SettingsPage }))); + +export function RouteLoader() { + return ( +
    +
    + + Preparing your workspace +
    +
    + ); +} ReactDOM.createRoot(document.getElementById('root')!).render( @@ -22,20 +35,29 @@ ReactDOM.createRoot(document.getElementById('root')!).render( - - } /> - }> - } /> - } /> - } /> - } /> - } /> - - + }> + + } /> + } /> + + + + } + > + } /> + } /> + } /> + } /> + } /> + + + - , ); diff --git a/frontend/src/pages/IntakePage.tsx b/frontend/src/pages/IntakePage.tsx index 9c61175..baa895c 100644 --- a/frontend/src/pages/IntakePage.tsx +++ b/frontend/src/pages/IntakePage.tsx @@ -1,140 +1,115 @@ import { useEffect, useMemo, useState } from 'react'; import { useQuery } from '@tanstack/react-query'; import { - AlertTriangle, ArrowRight, - BarChart3, - Building2, - Clock, - FileSpreadsheet, - TrendingUp, - Users, + CheckCircle2, + Clock3, + FileText, + Loader2, + MessageSquareText, + RefreshCw, + Search, + ShieldCheck, + Sparkles, } from 'lucide-react'; import { Link } from 'react-router-dom'; import { fetchDashboardInsights, fetchDocuments } from '../api/client'; import type { DashboardInsights, DocumentEntry } from '../api/types'; import { ActivityFeed } from '../components/notifications/ActivityFeed'; -import { Button } from '../components/ui/button'; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../components/ui/card'; import { Badge } from '../components/ui/badge'; +import { Button } from '../components/ui/button'; import { cn } from '../lib/utils'; -import { ROLE_ORDER, inferDueDate, inferRole, inferStatus, formatDateTime } from '../lib/routing'; - -type RoleSummary = Record; +import { formatDateTime, inferDueDate, inferRole, inferStatus } from '../lib/routing'; + +const FALLBACK_METRICS: DashboardInsights = { + total_documents: 0, + summarised_documents: 0, + chunk_count: 0, + embedded_count: 0, + queue_latency: '—', + estimated_savings: 0, + hours_saved: 0, + analyst_rate: 65, + sla_risk_count: 0, + sla_risk_message: 'Everything is on track.', + throughput_series: [], + compliance_series: [], + delta_processed: '—', + delta_processed_tone: 'neutral', + delta_summaries: '—', + delta_summaries_tone: 'neutral', + today_total: 0, + today_summaries: 0, + yesterday_total: 0, + yesterday_summaries: 0, +}; export function IntakePage() { const [documents, setDocuments] = useState([]); - const [error, setError] = useState(null); - const [isLoading, setIsLoading] = useState(false); - const [attentionView, setAttentionView] = useState<'overdue' | 'upcoming'>('overdue'); + const [documentsError, setDocumentsError] = useState(null); + const [isLoadingDocuments, setIsLoadingDocuments] = useState(false); + const currentTime = useMemo(() => Date.now(), []); const { data: insights, isFetching: isFetchingInsights, refetch: refetchInsights, error: insightsError, - } = useQuery({ - queryKey: ['dashboard-insights'], - queryFn: fetchDashboardInsights, - }); - - const metrics: DashboardInsights = - insights ?? { - total_documents: documents.length, - summarised_documents: documents.filter((doc) => Boolean(doc.summary)).length, - chunk_count: documents.reduce((acc, doc) => acc + (doc.chunk_count ?? 0), 0), - embedded_count: documents.reduce((acc, doc) => acc + (doc.embedded_chunk_count ?? 0), 0), - queue_latency: '—', - estimated_savings: 0, - hours_saved: 0, - analyst_rate: 65, - sla_risk_count: 0, - sla_risk_message: 'All documents are within SLA thresholds.', - throughput_series: [], - compliance_series: [], - delta_processed: '—', - delta_processed_tone: 'neutral', - delta_summaries: '—', - delta_summaries_tone: 'neutral', - today_total: 0, - today_summaries: 0, - yesterday_total: 0, - yesterday_summaries: 0, - }; + } = useQuery({ queryKey: ['dashboard-insights'], queryFn: fetchDashboardInsights }); + + const metrics = insights ?? { + ...FALLBACK_METRICS, + total_documents: documents.length, + summarised_documents: documents.filter((document) => Boolean(document.summary)).length, + chunk_count: documents.reduce((total, document) => total + (document.chunk_count ?? 0), 0), + embedded_count: documents.reduce((total, document) => total + (document.embedded_chunk_count ?? 0), 0), + }; const loadDocuments = () => { - setIsLoading(true); - setError(null); + setIsLoadingDocuments(true); + setDocumentsError(null); fetchDocuments(100) .then((items) => setDocuments( - items.filter((doc) => { - const status = inferStatus(doc).toLowerCase(); - return status !== 'archived' && status !== 'deleted'; - }), + items.filter((document) => !['archived', 'deleted'].includes(inferStatus(document).toLowerCase())), ), ) - .catch((err: unknown) => { - const message = err instanceof Error ? err.message : 'Failed to load documents'; - setError(message); - }) - .finally(() => setIsLoading(false)); + .catch((error: unknown) => + setDocumentsError(error instanceof Error ? error.message : 'Unable to load workspace documents.'), + ) + .finally(() => setIsLoadingDocuments(false)); }; useEffect(() => { loadDocuments(); }, []); - const roleSummary = useMemo(() => { - const summary: RoleSummary = Object.fromEntries(ROLE_ORDER.map((role) => [role, { count: 0, overdue: 0 }])); - const now = Date.now(); - for (const doc of documents) { - const role = inferRole(doc); - const dueAt = inferDueDate(doc); - const status = inferStatus(doc).toLowerCase(); - summary[role].count += 1; - if (dueAt) { - const dueTime = Date.parse(dueAt); - if (!Number.isNaN(dueTime) && dueTime < now && status !== 'completed') { - summary[role].overdue += 1; - } - } - } - return summary; - }, [documents]); - - const totalQueued = useMemo( - () => ROLE_ORDER.reduce((acc, role) => acc + (roleSummary[role]?.count ?? 0), 0), - [roleSummary], - ); - - const recentUploads = useMemo(() => documents.slice(0, 6), [documents]); - - const now = Date.now(); - const twoDays = 48 * 60 * 60 * 1000; - const overdueDocs = useMemo( - () => - documents.filter((doc) => { - const due = inferDueDate(doc); - if (!due) return false; - const dueTime = Date.parse(due); - return !Number.isNaN(dueTime) && dueTime < now; - }), - [documents, now], - ); - const dueSoonDocs = useMemo( + const recentDocuments = useMemo( () => - documents.filter((doc) => { - const due = inferDueDate(doc); - if (!due) return false; - const dueTime = Date.parse(due); - return !Number.isNaN(dueTime) && dueTime >= now && dueTime <= now + twoDays; - }), - [documents, now, twoDays], + documents + .slice() + .sort((first, second) => Date.parse(second.uploaded_at) - Date.parse(first.uploaded_at)) + .slice(0, 5), + [documents], ); - const attentionDocs = attentionView === 'overdue' ? overdueDocs : dueSoonDocs; + const attentionDocuments = useMemo(() => { + const windowEnd = currentTime + 48 * 60 * 60 * 1000; + return documents + .filter((document) => { + const dueAt = inferDueDate(document); + if (!dueAt || ['ready', 'completed', 'archived'].includes(inferStatus(document).toLowerCase())) return false; + const dueTime = Date.parse(dueAt); + return !Number.isNaN(dueTime) && dueTime <= windowEnd; + }) + .sort((first, second) => Date.parse(inferDueDate(first) ?? '') - Date.parse(inferDueDate(second) ?? '')) + .slice(0, 3); + }, [currentTime, documents]); + + const summaryCoverage = percentage(metrics.summarised_documents, metrics.total_documents); + const embeddingCoverage = percentage(metrics.embedded_count, metrics.chunk_count); + const isRefreshing = isLoadingDocuments || isFetchingInsights; const handleRefresh = () => { loadDocuments(); @@ -142,672 +117,190 @@ export function IntakePage() { }; return ( -
    -
    -
    -

    Operations centre

    -
    -

    Intake performance

    - - Updated {new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} - -
    -
    - -
    -
    - - -
    - - - - -
    - - {metrics.sla_risk_message ? ( -
    0 - ? 'border-destructive/40 bg-destructive/5 text-destructive' - : 'border-sky-300/60 bg-sky-50 text-sky-800', - )} - > - -
    -

    SLA update

    -

    - {metrics.sla_risk_message} -

    -
    - {metrics.sla_risk_count > 0 ? ( - - ) : null} -
    - ) : null} - - - -
    -
    - - - +
    +
    +
    +
    +
    +
    + Workspace pulse
    -
    - - +

    + {metrics.total_documents.toLocaleString()} documents are searchable. + + {metrics.sla_risk_count > 0 ? `${metrics.sla_risk_count} need a decision.` : 'Nothing is blocking your team.'} + +

    +

    + Ask a question, continue a review, or inspect the evidence behind any answer. +

    +
    + +
    +
    +

    Since yesterday

    +
    + + + +
    +
    -
    - - - - - Sync status - Pipeline heartbeat and manual refresh. - - -
    - {insightsError ? ( - Unable to refresh dashboard insights. - ) : error ? ( - {error} - ) : isLoading || isFetchingInsights ? ( - Refreshing data… - ) : ( - Everything looks healthy. - )} -
    - -     -     - - - - - - Recent uploads - Documents awaiting processing or routing. - - - {recentUploads.length === 0 ? ( -

    No documents ingested yet.

    - ) : ( - recentUploads.map((doc) => ( -
    -
    -
    -

    {doc.filename ?? doc.document_id}

    - {doc.doc_type ? ( - - {doc.doc_type} - - ) : null} -
    -

    - Uploaded {formatDateTime(doc.uploaded_at)} · Status {inferStatus(doc)} -

    -
    - -
    - )) - )} -     -     - - - +
    + + +
    -
    -
    -); -} - -type MetricVariant = 'lapis' | 'cerulean' | 'platinum' | 'sky'; - -function HeadlineMetric({ - icon: Icon, - label, - value, - helper, - variant, -}: { - icon: typeof Building2; - label: string; - value: string; - helper?: string; - variant: MetricVariant; -}) { - const palette: Record = { - lapis: { - bg: 'from-[#edf3f8] to-white', - accent: 'bg-[#2f6690] text-white', - }, - cerulean: { - bg: 'from-[#e9f4fb] to-white', - accent: 'bg-[#3a7ca5] text-white', - }, - platinum: { - bg: 'from-[#f5f6f4] to-white', - accent: 'bg-[#b7c0b7] text-[#1f2d32]', - }, - sky: { - bg: 'from-[#e8f2f8] to-white', - accent: 'bg-[#5189b3] text-white', - }, - }; - return ( - - -
    - - - - {label} + {(documentsError || insightsError) ? ( +
    + {documentsError ?? 'Workspace insights are temporarily unavailable.'} +
    - {value} - {helper ?

    {helper}

    : null} - - - ); -} - -function HeroTile({ label, value, delta }: { label: string; value: string; delta: string }) { - return ( - - -
    -

    {label}

    -

    {value}

    -
    -
    -
    - - Velocity + ) : null} + +
    +
    +
    +
    +
    +

    Continue working

    +

    Recently added and processed documents.

    +
    +
    + + +
    - {delta && delta !== '—' ? ( -

    Change vs last period: {delta}

    + + {isLoadingDocuments && !recentDocuments.length ? ( +
    Loading documents…
    + ) : !recentDocuments.length ? ( +

    Your workspace is ready.

    Upload a document to begin building searchable knowledge.

    ) : ( -

    No change recorded

    +
    + {recentDocuments.map((document) => )} +
    )} +
    +
    - - - ); -} -function HighlightCard({ title, value, helper }: { title: string; value: string; helper: string }) { - return ( - - -
    -
    -

    {title}

    -

    {value}

    -
    -
    - -
    +
    +
    +
    + +

    Ask your workspace

    +

    Get an answer grounded in the documents your team has already reviewed.

    + + Ask a question… + +
    +
    +

    Try asking

    +
    + {['Which contracts renew this quarter?', 'Summarize the open compliance risks'].map((prompt) => ( + + {prompt} + + ))} +
    +
    +
    + + +
    -

    {helper}

    - - +
    +
    ); } -function AttentionCard({ - documents, - overdueTotal, - dueSoonTotal, - view, - onChangeView, -}: { - documents: DocumentEntry[]; - overdueTotal: number; - dueSoonTotal: number; - view: 'overdue' | 'upcoming'; - onChangeView: (view: 'overdue' | 'upcoming') => void; -}) { +function PulseRow({ label, value, detail }: { label: string; value: string | number; detail: string }) { return ( - - -
    -
    - Documents needing attention - Route or follow up before SLAs slip. -
    -
    - - -
    -
    -     - - {documents.length === 0 ? ( -
    - {view === 'overdue' ? 'No overdue documents right now.' : 'No documents due in the next 48 hours.'} -
    - ) : ( - documents.slice(0, 6).map((doc) => { - const role = inferRole(doc); - const status = inferStatus(doc); - const dueAt = formatDateTime(inferDueDate(doc)); - return ( -
    -
    -
    -

    {doc.filename ?? doc.document_id}

    - - {role} - -
    -

    Status: {status}

    -

    Due: {dueAt}

    -
    - -
    - ); - }) - )} -     -     +
    + {label} + {value}{detail} +
    ); } -function SummaryProgressCard({ completed, total }: { completed: number; total: number }) { - const percentage = total > 0 ? Math.round((completed / total) * 100) : 0; +function HeroStat({ label, value, helper }: { label: string; value: string; helper: string }) { return ( - - - Summary coverage - - {total > 0 - ? `${completed.toLocaleString()} of ${total.toLocaleString()} documents have summaries ready.` - : 'No documents ingested yet.'} - - - -
    -
    -
    -
    - {percentage}% -
    -
    - {completed.toLocaleString()} ready - {Math.max(total - completed, 0).toLocaleString()} pending review - {total.toLocaleString()} total uploaded -
    - - +
    +

    {label}

    +
    {value}{helper}
    +
    ); } -function EmbeddingCoverageCard({ chunkCount, embeddedCount }: { chunkCount: number; embeddedCount: number }) { - const coverage = chunkCount > 0 ? Math.round((embeddedCount / chunkCount) * 100) : 0; +function DocumentRow({ document }: { document: DocumentEntry }) { + const status = inferStatus(document); + const role = inferRole(document); + const isReady = status.toLowerCase() === 'ready'; return ( - - -
    - Embedding coverage - {coverage}% -
    - Share of chunks with embeddings for search. -     - -
    -
    -
    -
    -
    -
    - {embeddedCount.toLocaleString()} embedded chunks - {chunkCount.toLocaleString()} total processed -
    - - + +
    + + {document.filename ?? document.document_id}{document.doc_type ?? 'Unclassified'} · Added {formatDateTime(document.uploaded_at)} +
    +
    + {role} + {status} + +
    + ); } -function QueueHealthCard({ totalQueued, overdue, dueSoon }: { totalQueued: number; overdue: number; dueSoon: number }) { - const onTrack = Math.max(totalQueued - overdue - dueSoon, 0); - const totalBar = Math.max(totalQueued, 1); - const segments = [ - { label: 'Overdue', value: overdue, color: '#d97706' }, - { label: 'Due soon', value: dueSoon, color: '#3a7ca5' }, - { label: 'On track', value: onTrack, color: '#2f6690' }, - ]; - +function AttentionPanel({ documents, riskCount }: { documents: DocumentEntry[]; riskCount: number }) { return ( - - - Queue status - Breakdown of items awaiting action. - - - {totalQueued === 0 ? ( -
    - All caught up—no queue today. -
    - ) : ( - <> -
    - {segments - .filter(({ value }) => value > 0) - .map(({ label, value, color }) => ( -
    - ))} -
    -
    - {segments.map(({ label, value, color }) => ( -
    -
    - - {label} -
    - - {value.toLocaleString()} ({Math.round((value / totalBar) * 100)}%) - -
    - ))} -
    - - )} - - +
    +

    Needs attention

    {riskCount > 0 ? `${riskCount} ${riskCount === 1 ? 'review is' : 'reviews are'} approaching SLA` : 'No urgent reviews'}

    +
    + {documents.length ? documents.map((document) => ( + + {document.filename ?? document.document_id}Due {formatDateTime(inferDueDate(document))} + + )) :

    The queue is clear. New SLA risks will appear here.

    } +
    + +
    ); } -function TrendCard({ title, series }: { title: string; series: Array<{ label: string; value: number }> }) { - const values = series.length ? series.map((item) => item.value) : [10, 18, 12, 20, 17, 24, 19]; - const max = Math.max(...values, 1); - const chartPoints = values - .map((value, index) => { - const x = (index / Math.max(values.length - 1, 1)) * 100; - const y = 100 - (value / max) * 80 - 10; - return `${x},${y}`; - }) - .join(' '); - - const labels = series.length ? series.map((item) => item.label) : ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']; - +function CoveragePanel({ summaryCoverage, embeddingCoverage, metrics }: { summaryCoverage: number; embeddingCoverage: number; metrics: DashboardInsights }) { return ( - - - {title} - Volume of documents processed over time. - - - - - - - - - - - - -
    - {labels.slice(-4).map((label) => ( - {label} - ))} -
    -     -     +
    +

    Knowledge readiness

    How much of your workspace can answer questions today.

    +
    + + +
    +
    Citations and page provenance enabled for indexed sources.
    +
    ); } -function WorkflowOverviewCard({ summary }: { summary: RoleSummary }) { - const total = ROLE_ORDER.reduce((acc, role) => acc + (summary[role]?.count ?? 0), 0); - const palette = ['#2f6690', '#3a7ca5', '#16425b', '#81c3d7', '#1b4965', '#0d1f2d']; - const radius = 15; - const circumference = 2 * Math.PI * radius; - let cumulative = 0; - - const segments = ROLE_ORDER.map((role, index) => { - const value = summary[role]?.count ?? 0; - const dash = total > 0 ? (value / total) * circumference : 0; - const percentage = total > 0 ? Math.round((value / total) * 100) : 0; - const segment = { - role, - value, - dash, - offset: cumulative, - color: palette[index % palette.length], - percentage, - }; - cumulative += dash; - return segment; - }); - +function CoverageRow({ label, value, helper }: { label: string; value: number; helper: string }) { return ( - - - Workflow by role - Current queue mix and ownership across teams. - - -
    - {ROLE_ORDER.map((role) => ( -
    -

    {role}

    -
    -

    {summary[role].count.toLocaleString()} in queue

    -

    {summary[role].overdue.toLocaleString()} overdue

    -
    -
    - ))} -
    -
    - {total === 0 ? ( -
    - No active queue items. -
    - ) : ( - <> - - - - {segments.map(({ role, dash, offset, color }) => - dash > 0 ? ( - - ) : null, - )} - - -
    - {segments.map(({ role, value, color, percentage }) => ( -
    -
    - - {role} -
    - - {value.toLocaleString()} ({percentage}%) - -
    - ))} -
    - - )} -
    -     -     +
    {label}{helper} · {value}%
    ); } -function OpsInsightsCard({ - total, - completed, - overdue, - dueSoon, -}: { - total: number; - completed: number; - overdue: number; - dueSoon: number; -}) { - const completionRate = total > 0 ? Math.round((completed / total) * 100) : 0; - const remaining = Math.max(total - completed, 0); - const automationRate = total > 0 ? Math.round((completed / Math.max(total, 1)) * 100) : 0; - const tiles = [ - { - label: 'Summary completion', - value: `${completionRate}%`, - detail: `${completed.toLocaleString()} ready · ${remaining.toLocaleString()} pending`, - }, - { - label: 'Routing backlog', - value: overdue > 0 ? `${overdue}` : 'On track', - detail: - overdue > 0 - ? dueSoon > 0 - ? `${dueSoon} due within 48h` - : 'Escalate overdue items' - : dueSoon > 0 - ? `${dueSoon} due within 48h` - : 'Nothing queued', - }, - { - label: 'Automation coverage', - value: `${automationRate}%`, - detail: total > 0 ? 'Completed via AI summarisation' : 'Awaiting uploads', - }, - ]; - - return ( - - - Operations insights - Quick health check on throughput and backlog. - - - {tiles.map(({ label, value, detail }) => ( -
    -
    - - {label} - - {detail} -
    - {value} -
    - ))} -     -     - ); +function percentage(value: number, total: number) { + return total > 0 ? Math.round((value / total) * 100) : 0; } diff --git a/frontend/src/pages/LandingPage.tsx b/frontend/src/pages/LandingPage.tsx new file mode 100644 index 0000000..38aa034 --- /dev/null +++ b/frontend/src/pages/LandingPage.tsx @@ -0,0 +1,291 @@ +import { type ReactNode, useEffect, useState } from 'react'; +import { Link } from 'react-router-dom'; +import { + ArrowRight, + BrainCircuit, + Check, + ChevronRight, + CircleCheck, + Database, + FileCheck2, + FileSearch, + Github, + Layers3, + MessageSquareText, + Network, + ScanLine, + Search, + ShieldCheck, + Sparkles, + UploadCloud, + UsersRound, + WalletCards, + Workflow, + Zap, +} from 'lucide-react'; + +import { Logo } from '../components/brand/Logo'; +import { Button } from '../components/ui/button'; +import { ThemeToggle } from '../components/ui/theme-toggle'; + +const capabilities = [ + { icon: ScanLine, title: 'Structure-aware extraction', copy: 'Keep headings, tables, page numbers, and reading order intact through ingestion.' }, + { icon: Search, title: 'Retrieval with intent', copy: 'Find the passages that answer a question, not merely the pages sharing its keywords.' }, + { icon: MessageSquareText, title: 'Grounded document chat', copy: 'Reason across long files while keeping every claim tied to its supporting context.' }, + { icon: FileCheck2, title: 'Evidence by default', copy: 'Expose source document, page, chunk, and confidence alongside every generated answer.' }, + { icon: Layers3, title: 'Multi-document synthesis', copy: 'Compare contracts, policies, reports, and research without losing provenance.' }, + { icon: ShieldCheck, title: 'Operational control', copy: 'Track ingestion, classifications, reviews, retries, and lifecycle events in one workspace.' }, +]; + +const useCases = [ + { icon: FileCheck2, team: 'Legal', question: 'Where do renewal terms conflict across our vendor agreements?', outcome: 'Clause review with page-level evidence' }, + { icon: ShieldCheck, team: 'Compliance', question: 'Which controls are incomplete, and what source proves it?', outcome: 'Auditable gap analysis' }, + { icon: BrainCircuit, team: 'Research', question: 'What do these studies agree on—and where do they diverge?', outcome: 'Cross-document synthesis' }, + { icon: WalletCards, team: 'Finance', question: 'What changed between the forecast and the board report?', outcome: 'Traceable variance analysis' }, + { icon: UsersRound, team: 'People', question: 'Which policy applies to this employee request?', outcome: 'Fast policy navigation' }, +]; + +const stack = ['React 19', 'TypeScript', 'FastAPI', 'Celery', 'Docling', 'PostgreSQL', 'pgvector', 'OpenAI']; + +export default function LandingPage() { + const [scrolled, setScrolled] = useState(false); + + useEffect(() => { + const update = () => setScrolled(window.scrollY > 12); + update(); + window.addEventListener('scroll', update, { passive: true }); + return () => window.removeEventListener('scroll', update); + }, []); + + return ( +
    +
    +
    + + +
    + + + +
    +
    +
    + +
    +
    +
    + +
    +
    +
    + One evidence layer +

    Built for the moment an answer matters.

    +

    Extraction, retrieval, generation, and review belong in one coherent workflow. DocuLens keeps them connected through source provenance.

    + +
    +
    + {capabilities.map(({ icon: Icon, title, copy }, index) => ( +
    + +

    0{index + 1}{title}

    {copy}

    +
    + ))} +
    +
    +
    + +
    +
    +
    +

    From file to finding

    A pipeline you can explain.

    +

    Every stage produces typed, inspectable outputs instead of hiding work inside one model call.

    +
    +
    + } title="Ingest" copy="Extract structure, OCR scans, and preserve page provenance." detail="Docling + validation" /> + } title="Understand" copy="Create bounded chunks, batch embeddings, and retrieve with filters." detail="pgvector + evaluation" /> + } title="Answer" copy="Construct grounded context and return evidence with confidence." detail="Structured model output" /> +
    +
    +
    + +
    +
    + +
    + Trust is a product feature +

    Answers that show their work.

    +

    A fluent answer is not enough. DocuLens pairs each claim with source context, page metadata, retrieval confidence, and the original document.

    +
    + + + +
    +
    +
    +
    + +
    +
    +
    +
    + Production-minded architecture +

    Fast at the edge. Durable underneath.

    +

    FastAPI accepts work quickly. Celery handles retryable document and model tasks. PostgreSQL remains the durable boundary for events, metadata, and vectors.

    +
    {stack.map((item) => {item})}
    +
    + +
    +
    +
    + +
    +
    +
    Evidence-heavy work

    Useful wherever the source matters.

    +
    + {useCases.map(({ icon: Icon, team, question, outcome }) => ( +
    +

    {team}

    +

    “{question}”

    +

    {outcome}

    +
    + ))} +
    +
    +
    + +
    +
    +
    +

    Make documents operational

    Know what your documents know.

    Run DocuLens locally, inspect the architecture, and ask your first evidence-grounded question.

    + +
    +
    +
    +
    + + +
    + ); +} + +function EvidenceCanvas() { + return ( +
    +
    +
    +
    +
    +
    Private workspace
    +
    +
    +
    +
    +

    Master services agreement

    Acme Cloud, Inc.

    18 / 28
    +

    12. Term and termination

    +

    This Agreement begins on the Effective Date and continues for the Initial Term.

    Thereafter, the Agreement automatically renews for successive one-year periods unless either party provides written notice at least sixty (60) days before renewal.

    Either party may terminate for material breach following a thirty-day cure period.

    +
    +
    +

    Answer

    94% confidence

    The agreement renews annually unless either party gives 60 days’ written notice. Material breach has a 30-day cure period.

    +
    +
    + +
    +
    +
    + ); +} + +function ProofPoint({ value, label }: { value: string; label: string }) { + return

    {value}

    {label}

    ; +} + +function Eyebrow({ children }: { children: ReactNode }) { + return

    {children}

    ; +} + +function WorkflowStep({ index, icon, title, copy, detail }: { index: string; icon: ReactNode; title: string; copy: string; detail: string }) { + return
    {icon}{index}

    {title}

    {copy}

    {detail}

    ; +} + +function EvidenceStory() { + return ( +
    +
    +

    Q2 Board Report.pdf

    41 pages · indexed 2 minutes ago

    +

    Question

    What are the top risks to the forecast?

    +
    +

    Grounded answer

    High confidence

    The report identifies enterprise concentration, delayed European expansion, and model inference costs as the three material forecast risks.

    +
    + ); +} + +function TrustRow({ title, copy }: { title: string; copy: string }) { + return

    {title}

    {copy}

    ; +} + +function ArchitectureMap() { + const nodes = [ + { icon: Workflow, title: 'FastAPI', copy: 'Validated, versioned API' }, + { icon: Zap, title: 'Celery', copy: 'Durable async workflows' }, + { icon: Network, title: 'RAG pipeline', copy: 'Extract, chunk, retrieve' }, + { icon: Database, title: 'Postgres + vectors', copy: 'Events and evidence' }, + ]; + return
    {nodes.map(({ icon: Icon, title, copy }, index) =>
    0{index + 1}

    {title}

    {copy}

    )}
    ; +} + +function SourcePill({ label }: { label: string }) { + return {label}; +} + +function SourceCard({ index, title, page, score, active = false }: { index: string; title: string; page: string; score: string; active?: boolean }) { + return
    {index}

    {title}

    {page}{score}
    ; +} + +function EvidenceMetric({ value, label }: { value: string; label: string }) { + return

    {value}

    {label}

    ; +} diff --git a/frontend/src/pages/LoginPage.tsx b/frontend/src/pages/LoginPage.tsx index 02f90a5..3653af4 100644 --- a/frontend/src/pages/LoginPage.tsx +++ b/frontend/src/pages/LoginPage.tsx @@ -1,35 +1,29 @@ import { type FormEvent, useState } from 'react'; -import { Navigate, useLocation, useNavigate } from 'react-router-dom'; -import { Lock, LogIn, Mail } from 'lucide-react'; +import { Link, Navigate, useLocation, useNavigate } from 'react-router-dom'; +import { ArrowLeft, ArrowRight, Check, Eye, EyeOff, FileCheck2, LockKeyhole, Sparkles } from 'lucide-react'; import { useAuth } from '../auth/useAuth'; +import { Logo } from '../components/brand/Logo'; import { Button } from '../components/ui/button'; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../components/ui/card'; import { Input } from '../components/ui/input'; -import { Badge } from '../components/ui/badge'; +import { ThemeToggle } from '../components/ui/theme-toggle'; +import { useSettings } from '../settings/useSettings'; -const DEMO_ACCOUNTS = [ - { email: 'admin@doculens.ai', role: 'Admin', password: 'Admin!234' }, - { email: 'analyst@doculens.ai', role: 'Analyst', password: 'Analyst!234' }, - { email: 'reviewer@doculens.ai', role: 'Reviewer', password: 'Reviewer!234' }, - { email: 'manager@doculens.ai', role: 'Manager', password: 'Manager!234' }, - { email: 'developer@doculens.ai', role: 'Developer', password: 'Developer!234' }, - { email: 'viewer@doculens.ai', role: 'Viewer', password: 'Viewer!234' }, -]; +const DEMO_ACCOUNT = { email: 'analyst@doculens.ai', password: 'Analyst!234' }; export function LoginPage() { const { user, login } = useAuth(); + const { serverConfig } = useSettings(); const navigate = useNavigate(); const location = useLocation(); const fromState = (location.state as { from?: { pathname?: string } } | undefined)?.from; const [email, setEmail] = useState(''); const [password, setPassword] = useState(''); + const [showPassword, setShowPassword] = useState(false); const [error, setError] = useState(null); const [isSubmitting, setIsSubmitting] = useState(false); - if (user) { - return ; - } + if (user || serverConfig?.showcase_read_only) return ; const handleSubmit = async (event: FormEvent) => { event.preventDefault(); @@ -37,93 +31,50 @@ export function LoginPage() { setIsSubmitting(true); try { await login(email, password); - const redirectTo = fromState?.pathname ?? '/'; - navigate(redirectTo, { replace: true }); - } catch (err) { - console.error(err); - setError('We could not sign you in with those credentials.'); + navigate(fromState?.pathname ?? '/app', { replace: true }); + } catch { + setError('Those credentials do not match this workspace.'); setIsSubmitting(false); } }; - return ( -
    - - - Sign in to DocuLens - - Secure access for analysts, reviewers, managers, and administrators. - - - -
    -
    - -
    - - setEmail(event.target.value)} - className="pl-9" - /> -
    -
    - -
    - -
    - - setPassword(event.target.value)} - className="pl-9" - /> -
    -
    + const useDemo = () => { + setEmail(DEMO_ACCOUNT.email); + setPassword(DEMO_ACCOUNT.password); + setError(null); + }; - {error ?

    {error}

    : null} + return ( +
    +
    +
    +
    +
    + Evidence-first document AI +

    Turn documents into decisions.

    +

    Search, summarize, classify, and ask complex questions—with every insight grounded in the source.

    +
    + {['Precise semantic retrieval', 'Page-level citations', 'Structured AI outputs', 'Auditable event history'].map((item) =>
    {item}
    )} +
    +
    +
    Built for teams where evidence matters.
    +
    - +
    +
    +
    +

    Workspace access

    Welcome back

    Sign in to continue to your DocuLens workspace.

    + +
    setEmail(event.target.value)} placeholder="you@company.com" className="h-11 rounded-xl bg-background" />
    +
    setPassword(event.target.value)} placeholder="Enter your password" className="h-11 rounded-xl bg-background pr-10" />
    + {error ?
    {error}
    : null} + - -
    -

    Role directory

    -

    - Demo credentials for each persona are available below. Rotate passwords in production and manage accounts from the admin - console. -

    -
      - {DEMO_ACCOUNTS.map((account) => ( -
    • -
      - {account.role} - {account.email} -
      -

      Password: {account.password}

      -
      • - ))} -
    -
    - - +
    Portfolio demo
    + +

    By continuing, you agree to the workspace security policy.
    Demo accounts are disabled in production.

    +
    +
    ); } diff --git a/frontend/src/pages/PipelinePage.tsx b/frontend/src/pages/PipelinePage.tsx index 386ccbc..3803414 100644 --- a/frontend/src/pages/PipelinePage.tsx +++ b/frontend/src/pages/PipelinePage.tsx @@ -130,14 +130,14 @@ export function PipelinePage() {
    - Pipeline overview - Quick snapshot of throughput and bottlenecks. + Document status + What is ready, processing, or needs your attention. - - - - + + + +
    @@ -151,8 +151,8 @@ export function PipelinePage() { - Document inventory - Select an item to inspect its pipeline details. + All documents + Select a document to review its summary, classification, and sources.
    - ) : ( -
    -
    -
    - {filteredDocuments.map((doc) => { - const isActive = doc.document_id === selectedDocumentId; - const role = inferRole(doc); - const status = inferStatus(doc); - const uploadedLabel = doc.uploaded_at ? new Date(doc.uploaded_at).toLocaleDateString() : '—'; - return ( - - ); - })} -
    -
    -
    - )} - -     + +
    setDocumentFilter(event.target.value)} placeholder="Search sources…" aria-label="Search sources" className="h-9 rounded-lg bg-background pl-8 text-xs" />
    + + + {isLoadingDocuments ?
    Loading sources…
    : documentsError ?
    {documentsError}
    : !filteredDocuments.length ?
    No sources match your filter.
    : ( +
    {filteredDocuments.map((doc) => { + const isActive = doc.document_id === selectedDocumentId; + return ; + })}
    + )} +     + + ) : null} - - + +
    -
    - - QA Studio - - - Conversational RAG grounded in {selectedDocument ? selectedDocument.filename ?? selectedDocument.document_id : 'your documents'}. - + +
    + + +
    -
    - {selectedDocument ? ( -
    - Document: - {selectedDocument.filename ?? selectedDocument.document_id} - {inferRole(selectedDocument)} - {inferStatus(selectedDocument)} - Uploaded {formatDateTime(selectedDocument.uploaded_at)} -
    - ) : null} - {historyError ? ( -
    {historyError}
    - ) : null} + {historyError ?
    {historyError}
    : null} - -
    - {!selectedDocument ? ( - - ) : !chatMessages.length ? ( - - ) : ( - chatMessages.map((message) => ) - )} + +
    +
    + {!selectedDocument ? : !chatMessages.length ? : chatMessages.map((message) => )} +
    -
    -
    -