diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d5e38db --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.venv +venv +**/__pycache__ +*.pyc +*.log +.env +.env.* +!.env.example +model-Thevindu +integrated-backend/data +integrated-backend/build +integrated-backend/tests +integrated-frontend/node_modules +integrated-frontend/dist diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..537b619 --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Public hostname or IP used by the browser and Keycloak. +PUBLIC_HOST=learnmateai.dinurag.dev +PUBLIC_ORIGIN=https://learnmateai.dinurag.dev + +# Required secrets. Generate with: openssl rand -hex 32 +JWT_SECRET_KEY=replace-with-a-long-random-secret +KEYCLOAK_ADMIN_PASSWORD=replace-with-a-strong-admin-password + +# Optional model settings. The backend downloads missing GGUF files into the persistent +# learnmate_models volume from these public Hugging Face files. +LEARNMATE_GENERATOR_REPO=Qwen/Qwen2.5-3B-Instruct-GGUF +LEARNMATE_GENERATOR_FILE=qwen2.5-3b-instruct-q4_k_m.gguf +LEARNMATE_JUDGE_REPO=bartowski/Llama-3.2-3B-Instruct-GGUF +LEARNMATE_JUDGE_FILE=Llama-3.2-3B-Instruct-Q4_K_M.gguf + +# Leave these same-origin production defaults unless the architecture changes. +LEARNMATE_GENERATOR_BACKEND=llamacpp +LEARNMATE_JUDGE_BACKEND=llamacpp +LEARNMATE_VECTOR_BACKEND=qdrant +LEARNMATE_MAX_PDF_MB=10 +LEARNMATE_MAX_PAGE_COUNT=300 +API_WARM_UP=1 +API_WARM_MODELS=0 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 50d918f..8a0dffa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,9 +3,13 @@ name: Deploy Application on: push: branches: - - dinura-deployment + - deployment workflow_dispatch: +concurrency: + group: deploy-production + cancel-in-progress: false + jobs: ci: runs-on: ubuntu-latest @@ -24,59 +28,112 @@ jobs: node-version: '20' - name: Backend syntax check + run: python -m compileall integrated-backend/app integrated-backend/learnmate + + - name: Frontend lint run: | - python -m compileall integrated-backend + npm ci --prefix integrated-frontend + npm run lint --prefix integrated-frontend - name: Frontend build + run: npm run build --prefix integrated-frontend + + - name: Compose configuration check run: | - npm ci --prefix integrated-frontend - npm run build --prefix integrated-frontend + cp .env.example .env + docker compose config >/dev/null deploy: needs: ci runs-on: ubuntu-latest steps: - - name: Deploy to EC2 via SSH - uses: webfactory/ssh-agent@v0.9.0 + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - ssh-private-key: ${{ secrets.EC2_SSH_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} - - name: SSH and deploy latest app + - name: Deploy via AWS SSM env: - EC2_HOST: ${{ secrets.EC2_HOST }} - EC2_USERNAME: ${{ secrets.EC2_USERNAME }} + EC2_INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }} + EC2_APP_DIR: ${{ secrets.EC2_APP_DIR }} + APP_ENV_FILE_B64: ${{ secrets.APP_ENV_FILE_B64 }} + GIT_DEPLOY_TOKEN: ${{ secrets.GIT_DEPLOY_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} run: | - set -e - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${EC2_USERNAME}@${EC2_HOST} <<'EOF' - set -e - cd ~/app - git fetch origin - git checkout dinura-deployment - git reset --hard origin/dinura-deployment - - if [ ! -f .env ]; then - cp .env.example .env - fi - - chmod 600 .env - if docker info >/dev/null 2>&1; then - DOCKER_CMD="docker" - elif sudo -n docker info >/dev/null 2>&1; then - DOCKER_CMD="sudo -n docker" - else - echo "Docker daemon is not accessible for deployment user" - exit 1 - fi - - $DOCKER_CMD compose build - $DOCKER_CMD compose up -d - $DOCKER_CMD compose ps - - if curl -fsS http://127.0.0.1/api/health >/dev/null 2>&1; then - echo "Backend health check passed" - else - echo "Backend health check failed" - $DOCKER_CMD compose logs --tail 200 backend - exit 1 - fi - EOF + set -euo pipefail + + if [ -z "${EC2_INSTANCE_ID:-}" ] || [ -z "${EC2_APP_DIR:-}" ]; then + echo "Missing required secrets: EC2_INSTANCE_ID and EC2_APP_DIR must be set." >&2 + exit 1 + fi + + echo "Checking that instance ${EC2_INSTANCE_ID} is registered with SSM..." + ONLINE=$(aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=${EC2_INSTANCE_ID}" \ + --query "length(InstanceInformationList)" \ + --output text) + if [ "${ONLINE}" != "1" ]; then + echo "Instance ${EC2_INSTANCE_ID} is not online in SSM." >&2 + exit 1 + fi + + PARAMS_FILE=$(mktemp) + python3 scripts/build-ssm-deploy-params.py > "${PARAMS_FILE}" + + echo "Sending SSM command to instance ${EC2_INSTANCE_ID}" + COMMAND_ID=$(aws ssm send-command \ + --instance-ids "${EC2_INSTANCE_ID}" \ + --document-name "AWS-RunShellScript" \ + --comment "Deploy LearnMate via GitHub Actions (${GITHUB_REPOSITORY}@${GITHUB_SHA})" \ + --timeout-seconds 3600 \ + --parameters "file://${PARAMS_FILE}" \ + --query "Command.CommandId" \ + --output text) + rm -f "${PARAMS_FILE}" + + echo "Waiting for SSM command ${COMMAND_ID} to complete..." + + STATUS="InProgress" + while [ "$STATUS" = "InProgress" ] || [ "$STATUS" = "Pending" ] || [ "$STATUS" = "Delayed" ]; do + sleep 15 + STATUS=$(aws ssm get-command-invocation \ + --instance-id "${EC2_INSTANCE_ID}" \ + --command-id "${COMMAND_ID}" \ + --query "Status" \ + --output text) + echo "Current status: $STATUS" + done + + if [ "$STATUS" != "Success" ]; then + echo "SSM command did not finish successfully. Fetching logs..." + fi + + echo "=== STANDARD ERROR ===" + aws ssm get-command-invocation \ + --instance-id "${EC2_INSTANCE_ID}" \ + --command-id "${COMMAND_ID}" \ + --query "StandardErrorContent" \ + --output text + + echo "=== STANDARD OUTPUT ===" + aws ssm get-command-invocation \ + --instance-id "${EC2_INSTANCE_ID}" \ + --command-id "${COMMAND_ID}" \ + --query "StandardOutputContent" \ + --output text + + STATUS=$(aws ssm get-command-invocation \ + --instance-id "${EC2_INSTANCE_ID}" \ + --command-id "${COMMAND_ID}" \ + --query "Status" \ + --output text) + + if [ "$STATUS" != "Success" ]; then + echo "Deployment failed with status: ${STATUS}" >&2 + exit 1 + fi \ No newline at end of file diff --git a/AWS_DEPLOYMENT.md b/AWS_DEPLOYMENT.md new file mode 100644 index 0000000..5b82ec5 --- /dev/null +++ b/AWS_DEPLOYMENT.md @@ -0,0 +1,195 @@ +# AWS Deployment Guide + +## Architecture + +``` +GitHub push -> GitHub Actions CI -> AWS SSM -> EC2 (docker compose) -> Nginx -> Browser +``` + +- **EC2**: Ubuntu, Docker, Git, SSM Agent +- **Compose**: Nginx (80/443) → Frontend + Backend + Keycloak, MongoDB, Qdrant (internal only) +- **CI/CD**: GitHub Actions runs tests, then deploys via AWS Systems Manager (no SSH needed) +- **Domain**: `learnmateai.dinurag.dev` with Let's Encrypt certs on Nginx + +--- + +## 1. One-time EC2 Setup + +### 1.1 Launch Instance + +```bash +# Recommended: Ubuntu 22.04 LTS +# Instance type: t3.large or larger (needs RAM for two ~2GB GGUF models) +# Storage: 30GB+ GP3 (models + vector data + MongoDB) +# Security group: allow inbound TCP 22 (SSH), 80, 443 from 0.0.0.0/0 +# everything else is internal-only +``` + +### 1.2 SSH and Bootstrap + +```bash +ssh ubuntu@ + +# Install Docker +curl -fsSL https://get.docker.com | sh +sudo systemctl enable --now docker +sudo usermod -aG docker ubuntu + +# Install Git +sudo apt update && sudo apt install -y git + +# Verify SSM Agent (preinstalled on Ubuntu AMIs) +sudo systemctl status snap.amazon-ssm-agent.amazon-ssm-agent.service +# If not running: sudo snap install amazon-ssm-agent --classic +``` + +### 1.3 Attach IAM Instance Profile + +1. Go to AWS Console → IAM → Roles → **Create role** +2. Trusted entity: **AWS service** → **EC2** +3. Attach policy: `AmazonSSMManagedInstanceCore` +4. Name it e.g. `LearnMateEC2SSM` +5. EC2 Console → Instances → select your instance → **Actions → Security → Modify IAM role** +6. Attach the `LearnMateEC2SSM` role + +Verify from your local machine: + +```bash +aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=" \ + --query "InstanceInformationList[].InstanceId" \ + --output text +# Should return your instance ID +``` + +### 1.4 Create CI IAM User for GitHub Actions + +1. IAM → Users → **Add user** → name: `LearnMateCICD` +2. Access type: **Programmatic access** +3. Attach policy: **Create inline policy** (JSON): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ssm:SendCommand", + "ssm:GetCommandInvocation", + "ssm:ListCommandInvocations", + "ssm:DescribeInstanceInformation" + ], + "Resource": [ + "arn:aws:ec2:::instance/", + "arn:aws:ssm:::managed-instance/*" + ] + } + ] +} +``` + +4. Save the **Access Key ID** and **Secret Access Key** + +--- + +## 2. GitHub Repository Secrets + +Go to: **Settings → Secrets and variables → Actions → New repository secret** + +| Secret | Value | Required | +|--------|-------|----------| +| `AWS_ACCESS_KEY_ID` | CI IAM user access key | Yes | +| `AWS_SECRET_ACCESS_KEY` | CI IAM user secret key | Yes | +| `AWS_REGION` | e.g. `ap-southeast-1` | Yes | +| `EC2_INSTANCE_ID` | e.g. `i-0123456789abcdef0` | Yes | +| `EC2_APP_DIR` | e.g. `/home/ubuntu/app` | Yes | +| `APP_ENV_FILE_B64` | base64-encoded `.env` (see below) | Yes (first deploy) | +| `GIT_DEPLOY_TOKEN` | GitHub PAT with repo read access | If repo is private | + +### Generate `APP_ENV_FILE_B64` + +```bash +cd ~/app # your local project root +base64 -w 0 .env +# On macOS: base64 .env | tr -d '\n' +``` + +Copy the output string and paste it as the `APP_ENV_FILE_B64` secret value. + +--- + +## 3. Deploy + +### Option A: Automatic (push to `deployment` branch) + +```bash +git checkout deployment +git merge main # or whatever branch has your changes +git push origin deployment +``` + +GitHub Actions will: +1. Run Python syntax check +2. Run frontend lint + build +3. Validate `docker compose config` +4. Send SSM deploy command to EC2 +5. EC2 pulls code, builds containers, configures Keycloak, verifies health + +### Option B: Manual trigger + +GitHub Console → Actions → **Deploy Application** → **Run workflow** → branch: `deployment` + +--- + +## 4. Post-Deploy Verification + +```bash +# Check health +curl https://learnmateai.dinurag.dev/api/health + +# Check containers (via SSM) +aws ssm send-command \ + --instance-ids \ + --document-name "AWS-RunShellScript" \ + --parameters 'commands=["docker compose -f /home/ubuntu/app/docker-compose.yml ps"]' +``` + +--- + +## 5. Subsequent Updates + +Just push to the `deployment` branch. The workflow handles everything. + +**Do not run `docker compose down -v`** — that deletes MongoDB, Qdrant, Keycloak, and model volumes. + +--- + +## 6. Troubleshooting + +| Issue | Fix | +|-------|-----| +| SSM "instance not online" | Check IAM role, SSM agent status, security group allows outbound HTTPS | +| Keycloak redirect mismatch | `configure-keycloak.sh` runs after deploy; verify `PUBLIC_ORIGIN` matches domain | +| Models downloading slowly | First request downloads ~4GB GGUF files; subsequent requests use cached volume | +| Nginx 502 | Backend still starting; wait 60s and retry; check `docker compose ps` | +| OOM kills | Instance too small; upgrade to t3.xlarge or add swap | + +--- + +## 7. Local Testing Before Push + +```bash +# Validate compose config +docker compose config + +# Full stack +docker compose up -d --build +docker compose ps +curl -fsS http://127.0.0.1/api/health + +# Keycloak setup +PUBLIC_ORIGIN="https://learnmateai.dinurag.dev" \ + KEYCLOAK_ADMIN_PASSWORD="" \ + ./scripts/configure-keycloak.sh +``` diff --git a/README-APPLICATION.md b/README-APPLICATION.md deleted file mode 100644 index fba8f39..0000000 --- a/README-APPLICATION.md +++ /dev/null @@ -1,238 +0,0 @@ -# LearnMateAI — the application - -Upload a PDF. Ask it questions and get answers that cite the pages they came from, or -generate study material from it — summaries, key points, multiple-choice questions and -short-answer practice questions. Everything the system produces is graded by a second -model before it reaches you. - -This document describes the **software**: how the pieces fit together and why they are -arranged the way they are. Two companion documents cover the rest: - -| Document | Covers | -|---|---| -| `README-USAGE.md` | running it, and using it as a student | -| `README-MACHINE-LEARNING.md` | the models, retrieval, prompting and evaluation | -| `integrated-backend/README.md` | backend detail: layout, configuration, endpoints | - ---- - -## Shape of the system - -``` -┌────────────────────────┐ ┌──────────────────────────────────────────┐ -│ integrated-frontend │ HTTP │ integrated-backend │ -│ React 19 + Vite 8 │◄───────►│ │ -│ Tailwind, React │ JSON │ server.py FastAPI: CORS, routers │ -│ Router │ + JWT │ app/ the web layer │ -└────────────────────────┘ │ learnmate/ the engine │ - └───────────────┬──────────────────────────┘ - │ - ┌────────────────────┼────────────────────┐ - ▼ ▼ ▼ - ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ - │ MongoDB │ │ Qdrant │ │ Local models │ - │ :27018 │ │ :6335 │ │ two GGUFs │ - │ PDFs (GridFS), │ │ chunk vectors, │ │ in-process │ - │ text, accounts,│ │ HNSW index │ │ via llama.cpp │ - │ history, jobs │ │ │ │ │ - └────────────────┘ └────────────────┘ └────────────────┘ -``` - -Nothing leaves the machine. The models run locally through `llama.cpp`, and the two -databases run in Docker containers alongside the server. - -### Why the ports are unusual - -MongoDB is on **27018** and Qdrant on **6335**, not their conventional 27017 and 6333. -Both defaults are already answered on the development machine by services belonging to -other projects, and sharing a database server means sharing a failure — another project's -`docker compose down -v` would take this one's corpus with it. Each has its own container -and its own named volume. - ---- - -## The two halves of the backend - -The single most important boundary in the codebase: - -``` -learnmate/ the engine. Knows nothing about HTTP, users or requests. - A library that ingests PDFs and generates from them. - -app/ the web layer. Accounts, access control, endpoints, the job queue. -``` - -`learnmate/` is single-user and synchronous: give it a PDF and it will ingest it, give it a -question and it will answer. `app/` is what turns that into a service several students can -use at once. The rule the layering exists to enforce is that **a router never touches the -engine or the database directly, and a service never raises an HTTPException**. - -``` -integrated-backend/ -├── server.py FastAPI app: CORS, routers, /api/health -├── app/ -│ ├── auth/ bcrypt hashing, JWT issue and verify -│ ├── routers/ endpoints — validate input, delegate, return -│ ├── services/ the work, called by routers and the job worker alike -│ ├── jobs/ the background queue (worker.py, runners.py) -│ ├── deps.py get_current_user, the dependency every protected route uses -│ └── errors.py engine exceptions → HTTP status codes, in one place -└── learnmate/ - ├── ingestion/ validate → extract → clean → chunk → embed - ├── chat_agent/ a LangGraph state machine, one pass per message - ├── resource_agent/ a LangGraph state machine, one pass per resource - ├── evaluator/ two gates: structural validators, then an LLM judge - ├── llm/ three interchangeable model backends - └── storage/ MongoDB, GridFS and the vector store -``` - ---- - -## Everything slow is a job - -Local inference on a 3B model takes tens of seconds to minutes. No browser holds a -connection that long and no proxy allows it, so every slow endpoint answers immediately -with a job id and the client polls: - -``` -POST /api/documents/upload → 202 {document, job_id} -POST /api/resources/generate → 202 {job_id} -POST /api/chat/sessions/{id}/messages → 202 {job_id} - -GET /api/jobs/{job_id} → queued | running | done | failed - + progress, then result -``` - -A job record lives in MongoDB, not in memory, so a poll works regardless of which process -answers it and a client that reloads the page does not lose its place. - -**One worker thread, and that is a correctness requirement rather than a resource one.** -`llama_cpp.Llama` holds a single mutable context: two threads generating at once interleave -their tokens and corrupt both replies. Locks sit next to the things they protect — -generation in `llm/llamacpp.py`, model loading in `llm/runtime.py`, `llm/embeddings.py` and -`llm/rerank.py` — so a job's database and network work no longer blocks anything. - -### The chat turn has two milestones - -Writing an answer is the fast half of a turn; judging it, and regenerating when the judge -says no, is the slow half. So the reply is handed over as soon as it exists rather than -when the turn ends: - -``` -tokens stream → progress.partial the answer being typed -reply complete → progress.reply_ready a finished answer, readable now -judge runs, may retry → (text frozen, off-screen) -job done → result the winning attempt -``` - -A regeneration is deliberately **not** streamed over the visible answer — replacing a -finished paragraph with a half-written one reads as the assistant having second thoughts in -public. If the retry scores better, the finished result swaps it in once, cleanly. - ---- - -## Data model - -Ten MongoDB collections plus one GridFS bucket, and one Qdrant collection. - -| Collection | Holds | -|---|---| -| `users` | accounts: email, bcrypt hash, name | -| `user_documents` | who may see which document | -| `documents` | one row per **distinct file**, keyed by the SHA-256 of its bytes | -| `pages` | cleaned page text — what the models actually read | -| `chunks` | chunk records (vectors live in Qdrant) | -| `sessions` | which PDF a chat session is bound to | -| `chat_turns` | one row per turn, with mode, score, pages on the assistant's | -| `resources` | generated material with its verdict and attempt trail | -| `evaluations` | every verdict, passes included — the quality log | -| `jobs` | the background queue | -| `pdfs` (GridFS) | the original files | - -Documents are keyed by content hash, so one PDF is stored and embedded **once** however -many people upload it. That is also why ownership cannot be a field on the document and -lives in `user_documents` instead. - ---- - -## The frontend - -React 19 with Vite, React Router and Tailwind. `src/api/` wraps every endpoint on one -axios instance with two interceptors: attach the token on the way out, and on a 401 clear -the session and redirect to `/login`. - -``` -src/ -├── api/ one module per resource, over a shared axios client -├── context/ AuthProvider — token, user, and a startup /me verification -├── components/ Layout, Sidebar, Topbar, PublicLayout, chat and chart pieces -├── hooks/ useJob — run a 202 endpoint and poll it to completion -└── pages/ one per route -``` - -Routes fall into three kinds, and the middle one is the reason the router is not just a -list: - -| Kind | Routes | Frame | -|---|---|---| -| open | `/login`, `/register` | none | -| explore | `/` (Home), `/about`, `/tour` | public header when signed out, app rail when signed in | -| protected | `/dashboard`, `/documents`, `/resources`, `/chat`, `/analytics`, `/account` | app rail, redirect to `/login` | - -`/` is Home rather than a redirect to the dashboard: a first-time visitor landing on a login -form has been asked to commit before being told what to. - -`useJob` is the hook the whole app is shaped around — it takes the function that produces -the 202, reads the job id off it, polls until the job finishes, and exposes `status`, -`progress` and finally `result` or `error`. It aborts on unmount, so navigating away -mid-generation stops the polling instead of setting state on a dead tree. - ---- - -## Request path, end to end - -One chat message, from click to answer: - -``` -1 Chat page POST /api/chat/sessions/{id}/messages -2 router checks ownership, enqueues, returns 202 {job_id} -3 worker thread picks it up, marks the job running -4 service builds a ChatAgent for this session and user -5 chat_agent graph rewrite → retrieve → generate → evaluate → decide → persist -6 runner streams tokens onto progress.partial as they arrive -7 frontend useJob polls; StreamingMessage renders the text -8 worker writes the result, marks the job done -9 frontend swaps the streaming bubble for the stored turn -``` - -The graph in step 5 is one pass of a LangGraph state machine with a single conditional -edge — accept the reply, or regenerate it with the judge's instruction. It is described in -`README-MACHINE-LEARNING.md`. - ---- - -## Access control - -Every protected route depends on `get_current_user`, which verifies the JWT and loads the -account. Ownership is then checked **in the service layer**, on every path that names a -document, session or resource — `services/ownership.py` is the only place that decision is -made, so there is no route that forgot to ask. - -Checks happen *before* work is queued, so posting into somebody else's session is a 403 -immediately rather than a job that fails a minute later. - ---- - -## Configuration - -Two files, both read from `integrated-backend/.env`: - -- `learnmate/config.py` — the engine: which models, which databases, which thresholds -- `app/config.py` — the web layer: JWT secret and expiry, CORS origin, password rules - -`.env.example` documents every setting with the value the code uses when it is absent. -`JWT_SECRET_KEY` is the only one with no default, deliberately: a fallback secret works in -development, ships unnoticed, and makes every token it ever signed forgeable. - -Swapping the generator for a different model — a finetune, a served endpoint, or a cloud -API — is two lines of `.env` and no code change. diff --git a/README-MACHINE-LEARNING.md b/README-MACHINE-LEARNING.md deleted file mode 100644 index c6f65bd..0000000 --- a/README-MACHINE-LEARNING.md +++ /dev/null @@ -1,125 +0,0 @@ -# LearnMateAI — models, retrieval, evaluation - -Two separate ML stories share this repository. They must not be confused. - -| Path | When it runs | What it is | -|---|---|---| -| **Live** (`integrated-backend/learnmate/`) | Every chat / resource request | Local GGUF generator + a different-family judge + RAG | -| **Offline** (`model-Thevindu/`) | Manual Colab / scripts, never in the request path | LoRA fine-tune of Qwen 2.5 on Sri Lankan legal text, then a promotion gate | - -Swapping the live generator for a served finetune or a cloud API is two lines in -`integrated-backend/.env` and no code change. That pointer is the only thing the app -should learn from the offline track. - -Companion docs: [README-APPLICATION.md](README-APPLICATION.md) (software shape), -[README-USAGE.md](README-USAGE.md) (how to run it), -[README-TECHNOLOGIES.md](README-TECHNOLOGIES.md) (library list). - ---- - -## 1. Live models - -| Role | Model | Why | -|---|---|---| -| Generator | Qwen2.5-3B-Instruct (Q4 GGUF) | Writes chat replies and study material | -| Judge | Llama-3.2-3B-Instruct (Q4 GGUF) | Grades the generator; different family on purpose | -| Embeddings | `all-MiniLM-L6-v2` | Chunk and query vectors | -| Reranker | `ms-marco-MiniLM-L-6-v2` | Reorders the top retrieved chunks with the question | - -The judge is a different family so it cannot just praise its own style. Q4 quantisation -is what makes both 3B models fit a laptop. - -Optional and off by default: Gemini (`google-genai`) if `.env` points the generator or -fallback at a cloud API. - ---- - -## 2. Retrieval and the chat graph - -A chat turn is one LangGraph pass: - -`rewrite → retrieve → generate → evaluate → decide → persist` - -- **Retrieve:** embed the question, nearest chunks in Qdrant, then cross-encoder rerank. -- **Generate:** answer from those chunks. The live Verification Agent treats a claim not - supported by retrieved context as a hallucination. -- **Evaluate / decide:** structural checks, then the judge. One retry on the same - generator if the judge rejects; the live path does **not** silently swap in Gemini - because an answer scored poorly. Gemini (or another API) is an *availability* - fallback — process down, timeout, missing pointer — not a per-answer quality router. - -Resource generation (summary, MCQ, keypoints, practice questions) is a second graph in -`learnmate/resource_agent/`. Same rule: grounded in the uploaded PDF, then judged. - ---- - -## 3. Offline domain fine-tune (`model-Thevindu/`) - -``` -Sri Lankan legal PDFs - → Stage 1 parse / clean / section-aware chunks - → Stage 2 instruction pairs (Q&A, summary, MCQ) - → Stage 3 chapter-group train/val/test + a small whole-document holdout - → LoRA / QLoRA on Qwen2.5-1.5B-Instruct (Colab T4) - → Eval vs acceptance_thresholds.yaml + fallback comparison - → staging → teammate sign-off → promote a live pointer → rollback kept -``` - -| Part | Path | -|---|---| -| Pipeline | `model-Thevindu/01_dataset_pipeline/` | -| Fine-tune notebook | `model-Thevindu/02_finetuning/finetune_qwen25_lora.ipynb` | -| Eval, registry, checklist | `model-Thevindu/03_testing_and_versioning/` | -| Lineage, training log, mentor note | `model-Thevindu/04_docs/` | -| Promote / rollback process | `model-Thevindu/05_mlops_workflow/` | - -Weights (`*.safetensors`, adapters, checkpoints) are gitignored. The auditable record of -a run is `02_finetuning/run_records/.json`. - -### Dataset issues already found - -- **GI-001** — Stage 2 was writing ungrounded section citations (~38% on a spot check). - Fixed in the generator + `validate_pairs.py`. Full-corpus reject rate after the fix: 1.0%. -- **GI-002** — Whole-document split left subjects with zero training pairs. Split is now - by `(doc_id, chapter)`. That number is **`in_corpus_accuracy (chapter-held-out)`**, not - true generalisation. `test_strict.jsonl` holds out one full document per multi-document - subject (`accuracy (document-held-out)`). Six subjects still have only one source - document, so they have no true generalisation test until the corpus grows. - -Pilot corpus: 21 files ingested, 19 parsed; 13 Tier A / 19 Tier B in the target manifest. -See `04_docs/mentor_pilot_disclosure.md`. - -### First real candidate — do not promote - -`qwen25-lora-20260815-090709` on `lm-legal-v0.1` (1590 train / 339 val). Logged in -`version_registry.csv`. Both splits **FAIL**. - -| Metric | chapter `test` | strict `test_strict` | Gate | -|---|---|---|---| -| Accuracy (token-F1, proxy) | 0.717 | 0.836 | looked like a pass | -| Accuracy (LLM-as-judge, `gpt-4o-mini`) | **0.557** | **0.621** | ≥ 0.70 — **fail** | -| Groundedness (`validate_pairs`) | 0.877 | 0.921 | ≥ 0.85 — pass | -| Hallucination | 0.123 | 0.079 | ≤ 0.15 — pass | -| Latency p95 | 16.4 s | 14.9 s | ≤ 8 s — fail (Colab T4 sequential 4-bit, not serving hardware) | -| vs gpt-4o-mini (token-F1) | 0.717 vs 0.871 | 0.836 vs 0.918 | lose | - -Token-F1 overstated correctness. The judge that matches the live “unsupported claim = -fail” idea is the number to read. Keep the API / local GGUF as the live generator. Treat -this adapter as a pilot, not a replacement. - -Promotion requires `passed=True` on **document-held-out**, a filled -`promotion_checklist.md`, and a teammate who did not train the run. None of that is done. - ---- - -## 4. What “evaluation” means in each path - -| | Live app | Offline gate | -|---|---|---| -| Correctness | Llama 3.2 judge + structural validators | LLM-as-judge preferred; token-F1 only if no judge API | -| Groundedness | Claim must be supported by retrieved chunks | Same citation checker as Stage 2 (`validate_pairs.py`) | -| Fallback | Availability (down / timeout / no pointer) | Candidate must beat or nearly match the configured API on the same test set | - -If those two groundedness definitions diverge, a green offline gate says nothing about -production behaviour. The current contract is in -`model-Thevindu/03_testing_and_versioning/acceptance_thresholds.yaml` (version 2). diff --git a/README-TECHNOLOGIES.md b/README-TECHNOLOGIES.md deleted file mode 100644 index 2b8b3d8..0000000 --- a/README-TECHNOLOGIES.md +++ /dev/null @@ -1,108 +0,0 @@ -# LearnMateAI — technologies used - -Every technology, library and model in the project, with a plain-English reason for each. - -Nothing here is a cloud service. The whole system — including the language models — runs on -one machine, and no uploaded document ever leaves it. - -**At a glance** - -| Layer | Choice | -|---|---| -| Frontend | React 19, Vite 8, Tailwind CSS 4, React Router 7, Axios | -| Backend | Python 3.13, FastAPI, Uvicorn | -| AI orchestration | LangChain Core 1.x, LangGraph 1.x | -| Language models | Qwen2.5-3B (writes), Llama-3.2-3B (grades) — local, via llama.cpp | -| Search | sentence-transformers embeddings + a cross-encoder reranker | -| Databases | MongoDB 8, Qdrant 1.18 (both in Docker) | -| Security | PyJWT, bcrypt | - ---- - -## 1. Frontend - -| Technology | Version | Why it is used | -|---|---|---| -| **React** | 19.2 | Builds the interface out of reusable components. The screen changes constantly here — a reply arriving word by word, a document turning from Processing to Ready — and React redraws only the part that changed. | -| **Vite** | 8.2 | The build tool and development server. Saves show up in the browser instantly, and it produces one small optimised bundle for release. | -| **React Router** | 7.18 | Gives each page its own web address, so `/chat` and `/documents` can be bookmarked, shared and reloaded. Also enforces which pages need a login. | -| **Tailwind CSS** | 4.3 | Styling written directly on the element instead of in separate stylesheets. Keeps every page visually consistent without a growing pile of CSS files nobody dares delete. | -| **Axios** | 1.19 | Talks to the backend. Chosen over the browser's built-in `fetch` for one feature: interceptors. The login token is attached to every request automatically, and an expired session is caught in one place instead of in forty. | -| **ESLint** | 10.8 | Catches mistakes — unused variables, misused React hooks — before they reach the browser. | - ---- - -## 2. Backend — the web layer - -| Technology | Version | Why it is used | -|---|---|---| -| **FastAPI** | 0.115+ | The web framework. Chosen because it validates every incoming request against a declared shape, so bad input is rejected with a clear message rather than crashing somewhere deeper. It also generates live API documentation at `/docs` for free. | -| **Uvicorn** | 0.30+ | The server that actually runs FastAPI and handles the network connections. | -| **Pydantic** | (with FastAPI) | Defines what a valid request looks like. If a field is missing or the wrong type, the request never reaches our code. | -| **PyJWT** | 2.8 | Issues and checks login tokens. After signing in, the browser holds a signed token instead of the server keeping a session — which means the server stays stateless and can be restarted without logging everyone out. | -| **bcrypt** | 4.1 | Hashes passwords. Deliberately slow, which is exactly what you want: it makes guessing passwords in bulk impractical even if the database is stolen. Passwords are never stored or logged in readable form. | -| **python-multipart** | 0.0.9 | Handles file uploads. FastAPI cannot accept an uploaded PDF without it. | -| **email-validator** | 2.1 | Checks that a registration email is really an email address. | -| **python-dotenv** | 1.0 | Reads settings from a `.env` file, so passwords and model paths are configuration rather than code. | - ---- - -## 3. The AI engine - -| Technology | Version | Why it is used | -|---|---|---| -| **langchain-core** | 1.5 | The common vocabulary for talking to language models — messages, prompts, streaming, and the `BaseChatModel` interface our local-model wrapper implements. Because our code speaks this interface, swapping the local model for a served one or a cloud API is a configuration change instead of a rewrite. | -| **langchain-text-splitters** | 1.1 | Cuts a document into overlapping chunks, breaking at sentence and paragraph boundaries rather than mid-word, so a chunk is still readable on its own. | -| **LangGraph** | 1.2 | Runs the two agents as **state machines** rather than one long function. A chat turn is `rewrite → retrieve → generate → evaluate → decide`, where `decide` either accepts the answer or sends it back to be rewritten. Loops like that are awkward to write by hand and are what this library exists for. | -| **langchain**, **langchain-community** | 1.3, 0.4 | Listed in `requirements.txt` for compatibility with the LangChain family, but **not imported anywhere in this codebase** — the project uses `langchain-core`, `langchain-text-splitters` and `langgraph` directly. Noted here rather than left to be discovered. | -| **llama-cpp-python** | 0.3 | Runs the language models **on this machine** — no API key, no internet, no data leaving the computer. It also supports *grammar-constrained decoding*, which is the important part: the model is forced to produce valid JSON. Without it, a 3B model asked politely for JSON returns prose about half the time. | -| **sentence-transformers** | 5.0 | Turns text into vectors (lists of numbers) so passages can be found by meaning rather than by exact words. Runs both the search model and the reranker. | -| **PyTorch** | 2.6 | The numerical engine underneath sentence-transformers. Not used directly. | -| **transformers** | 5.0 | Model loading and tokenisation, used by sentence-transformers. Not used directly. | -| **PyMuPDF** | 1.26 | Reads PDFs — page text, page count, and whether a file is encrypted or corrupt. One library for extraction, cleaning and upload validation, so a PDF behaves the same at every step. | -| **NumPy** | 2.0 | Vector arithmetic, used when comparing embeddings. | -| **huggingface_hub** | 0.20 | Downloads the models on first run, so a fresh install needs no manual file copying. | -| **google-genai** | 1.0 | **Optional and unused by default.** Only loaded if the project is pointed at Google's Gemini API instead of the local models. Safe to uninstall for a fully offline setup. | - -### The models themselves - -| Model | Size | Job | -|---|---|---| -| **Qwen2.5-3B-Instruct** (Q4) | ~2 GB | The **generator** — writes chat replies and study material. | -| **Llama-3.2-3B-Instruct** (Q4) | ~2 GB | The **judge** — grades what the generator wrote and says what to fix. | -| **all-MiniLM-L6-v2** | ~90 MB | Turns document chunks and questions into vectors for searching. | -| **ms-marco-MiniLM-L-6-v2** | ~90 MB | Re-reads the top search results *together with* the question and reorders them properly. | - -Two deliberate choices worth explaining to a reader: - -- **The judge is a different model family from the generator.** A model grading its own writing rates its own style highly, and the quality check stops catching anything. Different weights means a genuine second opinion. -- **"Q4" means quantised.** The models are compressed from 16 bits per number to about 4, which makes them roughly a quarter of the size and fast enough to run on an ordinary laptop, for a small loss in quality. - ---- - -## 4. Databases - -| Technology | Version | Why it is used | -|---|---|---| -| **MongoDB** | 8 | Stores everything that is not a vector: accounts, documents, page text, chat history, generated material, the evaluation log, the job queue. Chosen because the records have genuinely different shapes — an MCQ, a summary and a chat turn are not the same thing — and a document database stores them without inventing a table for each. | -| **GridFS** | (part of MongoDB) | Stores the original PDF files inside MongoDB, so the uploaded file and its extracted text cannot be separated by a stray file deletion. | -| **PyMongo** | 4.9 | The Python driver for MongoDB. | -| **Qdrant** | 1.18 | A purpose-built **vector database**. It holds the embeddings of every document chunk and answers "which passages are closest in meaning to this question" in milliseconds, using a proper index rather than comparing against every chunk one at a time. | -| **qdrant-client** | 1.15 | The Python driver for Qdrant. Kept within one minor version of the server, which is as far apart as it tolerates. | -| **dnspython** | 2.7 | Only needed if MongoDB is moved to a hosted cluster (`mongodb+srv://` addresses). Harmless locally. | -| **Docker Compose** | — | Runs both databases as containers with one command, each with its own storage volume, so no manual database installation is required and neither can be wiped by another project on the same machine. | - ---- - -## 5. Development and testing - -| Technology | Why it is used | -|---|---| -| **Git** | Version control. | -| **Python 3.13** | The version this project was developed and verified on. | -| **Python venv** | Keeps this project's Python packages separate from everything else on the machine. | -| **requests** | Used only by `scripts/smoke_test.py`, which drives the whole system end to end — register, upload, generate, chat, read analytics — and prints a pass or fail line per step. | -| **`/api/health`** | A built-in endpoint that reports both databases and both model files separately, so a problem can be located in seconds rather than guessed at. | - ---- - diff --git a/README-USAGE.md b/README-USAGE.md deleted file mode 100644 index e526798..0000000 --- a/README-USAGE.md +++ /dev/null @@ -1,146 +0,0 @@ -# LearnMateAI — running and using it - -Two parts: getting it running on a machine, and what to do with it once it is. - -For how the software is put together see `README-APPLICATION.md`; for the models and how -their output is judged see `README-MACHINE-LEARNING.md`. - ---- - -# Part 1 — Running it - -## What you need - -| | | -|---|---| -| Python | 3.13 (what it was developed and verified on) | -| Node | 18+ | -| Docker | for MongoDB and Qdrant | -| Disk | ~4 GB for the two models, plus your PDFs | -| RAM | 8 GB works; 16 GB is comfortable | - -No API keys and no internet connection are needed once the models are downloaded. Nothing -you upload leaves the machine. - -## First run - -```bash -# 1. databases — MongoDB on 27018, Qdrant on 6335, each with its own volume -cd integrated-backend -docker compose up -d - -# 2. python dependencies -python -m venv venv -source venv/Scripts/activate -pip install -r requirements.txt # Windows -# source venv/bin/activate && pip install -r requirements.txt # macOS/Linux/git bash - -# 3. settings -copy .env.example .env -python -c "import secrets; print(secrets.token_hex(32))" -# paste the output into JWT_SECRET_KEY — it is the only setting with no default - -# 4. run the Backend -python -m uvicorn server:app --reload --port 8010 -``` - -```bash -# 5. the frontend, in a second terminal -cd integrated-frontend -npm install -npm run dev -``` - -Open . - -The models are ~4 GB and download on first use. - -## Checking it end to end - -```bash -Use the Frontend UI : Open . -``` - - -## Moving it to a machine with a GPU - -The models run on the CPU by default, which is why a reply takes tens of seconds. On a GPU Server, offloading can make faster. - -In `integrated-backend/.env`: - -```ini -LEARNMATE_N_GPU_LAYERS=-1 # all layers on the GPU. This is the line that matters. -LEARNMATE_N_THREADS= # sysctl -n hw.perflevel0.physicalcpu -LEARNMATE_N_THREADS_BATCH= -LEARNMATE_N_BATCH=512 -LEARNMATE_FLASH_ATTN=1 -LEARNMATE_USE_MLOCK=1 -API_WARM_MODELS=1 # load both models at boot, not inside the first question -``` - -`llama-cpp-python` must be built for Metal, or `N_GPU_LAYERS` does nothing: - -```bash -CMAKE_ARGS="-DGGML_METAL=on" pip install --force-reinstall --no-cache-dir llama-cpp-python -``` - -Two things are gitignored and have to be copied by hand: `integrated-backend/models/` (the two GGUFs) and `.env` itself. - - ---- - -# Part 2 — Using it - -## The idea - -A conversation and a set of study materials are always **about one PDF**. You upload a document, the system reads it, and everything after that is grounded to it. - -## 1. Make an account - -Sign up at . Accounts are local to this installation. Before signing up you can read **Home**, **About** and **Take a Tour** without an account. - -## 2. Upload a document - -**Documents**, then drop a PDF on the upload panel and pick a subject. - -The row appears immediately as **Processing** and becomes **Ready** on its own — the page watches it, so there is nothing to refresh. Behind that: the text is extracted, cleaned (running heads stripped, ligatures repaired, hyphenation rejoined), split into overlapping chunks, and embedded into the vector index. - - - -Limits: PDF only, 10 MB, 300 pages. Uploading a file somebody else already uploaded is very fast, because documents are stored by content hash and embedded once. - -## 3. Ask it questions - -**Chat**, then choose a document to start a conversation. - - -- **A question the document does not cover is answered from general knowledge instead.** It is a different kind of answer and it carries no page citations. -- **Conversations survive.** History is stored, so you can close the tab and pick up a conversation later. - -## 4. Generate study material - -Open a document in **Documents** and use the panel beside it. Four kinds: - -| | | -|---|---| -| **Summary** | the passage in a few sentences | -| **Key points** | the points the passage treats as important | -| **MCQs** | four options, one right, three plausible | -| **Practice questions** | short-answer, with answers | - -Two scopes for generation: - -- **Passage** — one extract, optionally the pages best matching a topic you name. -- **Whole document** — read in groups and pooled. - -Generation runs in the background. **Resources** lists what is still running above what is finished, with live progress, so you can leave the page and come back. - -Every generation is checked before you see it: structural checks first (an MCQ really has four options and exactly one marked answer), then a second model grades it against the -source passage. A rejected attempt is regenerated once with the judge LLM's instruction, and the better of the two attempts is what you get. - -## 5. Track it - -**Analytics** shows what you have generated and your activity over the last seven days. - ---- - diff --git a/README.md b/README.md deleted file mode 100644 index a78fe1e..0000000 --- a/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# LearnMateAI - -AI-powered study platform for Sri Lankan legal education. Upload a PDF, ask grounded -questions, and generate study resources. A second model grades what the first wrote. - -This file is an index. The project docs were split on `main`; the offline fine-tuning -track lives in `model-Thevindu/`. - -| Document | Covers | -|---|---| -| [README-APPLICATION.md](README-APPLICATION.md) | How the software is put together | -| [README-USAGE.md](README-USAGE.md) | Running it, and using it as a student | -| [README-TECHNOLOGIES.md](README-TECHNOLOGIES.md) | Libraries and models, with reasons | -| [README-MACHINE-LEARNING.md](README-MACHINE-LEARNING.md) | Live retrieval / judge, and the offline LoRA track | -| [model-Thevindu/README.md](model-Thevindu/README.md) | Fine-tuning folder layout and honesty board | -| [integrated-backend/README.md](integrated-backend/README.md) | Backend layout, config, endpoints | -| [integrated-frontend/README.md](integrated-frontend/README.md) | React app | - -**Live path today:** `integrated-frontend` + `integrated-backend` (local Qwen 2.5 + Llama 3.2 via llama.cpp). -**Offline path:** `model-Thevindu/` — corpus → pairs → LoRA → eval → promote a pointer. The first real candidate (`qwen25-lora-20260815-090709`) **failed the gate**. Do not promote it. The app must keep its generator / API fallback. - -Semester 5 group project. Contributions land on feature branches and pull requests into `main`. diff --git a/backend/.env.example b/backend/.env.example deleted file mode 100644 index e69de29..0000000 diff --git a/backend/README.md b/backend/README.md deleted file mode 100644 index b393193..0000000 --- a/backend/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Backend development - -Create and populate the virtual environment once: - -```bash -cd backend -python3 -m venv venv -./venv/bin/python -m pip install -r requirements.txt -``` - -Activate the environment and start the server on macOS or Linux: - -```bash -source venv/bin/activate -python -m uvicorn server:app --reload --port 8000 -``` - -Confirm which interpreter is active with: - -```bash -python -c "import sys; print(sys.executable)" -``` - -It should end with `/backend/venv/bin/python`. - -Press `Ctrl+C` to stop the server. Run `deactivate` when you want to leave an -activated virtual environment. diff --git a/backend/api/documents.py b/backend/api/documents.py deleted file mode 100644 index 14568e7..0000000 --- a/backend/api/documents.py +++ /dev/null @@ -1,66 +0,0 @@ -from fastapi.responses import Response -from bson import ObjectId -from bson.errors import InvalidId -from fastapi import APIRouter, HTTPException, UploadFile, File, Form, Depends, BackgroundTasks - -from database.db import documents_collection, fs, chunks_collection -from app_infrastructure.middleware import get_current_user -from document_processing_pipeline.pdf_manager.upload import handle_upload -from document_processing_pipeline.pdf_manager.pdf_manager import process_document - -router = APIRouter(prefix="/api/documents", tags=["documents"]) - -@router.post("/upload", status_code=201) -async def upload_document( - background_tasks: BackgroundTasks, - file: UploadFile = File(...), - subject: str = Form("General"), - user: dict = Depends(get_current_user), -): - file_bytes = await file.read() - try: - document = handle_upload( - fs, documents_collection, file_bytes, file.content_type, file.filename, user["id"], subject - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) - - background_tasks.add_task( - process_document, file_bytes, documents_collection, chunks_collection, document["id"] - ) - return document - - -@router.get("") -def list_documents(user: dict = Depends(get_current_user)): - docs = documents_collection.find({"owner_id": user["id"]}) - return [ - { - "id": str(d["_id"]), - "filename": d["filename"], - "upload_date": d["upload_date"].isoformat(), - "subject": d["subject"], - "page_count": d["page_count"], - "file_size": d["file_size"], - "processing_status": d["processing_status"], - "chunk_count": d.get("chunk_count", 0), - } - for d in docs - ] - -@router.get("/{document_id}/file") -def get_document_file(document_id: str, user: dict = Depends(get_current_user)): - try: - doc= documents_collection.find_one({"_id": ObjectId(document_id)}) - except InvalidId: - raise HTTPException(status_code=400, detail="Invalid document id.") - - if not doc: - raise HTTPException(status_code=404, detail="Document not found.") - if doc["owner_id"] != user["id"]: - raise HTTPException(status_code=403, detail="You do not have access to this document.") - - grid_out = fs.get(doc["gridfs_file_id"]) - file_bytes = grid_out.read() - - return Response(content=file_bytes, media_type="application/pdf") \ No newline at end of file diff --git a/backend/api/resources.py b/backend/api/resources.py deleted file mode 100644 index 72e0e88..0000000 --- a/backend/api/resources.py +++ /dev/null @@ -1,66 +0,0 @@ -from fastapi import APIRouter, HTTPException, Depends -from pydantic import BaseModel -from bson import ObjectId -from bson.errors import InvalidId - -from database.db import documents_collection, chunks_collection, resources_collection -from app_infrastructure.middleware import get_current_user -from resource_generator_service.generator import generate_resource - -router = APIRouter(prefix="/api/resources", tags=["resources"]) - - -class GenerateRequest(BaseModel): - document_id: str - resource_type: str # "summary" or "key_points" for now - - -def _verify_document_ownership(document_id: str, user_id: str): - try: - doc = documents_collection.find_one({"_id": ObjectId(document_id)}) - except InvalidId: - raise HTTPException(status_code=400, detail="Invalid document id.") - if not doc: - raise HTTPException(status_code=404, detail="Document not found.") - if doc["owner_id"] != user_id: - raise HTTPException(status_code=403, detail="You do not have access to this document.") - return doc - - -@router.post("/generate", status_code=201) -def generate(payload: GenerateRequest, user: dict = Depends(get_current_user)): - doc = _verify_document_ownership(payload.document_id, user["id"]) - - if doc["processing_status"] != "Ready": - raise HTTPException( - status_code=400, - detail=f"This document isn't ready yet (status: {doc['processing_status']}). Please wait for processing to finish.", - ) - - try: - resource = generate_resource( - chunks_collection, resources_collection, payload.document_id, payload.resource_type - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) - except Exception as e: - print(f"DEBUG generation error: {e!r}") - raise HTTPException(status_code=502, detail="The AI service failed to respond. Please try again.") - return resource - - -@router.get("") -def list_resources(document_id: str, user: dict = Depends(get_current_user)): - _verify_document_ownership(document_id, user["id"]) - resources = resources_collection.find({"document_id": document_id}).sort("created_at", -1) - return [ - { - "id": str(r["_id"]), - "document_id": r["document_id"], - "resource_type": r["resource_type"], - "content": r["content"], - "verification_status": r["verification_status"], - "created_at": r["created_at"].isoformat(), - } - for r in resources - ] \ No newline at end of file diff --git a/backend/api/routes.py b/backend/api/routes.py deleted file mode 100644 index f803eb5..0000000 --- a/backend/api/routes.py +++ /dev/null @@ -1,43 +0,0 @@ -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel, EmailStr - -from database.db import users_collection -from app_infrastructure.authentication import create_access_token -from app_infrastructure.register import register_new_user -from app_infrastructure.login import authenticate_user - -router = APIRouter(prefix="/api/auth", tags=["auth"]) - -class RegisterRequest(BaseModel): - name: str - email: EmailStr - password: str - -class LoginRequest(BaseModel): - email: EmailStr - password: str - -class AuthResponse(BaseModel): - token: str - user: dict - -@router.post("/register", response_model=AuthResponse, status_code=201) -def register(payload: RegisterRequest): - try: - user = register_new_user(users_collection, payload.name, payload.email, payload.password) - except ValueError as e: - if "already exists" in str(e): - status_code=409 - else: - status_code=400 - raise HTTPException(status_code=status_code, detail=str(e)) - token = create_access_token(user["id"], user["email"]) - return {"token": token, "user": user} - -@router.post("/login", response_model=AuthResponse) -def login(payload: LoginRequest): - user = authenticate_user(users_collection, payload.email, payload.password) - if not user: - raise HTTPException(status_code=401, detail="Invalid email or password") - token = create_access_token(user["id"], user["email"]) - return {"token": token, "user": user} \ No newline at end of file diff --git a/backend/app_infrastructure/authentication.py b/backend/app_infrastructure/authentication.py deleted file mode 100644 index f5eca7b..0000000 --- a/backend/app_infrastructure/authentication.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import jwt -import bcrypt -from datetime import datetime, timedelta, timezone - -JWT_SECRET_KEY = os.environ.get("JWT_SECRET_KEY") -JWT_EXPIRY_HOURS = int(os.environ.get("JWT_EXPIRY_HOURS", "24")) -JWT_ALGORITHM = "HS256" - -if not JWT_SECRET_KEY: - raise RuntimeError("JWT_SECRET_KEY is not set. Check your backend/.env file.") - -def hash_password(plain_password: str) -> str: - """Hash a plaintext password with bcrypt. Plaintext is never stored (SRS FR-1).""" - hashed = bcrypt.hashpw(plain_password.encode("utf-8"), bcrypt.gensalt()) - return hashed.decode("utf-8") - -def verify_password(plain_password: str, hashed_password: str) -> bool: - """Check a login attempt's password against the stored bcrypt hash.""" - return bcrypt.checkpw(plain_password.encode("utf-8"), hashed_password.encode("utf-8")) - -def create_access_token(user_id: str, email: str) -> str: - """Issue a JWT that expires after JWT_EXPIRY_HOURS (SRS FR-2).""" - expire = datetime.now(timezone.utc) + timedelta(hours=JWT_EXPIRY_HOURS) - payload = {"sub": user_id, "email": email, "exp": expire} - return jwt.encode(payload, JWT_SECRET_KEY, algorithm=JWT_ALGORITHM) - -def decode_access_token(token: str) -> dict: - """Decode and validate a JWT. Raises jwt exceptions if expired or tampered with.""" - return jwt.decode(token, JWT_SECRET_KEY, algorithms=[JWT_ALGORITHM]) - diff --git a/backend/app_infrastructure/login.py b/backend/app_infrastructure/login.py deleted file mode 100644 index 3fd7d50..0000000 --- a/backend/app_infrastructure/login.py +++ /dev/null @@ -1,13 +0,0 @@ -from app_infrastructure.authentication import verify_password - -def authenticate_user(users_collection, email: str, password: str): - """ - Returns the user dict on success, or None on failure. - Deliberately does not reveal which field was wrong (SRS FR-2). - """ - user = users_collection.find_one({"email": email.strip().lower()}) - if not user: - return None - if not verify_password(password, user["password_hash"]): - return None - return {"id": str(user["_id"]), "name": user["name"], "email": user["email"]} \ No newline at end of file diff --git a/backend/app_infrastructure/middleware.py b/backend/app_infrastructure/middleware.py deleted file mode 100644 index 7f1bfdc..0000000 --- a/backend/app_infrastructure/middleware.py +++ /dev/null @@ -1,21 +0,0 @@ -from fastapi import Header, HTTPException -import jwt -from app_infrastructure.authentication import decode_access_token - -def get_current_user(authorization: str = Header(None)): - """ - FastAPI dependency. Reads the 'Authorization: Bearer ' header, - validates the JWT, and returns {id, email}. - """ - if not authorization or not authorization.startswith("Bearer "): - raise HTTPException(status_code=401, detail="Missing or invalid Authorization header") - - token = authorization.split(" ")[1] - try: - payload = decode_access_token(token) - except jwt.ExpiredSignatureError: - raise HTTPException(status_code=401, detail="Session expired. Please log in again.") - except jwt.InvalidTokenError: - raise HTTPException(status_code=401, detail="Invalid authentication token.") - - return {"id": payload["sub"], "email": payload["email"]} diff --git a/backend/app_infrastructure/register.py b/backend/app_infrastructure/register.py deleted file mode 100644 index 7f79770..0000000 --- a/backend/app_infrastructure/register.py +++ /dev/null @@ -1,32 +0,0 @@ -import re -from datetime import datetime, timezone -from pymongo.errors import DuplicateKeyError - -from app_infrastructure.authentication import hash_password - -PASSWORD_MIN_LENGTH = 8 - -def validate_password_strength(password: str) -> None: - """SRS FR-1: minimum length + at least one number, enforced server-side.""" - if len(password) < PASSWORD_MIN_LENGTH: - raise ValueError(f"Password must be at least {PASSWORD_MIN_LENGTH} characters long.") - if not re.search(r"\d", password): - raise ValueError("Password must contain at least one number.") - -def register_new_user(users_collection, name, email, password) -> dict: - """Creates a new user document. Raises ValueError on any validation failure.""" - validate_password_strength(password) - - user_doc = { - "name" : name.strip(), - "email": email.strip().lower(), - "password_hash": hash_password(password), - "created_at": datetime.now(timezone.utc), - } - - try: - result = users_collection.insert_one(user_doc) - except DuplicateKeyError: - raise ValueError("An account with this email already exists") - - return{"id": str(result.inserted_id), "name": user_doc["name"], "email": user_doc["email"]} diff --git a/backend/chat-service/LangChain.py b/backend/chat-service/LangChain.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/chat-service/RAG.py b/backend/chat-service/RAG.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/chat-service/manage-chats.py b/backend/chat-service/manage-chats.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/chat-service/storage.py b/backend/chat-service/storage.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/database/db.py b/backend/database/db.py deleted file mode 100644 index 15118f4..0000000 --- a/backend/database/db.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import certifi -from pymongo import MongoClient -from dotenv import load_dotenv -from gridfs import GridFS - -load_dotenv() - -MONGODB_URI = os.environ.get("MONGODB_URI") - -if not MONGODB_URI: - raise RuntimeError("MONGODB_URI is not set. Check your backend/.env file.") - -client = MongoClient(MONGODB_URI, tlsCAFile=certifi.where()) -db = client["learnmateai"] -documents_collection = db["documents"] -chunks_collection = db["chunks"] -resources_collection = db["resources"] -fs = GridFS(db) -def check_connection(): - """Quick check used at startup to confirm Mongo Atlas is reachable.""" - try: - client.admin.command("ping") - return True - except Exception as e: - print(f"[db.py] MongoDB connection failed: {e}") - return False - -users_collection = db["users"] -users_collection.create_index("email", unique=True) \ No newline at end of file diff --git a/backend/document_processing_pipeline/chunker/chunking.py b/backend/document_processing_pipeline/chunker/chunking.py deleted file mode 100644 index fe120c9..0000000 --- a/backend/document_processing_pipeline/chunker/chunking.py +++ /dev/null @@ -1,28 +0,0 @@ -def chunk_document(pages: list[str], chunk_size: int = 1000, overlap: int = 150) -> list[dict]: - """ - Splits each page's cleaned text into fixed-size, overlapping character - windows, tagged with the page they came from. - The overlap exists so that a sentence sitting right at a chunk boundary - still appears in full inside at least one chunk, rather than being cut - in half and lost from both. - """ - chunks = [] - chunk_index = 0 - step = chunk_size - overlap - - for page_number, page_text in enumerate(pages, start=1): - text = page_text.strip() - if not text: - continue - for start in range(0,len(text), step): - piece = text[start:start + chunk_size].strip() - if piece: - chunks.append({ - "chunk_index": chunk_index, - "page_number": page_number, - "text": piece - }) - chunk_index += 1 - if start + chunk_size >= len(text): - break - return chunks \ No newline at end of file diff --git a/backend/document_processing_pipeline/doc_cleaner/main.py b/backend/document_processing_pipeline/doc_cleaner/main.py deleted file mode 100644 index 8db0334..0000000 --- a/backend/document_processing_pipeline/doc_cleaner/main.py +++ /dev/null @@ -1,18 +0,0 @@ -import re - -def clean_pages(pages: list[str]) -> list[str]: - """ - Light per-page cleaning: strips lines that are just a page number, and - collapses excess whitespace. - """ - cleaned = [] - for page_text in pages: - lines = page_text.split("\n") - kept_lines = [ - line.strip() for line in lines - if line.strip() and not re.fullmatch(r"[-\s]*\d{1,4}[-\s]*", line.strip()) - ] - cleaned_text = " ".join(kept_lines) - cleaned_text = re.sub(r"\s{2,}", " ", cleaned_text) - cleaned.append(cleaned_text.strip()) - return cleaned \ No newline at end of file diff --git a/backend/document_processing_pipeline/doc_parser/main.py b/backend/document_processing_pipeline/doc_parser/main.py deleted file mode 100644 index 96edff4..0000000 --- a/backend/document_processing_pipeline/doc_parser/main.py +++ /dev/null @@ -1,14 +0,0 @@ -import io -import pdfplumber - -def extract_text_from_pdf(file_bytes: bytes) -> list[str]: - """ - Extracts plain text from each page of a PDF. - Returns one string per page, in page order (index 0 = page 1). - """ - pages=[] - with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: - for page in pdf.pages: - text = page.extract_text() or "" - pages.append(text) - return pages \ No newline at end of file diff --git a/backend/document_processing_pipeline/doc_reconstructor/main.py b/backend/document_processing_pipeline/doc_reconstructor/main.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/document_processing_pipeline/pdf_manager/pdf_manager.py b/backend/document_processing_pipeline/pdf_manager/pdf_manager.py deleted file mode 100644 index 3395612..0000000 --- a/backend/document_processing_pipeline/pdf_manager/pdf_manager.py +++ /dev/null @@ -1,37 +0,0 @@ -from bson import ObjectId - -from document_processing_pipeline.doc_parser.main import extract_text_from_pdf -from document_processing_pipeline.doc_cleaner.main import clean_pages -from document_processing_pipeline.chunker.chunking import chunk_document - -def process_document(file_bytes: bytes, documents_collection, chunks_collection, document_id: str) -> int: - """ - Runs the (simplified) processing pipeline for one uploaded PDF: - extract -> clean -> chunk -> store chunks -> update document status. - Returns the number of chunks created. - """ - try: - pages = extract_text_from_pdf(file_bytes) - cleaned_pages = clean_pages(pages) - chunks = chunk_document(cleaned_pages) - - for chunk in chunks: - chunks_collection.insert_one({ - "document_id": document_id, - "chunk_index": chunk["chunk_index"], - "page_number": chunk["page_number"], - "text": chunk["text"], - }) - - documents_collection.update_one( - {"_id": ObjectId(document_id)}, - {"$set": {"processing_status": "Ready", "chunk_count": len(chunks)}}, - ) - return len(chunks) - - except Exception as e: - documents_collection.update_one( - {"_id": ObjectId(document_id)}, - {"$set": {"processing_status": "Failed Processing", "processing_error": str(e)}}, - ) - raise \ No newline at end of file diff --git a/backend/document_processing_pipeline/pdf_manager/storage.py b/backend/document_processing_pipeline/pdf_manager/storage.py deleted file mode 100644 index 971085d..0000000 --- a/backend/document_processing_pipeline/pdf_manager/storage.py +++ /dev/null @@ -1,35 +0,0 @@ -from datetime import datetime, timezone - -def store_pdf(fs, documents_collection, file_bytes: bytes, filename:str, owner_id: str, subject: str, page_count:int) -> dict: - """ - Saves the raw PDF into GridFS, then creates a metadata record in the - documents collection referencing it (SRS §3.10 Document entity). - """ - gridfs_id = fs.put(file_bytes, filename=filename, content_type="application/pdf") - - document = { - "owner_id": owner_id, - "filename": filename, - "upload_date": datetime.now(timezone.utc), - "subject": subject, - "page_count": page_count, - "file_size": len(file_bytes), - "gridfs_file_id": gridfs_id, - # SRS FR-5 specifies status "Processing" immediately after upload. - # Simplified to "Uploaded" here since the real parsing pipeline - # (FR-6 cleaning / FR-7 parsing) isn't built until Day 7 — the full - # Processing -> Ready / Failed Validation lifecycle starts then. - "processing_status": "Uploaded", - } - - result = documents_collection.insert_one(document) - - return { - "id": str(result.inserted_id), - "filename": filename, - "upload_date": document["upload_date"].isoformat(), - "subject": subject, - "page_count": page_count, - "file_size": document["file_size"], - "processing_status": document["processing_status"], - } \ No newline at end of file diff --git a/backend/document_processing_pipeline/pdf_manager/upload.py b/backend/document_processing_pipeline/pdf_manager/upload.py deleted file mode 100644 index dc80fb5..0000000 --- a/backend/document_processing_pipeline/pdf_manager/upload.py +++ /dev/null @@ -1,10 +0,0 @@ -from document_processing_pipeline.pdf_manager.validation import validate_pdf -from document_processing_pipeline.pdf_manager.storage import store_pdf - -def handle_upload(fs, documents_collection, file_bytes: bytes, content_type: str, filename: str, owner_id: str, subject: str) -> dict: - """ - Orchestrates one uploaded file: validate first, then store. - Raises ValueError (caught in api/documents.py) on any validation failure. - """ - page_count = validate_pdf(file_bytes, content_type, filename) - return store_pdf(fs, documents_collection, file_bytes, filename, owner_id, subject, page_count) \ No newline at end of file diff --git a/backend/document_processing_pipeline/pdf_manager/validation.py b/backend/document_processing_pipeline/pdf_manager/validation.py deleted file mode 100644 index 02a5d75..0000000 --- a/backend/document_processing_pipeline/pdf_manager/validation.py +++ /dev/null @@ -1,34 +0,0 @@ -import io -import pdfplumber - -"targeted at 50 MB / 300 pages per document for the pilot" -MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 # 50 MB -MAX_PAGE_COUNT = 300 # 300 pages -ALLOWED_FILE_TYPES = ["application/pdf"] - -def validate_pdf(file_bytes: bytes, content_type: str, file_name: str) -> int: - """ - Validates an uploaded file against SRS FR-5 / UI-03 / FR-6 rules. - Returns the page count on success. Raises ValueError with a - human-readable message on failure (SRS DOC-04). - """ - if not file_name.lower().endswith(".pdf"): - raise ValueError("Only PDF files are allowed.") - if len(file_bytes) > MAX_FILE_SIZE_BYTES: - raise ValueError(f"File size exceeds the maximum allowed size of {MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB.") - try: - with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: - page_count = len(pdf.pages) - except Exception: - # Covers corrupted files and password-protected PDFs (FR-6) - raise ValueError( - "This PDF could not be read. It may be corrupted or password-protected — " - "please upload an unprotected copy." - ) - - if page_count == 0: - raise ValueError("This PDF appears to have no pages.") - if page_count > MAX_PAGE_COUNT: - raise ValueError(f"Page count exceeds the maximum allowed count of {MAX_PAGE_COUNT} pages.") - - return page_count \ No newline at end of file diff --git a/backend/document_processing_pipeline/structured_document_manager/doc-representation.py b/backend/document_processing_pipeline/structured_document_manager/doc-representation.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/document_processing_pipeline/structured_document_manager/storage.py b/backend/document_processing_pipeline/structured_document_manager/storage.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/multi-agents/chat-agent/llm-connect.py b/backend/multi-agents/chat-agent/llm-connect.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/multi-agents/embedding-agent/llm-connect.py b/backend/multi-agents/embedding-agent/llm-connect.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/multi-agents/resource-gen-agent/llm-connect.py b/backend/multi-agents/resource-gen-agent/llm-connect.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index 1af383d..0000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,45 +0,0 @@ -annotated-doc==0.0.5 -annotated-types==0.8.0 -anyio==4.14.2 -bcrypt==5.0.0 -certifi==2026.7.22 -cffi==2.1.0 -charset-normalizer==3.4.9 -click==8.4.2 -cryptography==50.0.0 -distro==1.9.0 -dnspython==2.8.0 -email-validator==2.3.0 -fastapi==0.141.1 -google-auth==2.56.2 -google-genai==2.16.0 -h11==0.16.0 -httpcore==1.0.9 -httptools==0.8.0 -httpx==0.28.1 -idna==3.18 -pdfminer.six==20260107 -pdfplumber==0.11.10 -pillow==12.3.0 -pyasn1==0.6.4 -pyasn1_modules==0.4.2 -pycparser==3.0 -pydantic==2.13.4 -pydantic_core==2.46.4 -PyJWT==2.13.0 -pymongo==4.17.0 -pypdfium2==5.12.1 -python-dotenv==1.2.2 -python-multipart==0.0.32 -PyYAML==6.0.3 -requests==2.34.2 -sniffio==1.3.1 -starlette==1.3.1 -tenacity==9.1.4 -typing-inspection==0.4.2 -typing_extensions==4.16.0 -urllib3==2.7.0 -uvicorn==0.52.0 -uvloop==0.22.1 -watchfiles==1.2.0 -websockets==16.1.1 diff --git a/backend/resource_generator_service/explanation_summary.py b/backend/resource_generator_service/explanation_summary.py deleted file mode 100644 index 145352b..0000000 --- a/backend/resource_generator_service/explanation_summary.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -from google import genai -from dotenv import load_dotenv - -load_dotenv() - -GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY") -GEMINI_MODEL = os.environ.get("GEMINI_MODEL") # set to whatever model string worked in your Phase 0 practice - -client = genai.Client(api_key=GEMINI_API_KEY) - - -def generate_summary(document_text: str) -> str: - """ - Generates a plain-language summary of a document's content using Gemini. - - Simplified version of FR-13/FR-11: the SRS specifies routing this - generation request to a fine-tuned Qwen 2.5 model first, falling back to - Gemini only if that endpoint is unavailable. This MVP calls Gemini - directly — the fine-tuned model and routing logic are Iteration 2 work. - """ - prompt = ( - "You are a study assistant helping a law student review their own " - "lecture notes. Based only on the document text below, write a clear, " - "well-organized summary in plain English, covering the main topics " - "and arguments. Do not add legal advice or information not present " - "in the text.\n\n" - f"DOCUMENT TEXT:\n{document_text}" - ) - - response = client.models.generate_content( - model=GEMINI_MODEL, - contents=prompt, - ) - return response.text.strip() \ No newline at end of file diff --git a/backend/resource_generator_service/gen_compatibility.py b/backend/resource_generator_service/gen_compatibility.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/resource_generator_service/generator.py b/backend/resource_generator_service/generator.py deleted file mode 100644 index dc9bf02..0000000 --- a/backend/resource_generator_service/generator.py +++ /dev/null @@ -1,41 +0,0 @@ -from resource_generator_service.explanation_summary import generate_summary -from resource_generator_service.key_points import generate_key_points -from resource_generator_service.storage import store_generated_resource - -MAX_CONTEXT_CHARS = 15000 # keeps the prompt a reasonable size and cost - - -def build_document_text(chunks_collection, document_id: str) -> str: - """ - Reassembles a document's stored chunks (Day 7) back into one text block, - in original order, capped to a reasonable length for the prompt. - """ - chunks = list( - chunks_collection.find({"document_id": document_id}).sort("chunk_index", 1) - ) - full_text = "\n\n".join(c["text"] for c in chunks) - return full_text[:MAX_CONTEXT_CHARS] - - -def generate_resource(chunks_collection, resources_collection, document_id: str, resource_type: str) -> dict: - """ - Orchestrates one resource-generation request: build context from stored - chunks, call the appropriate generator, do a basic verification check, - and store the result. Raises ValueError on any failure (caught in - api/resources.py). - """ - document_text = build_document_text(chunks_collection, document_id) - if not document_text.strip(): - raise ValueError("This document has no processed content yet. Please wait for processing to finish.") - - if resource_type == "summary": - content = generate_summary(document_text) - elif resource_type == "key_points": - content = generate_key_points(document_text) - else: - raise ValueError(f"Unsupported resource type: {resource_type}") - - if not content or (isinstance(content, list) and len(content) == 0): - raise ValueError("Generation produced an empty result. Please try again.") - - return store_generated_resource(resources_collection, document_id, resource_type, content) \ No newline at end of file diff --git a/backend/resource_generator_service/key_points.py b/backend/resource_generator_service/key_points.py deleted file mode 100644 index 3875bdc..0000000 --- a/backend/resource_generator_service/key_points.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import json -from google import genai -from google.genai import types -from dotenv import load_dotenv - -load_dotenv() - -GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY") -GEMINI_MODEL = os.environ.get("GEMINI_MODEL") - -client = genai.Client(api_key=GEMINI_API_KEY) - - -def generate_key_points(document_text: str) -> list[str]: - """ - Generates a list of key points from a document using Gemini, requesting - structured JSON output so the result is directly usable as a list - rather than free-form text needing fragile manual parsing. - """ - prompt = ( - "You are a study assistant helping a law student review their own " - "lecture notes. Based only on the document text below, extract the " - "8 to 12 most important key points a student should remember. " - "Return ONLY a JSON array of strings, one per key point, with no " - "other text.\n\n" - f"DOCUMENT TEXT:\n{document_text}" - ) - - response = client.models.generate_content( - model=GEMINI_MODEL, - contents=prompt, - config=types.GenerateContentConfig(response_mime_type="application/json"), - ) - - try: - points = json.loads(response.text) - if isinstance(points, list): - return [str(p).strip() for p in points if str(p).strip()] - except (json.JSONDecodeError, TypeError): - pass - - return [line.strip("-• \t") for line in response.text.split("\n") if line.strip()] \ No newline at end of file diff --git a/backend/resource_generator_service/mcq.py b/backend/resource_generator_service/mcq.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/resource_generator_service/practise_qsn.py b/backend/resource_generator_service/practise_qsn.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/resource_generator_service/short_qsn.py b/backend/resource_generator_service/short_qsn.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/resource_generator_service/storage.py b/backend/resource_generator_service/storage.py deleted file mode 100644 index 7104e26..0000000 --- a/backend/resource_generator_service/storage.py +++ /dev/null @@ -1,29 +0,0 @@ -from datetime import datetime, timezone - - -def store_generated_resource(resources_collection, document_id: str, resource_type: str, content) -> dict: - """ - Persists one generated resource (SRS §3.10 Generated Resource entity). - `content` is a string for summaries, a list of strings for key points. - """ - resource = { - "document_id": document_id, - "resource_type": resource_type, - "content": content, - # Simplified version of FR-14 (Content Verification): a full - # implementation runs a second LLM call (Evaluator Agent) to grade - # output quality and trigger regeneration if it fails a threshold - # (Dinura's components-Dinura/learnmate/evaluator/ does this - # properly — not yet integrated into this shared backend). - "verification_status": "Unverified", - "created_at": datetime.now(timezone.utc), - } - result = resources_collection.insert_one(resource) - return { - "id": str(result.inserted_id), - "document_id": document_id, - "resource_type": resource_type, - "content": content, - "verification_status": resource["verification_status"], - "created_at": resource["created_at"].isoformat(), - } \ No newline at end of file diff --git a/backend/server.py b/backend/server.py deleted file mode 100644 index 485c798..0000000 --- a/backend/server.py +++ /dev/null @@ -1,41 +0,0 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -import os -from dotenv import load_dotenv -from database.db import check_connection -from api.routes import router as auth_router -from api.documents import router as documents_router -from api.resources import router as resources_router - -load_dotenv() - -app = FastAPI(title="LearnMateAI Backend") -app.include_router(documents_router) - -FRONTEND_ORIGIN = os.environ.get("FRONTEND_ORIGIN", "http://localhost:5173") - -app.add_middleware( - CORSMiddleware, - allow_origins=[FRONTEND_ORIGIN], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -app.include_router(auth_router) -app.include_router(resources_router) - -@app.get("/") -def root(): - return { - "message": "LearnMateAI Backend", - "health": "/api/health", - } - -@app.get("/api/health") -def health_check(): - db_ok = check_connection() - return { - "status": "ok" if db_ok else "degraded", - "database": "connected" if db_ok else "unreachable", - } diff --git a/backend/user-analytics-service/user-stats.py b/backend/user-analytics-service/user-stats.py deleted file mode 100644 index e69de29..0000000 diff --git a/components-Dinura/.env.example b/components-Dinura/.env.example deleted file mode 100644 index 68891ee..0000000 --- a/components-Dinura/.env.example +++ /dev/null @@ -1,115 +0,0 @@ -# Copy to .env and edit. Every value here is optional; the defaults shown are what the -# code uses when the variable is absent. Commented-out lines are the less common knobs, -# shown with their default so you can see what they are before changing them. - -# Where the GGUF model files live, and where a missing one is downloaded to. -# LEARNMATE_MODELS_DIR=models - -# --- MongoDB ------------------------------------------------------------------------- -# Holds the PDFs (GridFS), page text, session bindings, chat history, generated resources -# and the evaluation log. Runs as its own container: `docker compose up -d mongo`. -# -# 27018 rather than 27017, because 27017 on this machine is answered by a native MongoDB -# service that also hosts two other projects. Point this at 27017 only if you deliberately -# want to share that server. -LEARNMATE_MONGODB_URI=mongodb://localhost:27018 -LEARNMATE_MONGODB_DB=learnmate - -# --- Vector database ----------------------------------------------------------------- -# qdrant = a Qdrant server over HTTP (default). Start it with `docker compose up -d qdrant`. -# mongodb = keep the vectors in MongoDB instead, so there is no second service to run. -LEARNMATE_VECTOR_BACKEND=qdrant - -# Server mode: a URL, never a directory. Passing a path would run Qdrant inside the -# process and lock that directory, which is the embedded mode this project moved off. -# -# 6335 rather than Qdrant's usual 6333 because this machine already runs another Qdrant -# on 6333 for a different project; docker-compose.yml publishes the matching port. -LEARNMATE_QDRANT_URL=http://localhost:6335 -LEARNMATE_QDRANT_API_KEY= -LEARNMATE_QDRANT_COLLECTION=learnmate_chunks -# Seconds before a Qdrant request gives up, and points per upsert/scroll request. -# LEARNMATE_QDRANT_TIMEOUT=30 -# LEARNMATE_QDRANT_BATCH_SIZE=128 - -# Only used when LEARNMATE_VECTOR_BACKEND=mongodb: the name of the Atlas vector index. -# Ignored off Atlas, where scoring falls back to exact NumPy cosine. -LEARNMATE_VECTOR_INDEX=chunk_vector_index - -# --- Generator model ----------------------------------------------------------------- -# This is what you change to swap Qwen2.5 for the finetuned model. -# -# Shipping as a GGUF file: -LEARNMATE_GENERATOR_BACKEND=llamacpp -LEARNMATE_GENERATOR_MODEL=models/qwen2.5-3b-instruct-q4_k_m.gguf -# -# Served over an OpenAI-compatible HTTP API instead: -# LEARNMATE_GENERATOR_BACKEND=http -# LEARNMATE_GENERATOR_API_URL=http://localhost:8001/v1 -# LEARNMATE_GENERATOR_MODEL=learnmate-finetuned -# LEARNMATE_GENERATOR_API_KEY= - -# Only needed if the finetune uses a prompt template llama.cpp cannot read from the GGUF -# metadata. Leave empty for anything finetuned from a standard base. -LEARNMATE_GENERATOR_CHAT_FORMAT= -LEARNMATE_GENERATOR_N_CTX=4096 - -# Where a missing GGUF is downloaded from, on first run only. Ignored once the file at -# LEARNMATE_GENERATOR_MODEL exists, so a hand-placed finetune is never overwritten. -# LEARNMATE_GENERATOR_REPO=Qwen/Qwen2.5-3B-Instruct-GGUF -# LEARNMATE_GENERATOR_FILE=qwen2.5-3b-instruct-q4_k_m.gguf - -# --- Judge model --------------------------------------------------------------------- -# Keep this a different family from the generator: a judge sharing the generator's -# weights rates its own output style highly and the retry loop stops firing. -LEARNMATE_JUDGE_BACKEND=llamacpp -LEARNMATE_JUDGE_MODEL=models/Llama-3.2-3B-Instruct-Q4_K_M.gguf -# Larger than the generator's on purpose: the judge reads a resource *and* the passage -# it must be faithful to, so its input is the longer of the two. -LEARNMATE_JUDGE_N_CTX=8192 -# LEARNMATE_JUDGE_REPO=bartowski/Llama-3.2-3B-Instruct-GGUF -# LEARNMATE_JUDGE_FILE=Llama-3.2-3B-Instruct-Q4_K_M.gguf -# LEARNMATE_JUDGE_CHAT_FORMAT= -# -# Serve the judge over HTTP instead of loading it in-process: -# LEARNMATE_JUDGE_BACKEND=http -# LEARNMATE_JUDGE_API_URL=http://localhost:8002/v1 -# LEARNMATE_JUDGE_API_KEY= - -# --- Inference ----------------------------------------------------------------------- -# 0 lets llama.cpp auto-detect physical cores. Raise N_GPU_LAYERS to offload to a GPU. -LEARNMATE_N_THREADS=0 -LEARNMATE_N_GPU_LAYERS=0 - -# --- Uploads ------------------------------------------------------------------------- -# A session is about exactly one PDF: a second, different PDF ingested into the same -# session is refused, because embedding a document is the expensive step here. Set to 0 -# to allow a session to hold several. -LEARNMATE_ONE_PDF_PER_SESSION=1 - -# Largest PDF accepted, in MB. -LEARNMATE_MAX_PDF_MB=10 - -# --- Retrieval ----------------------------------------------------------------------- -LEARNMATE_EMBEDDING_MODEL=all-MiniLM-L6-v2 -LEARNMATE_CHUNK_SIZE=900 -LEARNMATE_CHUNK_OVERLAP=150 -LEARNMATE_TOP_K=4 - -# Shortest chunk worth embedding, measured on alphanumerics only; below this it is a -# running head or a stray caption. -# LEARNMATE_MIN_CHUNK_CHARS=80 - -# Cosine similarity below which the chat agent stops trusting the PDF and answers from -# general knowledge instead. -LEARNMATE_RELEVANCE_THRESHOLD=0.25 - -# --- Agents -------------------------------------------------------------------------- -# Judge scores below this trigger exactly one regeneration. -LEARNMATE_EVALUATOR_THRESHOLD=70 -LEARNMATE_MAX_ATTEMPTS=2 -LEARNMATE_MAX_HISTORY_TURNS=6 -LEARNMATE_MAX_SOURCE_CHARS=6000 - -# Lifts anonymous rate limits when a model has to be downloaded from Hugging Face. -HF_TOKEN= diff --git a/components-Dinura/README.md b/components-Dinura/README.md deleted file mode 100644 index f1f70d3..0000000 --- a/components-Dinura/README.md +++ /dev/null @@ -1,558 +0,0 @@ -# LearnMate - -A study assistant that runs entirely on your own machine. You give it a PDF; it answers -questions about that PDF, and generates study material from it — multiple-choice -questions, short-answer practice questions, key points and summaries. - -Nothing is sent to an external API. Three local models do the work: - -| Model | Job | Size | -|---|---|---| -| Qwen2.5-3B-Instruct | writes chat replies and study resources | 2.0 GB | -| Llama-3.2-3B-Instruct | grades what the generator wrote | 1.9 GB | -| all-MiniLM-L6-v2 | turns text into vectors for retrieval | 90 MB | - -The generator and the judge are **deliberately different model families**. A judge sharing -the generator's weights rates its own writing style highly, and the quality gate stops -firing. - ---- - -## Quickstart - -```bash -cd components-Dinura - -# 1. dependencies -python -m venv venv -venv\Scripts\pip install -r requirements.txt - -# 2. both databases, each in its own container with its own volume -docker compose up -d - -# 3. check models, databases and settings before anything slow runs -venv\Scripts\python cli.py doctor - -# 4. either entry point: -venv\Scripts\python cli.py ingest data\constitution.pdf --session s1 # one command at a time -python learnmate\full_program.py # guided walkthrough -``` - -The two GGUF models download from Hugging Face on first use (~4 GB, once). - -**Two ways in**, and they suit different jobs: - -- **`cli.py`** — one verb per command, scriptable. This is the day-to-day interface, and - the only one with the maintenance verbs: `stats`, `export`, `delete`, `docs`. -- **`learnmate/full_program.py`** — a guided menu that walks the whole workflow in one - session, plus `--demo` as a one-command smoke test. It finds the project's virtualenv by - itself, so plain `python` works without activating it. - -`cli.py doctor` is the first thing to run whenever something misbehaves — it reports which -model files exist, whether each database is reachable, and what every threshold is set to. -`full_program.py` runs the same check on startup, so a stopped container is reported in one -line instead of surfacing minutes into an ingest. - -**Expect it to be slow.** Everything runs on CPU: a chat turn is 20–60 seconds and a -graded resource 60–120 seconds. `--no-eval` skips the quality gate and roughly halves -that. - ---- - -## The workflow - -Everything starts with one PDF and one **session**. - -``` - ┌─────────────────────────────────────────┐ - your PDF ───────►│ INGESTION │ - │ extract → clean → chunk → embed │ - └───────────┬─────────────────────────────┘ - │ - ┌──────────────┴───────────────┐ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ MongoDB :27018 │ │ Qdrant :6335 │ - │ the PDF itself │ │ chunk vectors │ - │ page text │ └────────┬────────┘ - │ sessions │ │ - │ chat history │ │ - │ resources │ │ - └────────┬────────┘ │ - │ │ - ┌──────────┴──────────┐ │ - ▼ ▼ │ -┌───────────────┐ ┌─────────────────┐ │ -│ CHAT AGENT │◄──┤ retrieval │◄────────┘ -│ ask a question│ └─────────────────┘ -└───────┬───────┘ - │ ┌─────────────────┐ - ▼ │ RESOURCE AGENT │ - ┌─────────┐ │ mcq / summary / │ - │EVALUATOR│◄──────┤ keypoints / │ - │ grades │ │ practice_qsn │ - │ + retry │ └─────────────────┘ - └─────────┘ -``` - -### Step 1 — Ingest a PDF - -You give a file path. The system then: - -1. **Validates** it — not empty, not over 10 MB, and this session doesn't already hold a - different PDF. All three checks happen **before** anything is written, because - embedding is the expensive part and a rejected upload must not cost you the PDF you - already had working. -2. **Stores the file whole** in MongoDB (GridFS), identified by the SHA-256 of its bytes. - Upload the same PDF twice under different names and it is recognised as one document — - no re-embedding. -3. **Extracts and cleans** each page: removes running heads and footers (any short line - appearing on more than half the pages), deletes standalone page numbers, rejoins words - hyphenated across a line break, and flattens line breaks — because a line break in a - PDF reflects the column, not the sentence. -4. **Splits** the cleaned pages into ~900-character overlapping chunks. -5. **Embeds** the chunks and stores the vectors in Qdrant. -6. **Stores the whole page text too**, separately from the chunks. -7. **Binds the session** to the document — but only now, once the document has proved - usable. - -### Step 2a — Chat - -One question is one pass through a five-node state machine: - -``` -rewrite ──► retrieve ──► generate ──► evaluate ──► decide ─┬─► persist ─► END - ▲ │ - └────────── regenerate ────────┘ -``` - -- **rewrite** turns a follow-up into a standalone question. *"What about its national - flag?"* becomes *"What is the national flag of the Republic of Sri Lanka?"*. This runs - **before** retrieval — searching on the pronoun would already have failed. -- **retrieve** searches the vectors and **decides the mode from the score**, not by asking - the model. Top score ≥ 0.25 → **PDF mode** (answer strictly from the retrieved chunks). - Below → **general mode** (answer from the model's own knowledge). -- **generate** writes the reply using whichever system prompt the mode calls for. -- **evaluate** hands it to the judge. In PDF mode the judge is given the chunks and - anything beyond them counts as a hallucination; in general mode it can only grade - relevance, coherence and informativeness. -- **decide** accepts, or sends it back once with the judge's fix instruction. -- **persist** saves the turn, so the next question has history to resolve against. - -### Step 2b — Generate study resources - -``` -generate ──► check ──► decide ─┬─► persist ─► END - ▲ │ - └───────── regenerate ─────┘ -``` - -First the system picks **which part of the PDF to use**. A whole book doesn't fit in a 4k -context window, so: - -- give it a **topic** → the pages whose text best matches it -- give it **page numbers** → exactly those pages -- give it **neither** → the opening of the document - -The unit is always a **whole page**, never the retrieved chunks. Chunks overlap by ~150 -characters, so joining them repeats text at every boundary and starts mid-sentence — a -generator handed that writes questions about the fragments. - -Then `check` runs **two gates, cheapest first**: - -1. **Structural validators** — plain Python, microseconds. Four options per question? Is - the correct answer actually one of them? Duplicate options? Is the answer always in - slot B, or always the longest? An empty summary? -2. **The LLM judge** — ~25 seconds, and only ever spent on content that is already - well-formed. - -If either rejects, the fix instruction goes back into the next prompt along with the -rejected attempt, so the model *revises* rather than starting over. - ---- - -## What's in `components-Dinura/` - -| Path | What it is | -|---|---| -| `learnmate/` | the library — all the logic lives here | -| `cli.py` | command line: `ingest`, `chat`, `generate`, `docs`, `resources`, `stats`, `export`, `delete`, `doctor` | -| `learnmate/full_program.py` | guided end-to-end walkthrough, and `--demo` as a smoke test | -| `docker-compose.yml` | both databases, each with a named volume | -| `requirements.txt` | pinned dependencies, with notes on which pins actually matter | -| `.env.example` | every setting, with its default; copy to `.env` | -| `models/` | the two GGUF files (gitignored, downloaded on first use) | -| `data/` | sample PDFs | - ---- - -## What's in `learnmate/` - -Six packages. Each one is decomposed into small single-purpose files, and every file -opens with a docstring saying what it does and why it is that way. - -``` -learnmate/ -├── config.py every tunable setting, all overridable by environment variable -├── full_program.py the guided end-to-end program -├── ingestion/ PDF → cleaned pages → chunks → vectors → a bound session -├── storage/ MongoDB and the vector database -├── llm/ access to the three models -├── chat_agent/ the question-answering state machine -├── resource_agent/ the study-material generator -└── evaluator/ the two quality gates -``` - -### `ingestion/` — getting a PDF into the system - -| File | Does | -|---|---| -| `clean.py` | extracts pages with PyMuPDF; strips running heads, page numbers, hyphenation | -| `chunking.py` | splits cleaned pages into the overlapping chunks that get embedded | -| `sessions.py` | session kinds, one-PDF-per-session, and the binding | -| `pipeline.py` | `ingest_pdf()` — the order all of the above happens in | -| `source_text.py` | `build_source_text()` — what a resource session reads back | - -**The distinction that matters:** `chunking.py` produces text sized for *retrieval*; -`source_text.py` reads back whole pages for *reading*. They are not interchangeable. - -**Sessions.** An upload always belongs to a session, and a session is opened for one -purpose: - -```bash -python cli.py ingest constitution.pdf --session s1 # --for chat (default) -python cli.py ingest constitution.pdf --session s2 --for resource -python cli.py ingest constitution.pdf --session s3 --for both -``` - -```python -from learnmate import ingest_pdf - -ingest_pdf("constitution.pdf", session_id="s1") # chat (default) -ingest_pdf("constitution.pdf", session_id="s2", session_for="resource") # generation -ingest_pdf("constitution.pdf", session_id="s3", session_for="both") # both -``` - -Both purposes need identical ingestion, so the kind is a statement of intent, not an -optimisation. Using a session for the other purpose is refused, with the call that fixes -it — and following that advice is nearly free, because an already-ingested PDF is not -re-embedded. (`full_program.py` always opens sessions `for="both"`, so you will not hit -this unless you use `cli.py` or the library directly.) - -### `storage/` — two databases - -``` -MongoDB (:27018) the PDFs, page text, sessions, chat history, resources, evaluations -Qdrant (:6335) the chunk vectors -``` - -**The split is deliberate and asymmetric.** Nothing in MongoDB can be derived from -anything else, so losing it loses the corpus. The vectors are computed *from* that page -text, so losing Qdrant only costs a re-ingest. That is why the vector backend is swappable -and MongoDB is not. - -| File | Does | -|---|---| -| `mongo.py` | the connection, `StorageUnavailable` | -| `indexes.py` | every index — including three that enforce rules, not speed | -| `ids.py` | ObjectId coercion, shared by everything that queries by id | -| `pdf_files.py` | the PDF bytes, in GridFS | -| `documents.py` | the document record: store, look up, resolve, delete | -| `pages.py` | cleaned page text | -| `pdf_store.py` | one facade over those three | -| `sessions.py` | which PDF a session is about, and what for | -| `history.py` | chat turns | -| `resources.py` | generated resources, with their whole attempt trail | -| `evaluations.py` | the verdict log and its statistics | -| `content_store.py` | one facade over those four | -| `vectors.py` | picks the vector backend | -| `qdrant_vectors.py` | Qdrant: real HNSW index, filtering server-side | -| `mongo_vectors.py` | the same interface over MongoDB, when you don't want a second service | - -Three indexes are `unique` because they enforce a rule structurally: one document per set -of bytes, re-ingesting overwrites a chunk in place, and one PDF per session. - -Both databases run in containers with **named volumes**, so `docker compose down` keeps -your data and only `docker compose down -v` clears it. - -### `llm/` — reaching the three models - -| File | Does | -|---|---| -| `registry.py` | `get_generator_llm()` and `get_judge_llm()` — the entry points | -| `llamacpp.py` | backend 1: a local GGUF, in-process, with JSON grammars | -| `http_api.py` | backend 2: a served OpenAI-compatible endpoint | -| `messages.py` | LangChain messages ↔ the role/content dicts both backends want | -| `runtime.py` | the GGUF weight cache, released cleanly at exit | -| `download.py` | fetches a missing GGUF from Hugging Face | -| `json_output.py` | recovers JSON from an unconstrained reply | -| `embeddings.py` | MiniLM behind LangChain's `Embeddings` interface | - -**There is no Qwen class and no Llama class.** Both chat models share every line of code; -which family loads is the GGUF path in config. Swapping in a finetuned model is two lines -of `.env`. - -**Why these are custom classes** rather than a stock integration: the `response_schema` -argument. A 3B model politely asked for JSON returns prose about half the time. llama.cpp -can instead compile a JSON schema into a *decoding grammar*, making malformed output -impossible. Every structured thing in this project depends on it. - -Caching happens at three separate layers — wrapper objects by role and temperature, actual -weights by file, and the embedding model. Two wrappers can share one set of weights. - -### `chat_agent/` — answering a question - -| File | Does | -|---|---| -| `state.py` | `ChatState` — what flows between nodes | -| `rewrite.py` | node 1 — resolve the follow-up into a standalone question | -| `retrieve.py` | node 2 — search the vectors, pick PDF or general mode | -| `generate.py` | node 3 — write the reply | -| `evaluate.py` | node 4 — judge it | -| `routing.py` | the accept-or-retry branch | -| `persist.py` | node 5 — save the turn | -| `prompts.py` | the three system prompts | -| `graph.py` | the LangGraph wiring | -| `agent.py` | `ChatAgent` — the public entry point | - -```python -from learnmate import ChatAgent - -agent = ChatAgent(session_id="s1", doc_id=doc_id) -result = agent.ask("What are the directors' duties?") -print(result["reply"], result["accepted"], result["mode"]) -``` - -The retry budget is one regeneration, and the regenerated reply is returned **whether or -not it passes** — a student mid-conversation needs an answer, and `accepted` says whether -it was reviewed clean. - -### `resource_agent/` — generating study material - -Four resource types, **one file each**: - -| File | Produces | -|---|---| -| `mcq.py` | `{question, options[4], correct_answer}` | -| `practice_qsn.py` | `{question, answer}` | -| `keypoints.py` | `["point", ...]` | -| `summary.py` | one block of connected prose | - -Each owns its prompt, JSON schema, how to read the reply and how to render it. Everything -else is shared, so a fifth type is one new file plus one line in `tasks.py`. - -| File | Does | -|---|---| -| `task.py` | the contract every resource type implements | -| `tasks.py` | the registry | -| `state.py` | `ResourceState` | -| `generate.py` | node 1 — ask the generator, folding in a critique on retry | -| `check.py` | node 2 — both gates | -| `routing.py` | the accept-or-retry branch | -| `persist.py` | node 3 — store the resource and its attempt trail | -| `graph.py` | the wiring | -| `agent.py` | `generate_resource()` — the public entry point | - -### `evaluator/` — the two gates - -``` -gate 1 structural validators plain Python, microseconds -gate 2 an LLM rubric grade ~25 seconds -``` - -| File | Gate | Does | -|---|---|---| -| `normalise.py` | 1 | comparison-safe text | -| `mcq_rules.py` | 1 | per-question faults **and** set-wide biases | -| `text_rules.py` | 1 | summary, keypoints, practice questions | -| `validators.py` | 1 | the dispatcher | -| `rubrics.py` | 2 | the grading criteria, one per task | -| `prompt.py` | 2 | system prompt and message assembly | -| `verdict.py` | 2 | the schema, parsing, and fail-closed verdicts | -| `judge.py` | 2 | orchestration | - -The set-wide MCQ rules are the interesting ones: every question can be individually -perfect while the set as a whole is guessable — the answer always in the same slot, or -always the longest option. No single-question check can see that. - -**Everything fails closed.** A judge that cannot be parsed or reached returns a *failing* -verdict, not an exception — the caller is mid-loop and needs a decision, and silently -passing unreviewed content through is the one outcome worth ruling out. - -A chat reply has no structural gate: free prose has nothing mechanical to check, so it -goes straight to gate 2. - ---- - -## Running it - -### `cli.py` — one command at a time - -``` -python cli.py doctor check models, databases, settings -python cli.py ingest [--session ID] [--force] store + index one PDF - [--for chat|resource|both] what the session is for -python cli.py docs [--limit N] list ingested documents -python cli.py chat [--session ID] [--doc X] interactive chat - [--threshold N] [--no-eval] [--quiet] -python cli.py generate [--session ID] [--doc X] generate a resource - [--count N] [--topic "..."] [--pages 3-7] - [--max-source-chars N] [--threshold N] [--no-eval] [--json] [--quiet] -python cli.py resources [--doc X] [--task T] list what has been generated - [--accepted] [--show] [--limit N] -python cli.py stats score distribution per task -python cli.py export write a stored PDF back to disk -python cli.py delete remove a document and its chunks -``` - -`` is one of `mcq`, `practice_qsn`, `keypoints`, `summary`. - -`--doc` accepts an id, an exact filename, or a unique fragment (`--doc constitution`). A -fragment matching several documents is rejected rather than guessed at. Omit it and the -session's own PDF is used; omit both and it falls back to the most recently ingested one. - -`stats`, `export`, `delete` and `docs` exist only here — they are maintenance verbs that -do not belong in a guided walkthrough. - -### `full_program.py` — the whole workflow in one session - -``` -python learnmate\full_program.py interactive menu -python learnmate\full_program.py --demo run everything without prompting -python learnmate\full_program.py --demo --no-eval same, roughly half the time -python learnmate\full_program.py --pdf notes.pdf --ask "..." -python learnmate\full_program.py --topic "fundamental rights" -``` - -The menu: - -| | | -|---|---| -| 1 | Upload a PDF (by file path) | -| 2 | Chat about it | -| 3–6 | Generate MCQs / summary / key points / practice questions individually | -| 7 | Generate all four | -| 8 | Show stored state — documents, session, resources, vector counts | -| 9 | Set a topic for generation | -| e | Toggle evaluation (the judge and its retry) | -| 0 | Quit | - -### Using it as a library - -This is the interface the backend will integrate against. - -```python -from learnmate import ChatAgent, build_source_text, generate_resource, ingest_pdf - -report = ingest_pdf("notes.pdf", session_id="s1", session_for="both") -doc_id = report["doc_id"] - -agent = ChatAgent(session_id="s1", doc_id=doc_id) -reply = agent.ask("What are the directors' duties?") -# {reply, mode, accepted, verdict, contexts, scores, attempts, ...} - -source = build_source_text(doc_id, topic="directors' duties") -result = generate_resource("mcq", source, count=5, doc_id=doc_id) -# {task, content, accepted, verdict, attempts, resource_id} -``` - -### Reading the evaluation log - -```bash -python cli.py stats -``` - -```python -from learnmate.storage import content_store - -content_store.evaluation_stats() # score distribution per task -content_store.stage_counts() # which gate decided each attempt -``` - -The `distinct` column is the important one. A judge whose scores cluster in a narrow band -cannot separate good from bad at **any** threshold — that is a rubric problem, not a -threshold problem. The stage counts show which gate decided each attempt: if the validator -is deciding most of them, the generation prompt needs work. - ---- - -## Configuration - -Everything lives in `config.py` and is overridable by environment variable. Copy -`.env.example` to `.env` and edit. The settings you are most likely to touch: - -| Variable | Default | Meaning | -|---|---|---| -| `LEARNMATE_GENERATOR_MODEL` | `models/qwen2.5-3b-…gguf` | swap in your finetuned model | -| `LEARNMATE_GENERATOR_BACKEND` | `llamacpp` | or `http` for a served model | -| `LEARNMATE_MONGODB_URI` | `mongodb://localhost:27018` | the container from `docker-compose.yml` | -| `LEARNMATE_VECTOR_BACKEND` | `qdrant` | or `mongodb`, to avoid a second service | -| `LEARNMATE_RELEVANCE_THRESHOLD` | `0.25` | below this, chat answers from general knowledge | -| `LEARNMATE_EVALUATOR_THRESHOLD` | `70` | judge score needed to accept | -| `LEARNMATE_MAX_ATTEMPTS` | `2` | one generation plus one retry | -| `LEARNMATE_MAX_PDF_MB` | `10` | upload limit | -| `LEARNMATE_ONE_PDF_PER_SESSION` | `1` | set `0` to lift the restriction | - -### Why the ports are unusual - -Both services publish on non-default host ports — MongoDB on **27018**, Qdrant on -**6335** — because this machine already runs another project's MongoDB on 27017 and -another Qdrant on 6333. Sharing a server means sharing a failure: another project's -`docker compose down -v` would take LearnMate's PDFs and generated resources with it. - ---- - -## Design decisions worth knowing - -**The mode is decided by a number, not by the model.** Whether chat answers from the PDF -or from general knowledge comes from the retrieval score. A number can be tuned and -logged; a second LLM call can be wrong and costs 25 seconds. - -**Rewrite runs before retrieval.** *"What about his powers?"* embeds to nothing useful. -Resolved first, it retrieves correctly. - -**Chunks for retrieval, whole pages for reading.** Two different products of one ingest, -not interchangeable. - -**The cheap gate runs first.** Most bad generations fail mechanically. Catching those in -microseconds means the 25-second judge is only spent on well-formed content. - -**Failed output is still returned, and still stored.** Marked `accepted: False`. Hiding it -would make the failure rate invisible. - -**The whole attempt trail is kept**, not just the winner. Whether the threshold is set -anywhere near right is unanswerable after the fact if you only keep what passed. - -**The retry budget is 2 on purpose.** A 3B judge tends to oscillate rather than converge -over more rounds; the third attempt is usually a worse version of the first. - ---- - -## Troubleshooting - -| Symptom | Cause and fix | -|---|---| -| `No module named 'bson'` / `'langchain_text_splitters'` | You used the system Python. Use `venv\Scripts\python`, or run `full_program.py`, which switches by itself. | -| `Cannot reach MongoDB at …` | `docker compose up -d mongo` | -| `Cannot reach the Qdrant server at …` | `docker compose up -d qdrant` | -| Qdrant container crash-loops after an image bump | Its on-disk format isn't backward compatible. `docker compose down -v` and re-ingest — nothing is lost, the vectors are derived from MongoDB. | -| `Session 'x' is already about y.pdf` | One PDF per session. Use a new session id. | -| `Session 'x' was opened for chat, not resource generation` | Open a session with `--for resource`; the PDF is not re-embedded. | -| `No extractable text in …` | A scanned PDF. It needs OCR before it can be indexed. | -| Everything is very slow | Expected on CPU. Use `--no-eval` to skip the judge, or set `LEARNMATE_N_GPU_LAYERS` to offload to a GPU. | - ---- - -## Known limitations - -- **Key-point grading is weak.** The `keypoints` rubric scores faithful content around 40 - against a threshold of 70 — the 3B judge produces one complaint per rubric criterion - regardless of the content. The other three resource types separate good from bad by - 50–79 points. Softening the rubric's "missing the central point is a serious fault" - clause is the place to start. -- **Generation quality depends heavily on which passage is used.** With no `--topic`, the - source is the opening of the document, which for a book-style PDF is the title page and - preamble. Pass a topic for anything substantive. -- **Scanned PDFs are not supported** — there is no OCR step. -- **One PDF per session by design.** Combining several documents into one corpus and - chatting across all of them is not supported. diff --git a/components-Dinura/cli.py b/components-Dinura/cli.py deleted file mode 100644 index 8255c42..0000000 --- a/components-Dinura/cli.py +++ /dev/null @@ -1,482 +0,0 @@ -""" -LearnMate command line. - - python cli.py doctor check models, MongoDB, config - python cli.py ingest constitution.pdf --session s1 upload for a chat session - python cli.py chat --session s1 chat about that session's PDF - - python cli.py ingest constitution.pdf --session s2 --for resource - python cli.py generate mcq --session s2 --count 5 - python cli.py generate summary --session s2 --topic "fundamental rights" - - python cli.py docs list ingested documents - python cli.py resources --task mcq show what has been generated - python cli.py stats evaluation score distribution - python cli.py export ./out write a stored PDF back to disk - -A session is about exactly one PDF, up to LEARNMATE_MAX_PDF_MB (10 MB by default), and is -opened for one purpose: `--for chat` (the default), `--for resource`, or `--for both`. -Using a session for the other purpose is refused, with the command to open the right kind -of session -- which costs nothing, because an already-ingested PDF is not re-embedded. - -Ingesting a second, different PDF into the same session is refused too: embedding is the -expensive step, so a new PDF means a new session id. `ingest` prints the session to use -when none is given. `--doc` overrides the session's PDF, and its kind check, for one -command. - -Run `python cli.py --help` for the options of any one command. -""" - -import argparse -import json -import sys -import uuid -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from learnmate import config -from learnmate.chat_agent import ChatAgent -from learnmate.ingestion import build_source_text, describe_kinds, ingest_pdf, require_kind -from learnmate.resource_agent import TASK_NAMES, generate_resource, render -from learnmate.storage import ( - QdrantUnavailable, - StorageUnavailable, - build_vector_store, - content_store, - export_pdf, - list_documents, - list_resources, - pdf_store, -) - -RULE = "=" * 68 - - -def _resolve_doc(reference, session=None, required=True): - """ - Work out which document a command is about. - - Three sources, most specific first: - - --doc an explicit id, filename or fragment - --session the PDF that session was bound to at ingest time - neither the most recently ingested PDF - - The session path is the normal one: a session holds exactly one PDF, so `chat - --session s1` never has to be told which document it means. - """ - if not reference: - document = pdf_store.get_document(content_store.session_doc_id(session)) \ - if session else None - document = document or pdf_store.get_active_document() - if document: - return document - if required: - raise SystemExit("No PDF ingested yet. Run: python cli.py ingest ") - return None - - document = pdf_store.resolve_document(reference) - if not document: - raise SystemExit(f"No document matches {reference!r}. " - "Run `python cli.py docs` to list them.") - return document - - -# --- Commands ------------------------------------------------------------------------ - -def cmd_doctor(args): - """Check every external dependency before anything slow is attempted.""" - from learnmate.storage.mongo import get_db - - print(RULE) - print("LearnMate environment check") - print(RULE) - - ok = True - - print("\nModels") - for label, backend, path in ( - ("generator", config.GENERATOR_BACKEND, config.GENERATOR_MODEL), - ("judge", config.JUDGE_BACKEND, config.JUDGE_MODEL), - ): - if backend == "http": - print(f" {label:10} http backend -> {path}") - continue - exists = Path(path).exists() - size = f"{Path(path).stat().st_size / 1_073_741_824:.1f} GB" if exists else "MISSING" - print(f" {label:10} {'ok ' if exists else '!! '}{path} ({size})") - if not exists: - print(f" will be downloaded from HF on first use") - - print("\nMongoDB (PDFs, page text, resources, evaluations, history)") - try: - database = get_db() - server = database.client.server_info() - print(f" ok {config.MONGODB_URI} (server {server['version']}, db " - f"{config.MONGODB_DB})") - print(f" documents={database[config.COLL_DOCUMENTS].count_documents({})} " - f"pages={database[config.COLL_PAGES].count_documents({})} " - f"resources={database[config.COLL_RESOURCES].count_documents({})}") - except StorageUnavailable as exc: - ok = False - print(f" !! {exc}") - - print(f"\nVector database (backend: {config.VECTOR_BACKEND})") - try: - store = build_vector_store() - print(f" ok {store.describe_backend()}") - print(f" vectors={store.count()}") - except QdrantUnavailable as exc: - ok = False - print(f" !! {exc}") - except StorageUnavailable as exc: - ok = False - print(f" !! {exc}") - except Exception as exc: - ok = False - print(f" !! {type(exc).__name__}: {exc}") - - print("\nSettings") - print(f" upload limit {config.MAX_PDF_MB:g} MB per PDF" - + (" (one PDF per session)" if config.ONE_PDF_PER_SESSION else "")) - print(f" embeddings {config.EMBEDDING_MODEL}") - print(f" chunk size/overlap {config.CHUNK_SIZE}/{config.CHUNK_OVERLAP}") - print(f" relevance threshold {config.RELEVANCE_THRESHOLD}") - print(f" judge threshold {config.EVALUATOR_THRESHOLD}") - print(f" max attempts {config.MAX_ATTEMPTS}") - print(RULE) - return 0 if ok else 1 - - -def cmd_ingest(args): - # A session is generated when none is given, so every ingest is bound to something - # and the one-PDF-per-session rule has an id to hold on to. It is printed below - # because the user needs it to work with what they just ingested. - session = args.session or f"cli-{uuid.uuid4().hex[:12]}" - - report = ingest_pdf(args.path, session_id=session, session_for=args.session_for, - force=args.force) - document = report["document"] - kinds = report["session_for"] - print(f" id={document['_id']} pages={report['n_pages']} " - f"chunks={report['n_chunks']}") - - # Print only the commands this session is actually for, so the next step is never a - # guess. A "both" session gets both lines. - print(f"\n session {session} is for {describe_kinds(kinds)}:") - if "chat" in kinds: - print(f" python cli.py chat --session {session}") - if "resource" in kinds: - print(f" python cli.py generate mcq --session {session} --count 5") - return 0 - - -def cmd_docs(args): - documents = list_documents(limit=args.limit) - if not documents: - print("No documents ingested yet. Try: python cli.py ingest ") - return 0 - - print(f"{'id':26} {'pages':>6} {'chunks':>7} filename") - print("-" * 68) - for document in documents: - print(f"{str(document['_id']):26} {str(document.get('n_pages') or '-'):>6} " - f"{str(document.get('n_chunks') or '-'):>7} {document['filename']}") - return 0 - - -def cmd_chat(args): - # Refuse a session opened for resource generation, unless --doc names a document - # explicitly and so overrides the session's scope anyway. - if not args.doc: - require_kind(args.session, "chat") - - document = _resolve_doc(args.doc, session=args.session, required=False) - doc_id = document["_id"] if document else None - - agent = ChatAgent( - session_id=args.session, - doc_id=doc_id, - threshold=args.threshold, - evaluate=not args.no_eval, - verbose=not args.quiet, - ) - - print(RULE) - scope = document["filename"] if document else "the whole corpus" - print(f"LearnMate chat | scope: {scope} | session: {agent.session_id}") - print("Type 'exit' to stop, 'reset' to clear this session's history.") - print(RULE) - - while True: - try: - query = input("\nYou: ").strip() - except (EOFError, KeyboardInterrupt): - print("\nBye.") - return 0 - - if not query: - continue - if query.lower() in ("exit", "quit"): - print("Bye.") - return 0 - if query.lower() == "reset": - print(f"Cleared {agent.reset()} turns.") - continue - - result = agent.ask(query) - - print("\n--- Answer ---") - print(result["reply"] or "(no answer produced)") - - if result["mode"] == "pdf": - print("\n--- Sources ---") - for i, (doc, score) in enumerate(zip(result["contexts"], result["scores"]), 1): - print(f" [{i}] {doc.metadata.get('filename', '?')} " - f"p.{doc.metadata.get('page_number')} (score {score:.4f})") - else: - print(f"\n--- Sources ---\n general knowledge " - f"(best retrieval score {result['top_score']:.4f} " - f"< {config.RELEVANCE_THRESHOLD})") - - verdict = result["verdict"] - if verdict: - trail = " -> ".join(str(a["verdict"]["score"]) for a in result["attempts"] - if a.get("verdict")) - status = "accepted" if result["accepted"] else "BELOW THRESHOLD (shown anyway)" - print(f"\n--- Evaluation ---\n {verdict['score']}/100 " - f"(threshold {verdict['threshold']}) - {status}") - if len(result["attempts"]) > 1: - print(f" regenerated once, scores: {trail}") - print(f" {verdict['reasoning']}") - if not result["accepted"] and verdict.get("regeneration_instruction"): - print(f" unresolved: {verdict['regeneration_instruction']}") - - -def cmd_generate(args): - if not args.doc: - require_kind(args.session, "resource") - - document = _resolve_doc(args.doc, session=args.session) - - source = build_source_text(document["_id"], topic=args.topic, - pages=args.pages, max_chars=args.max_source_chars) - print(f"[*] Source: {len(source)} chars from {document['filename']}" - + (f" matching {args.topic!r}" if args.topic else "")) - - result = generate_resource( - args.task, source, count=args.count, doc_id=document["_id"], - threshold=args.threshold, evaluate=not args.no_eval, verbose=not args.quiet, - ) - - if args.json: - print(json.dumps({"task": result["task"], "content": result["content"], - "accepted": result["accepted"], - "resource_id": result["resource_id"]}, - indent=2, ensure_ascii=False)) - return 0 if result["accepted"] else 1 - - print("\n" + RULE) - print(render(args.task, result["content"]) or "(nothing generated)") - print(RULE) - - verdict = result["verdict"] - if verdict: - status = "accepted" if result["accepted"] else "BELOW THRESHOLD (shown anyway)" - print(f"Score {verdict['score']}/100 (threshold {verdict['threshold']}) - {status}") - print(f"Reasoning: {verdict['reasoning']}") - if not result["accepted"] and verdict.get("regeneration_instruction"): - print(f"Unresolved: {verdict['regeneration_instruction']}") - elif result["attempts"]: - last = result["attempts"][-1] - if last["stage"] in ("validator", "parse"): - print(f"Rejected by the {last['stage']} gate: {'; '.join(last['reasons'])}") - - if result["resource_id"]: - print(f"Stored as resource {result['resource_id']}") - return 0 if result["accepted"] else 1 - - -def cmd_resources(args): - document = _resolve_doc(args.doc, required=False) - records = list_resources(doc_id=document["_id"] if document else None, - task=args.task, accepted_only=args.accepted, - limit=args.limit) - if not records: - print("No generated resources match.") - return 0 - - for record in records: - stamp = record["created_at"].strftime("%Y-%m-%d %H:%M") - score = record.get("score") - flag = "PASS" if record["accepted"] else "fail" - print(f"\n{RULE}\n{record['task']:13} {stamp} " - f"score={score if score is not None else '-':<5} {flag} " - f"attempts={record['n_attempts']} id={record['_id']}") - if args.show: - print(RULE) - print(render(record["task"], record["content"])) - return 0 - - -def cmd_stats(args): - stats = content_store.evaluation_stats() - stages = content_store.stage_counts() - - if not stats and not stages: - print("No evaluations recorded yet.") - return 0 - - print(RULE) - print("Which gate decided each attempt") - print(RULE) - for stage, count in sorted(stages.items(), key=lambda kv: -kv[1]): - print(f" {stage:12} {count}") - - if stats: - print("\n" + RULE) - print("Judge score distribution") - print(RULE) - print(f"{'task':14} {'n':>4} {'min':>4} {'med':>4} {'max':>4} {'mean':>6} " - f"{'distinct':>9} {'pass':>6}") - for task, row in stats.items(): - print(f"{task:14} {row['n']:>4} {row['min']:>4} {row['median']:>4} " - f"{row['max']:>4} {row['mean']:>6} {row['distinct']:>9} " - f"{row['pass_rate']:>6}") - print("\nA narrow `distinct` range means the judge cannot separate good from bad") - print("at any threshold; that is a rubric problem, not a threshold problem.") - return 0 - - -def cmd_export(args): - document = _resolve_doc(args.doc) - written = export_pdf(document["_id"], args.destination) - print(f"[+] Wrote {written}") - return 0 - - -def cmd_delete(args): - document = _resolve_doc(args.doc) - confirm = input(f"Delete {document['filename']} and all its chunks? [y/N] ").strip() - if confirm.lower() != "y": - print("Cancelled.") - return 1 - pdf_store.delete_document(document["_id"]) - print(f"[+] Deleted {document['filename']}") - return 0 - - -# --- Parser -------------------------------------------------------------------------- - -def build_parser(): - parser = argparse.ArgumentParser( - prog="cli.py", - description="LearnMate: ingest PDFs, chat about them, generate study resources.", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - doctor = subparsers.add_parser("doctor", help="check models, MongoDB and settings") - doctor.set_defaults(func=cmd_doctor) - - ingest = subparsers.add_parser( - "ingest", help=f"store and index one PDF (max {config.MAX_PDF_MB:g} MB)") - ingest.add_argument("path", help="the PDF to ingest") - ingest.add_argument("--session", help="bind the PDF to this session; " - "omit to start a new one") - # dest is spelled out because `--for` would otherwise become args.for, and `for` is - # a Python keyword. - ingest.add_argument("--for", dest="session_for", default="chat", - choices=["chat", "resource", "both"], - help="what the session is for (default: chat)") - ingest.add_argument("--force", action="store_true", - help="re-index even if this same PDF is already stored") - ingest.set_defaults(func=cmd_ingest) - - docs = subparsers.add_parser("docs", help="list ingested documents") - docs.add_argument("--limit", type=int, default=50) - docs.set_defaults(func=cmd_docs) - - chat = subparsers.add_parser("chat", help="ask questions about a document") - chat.add_argument("--doc", help="document id, filename, or fragment; " - "omit to use the session's PDF") - chat.add_argument("--session", help="the session to resume; its PDF is the scope") - chat.add_argument("--threshold", type=int, default=None) - chat.add_argument("--no-eval", action="store_true", - help="skip evaluation and the retry loop (much faster)") - chat.add_argument("--quiet", action="store_true", help="hide per-node progress") - chat.set_defaults(func=cmd_chat) - - generate = subparsers.add_parser("generate", help="generate a study resource") - generate.add_argument("task", choices=TASK_NAMES) - generate.add_argument("--doc", help="document id, filename, or fragment; " - "omit to use the session's PDF") - generate.add_argument("--session", help="generate from this session's PDF") - generate.add_argument("--count", type=int, default=5, - help="items to produce (sentence budget for summary)") - generate.add_argument("--topic", help="generate from the parts of the document most " - "relevant to this topic") - generate.add_argument("--pages", type=_page_list, - help="restrict the source to these pages, e.g. 3-7 or 2,5,9") - generate.add_argument("--max-source-chars", type=int, default=config.MAX_SOURCE_CHARS) - generate.add_argument("--threshold", type=int, default=None) - generate.add_argument("--no-eval", action="store_true", - help="generate without the evaluator gates") - generate.add_argument("--json", action="store_true", help="print raw JSON") - generate.add_argument("--quiet", action="store_true") - generate.set_defaults(func=cmd_generate) - - resources = subparsers.add_parser("resources", help="list generated resources") - resources.add_argument("--doc") - resources.add_argument("--task", choices=TASK_NAMES) - resources.add_argument("--accepted", action="store_true", help="only ones that passed") - resources.add_argument("--show", action="store_true", help="print the content too") - resources.add_argument("--limit", type=int, default=10) - resources.set_defaults(func=cmd_resources) - - stats = subparsers.add_parser("stats", help="evaluation score distribution") - stats.set_defaults(func=cmd_stats) - - export = subparsers.add_parser("export", help="write a stored PDF back to disk") - export.add_argument("doc") - export.add_argument("destination") - export.set_defaults(func=cmd_export) - - delete = subparsers.add_parser("delete", help="remove a document and its chunks") - delete.add_argument("doc") - delete.set_defaults(func=cmd_delete) - - return parser - - -def _page_list(value: str): - """Parse `3-7` or `2,5,9` into a list of page numbers.""" - pages = [] - for part in value.split(","): - part = part.strip() - if "-" in part: - start, end = part.split("-", 1) - pages.extend(range(int(start), int(end) + 1)) - elif part: - pages.append(int(part)) - return pages - - -def main(): - args = build_parser().parse_args() - try: - return args.func(args) - except (StorageUnavailable, QdrantUnavailable) as exc: - print(f"\n[!] {exc}", file=sys.stderr) - return 2 - except (ValueError, FileNotFoundError, KeyError) as exc: - print(f"\n[!] {exc}", file=sys.stderr) - return 1 - except KeyboardInterrupt: - print("\nInterrupted.", file=sys.stderr) - return 130 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/components-Dinura/data/Company-law-part1-notes.pdf b/components-Dinura/data/Company-law-part1-notes.pdf deleted file mode 100644 index 402db4b..0000000 Binary files a/components-Dinura/data/Company-law-part1-notes.pdf and /dev/null differ diff --git a/components-Dinura/data/company-law.pdf b/components-Dinura/data/company-law.pdf deleted file mode 100644 index 02651bb..0000000 Binary files a/components-Dinura/data/company-law.pdf and /dev/null differ diff --git a/components-Dinura/data/constitution.pdf b/components-Dinura/data/constitution.pdf deleted file mode 100644 index 8bbaba8..0000000 Binary files a/components-Dinura/data/constitution.pdf and /dev/null differ diff --git a/components-Dinura/docker-compose.yml b/components-Dinura/docker-compose.yml deleted file mode 100644 index 108710d..0000000 --- a/components-Dinura/docker-compose.yml +++ /dev/null @@ -1,87 +0,0 @@ -# Both of LearnMate's databases, each in its own container with its own named volume. -# -# docker compose up -d start both -# docker compose ps check them -# docker compose down stop them (the named volumes keep the data) -# docker compose down -v stop them AND delete the data -# -# Qdrant's dashboard is at http://localhost:6335/dashboard once it is up. -# -# Both services publish on non-default host ports. That is deliberate: this machine -# already runs other projects' Qdrant on 6333 and a MongoDB on 27017, and sharing a -# server means sharing a failure. Another project's `docker compose down -v` would take -# LearnMate's PDFs, sessions and generated resources with it. - -services: - qdrant: - # Pinned, and kept within one minor of the qdrant-client pin in requirements.txt -- - # the client refuses to run against a server more than one minor version away. - # - # Changing this tag across more than a minor version or two will crash-loop the - # container on startup: Qdrant's on-disk segment format is not backward compatible - # (1.18 cannot read 1.12 storage, and fails with `unknown variant 'on_disk'`). If that - # happens, `docker compose down -v` and re-ingest. Nothing is lost -- the vectors are - # derived from the PDFs and page text, which live in MongoDB. - image: qdrant/qdrant:v1.18.1 - container_name: learnmate-qdrant - restart: unless-stopped - ports: - # 6335, not the conventional 6333: this machine already runs a `helpmedai-qdrant` - # container bound to 6333/6334 for a different project. Publishing on a free port - # keeps the two corpora in separate servers rather than sharing one instance and - # relying on collection names not to collide. - - "6335:6333" # REST + dashboard - - "6336:6334" # gRPC - volumes: - # A named volume, not a bind mount into the repo: Qdrant's storage engine does not - # behave well on a Windows bind mount, and the vectors are rebuildable from the PDFs - # in MongoDB anyway. - - qdrant_storage:/qdrant/storage - environment: - # Uncomment to require an API key, then set LEARNMATE_QDRANT_API_KEY to match. - # QDRANT__SERVICE__API_KEY: change-me - QDRANT__LOG_LEVEL: INFO - healthcheck: - # The image is distroless, so there is no curl or wget to call. Qdrant ships a - # readiness probe on the REST port; exec'ing the binary itself is the supported way. - test: ["CMD", "/qdrant/qdrant", "--version"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - - mongo: - # Holds everything that is not a vector: the uploaded PDFs (GridFS), the cleaned page - # text, session bindings, chat history, generated resources and the evaluation log. - # Losing this loses the corpus; losing Qdrant only costs a re-ingest. - image: mongo:8 - container_name: learnmate-mongo - restart: unless-stopped - ports: - # 27018, not 27017. Port 27017 on this machine is already answered by a native - # MongoDB service that also hosts two other projects' databases -- publishing there - # would either fail to bind or silently be shadowed by it, which is exactly the - # confusion this port choice avoids. Set LEARNMATE_MONGODB_URI to match. - - "27018:27017" - volumes: - # Named volume, so `docker compose down` keeps the corpus and only `down -v` clears - # it. This is the "memory" of the system: PDFs and page text are not derivable from - # anything else. - - mongo_data:/data/db - command: ["--wiredTigerCacheSizeGB", "1"] - healthcheck: - # mongosh ships in the mongo:8 image; a ping is enough to know it is accepting - # connections rather than still recovering its journal. - test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping').ok"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 20s - -volumes: - # Both are named volumes managed by Docker, so the data survives `docker compose down`, - # a container upgrade, and a host reboot. Docker prefixes them with the project - # directory name, so they appear as components-dinura_qdrant_storage and - # components-dinura_mongo_data in `docker volume ls`. - qdrant_storage: - mongo_data: diff --git a/components-Dinura/learnmate/__init__.py b/components-Dinura/learnmate/__init__.py deleted file mode 100644 index b4b8c93..0000000 --- a/components-Dinura/learnmate/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -LearnMate: a local study assistant over your own PDFs. - -Three agents over one corpus: - - chat_agent answers questions about an ingested PDF, or from general knowledge - when nothing relevant is retrieved - resource_agent generates MCQs, practice questions, key points and summaries from a - document's text - evaluator grades what the other two produce and drives a single retry - -Both agents are LangGraph state machines over LangChain components, and both run entirely -on local models: Qwen2.5-3B generates, Llama-3.2-3B judges, MiniLM embeds. - -Two databases, each in its own container (`docker compose up -d`): - - MongoDB (:27018) the PDFs in GridFS, their cleaned page text, session bindings, - chat history, generated resources and the evaluation log - Qdrant (:6335) the chunk embeddings - -The asymmetry is deliberate. Nothing in MongoDB can be derived from anything else, so -losing it loses the corpus; the vectors are computed from that page text, so losing Qdrant -only costs a re-ingest. That is why the vector backend is swappable -(LEARNMATE_VECTOR_BACKEND=mongodb keeps everything in one service) and MongoDB is not. - -An upload belongs to a session, and a session is about exactly one PDF, opened either for -chat or for resource generation. - -Typical use: - - from learnmate import ChatAgent, build_source_text, generate_resource, ingest_pdf - - report = ingest_pdf("notes.pdf", session_id="s1", session_for="both") - doc_id = report["doc_id"] - - agent = ChatAgent(session_id="s1", doc_id=doc_id) - print(agent.ask("What are the directors' duties?")["reply"]) - - source = build_source_text(doc_id, topic="directors' duties") - result = generate_resource("mcq", source, count=5, doc_id=doc_id) -""" - -from . import config -from .chat_agent import ChatAgent -from .evaluator import Judge, get_judge -from .ingestion import build_source_text, ingest_pdf -from .resource_agent import TASK_NAMES, generate_resource, render - -__all__ = [ - "ChatAgent", - "Judge", - "TASK_NAMES", - "build_source_text", - "config", - "generate_resource", - "get_judge", - "ingest_pdf", - "render", -] - -__version__ = "1.0.0" diff --git a/components-Dinura/learnmate/chat_agent/__init__.py b/components-Dinura/learnmate/chat_agent/__init__.py deleted file mode 100644 index 2eef9ff..0000000 --- a/components-Dinura/learnmate/chat_agent/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -The chat agent: retrieval-grounded PDF chat with a general-knowledge fallback. - -One turn is one pass through a LangGraph state machine: - - rewrite -> retrieve -> generate -> evaluate -> decide -+-> persist -> END - ^ | - +--------- regenerate -------+ - -Two possible modes, decided by retrieval rather than by asking the model: - - pdf mode the top chunk scores at or above RELEVANCE_THRESHOLD, so the reply is - written from those chunks and judged strictly against them -- anything - they do not support is a hallucination - general mode nothing relevant was retrieved, so the reply comes from the model's own - knowledge and is judged only on relevance, coherence and informativeness - -`rewrite` runs before retrieval, not after. A follow-up like "what about his powers?" -embeds to nothing useful; resolved against the history into "what are the President's -powers?" it retrieves correctly. Doing it in the other order retrieves on the pronoun. - -The retry budget is one regeneration. The regenerated reply is returned whether or not it -clears the threshold -- a user mid-conversation needs an answer, and `accepted` reports -whether it was reviewed clean so a caller can flag it. - -Where things live, in reading order: - - state.py ChatState -- what flows between nodes, and the one reducer - prompts.py the three system prompts - helpers.py logging and history-to-messages conversion - rewrite.py node 1 resolve the follow-up into a standalone question - retrieve.py node 2 search the vectors, pick pdf or general mode - generate.py node 3 write the reply (re-entered on retry) - evaluate.py node 4 judge it, strictly if it was meant to be grounded - routing.py the accept-or-retry branch out of evaluate - persist.py node 5 save the turn to Mongo - graph.py the wiring that connects the above - agent.py ChatAgent -- the public entry point -""" - -from .agent import ChatAgent -from .graph import build_chat_graph, get_chat_graph - -__all__ = ["ChatAgent", "build_chat_graph", "get_chat_graph"] diff --git a/components-Dinura/learnmate/chat_agent/agent.py b/components-Dinura/learnmate/chat_agent/agent.py deleted file mode 100644 index 06da17f..0000000 --- a/components-Dinura/learnmate/chat_agent/agent.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -The public face of the chat agent. - -`ChatAgent` is the only thing callers touch. It holds the per-conversation settings, -loads history, runs one pass of the graph per message, and flattens the final state into -a plain dict. -""" - -import uuid -from typing import Dict, List - -from .. import config -from ..storage import content_store -from .graph import get_chat_graph -from .state import ChatState - - -class ChatAgent: - """ - One conversation. - - History lives in MongoDB keyed by session_id, so a conversation survives the process - and can be resumed by passing the same id back. - """ - - def __init__(self, session_id: str = None, doc_id=None, threshold: int = None, - max_attempts: int = None, evaluate: bool = True, verbose: bool = True): - # A generated id gives an anonymous CLI session somewhere to store history, - # without the caller having to invent one. - self.session_id = session_id or f"cli-{uuid.uuid4().hex[:12]}" - self.doc_id = doc_id - # `is not None` rather than `or`: threshold=0 is a legitimate "accept anything". - self.threshold = threshold if threshold is not None else config.EVALUATOR_THRESHOLD - self.max_attempts = max_attempts or config.MAX_ATTEMPTS - self.evaluate = evaluate - self.verbose = verbose - - def ask(self, query: str) -> Dict: - """ - Handle one user message end to end. - - Returns {query, standalone_query, mode, top_score, contexts, scores, reply, - verdict, accepted, attempts}. `reply` is the last attempt: once a regeneration has - run, its output is what the caller gets, pass or fail, and `accepted` says which. - """ - if not (query or "").strip(): - raise ValueError("Empty query.") - - # History is re-read from Mongo every turn rather than cached in the object, so - # two processes sharing a session_id stay consistent. - initial: ChatState = { - "query": query.strip(), - "session_id": self.session_id, - "doc_id": self.doc_id, - "history": content_store.load_history(self.session_id), - "threshold": self.threshold, - "max_attempts": self.max_attempts, - "evaluate": self.evaluate, - "verbose": self.verbose, - "persist": True, - "attempt": 0, - "attempts": [], - } - - # LangGraph counts every node execution against this limit and raises if it is - # exceeded. Two nodes per attempt (generate + evaluate) plus headroom for the - # linear nodes -- a safety net in case `decide` ever fails to terminate. - limit = 2 * self.max_attempts + 6 - final = get_chat_graph().invoke(initial, {"recursion_limit": limit}) - - # Flatten the state into a stable return shape. Callers depend on these keys, so - # they are listed explicitly rather than handing back the raw state dict. - return { - "query": query, - "standalone_query": final.get("standalone_query"), - "mode": final.get("mode"), - "top_score": final.get("top_score", 0.0), - "contexts": final.get("contexts", []), - "scores": final.get("scores", []), - "reply": final.get("reply", ""), - "verdict": final.get("verdict"), - "accepted": bool(final.get("passed")), - "attempts": final.get("attempts", []), - } - - def history(self) -> List[Dict[str, str]]: - """Every turn stored for this session, oldest first.""" - return content_store.load_history(self.session_id) - - def reset(self) -> int: - """Wipe this session's history. Returns how many turns were deleted.""" - return content_store.clear_history(self.session_id) diff --git a/components-Dinura/learnmate/chat_agent/evaluate.py b/components-Dinura/learnmate/chat_agent/evaluate.py deleted file mode 100644 index 93d18e1..0000000 --- a/components-Dinura/learnmate/chat_agent/evaluate.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Node 4 of 5: evaluate. - -Hands the reply to the judge (a separate, deliberately different model -- see -learnmate/evaluator) and records the verdict. - -Passing `contexts` through is what makes the grading strict: given contexts the judge -checks every claim against them and anything unsupported counts as a hallucination. -Given None it can only grade relevance, coherence and informativeness, which is the -right standard for a general-knowledge answer. - - reply (+ contexts) --> passed, verdict, critique, attempts -""" - -import time -from typing import Dict - -from ..evaluator.judge import get_judge -from ..storage import content_store -from .helpers import _log -from .state import ChatState - - -def evaluate_node(state: ChatState) -> Dict: - """Judge the reply, strictly when it was supposed to be grounded.""" - # Escape hatch for callers that want raw generation speed, or that are running - # without the judge model available. Auto-passing here means decide() routes - # straight to persist and the retry loop never runs. - if not state.get("evaluate", True): - return {"passed": True, "verdict": None, - "attempts": [{"attempt": state["attempt"], "reply": state.get("reply", ""), - "verdict": None}]} - - _log(state, "[*] Evaluating...") - started = time.time() - - verdict = get_judge().judge_chat_reply( - state["query"], # the original question, not the - state.get("reply", ""), # critique-padded one generate built - contexts=state.get("contexts") or None, # None here switches the judge's rubric - history=state.get("history"), - threshold=state["threshold"], - ) - - # Every judgement is logged, including the ones that pass, so score distributions - # and timings can be analysed later rather than only failures being visible. - content_store.log_evaluation( - "chat_msg", state["attempt"], verdict["score"], verdict["passed"], - state["threshold"], stage="judge", elapsed=time.time() - started, - doc_id=state.get("doc_id"), extra={"mode": state.get("mode")}) - - _log(state, f"[*] Score {verdict['score']}/100 -> " - f"{'PASS' if verdict['passed'] else 'REGENERATE'}") - - return { - "passed": verdict["passed"], - "verdict": verdict, - # Read by generate on the next pass through the loop. - "critique": verdict["regeneration_instruction"], - # Appended, not overwritten -- see the reducer on `attempts` in state.py. - "attempts": [{"attempt": state["attempt"], "reply": state.get("reply", ""), - "verdict": verdict}], - } diff --git a/components-Dinura/learnmate/chat_agent/generate.py b/components-Dinura/learnmate/chat_agent/generate.py deleted file mode 100644 index 02c644a..0000000 --- a/components-Dinura/learnmate/chat_agent/generate.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Node 3 of 5: generate. - -Writes the reply. This is the node the retry loop comes back to, so it has two jobs: -produce a first answer, and produce a corrected answer when the judge rejected the last -one. - -Which system prompt it uses is decided purely by whether `contexts` is non-empty, which -retrieve already settled. This node never re-decides the mode. - - contexts + query (+ critique) --> reply, attempt -""" - -from typing import Dict - -from langchain_core.messages import HumanMessage, SystemMessage - -from ..llm import get_generator_llm -from .helpers import _as_messages, _log -from .prompts import GENERAL_SYSTEM, GROUNDED_SYSTEM -from .state import ChatState - - -def generate_node(state: ChatState) -> Dict: - """Write the reply with whichever generator the mode calls for.""" - attempt = state.get("attempt", 0) + 1 - _log(state, f"[*] Generating (attempt {attempt}/{state['max_attempts']})...") - - contexts = state.get("contexts") or [] - query = state["query"] - - # --- Retry path ------------------------------------------------------------------ - # `critique` is only set once evaluate has rejected a reply, so this block is skipped - # on the first attempt. - if state.get("critique"): - # The judge's instruction rides inside the question so the generators stay - # single-purpose. The judge still grades against the original query. - query = ( - f"{query}\n\n" - "[REVISION REQUIRED] Your previous reply was rejected by an evaluator.\n" - f'Previous reply:\n"""\n{state.get("reply", "")}\n"""\n' - f"Required fix: {state['critique']}\n" - "Answer the original question again, corrected. Do not mention this " - "instruction or the fact that you are revising." - ) - - # --- Mode selection -------------------------------------------------------------- - if contexts: - # Page numbers are prefixed so the model can cite them and so the metadata - # survives into persist for the "which pages was this from" record. - context_text = "\n\n".join( - f"Page {doc.metadata.get('page_number', 'N/A')}: {doc.page_content}" - for doc in contexts) - system = GROUNDED_SYSTEM - user = f"Context:\n{context_text}\n\nQuestion: {query}" - else: - system = GENERAL_SYSTEM - user = query - - # History goes between the system prompt and the current question so the model reads - # the conversation in the order it happened. - messages = [SystemMessage(content=system), *_as_messages(state.get("history")), - HumanMessage(content=user)] - - try: - # Low but non-zero temperature: enough variation that a regeneration can differ - # from the reply the judge just rejected, not so much that it drifts. - reply = get_generator_llm().invoke(messages, temperature=0.3, max_tokens=512) - return {"attempt": attempt, "reply": (reply.content or "").strip()} - except Exception as exc: - # Return an empty reply rather than raising: the graph continues, the judge - # scores the emptiness badly, and the retry loop gets a chance to recover. - _log(state, f"[!] Generation failed: {exc}") - return {"attempt": attempt, "reply": ""} diff --git a/components-Dinura/learnmate/chat_agent/graph.py b/components-Dinura/learnmate/chat_agent/graph.py deleted file mode 100644 index 707a19c..0000000 --- a/components-Dinura/learnmate/chat_agent/graph.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Wiring: the nodes assembled into a LangGraph state machine. - -This file contains no logic of its own -- it only says how the pieces connect. The -behaviour lives in the node modules it imports. - - rewrite -> retrieve -> generate -> evaluate -> decide -+-> persist -> END - ^ | - +--------- regenerate -------+ - -Only one edge is conditional (evaluate -> decide). Everything else is a straight line, -which is deliberate: the sequencing is fixed and only the accept/retry choice depends on -what happened at runtime. -""" - -from langgraph.graph import END, StateGraph - -from .evaluate import evaluate_node -from .generate import generate_node -from .persist import persist_node -from .retrieve import retrieve_node -from .rewrite import rewrite_node -from .routing import decide -from .state import ChatState - - -def build_chat_graph(): - """Compile the one-turn chat graph.""" - # The state schema tells LangGraph which keys exist and which ones have reducers. - graph = StateGraph(ChatState) - - graph.add_node("rewrite", rewrite_node) - graph.add_node("retrieve", retrieve_node) - graph.add_node("generate", generate_node) - graph.add_node("evaluate", evaluate_node) - graph.add_node("persist", persist_node) - - graph.set_entry_point("rewrite") - graph.add_edge("rewrite", "retrieve") - graph.add_edge("retrieve", "generate") - graph.add_edge("generate", "evaluate") - - # `decide` returns "generate" or "persist"; this mapping turns those strings into - # the actual edges. The "generate" branch is what closes the retry loop. - graph.add_conditional_edges("evaluate", decide, - {"generate": "generate", "persist": "persist"}) - - graph.add_edge("persist", END) - return graph.compile() - - -# Compiling is cheap but not free, and the graph is stateless once built -- every turn -# passes its own state in -- so one process-wide instance is safe and saves the work on -# every message after the first. -_GRAPH = None - - -def get_chat_graph(): - """Process-wide compiled graph, built on first use.""" - global _GRAPH - if _GRAPH is None: - _GRAPH = build_chat_graph() - return _GRAPH diff --git a/components-Dinura/learnmate/chat_agent/helpers.py b/components-Dinura/learnmate/chat_agent/helpers.py deleted file mode 100644 index cdffcab..0000000 --- a/components-Dinura/learnmate/chat_agent/helpers.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Small shared utilities used by more than one node. - -Nothing here makes a decision -- these only format and print. -""" - -from typing import Dict, List - -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage - -from .state import ChatState - - -def _log(state: ChatState, message: str) -> None: - """ - Print progress, unless the caller asked for silence. - - Defaults to on: the CLI wants the running commentary, and a caller embedding the - agent in a server passes verbose=False once at construction. - """ - if state.get("verbose", True): - print(message) - - -def _as_messages(history: List[Dict[str, str]]) -> List[BaseMessage]: - """ - Convert stored history into LangChain message objects. - - History is persisted as plain dicts so it stays readable in MongoDB, but the chat - models want typed messages. Anything not explicitly tagged "assistant" is treated as - a user turn, so an unknown or missing role degrades to the safe interpretation - rather than raising mid-conversation. - """ - messages: List[BaseMessage] = [] - for turn in history or []: - if turn.get("role") == "assistant": - messages.append(AIMessage(content=turn.get("content", ""))) - else: - messages.append(HumanMessage(content=turn.get("content", ""))) - return messages diff --git a/components-Dinura/learnmate/chat_agent/persist.py b/components-Dinura/learnmate/chat_agent/persist.py deleted file mode 100644 index 92d5882..0000000 --- a/components-Dinura/learnmate/chat_agent/persist.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Node 5 of 5: persist. - -Records both halves of the turn in MongoDB. This is what makes the conversation outlive -the process: the next turn's `rewrite` node resolves pronouns against exactly what is -written here, so a session can be resumed later just by passing the same session_id. - - reply + verdict --> (two rows in Mongo) -""" - -from typing import Dict - -from ..storage import content_store -from .state import ChatState - - -def persist_node(state: ChatState) -> Dict: - """Record the turn so the next one has history to resolve against.""" - # Off for evaluation runs and tests, which need the answer without polluting a real - # conversation's history. - if not state.get("persist", True): - return {} - - session_id = state["session_id"] - - # The user turn is saved here rather than at the start of the graph so that a turn - # which crashes mid-way leaves no half-record behind. - content_store.save_turn(session_id, "user", state["query"], doc_id=state.get("doc_id")) - - # The assistant turn carries the audit trail alongside the text: how it was answered, - # what it scored, whether it was accepted, how many tries it took, and which pages - # of the PDF it drew on. - content_store.save_turn( - session_id, "assistant", state.get("reply", ""), doc_id=state.get("doc_id"), - meta={ - "mode": state.get("mode"), - "score": (state.get("verdict") or {}).get("score"), - "accepted": bool(state.get("passed")), - "attempts": len(state.get("attempts", [])), - "pages": [doc.metadata.get("page_number") for doc in state.get("contexts") or []], - }) - - # Nothing to merge back -- this node's effect is entirely in the database. - return {} diff --git a/components-Dinura/learnmate/chat_agent/prompts.py b/components-Dinura/learnmate/chat_agent/prompts.py deleted file mode 100644 index 88c013e..0000000 --- a/components-Dinura/learnmate/chat_agent/prompts.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -System prompts for the chat agent. - -Three prompts, one per job the agent does in a turn. They live together in one file so -the wording can be compared and tuned side by side -- these strings are the actual -behaviour of the agent far more than any of the surrounding Python is. -""" - -# Used in PDF mode, when retrieval found relevant chunks. The last sentence matters: -# without it the model narrates its own plumbing ("Based on the provided context...") -# which reads badly to a student who never saw a context block. -GROUNDED_SYSTEM = ( - "You are a precise study assistant. Answer the user's question strictly from the " - "provided context. If the context does not contain the answer, say so plainly instead " - "of guessing. Do not mention that you were given context." -) - -# Used in general mode, when nothing relevant was retrieved. The hedging instruction is -# the only defence against hallucination here, since there is no context to check against. -GENERAL_SYSTEM = ( - "You are a helpful, knowledgeable study assistant. Answer the user's question " - "accurately from your general knowledge. If you are not confident about a specific " - "fact, figure or date, say so rather than inventing one." -) - -# Used before retrieval to turn a follow-up into a standalone question. -# "Output only the rewritten question" is load-bearing: any preamble the model adds -# would be embedded along with the question and would blur the retrieval vector. -REWRITE_SYSTEM = ( - "Rewrite the follow-up question into a standalone question, resolving any pronouns or " - "references using the conversation history. Output only the rewritten question, " - "nothing else." -) diff --git a/components-Dinura/learnmate/chat_agent/retrieve.py b/components-Dinura/learnmate/chat_agent/retrieve.py deleted file mode 100644 index 7b56353..0000000 --- a/components-Dinura/learnmate/chat_agent/retrieve.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Node 2 of 5: retrieve. - -Searches the vector store and, from the result, decides which of the two modes this turn -runs in: - - pdf mode the top chunk scored at or above RELEVANCE_THRESHOLD. The reply will be - written from those chunks and judged strictly against them. - general mode nothing relevant came back. The reply will come from the model's own - knowledge and be judged only on relevance, coherence, informativeness. - -The mode is decided by the retrieval score, not by asking the model whether it thinks the -context is useful. A number that can be tuned and logged beats a second LLM call that can -be wrong -- and it costs nothing. - - standalone_query --> contexts, scores, mode, top_score -""" - -from typing import Dict - -from .. import config -from ..storage.vectors import get_vector_store -from .helpers import _log -from .state import ChatState - - -def retrieve_node(state: ChatState) -> Dict: - """Retrieve context and decide the mode from the top score.""" - # Fall back to the raw query if rewrite produced nothing -- see rewrite.py, which - # returns the original query on failure. - hits = get_vector_store().similarity_search_with_score( - state.get("standalone_query") or state["query"], - k=config.TOP_K, - doc_id=state.get("doc_id"), # None searches every ingested document - ) - - # Both vector backends return raw cosine similarity, so this threshold means the - # same thing whichever one is configured. Hits come back best-first. - top_score = hits[0][1] if hits else 0.0 - grounded = bool(hits) and top_score >= config.RELEVANCE_THRESHOLD - - if grounded: - _log(state, f"[*] PDF mode (top retrieval score {top_score:.4f})") - return {"contexts": [doc for doc, _ in hits], - "scores": [score for _, score in hits], - "mode": "pdf", "top_score": top_score} - - # Deliberately clear the contexts. Downstream nodes switch on "are there contexts", - # so leaving weak chunks in place would ground the answer on irrelevant text and - # then let the judge punish it for not matching. - _log(state, f"[*] General mode (top retrieval score {top_score:.4f})") - return {"contexts": [], "scores": [], "mode": "general", "top_score": top_score} diff --git a/components-Dinura/learnmate/chat_agent/rewrite.py b/components-Dinura/learnmate/chat_agent/rewrite.py deleted file mode 100644 index 2261645..0000000 --- a/components-Dinura/learnmate/chat_agent/rewrite.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Node 1 of 5: rewrite. - -Resolves a follow-up question into a standalone one, so that retrieval embeds something -meaningful. - -Why this runs *before* retrieval rather than after: "what about his powers?" embeds to -almost nothing useful, and retrieving on the pronoun returns noise. Resolved against the -history into "what are the President's powers?" it retrieves correctly. Doing it the -other way round means the retrieval has already failed by the time we fix the query. - - query + history --> standalone_query -""" - -from typing import Dict - -from langchain_core.messages import HumanMessage, SystemMessage - -from ..llm import get_generator_llm -from .helpers import _log -from .prompts import REWRITE_SYSTEM -from .state import ChatState - - -def rewrite_node(state: ChatState) -> Dict: - """Resolve follow-up references so retrieval sees a self-contained question.""" - query = state["query"] - history = state.get("history") or [] - - # First message of a conversation: there is nothing to resolve against, and calling - # the model anyway would just add latency to every new session. - if not history: - return {"standalone_query": query} - - history_text = "\n".join(f"{turn['role']}: {turn['content']}" for turn in history) - messages = [ - SystemMessage(content=REWRITE_SYSTEM), - HumanMessage(content=f"Conversation history:\n{history_text}\n\n" - f"Follow-up question: {query}"), - ] - - try: - # Temperature 0: this is a mechanical transformation, not a creative one. - # max_tokens is small because the output should be a single question -- capping - # it also stops a chatty model from appending an explanation. - reply = get_generator_llm().invoke(messages, temperature=0.0, max_tokens=100) - rewritten = (reply.content or "").strip() - - # Only log when it actually changed something, to keep the CLI output quiet - # for the common case of an already-standalone question. - if rewritten and rewritten != query: - _log(state, f"[*] Rewritten: {rewritten}") - - return {"standalone_query": rewritten or query} - except Exception: - # Rewriting is an optimisation; the raw query still retrieves something. - # A failure here must never cost the user their answer. - return {"standalone_query": query} diff --git a/components-Dinura/learnmate/chat_agent/routing.py b/components-Dinura/learnmate/chat_agent/routing.py deleted file mode 100644 index a2d2409..0000000 --- a/components-Dinura/learnmate/chat_agent/routing.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -The conditional edge out of evaluate -- the only branch in the graph. - -This is not a node: it returns the *name* of where to go next rather than a state -update. graph.py maps those names onto real nodes. - - passed? -> persist - out of attempts? -> persist anyway - otherwise -> back to generate, carrying the critique -""" - -from .helpers import _log -from .state import ChatState - - -def decide(state: ChatState) -> str: - """Choose between accepting the reply and regenerating it.""" - if state.get("passed"): - return "persist" - - # The budget is spent. Persist the failed reply anyway and let `accepted` tell the - # caller it was not reviewed clean -- a user mid-conversation needs an answer more - # than they need silence. - if state["attempt"] >= state["max_attempts"]: - return "persist" - - _log(state, f"[*] Feedback: {state.get('critique')}") - return "generate" diff --git a/components-Dinura/learnmate/chat_agent/state.py b/components-Dinura/learnmate/chat_agent/state.py deleted file mode 100644 index 68caf90..0000000 --- a/components-Dinura/learnmate/chat_agent/state.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -The state that flows between nodes across one turn. - -In LangGraph a node does not mutate state; it returns a dict of the keys it wants to -change, and the framework merges that into the running state. So every field below is -written by exactly one node and read by the ones downstream of it. - -By default a returned key *overwrites* whatever was there. A field can opt out of that -with `Annotated[type, reducer]`, which tells LangGraph to combine the old and new values -instead. `attempts` is the only field here that does -- see the note on it below. -""" - -from typing import Any, Dict, List, Optional - -from typing_extensions import Annotated, TypedDict - - -def _append(left: List, right: List) -> List: - """ - Reducer for `attempts`: concatenate instead of overwrite. - - Both sides are defaulted to `[]` because on the first node to write the key the - left side is still absent (None), not an empty list. - """ - return (left or []) + (right or []) - - -class ChatState(TypedDict, total=False): - """ - One turn's working memory. - - `total=False` means every key is optional, which is what lets nodes return partial - dicts and lets `ChatAgent.ask` seed only the fields it actually knows up front. - """ - - # --- Inputs: set by ChatAgent.ask before the graph runs, never changed after ------ - query: str # what the user actually typed, verbatim - session_id: str # conversation key; history is stored under it - doc_id: Any # restricts retrieval to one ingested PDF (None = all) - history: List[Dict[str, str]] # prior turns, as [{"role": ..., "content": ...}] - threshold: int # score out of 100 the judge must see to pass - max_attempts: int # total generations allowed, including the first - evaluate: bool # False skips the judge entirely (fast/offline mode) - verbose: bool # False silences the progress logging - persist: bool # False runs the turn without saving it to Mongo - - # --- Written by rewrite ---------------------------------------------------------- - standalone_query: str # the query with pronouns resolved; what we embed - - # --- Written by retrieve --------------------------------------------------------- - contexts: List[Any] # retrieved chunks as LangChain Documents; [] in general mode - scores: List[float] # cosine similarity per chunk, aligned with `contexts` - mode: str # "pdf" or "general" -- decided by score, not by the model - top_score: float # best similarity seen; the number the mode was decided on - - # --- Written by generate / evaluate (the retry loop) ------------------------------ - attempt: int # 1-based counter of generations so far - reply: str # the current candidate answer; overwritten on retry - critique: Optional[str] # judge's fix instruction, fed into the next generation - passed: bool # did the latest reply clear the threshold - verdict: Optional[Dict] # the judge's full structured result for the latest reply - - # The one accumulating field. `reply` keeps only the newest text, but `attempts` - # keeps every version and its verdict, so a caller can show the whole retry history. - attempts: Annotated[List[Dict], _append] diff --git a/components-Dinura/learnmate/config.py b/components-Dinura/learnmate/config.py deleted file mode 100644 index 3c7f2f3..0000000 --- a/components-Dinura/learnmate/config.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -Central configuration for every LearnMate component. - -Everything tunable lives here and is overridable through the environment, so switching -the generator model, pointing at a different MongoDB, or retuning a threshold never -means editing code. Values are read once at import. - -The two settings that matter most: - - LEARNMATE_GENERATOR_BACKEND "llamacpp" (in-process GGUF) or "http" (served model) - LEARNMATE_GENERATOR_MODEL path to the GGUF, or the model name for the HTTP backend - -Swapping Qwen2.5 for the finetuned model is a change to those two lines and nothing else. -""" - -import os -from pathlib import Path - -from dotenv import load_dotenv - -PACKAGE_DIR = Path(__file__).resolve().parent -ROOT_DIR = PACKAGE_DIR.parent - -# components-Dinura/.env holds HF_TOKEN and any override below. -load_dotenv(ROOT_DIR / ".env") - - -def _env(name: str, default: str) -> str: - """Read an env var, treating an empty string as unset.""" - return (os.getenv(name) or "").strip() or default - - -def _env_int(name: str, default: int) -> int: - try: - return int(_env(name, str(default))) - except ValueError: - return default - - -def _env_float(name: str, default: float) -> float: - try: - return float(_env(name, str(default))) - except ValueError: - return default - - -# --- Filesystem ---------------------------------------------------------------------- - -# Where the GGUF model files live, and where a missing one is downloaded to. -# (There is deliberately no DATA_DIR: PDFs are read from wherever the caller names and -# then stored in MongoDB, so the project never has a directory it owns for them.) -MODELS_DIR = Path(_env("LEARNMATE_MODELS_DIR", str(ROOT_DIR / "models"))) - -# --- Generator model ----------------------------------------------------------------- -# The model that writes chat replies and study resources. - -GENERATOR_BACKEND = _env("LEARNMATE_GENERATOR_BACKEND", "llamacpp").lower() -GENERATOR_MODEL = _env("LEARNMATE_GENERATOR_MODEL", - str(MODELS_DIR / "qwen2.5-3b-instruct-q4_k_m.gguf")) - -# Used only when the file above is missing and the backend is llamacpp. -GENERATOR_REPO = _env("LEARNMATE_GENERATOR_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF") -GENERATOR_FILE = _env("LEARNMATE_GENERATOR_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf") - -# A finetune with a non-standard prompt template needs its chat format named here -# (e.g. "chatml", "llama-3"). Empty lets llama.cpp read it from the GGUF metadata, -# which is correct for anything finetuned from a standard base. -GENERATOR_CHAT_FORMAT = _env("LEARNMATE_GENERATOR_CHAT_FORMAT", "") - -GENERATOR_N_CTX = _env_int("LEARNMATE_GENERATOR_N_CTX", 4096) - -# Base URL for the "http" backend, e.g. an OpenAI-compatible server in front of the -# finetuned model. Points at whatever local-model-api/ or finetuned-model-api/ serves. -GENERATOR_API_URL = _env("LEARNMATE_GENERATOR_API_URL", "http://localhost:8001/v1") -GENERATOR_API_KEY = _env("LEARNMATE_GENERATOR_API_KEY", "") - -# --- Judge model --------------------------------------------------------------------- -# Deliberately a different family from the generator: a judge sharing the generator's -# weights rates its own output style highly and the retry loop never fires. - -JUDGE_BACKEND = _env("LEARNMATE_JUDGE_BACKEND", "llamacpp").lower() -JUDGE_MODEL = _env("LEARNMATE_JUDGE_MODEL", - str(MODELS_DIR / "Llama-3.2-3B-Instruct-Q4_K_M.gguf")) -JUDGE_REPO = _env("LEARNMATE_JUDGE_REPO", "bartowski/Llama-3.2-3B-Instruct-GGUF") -JUDGE_FILE = _env("LEARNMATE_JUDGE_FILE", "Llama-3.2-3B-Instruct-Q4_K_M.gguf") -JUDGE_CHAT_FORMAT = _env("LEARNMATE_JUDGE_CHAT_FORMAT", "") - -# Judging is short-output / long-input: a resource plus its source text must fit. -JUDGE_N_CTX = _env_int("LEARNMATE_JUDGE_N_CTX", 8192) -JUDGE_API_URL = _env("LEARNMATE_JUDGE_API_URL", "http://localhost:8002/v1") -JUDGE_API_KEY = _env("LEARNMATE_JUDGE_API_KEY", "") - -# None lets llama.cpp auto-detect physical cores. -N_THREADS = int(_env("LEARNMATE_N_THREADS", "0")) or None -N_GPU_LAYERS = _env_int("LEARNMATE_N_GPU_LAYERS", 0) - -# --- Embeddings ---------------------------------------------------------------------- - -EMBEDDING_MODEL = _env("LEARNMATE_EMBEDDING_MODEL", "all-MiniLM-L6-v2") - -# --- MongoDB ------------------------------------------------------------------------- -# An external server, not an embedded file store. A plain mongodb:// URI works; an -# Atlas mongodb+srv:// URI additionally unlocks server-side $vectorSearch. -# -# 27018, not the conventional 27017: this project runs its own MongoDB container -# (`docker compose up -d mongo`) with its own named volume, because 27017 on this machine -# is answered by a native service shared with two other projects. Sharing a server means -# sharing a failure -- another project's `docker compose down -v` would take LearnMate's -# PDFs, sessions and generated resources with it. - -MONGODB_URI = _env("LEARNMATE_MONGODB_URI", "mongodb://localhost:27018") -MONGODB_DB = _env("LEARNMATE_MONGODB_DB", "learnmate") - -COLL_DOCUMENTS = "documents" -COLL_CHUNKS = "chunks" -COLL_PAGES = "pages" -COLL_RESOURCES = "resources" -COLL_EVALUATIONS = "evaluations" -COLL_CHAT_TURNS = "chat_turns" -# One record per session, holding the PDF that session is bound to. -COLL_SESSIONS = "sessions" -GRIDFS_BUCKET = "pdfs" - -# Name of the Atlas vector index over chunks.embedding. Only consulted by the "mongodb" -# vector backend, and ignored on a community server where it falls back to NumPy. -VECTOR_INDEX_NAME = _env("LEARNMATE_VECTOR_INDEX", "chunk_vector_index") - -# --- Vector database ----------------------------------------------------------------- -# Where the chunk embeddings live. MongoDB always holds the PDFs, page text, generated -# resources and history regardless of this setting; only the vectors move. -# -# qdrant a Qdrant server over HTTP -- a real HNSW index, filtering and scoring -# server-side. The default. -# mongodb vectors in the same MongoDB as everything else. Atlas $vectorSearch when -# available, otherwise exact NumPy scoring. Useful when you do not want a -# second service running. -VECTOR_BACKEND = _env("LEARNMATE_VECTOR_BACKEND", "qdrant").lower() - -# Server mode only. A URL, never a directory: `QdrantClient(path=...)` runs Qdrant inside -# the process and locks the directory, so only one process could use the corpus at a time. -# -# 6335 rather than Qdrant's conventional 6333, because this machine already runs a -# separate Qdrant on 6333 for another project. docker-compose.yml publishes the matching -# port; override this if you move it. -QDRANT_URL = _env("LEARNMATE_QDRANT_URL", "http://localhost:6335") -QDRANT_API_KEY = _env("LEARNMATE_QDRANT_API_KEY", "") -QDRANT_COLLECTION = _env("LEARNMATE_QDRANT_COLLECTION", "learnmate_chunks") -QDRANT_TIMEOUT = _env_int("LEARNMATE_QDRANT_TIMEOUT", 30) - -# Points per upsert/scroll request. Large enough to keep ingestion off the round-trip -# treadmill, small enough that one request stays well inside Qdrant's payload limit. -QDRANT_BATCH_SIZE = _env_int("LEARNMATE_QDRANT_BATCH_SIZE", 128) - -# --- Uploads --------------------------------------------------------------------------- -# One PDF per session. Embedding a document is the expensive part of this system -- a few -# thousand chunks through a CPU embedding model -- so a session is bound to the first PDF -# ingested into it and a second upload is refused rather than silently paying that cost -# again. A new PDF means a new session id. -# Set LEARNMATE_ONE_PDF_PER_SESSION=0 to lift the restriction. -ONE_PDF_PER_SESSION = _env("LEARNMATE_ONE_PDF_PER_SESSION", "1").lower() not in ( - "0", "false", "no", "off") - -# Largest PDF accepted, in MB. A 10 MB textbook is already a few thousand chunks and -# several minutes of embedding on CPU; past that the ingest looks hung rather than slow. -MAX_PDF_MB = _env_float("LEARNMATE_MAX_PDF_MB", 10.0) -MAX_PDF_BYTES = int(MAX_PDF_MB * 1_048_576) - -# --- Retrieval and chunking ---------------------------------------------------------- - -CHUNK_SIZE = _env_int("LEARNMATE_CHUNK_SIZE", 900) -CHUNK_OVERLAP = _env_int("LEARNMATE_CHUNK_OVERLAP", 150) - -# Shortest chunk worth embedding; below this it is a running head or a stray caption. -MIN_CHUNK_CHARS = _env_int("LEARNMATE_MIN_CHUNK_CHARS", 80) - -TOP_K = _env_int("LEARNMATE_TOP_K", 4) - -# Cosine similarity below which retrieved context is treated as irrelevant and the chat -# agent answers from general knowledge instead. -RELEVANCE_THRESHOLD = _env_float("LEARNMATE_RELEVANCE_THRESHOLD", 0.25) - -# --- Agent behaviour ----------------------------------------------------------------- - -EVALUATOR_THRESHOLD = _env_int("LEARNMATE_EVALUATOR_THRESHOLD", 70) - -# One generation plus at most one regeneration. Raising this is not just slower: a 3B -# judge tends to oscillate rather than converge over more rounds. -MAX_ATTEMPTS = _env_int("LEARNMATE_MAX_ATTEMPTS", 2) - -# Rolling chat history depth, in user+assistant pairs. -MAX_HISTORY_TURNS = _env_int("LEARNMATE_MAX_HISTORY_TURNS", 6) - -# How much document text a resource-generation run is allowed to use as its source. -# Must leave room in the context window for the prompt and the generated JSON. -MAX_SOURCE_CHARS = _env_int("LEARNMATE_MAX_SOURCE_CHARS", 6000) - -HF_TOKEN = _env("HF_TOKEN", "") or None diff --git a/components-Dinura/learnmate/evaluator/__init__.py b/components-Dinura/learnmate/evaluator/__init__.py deleted file mode 100644 index e24a733..0000000 --- a/components-Dinura/learnmate/evaluator/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -The evaluator: two gates over generated content. - - gate 1 structural validators plain Python, microseconds - gate 2 an LLM rubric grade ~25 seconds - -Cheapest first. Most bad generations fail mechanically -- three options instead of four, a -correct_answer in none of them, an empty summary -- so the judge is only ever spent on -content that is already well-formed. - -Two producers are graded, and they enter at different points: - - resource_agent a generated resource (mcq, summary, keypoints, practice_qsn) - -> gate 1 validate(), then gate 2 judge(), against the PDF passage - chat_agent one chat reply - -> gate 2 only, via judge_chat_reply(). Free prose has no mechanical - structure to check, so there is no "chat_msg" validator. - -Which rubric gate 2 applies is decided by the caller's *situation*, never by asking the -model: a chat reply is held to the retrieved chunks when retrieval found any, and only to -relevance and coherence when it did not. - -Everything fails closed. A judge that cannot be parsed or reached returns a failing -verdict, not an exception -- the caller is mid-loop and needs a decision, and silently -passing unreviewed content through is the one outcome worth ruling out. - -Where things live, in reading order: - - normalise.py norm() -- comparison-safe text, shared by every gate-1 rule - mcq_rules.py gate 1 for MCQs: per-question faults and set-wide biases - text_rules.py gate 1 for summary, keypoints and practice questions - validators.py gate 1 dispatcher: validate(task, content) -> (ok, reasons) - rubrics.py gate 2 criteria, one per task - prompt.py gate 2 system prompt and message assembly - verdict.py gate 2 output: the schema, parsing, and the fail-closed verdicts - judge.py gate 2 orchestration: Judge.judge / judge_chat_reply / get_judge -""" - -from . import mcq_rules, prompt, rubrics, text_rules, validators -from .judge import Judge, get_judge -from .validators import validate -from .verdict import VERDICT_SCHEMA, failed_verdict, parse_verdict, usable_instruction - -__all__ = [ - "Judge", - "VERDICT_SCHEMA", - "failed_verdict", - "get_judge", - "mcq_rules", - "parse_verdict", - "prompt", - "rubrics", - "text_rules", - "usable_instruction", - "validate", - "validators", -] diff --git a/components-Dinura/learnmate/evaluator/judge.py b/components-Dinura/learnmate/evaluator/judge.py deleted file mode 100644 index af136a3..0000000 --- a/components-Dinura/learnmate/evaluator/judge.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Gate 2: the LLM judge. - -Scores one piece of generated content from 1-100 against a rubric and says what to fix. -The verdict is grammar-constrained JSON, so the model cannot answer with prose about how -it would grade if it were grading. - -Two entry points, one per producer in this system: - - judge() a generated resource, graded against the PDF passage it came from - -- called by resource_agent/graph.py's check node - judge_chat_reply() one chat turn, graded against the retrieved chunks if there were - any -- called by chat_agent/evaluate.py - -The second is a thin wrapper over the first. What differs is only how the "source -material" is assembled and which rubric applies, and both of those decisions are made -here rather than by the caller. - -This file is orchestration only. The prompt is in prompt.py, the criteria in rubrics.py, -the parsing and the fail-closed verdicts in verdict.py. -""" - -from typing import Dict, List, Optional - -from .. import config -from ..llm import get_judge_llm -from . import prompt, rubrics -from .verdict import VERDICT_SCHEMA, failed_verdict, parse_verdict - - -class Judge: - """Grades generated content against a rubric.""" - - def __init__(self, llm=None, threshold: int = None): - self._llm = llm - self.threshold = threshold if threshold is not None else config.EVALUATOR_THRESHOLD - - @property - def llm(self): - # Lazy so constructing a Judge does not load 2 GB of weights; the resource graph - # builds one even when evaluation is switched off. - if self._llm is None: - self._llm = get_judge_llm() - return self._llm - - def judge(self, task: str, generated: str, source: str = None, criteria: str = None, - threshold: int = None, grounded: bool = False) -> Dict: - """ - Score one artefact and say how to regenerate it. - - task -- "mcq", "summary", "chat_msg", ...; selects the default rubric - generated -- the content being graded, as text - source -- the material it must be faithful to; optional - criteria -- overrides the default rubric for the task - grounded -- for chat, whether retrieved context applies (selects the strict rubric) - - Returns {task, score, passed, reasoning, regeneration_instruction, threshold}. - `regeneration_instruction` is emptied when the score passes, so callers can check - either `passed` or the instruction's truthiness. - """ - threshold = threshold if threshold is not None else self.threshold - criteria = criteria or rubrics.for_task(task, grounded=grounded) - - messages = prompt.build_messages(task, generated, criteria, source=source) - - try: - reply = self.llm.invoke(messages, response_schema=VERDICT_SCHEMA) - verdict = parse_verdict(reply.content) - except Exception as exc: - # Context overflow, OOM, a bad model file, an unreachable endpoint. Fail - # closed: a failing verdict, never an exception into the caller's loop. - return failed_verdict( - task, threshold, - f"Evaluator could not run: {type(exc).__name__}: {exc}", - "Regenerate the content; the evaluator failed to return a verdict.", - ) - - verdict["task"] = task - verdict["threshold"] = threshold - verdict["passed"] = verdict["score"] >= threshold - if verdict["passed"]: - verdict["regeneration_instruction"] = "" - return verdict - - # --- Chat-specific entry point --------------------------------------------------- - - def judge_chat_reply(self, query: str, reply: str, contexts: Optional[List] = None, - history: Optional[List[Dict]] = None, - threshold: int = None) -> Dict: - """ - Score one chat turn. - - Passing `contexts` is what selects the strict grounded rubric: the reply is then - held to the retrieved chunks and anything beyond them is a hallucination. Without - contexts the reply came from general knowledge and there is nothing to check it - against, so only relevance, coherence and informativeness are judged. - """ - threshold = threshold if threshold is not None else self.threshold - reply = (reply or "").strip() - - # Don't burn ~25s of CPU judging an empty reply; it is a failure by definition. - if not reply: - return failed_verdict( - "chat_msg", threshold, - "The chat agent returned an empty reply.", - "Regenerate the reply; the previous attempt produced no answer.", - ) - - source = prompt.build_chat_source(query, contexts=contexts, history=history) - return self.judge("chat_msg", reply, source=source, threshold=threshold, - grounded=bool(contexts)) - - -_JUDGE: Optional[Judge] = None - - -def get_judge() -> Judge: - """Process-wide judge.""" - global _JUDGE - if _JUDGE is None: - _JUDGE = Judge() - return _JUDGE diff --git a/components-Dinura/learnmate/evaluator/mcq_rules.py b/components-Dinura/learnmate/evaluator/mcq_rules.py deleted file mode 100644 index 8a1bf45..0000000 --- a/components-Dinura/learnmate/evaluator/mcq_rules.py +++ /dev/null @@ -1,115 +0,0 @@ -""" -Gate 1 for multiple-choice questions. - -By far the largest structural check, because an MCQ set can be broken in two different -ways: - - per question three options instead of four, a correct_answer matching none of them, - duplicate or blank options, an "all of the above" giveaway - per set biases only visible across questions -- the answer always in slot B, - the correct option always the longest, the same question asked twice - -The set-level rules are the interesting ones. Each question can be individually perfect -while the set as a whole is still guessable without reading the passage, and no -single-item check can see that. - -Every checker returns (ok, reasons). `reasons` is phrased as faults so it can be handed -straight to the generator as a regeneration instruction when the LLM judge never runs. -""" - -import re -from typing import List, Tuple - -from .normalise import norm - -# A distractor that gives the answer away by referring to the other options. -_META_OPTION = re.compile(r"\b(all|none|both)\s+of\s+the\s+(above|these|following)\b", - re.IGNORECASE) - -# Below this many questions, a "bias" is just a coincidence: with three questions the -# answer landing in the same slot each time happens by chance often enough to be useless -# as a signal. -_MIN_FOR_BIAS = 4 - - -def validate_mcq(item: dict) -> Tuple[bool, List[str]]: - """Check one multiple-choice question.""" - reasons = [] - - question = str(item.get("question", "")).strip() - options = item.get("options") or [] - correct = item.get("correct_answer", "") - - if not question: - reasons.append("a question has empty question text") - - if not isinstance(options, list) or len(options) != 4: - count = len(options) if isinstance(options, list) else 0 - reasons.append(f"question {question[:40]!r} has {count} options instead of exactly 4") - # Without four options the remaining checks would report noise. - return (not reasons), reasons - - normalised = [norm(option) for option in options] - - if any(not option for option in normalised): - reasons.append(f"question {question[:40]!r} has a blank option") - if len(set(normalised)) != len(normalised): - reasons.append(f"question {question[:40]!r} has duplicate options") - if norm(correct) not in normalised: - reasons.append(f"question {question[:40]!r} has a correct_answer that is not one " - "of its options") - if any(_META_OPTION.search(str(option)) for option in options): - reasons.append(f"question {question[:40]!r} uses an 'all/none of the above' option") - - return (not reasons), reasons - - -def _answer_positions(items) -> List[int]: - """Which slot the correct answer sits in, per question that has a findable answer.""" - positions = [] - for item in items: - options = [norm(option) for option in (item.get("options") or [])] - correct = norm(item.get("correct_answer", "")) - if correct in options: - positions.append(options.index(correct)) - return positions - - -def _correct_is_longest(items) -> List[bool]: - """Per question, whether the correct option is also the longest one.""" - flags = [] - for item in items: - options = [str(option) for option in (item.get("options") or [])] - correct = norm(item.get("correct_answer", "")) - if len(options) == 4 and correct: - flags.append(norm(max(options, key=len)) == correct) - return flags - - -def validate_mcq_set(items) -> Tuple[bool, List[str]]: - """Check a whole MCQ set, including biases only visible across questions.""" - if not items: - return False, ["no questions were generated"] - - reasons = [] - for item in items: - _, item_reasons = validate_mcq(item) - reasons.extend(item_reasons) - - # Position bias: a set where the answer is always in the same slot is guessable - # without reading the question at all. - positions = _answer_positions(items) - if len(positions) >= _MIN_FOR_BIAS and len(set(positions)) == 1: - reasons.append(f"the correct answer is in position {positions[0] + 1} for every question") - - # Length bias: if the correct option is always the longest, length alone gives it away. - longest = _correct_is_longest(items) - if len(longest) >= _MIN_FOR_BIAS and all(longest): - reasons.append("the correct answer is the longest option in every question") - - # The same fact asked twice with different wording still halves the value of the set. - stems = [norm(item.get("question", "")) for item in items] - if len(set(stems)) != len(stems): - reasons.append("two questions are identical") - - return (not reasons), reasons diff --git a/components-Dinura/learnmate/evaluator/normalise.py b/components-Dinura/learnmate/evaluator/normalise.py deleted file mode 100644 index de7b7a2..0000000 --- a/components-Dinura/learnmate/evaluator/normalise.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Comparison-safe text, shared by every structural check. - -Almost every rule in gate 1 is a comparison: is the correct answer one of the options, -are two key points duplicates, does an answer merely restate its question. A generator -that writes "The Cabinet." in one place and "the cabinet" in another means the same -thing, and a check that says otherwise reports a fault that is not there. -""" - -import re - - -def norm(text) -> str: - """ - Collapse whitespace, strip trailing punctuation, lowercase. - - Applied to both sides of every comparison, so the checks are about content rather - than about how the model happened to punctuate. - """ - return re.sub(r"\s+", " ", str(text or "")).strip().strip(".?!,;:").lower() diff --git a/components-Dinura/learnmate/evaluator/prompt.py b/components-Dinura/learnmate/evaluator/prompt.py deleted file mode 100644 index 5e90c50..0000000 --- a/components-Dinura/learnmate/evaluator/prompt.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -What the judge is actually shown. - -Two pieces: the system prompt that sets the grading behaviour, and the assembly of one -user message out of the rubric, the source material and the content being graded. - -The scoring bands and the "below 50" instruction matter more than they look. A 3B model -told "rate the quality" returns 75 for everything, and a grade that never varies cannot -gate anything -- so the prompt names what should push a score down, and the rubrics -(rubrics.py) name it again per task. -""" - -from typing import List, Optional - -from langchain_core.messages import HumanMessage, SystemMessage - -SYSTEM_PROMPT = """You are a strict evaluation model. You grade content produced by another AI. - -Rules: -- Score from 1 to 100. Be harsh: 90+ means flawless, 70-89 usable with minor faults, 50-69 clearly weak, below 50 unusable. -- Judge only against the criteria and the material given. Never use outside knowledge to fill gaps. -- If the material is factually unsupported by the source, the score must be below 50. -- "reasoning" is at most 3 short sentences naming the concrete faults you found. -- "regeneration_instruction" is a direct order to the generator model telling it exactly what to fix and keep. Write one complete sentence of at least ten words that names the specific fault and the specific correction. Begin with an imperative verb such as Remove, Rewrite, Add or Replace, but never emit a bare verb or trailing dots: "Rewrite..." is not an instruction and will be rejected. Never write the corrected content itself, and never state a fact that is not in the material you were given: you are instructing a rewrite, not performing it. Write it even when the score is high; it is discarded when the score passes the threshold. -- Reply with JSON only.""" - - -def build_messages(task: str, generated: str, criteria: str, - source: Optional[str] = None) -> List: - """ - Assemble the judge's prompt. - - Order matters: criteria first so the model knows what it is looking for, then the - source it must check against, then the content itself, then the output contract. A - model shown the content before the criteria starts forming an opinion first. - """ - parts = [f"Evaluation task: {task}", f"Criteria:\n{criteria}"] - - if source: - parts.append(f'SOURCE MATERIAL:\n"""\n{source}\n"""') - - parts.append(f'CONTENT TO EVALUATE:\n"""\n{generated}\n"""') - parts.append('Return JSON with exactly these keys: "score" (integer 1-100), ' - '"reasoning" (string), "regeneration_instruction" (string).') - - return [SystemMessage(content=SYSTEM_PROMPT), - HumanMessage(content="\n\n".join(parts))] - - -def build_chat_source(query: str, contexts: Optional[List] = None, - history: Optional[List[dict]] = None) -> str: - """ - Turn one chat turn into the "source material" block the judge checks against. - - A chat reply has no single source passage the way a generated resource does, so the - conversation, the retrieved chunks and the question are assembled into one. The - parenthetical on the context block is deliberate: it tells the judge the retrieved - text is a *ceiling*, not merely a hint. - """ - blocks = [] - - if history: - lines = [f"{turn.get('role', 'user').capitalize()}: {turn.get('content', '')}" - for turn in history] - blocks.append("CONVERSATION SO FAR:\n" + "\n".join(lines)) - - if contexts: - chunks = [f"Page {getattr(c, 'metadata', {}).get('page_number', 'N/A')}: " - f"{getattr(c, 'page_content', c)}" for c in contexts] - blocks.append("RETRIEVED CONTEXT (the reply may not go beyond this):\n" - + "\n\n".join(chunks)) - - blocks.append(f"USER MESSAGE THE REPLY MUST ANSWER:\n{query}") - return "\n\n".join(blocks) diff --git a/components-Dinura/learnmate/evaluator/rubrics.py b/components-Dinura/learnmate/evaluator/rubrics.py deleted file mode 100644 index a4d802b..0000000 --- a/components-Dinura/learnmate/evaluator/rubrics.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -The rubrics the judge grades against, one per kind of content. - -Each is written to be decidable from the material given, and to name what should push a -score below 50 -- a 3B judge handed "rate the quality" returns 75 for everything, and a -grade that never varies cannot gate anything. - -Each rubric deliberately leaves out what validators.py already checks mechanically. There -is no point spending 25 seconds of CPU asking a model to count to four. -""" - -MCQ = """Judge these multiple-choice questions against the source passage. -- Answerability: each question must be answerable from the passage alone. A question needing outside knowledge scores below 50. -- Correctness: the marked correct answer must be the right one according to the passage. Any question whose marked answer is wrong or unsupported puts the whole set below 50. -- Distractors: the three wrong options should be plausible to someone who has not read carefully, but clearly wrong to someone who has. Options that are obviously absurd, or that are also defensible as correct, lower the score. -- Coverage: the questions should test different parts of the passage rather than asking the same fact repeatedly. -- Wording: a question that gives away its own answer, or that quotes the passage so completely that no understanding is needed, lowers the score.""" - -PRACTICE_QSN = """Judge these short-answer practice questions against the source passage. -- Correctness: each answer must be right according to the passage. One wrong answer puts the set below 50. -- Answerability: each question must be answerable from the passage alone. A question requiring outside knowledge scores below 50. -- Completeness: an answer that is correct but omits a qualifying condition the passage states is only partly right and should score in the middle. -- Non-triviality: a question whose answer is a single word lifted verbatim from the passage tests recall, not understanding, and lowers the score. -- Coverage: the questions should test different parts of the passage rather than circling one fact.""" - -KEYPOINTS = """Judge these key points against the source passage. -- Groundedness: every point must be supported by the passage. One invented point puts the set below 50. -- Significance: the points should capture what the passage treats as important, not incidental asides. Missing the central point is a serious fault even when every point listed is true. -- Distinctness: two points that say the same thing in different words count as one. Near-duplicates lower the score. -- Self-containment: each point must make sense read on its own, without a dangling "this" or "it" referring to another point. -- Granularity: points should be comparable in scope. A set mixing one sweeping statement with four trivia items scores poorly.""" - -SUMMARY = """Judge this summary against the source passage. -- Faithfulness: every claim must be supported by the passage. A single invented fact, name, number or date puts the summary below 50. -- Coverage: the main points of the passage should be present. A summary that captures only the opening and drops what follows scores poorly even when nothing in it is false. -- Proportion: emphasis should match the passage. Dwelling on an incidental detail while omitting the central point lowers the score. -- Concision: no padding, no restating the same point in different words, no meta-commentary such as "this passage discusses". -- Standalone sense: it should be readable by someone who has not seen the passage, with no dangling references.""" - -# The chat agent answers one of two ways per turn and the two are held to different -# standards. Which rubric applies is decided by whether context was actually retrieved, -# not by anything the reply says about itself. - -CHAT_GROUNDED = """The reply answers the user's message using ONLY the retrieved context. -- Any claim not supported by the retrieved context is a hallucination: score below 50. -- The reply must actually answer the question, not merely restate the context. -- It must stay coherent with the conversation so far and resolve any follow-up reference. -- If the context genuinely does not contain the answer, saying so plainly is correct and scores well; inventing an answer does not.""" - -CHAT_GENERAL = """The reply answers the user's message from general knowledge; there is no source text to check it against. -- Relevance: it addresses what was actually asked, including any follow-up reference to earlier turns. -- Coherence: it does not contradict itself or the conversation so far. -- Informativeness: it is specific and useful, not padding, hedging, or a restatement of the question. -- Confident claims about niche facts, figures or dates that a small model is likely to get wrong should pull the score down.""" - -GENERIC = """Correctness, relevance to the given material, completeness, and clarity.""" - -RUBRICS = { - "mcq": MCQ, - "practice_qsn": PRACTICE_QSN, - "keypoints": KEYPOINTS, - "summary": SUMMARY, - "chat_msg": CHAT_GENERAL, - "generic": GENERIC, -} - - -def for_task(task: str, grounded: bool = False) -> str: - """The rubric for a task. `grounded` selects the strict chat rubric.""" - if task == "chat_msg": - return CHAT_GROUNDED if grounded else CHAT_GENERAL - return RUBRICS.get(task, GENERIC) diff --git a/components-Dinura/learnmate/evaluator/text_rules.py b/components-Dinura/learnmate/evaluator/text_rules.py deleted file mode 100644 index 8268d30..0000000 --- a/components-Dinura/learnmate/evaluator/text_rules.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Gate 1 for the three prose resource types. - -Short by design. There is little structure to check in a summary or a key point, so these -only catch the failures that need no judgement at all -- empty, too thin to be worth -reading, or duplicated. Everything else about them is a quality question, which is gate -2's job. - - practice_qsn question and answer both present, the answer not a restatement - keypoints enough points, none empty, none duplicated - summary present, and long enough to have covered anything - -Every checker returns (ok, reasons), phrased as faults so it can be handed straight to the -generator when the LLM judge never runs. -""" - -from typing import List, Tuple - -from .normalise import norm - - -def validate_practice_qsn(items) -> Tuple[bool, List[str]]: - """Check short-answer questions.""" - if not items: - return False, ["no questions were generated"] - - reasons = [] - for item in items: - question = str(item.get("question", "")).strip() - answer = str(item.get("answer", "")).strip() - - if not question: - reasons.append("a question has empty question text") - if not answer: - reasons.append(f"question {question[:40]!r} has an empty answer") - elif norm(answer) == norm(question): - # A model padding out a set will echo the question back as its own answer. - reasons.append(f"question {question[:40]!r} has an answer that just restates " - "the question") - return (not reasons), reasons - - -def validate_keypoints(items, min_points: int = 2) -> Tuple[bool, List[str]]: - """Check extracted key points.""" - if not items: - return False, ["no key points were generated"] - - reasons = [] - points = [str(point).strip() for point in items] - - if len(points) < min_points: - reasons.append(f"only {len(points)} key point(s) were produced, fewer than {min_points}") - if any(not point for point in points): - reasons.append("a key point is empty") - if len({norm(point) for point in points}) != len(points): - reasons.append("two key points are duplicates of each other") - return (not reasons), reasons - - -def validate_summary(text, min_chars: int = 80) -> Tuple[bool, List[str]]: - """ - Check a summary. - - Only length: a summary shorter than a sentence or two cannot have covered a page of - source, whatever it says. Whether it is *faithful* is a judgement, and belongs to the - rubric rather than here. - """ - text = str(text or "").strip() - reasons = [] - - if not text: - reasons.append("the summary is empty") - elif len(text) < min_chars: - reasons.append(f"the summary is only {len(text)} characters, too short to cover " - "the source") - return (not reasons), reasons diff --git a/components-Dinura/learnmate/evaluator/validators.py b/components-Dinura/learnmate/evaluator/validators.py deleted file mode 100644 index 5077445..0000000 --- a/components-Dinura/learnmate/evaluator/validators.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Gate 1: the structural dispatcher. - - validate(task, content) -> (ok, reasons) - -Most bad generations fail mechanically rather than qualitatively: three options instead of -four, a correct_answer that appears in no option, an empty summary. Those are decidable in -plain Python in microseconds, and catching them here means the ~25 second judge call is -only ever spent on content that is already well-formed. - -The rules themselves live next door -- mcq_rules.py and text_rules.py -- so this file is -only the map from a task name to its checker. -""" - -from typing import Callable, Dict, List, Tuple - -from .mcq_rules import validate_mcq, validate_mcq_set -from .text_rules import validate_keypoints, validate_practice_qsn, validate_summary - -# task name -> the checker to run before calling the judge. -VALIDATORS: Dict[str, Callable] = { - "mcq": validate_mcq_set, - "practice_qsn": validate_practice_qsn, - "keypoints": validate_keypoints, - "summary": validate_summary, -} - -__all__ = [ - "VALIDATORS", - "validate", - "validate_keypoints", - "validate_mcq", - "validate_mcq_set", - "validate_practice_qsn", - "validate_summary", -] - - -def validate(task: str, content) -> Tuple[bool, List[str]]: - """ - Run the structural checker for `task`. - - An unknown task passes through rather than failing: a new resource type should reach - the judge and be graded on its rubric, not be rejected for having no structural rules - written for it yet. Note there is no "chat_msg" entry -- a chat reply is free prose - with nothing mechanical to check, so it goes straight to gate 2. - """ - checker = VALIDATORS.get(task) - if checker is None: - return True, [] - return checker(content) diff --git a/components-Dinura/learnmate/evaluator/verdict.py b/components-Dinura/learnmate/evaluator/verdict.py deleted file mode 100644 index 8fb65be..0000000 --- a/components-Dinura/learnmate/evaluator/verdict.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -The shape of a verdict, and how a model reply becomes one. - - {task, score, passed, reasoning, regeneration_instruction, threshold} - -Everything here fails closed. A judge that cannot be parsed, cannot be reached, or answers -with something unusable produces a *failing* verdict rather than an exception -- the caller -is mid-loop and needs a decision, and silently passing unreviewed content through is the -one outcome worth ruling out. The reason always travels in `reasoning`, so a failure stays -visible instead of looking like a bad score. -""" - -import re -from typing import Any, Dict - -from ..llm import parse_json_reply - -# Constrained decoding: llama.cpp compiles this into a grammar, so the model cannot emit -# prose around the JSON or invent extra keys. -VERDICT_SCHEMA = { - "type": "object", - "properties": { - "score": {"type": "integer", "minimum": 1, "maximum": 100}, - "reasoning": {"type": "string"}, - "regeneration_instruction": {"type": "string"}, - }, - "required": ["score", "reasoning", "regeneration_instruction"], -} - -# A bare verb, optionally trailing dots: the judge echoing the prompt's example rather -# than writing an order. Anything this matches carries no information for the generator. -_BARE_VERB = re.compile(r"^(remove|rewrite|add|replace|revise|correct|fix)?[\s.…]*$", - re.IGNORECASE) - -# Shorter than this and an "instruction" cannot name both a fault and a correction. -_MIN_INSTRUCTION_CHARS = 20 - - -def usable_instruction(instruction: str, reasoning: str) -> str: - """ - Guarantee the retry loop has something to act on. - - A 3B judge intermittently answers with the example verb from the system prompt - ("Rewrite...") instead of an actual order. Regenerating on that is strictly worse than - not regenerating: the generator gets no signal and tends to collapse into a vaguer - answer. The reasoning field does describe the fault, so fall back to it. - """ - instruction = (instruction or "").strip() - if len(instruction) >= _MIN_INSTRUCTION_CHARS and not _BARE_VERB.match(instruction): - return instruction - - reasoning = (reasoning or "").strip() - if reasoning: - return f"Fix exactly this problem and change nothing else: {reasoning}" - return "Regenerate the content; the evaluator returned no usable instruction." - - -def failed_verdict(task: str, threshold: int, reasoning: str, instruction: str) -> Dict: - """A complete, failing verdict for when the judge could not produce one itself.""" - return { - "task": task, - "score": 1, - "passed": False, - "reasoning": reasoning, - "regeneration_instruction": instruction, - "threshold": threshold, - } - - -def parse_verdict(raw: Any) -> Dict: - """ - Turn the model's reply into a verdict dict, tolerating stray prose or code fences. - - Returns only {score, reasoning, regeneration_instruction}; the Judge adds task, - threshold and passed. Every failure path yields score 1 rather than raising. - """ - try: - data = parse_json_reply(raw) - except ValueError: - data = None - - if not isinstance(data, dict): - # Unparseable, or valid JSON of the wrong shape (a list, a bare string). - text = raw if isinstance(raw, str) else str(raw) - return { - "score": 1, - "reasoning": f"Could not parse evaluator output: {text.strip()[:200]}", - "regeneration_instruction": "Regenerate the content; the evaluator returned " - "an unreadable verdict.", - } - - try: - # float() first so "85" and 85.0 both survive; a model ignoring the integer - # constraint should not cost a verdict. - score = int(float(data.get("score", 1))) - except (TypeError, ValueError): - score = 1 - - reasoning = str(data.get("reasoning", "")).strip() - - return { - # Clamped: a schema-free backend can return 0 or 120, and a score outside the - # range would make every threshold comparison meaningless. - "score": max(1, min(100, score)), - "reasoning": reasoning, - "regeneration_instruction": usable_instruction( - str(data.get("regeneration_instruction", "")), reasoning), - } diff --git a/components-Dinura/learnmate/full_program.py b/components-Dinura/learnmate/full_program.py deleted file mode 100644 index 207b289..0000000 --- a/components-Dinura/learnmate/full_program.py +++ /dev/null @@ -1,739 +0,0 @@ -""" -LearnMate end to end, in one terminal program. - - python full_program.py interactive menu - python full_program.py --demo run the whole workflow start to finish - python full_program.py --pdf x.pdf --ask "..." upload and ask one question - python full_program.py --demo --no-eval same, but skip the judge (faster) - -Every package in learnmate/ is exercised, in the order the real system uses them: - - ingestion a PDF path -> stored PDF, cleaned pages, chunks, embeddings, and a - session bound to it - storage MongoDB holds the PDF, its page text and everything produced from it; - Qdrant holds the chunk vectors - llm Qwen2.5-3B generates, Llama-3.2-3B judges, MiniLM embeds - chat_agent a question -> rewrite -> retrieve -> generate -> evaluate -> reply - resource_agent + evaluator - a passage -> mcq / summary / keypoints / practice_qsn, each through a - structural gate and then the judge, with one retry - -Both databases run in containers: `docker compose up -d` before starting this. - -A note on speed. Everything runs locally on CPU, so a chat turn is roughly 20-60s and a -judged resource 60-120s. `--no-eval` skips the judge and roughly halves that, at the cost -of the quality gate. The first action of a session also loads ~4 GB of model weights. -""" - -import argparse -import os -import subprocess -import sys -import time -import uuid -from pathlib import Path - -# Works whether this is run from learnmate/ or from components-Dinura/: put the package's -# parent on the path so `import learnmate` resolves either way. -HERE = Path(__file__).resolve().parent -PROJECT = HERE.parent -sys.path.insert(0, str(PROJECT)) - - -def _use_project_venv() -> None: - """ - Re-run under the project's virtualenv if this interpreter cannot import the deps. - - `python full_program.py` on a machine where `python` is the system interpreter fails - deep inside an import chain with something like "No module named 'bson'", which says - nothing about the real problem. Checking here turns that into either a silent - hand-off to the right interpreter or one clear sentence. - - Guarded by an environment variable so a venv that is itself missing a dependency - reports that honestly instead of looping. - """ - if os.environ.get("LEARNMATE_VENV_REEXEC"): - return # already handed off once; let the real ImportError surface - - try: - # Import the package itself rather than a hand-picked list of third-party names. - # Guessing which dependency is missing gets it wrong -- an interpreter can have - # pymongo but not langchain_text_splitters -- and this exercises the exact chain - # that is about to run. It is free on success: Python caches the module. - import learnmate # noqa: F401 - return - except ImportError as exc: - missing = getattr(exc, "name", None) or "a dependency" - - candidates = [PROJECT / "venv" / "Scripts" / "python.exe", # Windows - PROJECT / "venv" / "bin" / "python"] # macOS / Linux - venv_python = next((p for p in candidates if p.exists()), None) - - if venv_python is None: - pip = "venv\\Scripts\\pip" if os.name == "nt" else "venv/bin/pip" - print( - f"This interpreter cannot import {missing}, and there is no virtualenv at " - f"{PROJECT / 'venv'}.\n\n" - "Create one and install the dependencies:\n" - f" cd {PROJECT}\n" - " python -m venv venv\n" - f" {pip} install -r requirements.txt\n", - file=sys.stderr, - ) - raise SystemExit(2) - - # flush: this process's stdout is buffered until it exits, which is *after* the child - # has finished, so without it the hand-off message prints last and reads as if the - # switch happened at the end. - print(f"[*] This interpreter cannot import {missing}; " - f"switching to {venv_python}", flush=True) - # subprocess rather than os.execv: it inherits stdin, so the interactive menu still - # works, and it avoids Windows' argument-quoting quirks in exec. - os.environ["LEARNMATE_VENV_REEXEC"] = "1" - completed = subprocess.run([str(venv_python), str(Path(__file__).resolve()), - *sys.argv[1:]]) - raise SystemExit(completed.returncode) - - -_use_project_venv() - -from learnmate import config -from learnmate.chat_agent import ChatAgent -from learnmate.ingestion import build_source_text, ingest_pdf -from learnmate.resource_agent import ( - DEFAULT_PER_PAGE, - MCQ_COUNT_CHOICES, - MCQ_DEFAULT_COUNT, - PER_PAGE_CHOICES, - generate_document_items, - generate_resource, - get_task, - render, - summarize_document, -) -from learnmate.storage import ( - QdrantUnavailable, - StorageUnavailable, - build_vector_store, - content_store, - list_resources, - pdf_store, -) - -def _find_sample_pdf(): - """ - A PDF to offer as the default, if one is lying around. - - Two fixed locations are tried because the sample PDFs have lived in both, then any - PDF under data/ -- so moving or renaming them does not silently break the default. - Returns None when there is nothing, and the menu then simply requires a path. - """ - for candidate in (PROJECT / "data" / "constitution.pdf", - PROJECT / "data" / "raw_pdfs" / "constitution.pdf"): - if candidate.exists(): - return candidate - return next((PROJECT / "data").glob("**/*.pdf"), None) - - -DEFAULT_PDF = _find_sample_pdf() -RULE = "=" * 74 -THIN = "-" * 74 - -# What each task is called on screen, and a sensible count for it. -TASK_LABELS = { - "mcq": ("Multiple-choice questions", 5), - "summary": ("Summary", 5), - "keypoints": ("Key points", 5), - "practice_qsn": ("Short-answer practice questions", 4), -} -TASK_ORDER = ("mcq", "summary", "keypoints", "practice_qsn") - - -def banner(title: str) -> None: - print(f"\n{RULE}\n{title}\n{RULE}") - - -# --- Step 0: is the system actually ready? --------------------------------------------- - -def preflight() -> bool: - """ - Check both databases and both model files before anything slow is attempted. - - Worth doing first: without it, a missing Qdrant surfaces several minutes into an - ingest, after the embedding model has already loaded. - """ - banner("Checking the system") - ready = True - - print("Models") - for label, backend, path in ( - ("generator", config.GENERATOR_BACKEND, config.GENERATOR_MODEL), - ("judge", config.JUDGE_BACKEND, config.JUDGE_MODEL), - ): - if backend == "http": - print(f" {label:10} served over HTTP -> {path}") - continue - exists = Path(path).exists() - size = f"{Path(path).stat().st_size / 1_073_741_824:.1f} GB" if exists else "MISSING" - print(f" {label:10} {'ok ' if exists else '!! '}{Path(path).name} ({size})") - if not exists: - print(" will be downloaded from Hugging Face on first use (~2 GB)") - - print("\nMongoDB (PDFs, page text, sessions, history, resources, evaluations)") - try: - from learnmate.storage.mongo import get_db - - db = get_db() - version = db.client.server_info()["version"] - print(f" ok {config.MONGODB_URI} (server {version}, db {config.MONGODB_DB})") - print(f" documents={db[config.COLL_DOCUMENTS].count_documents({})} " - f"pages={db[config.COLL_PAGES].count_documents({})} " - f"sessions={db[config.COLL_SESSIONS].count_documents({})} " - f"resources={db[config.COLL_RESOURCES].count_documents({})}") - except StorageUnavailable as exc: - ready = False - print(f" !! {exc}") - - print(f"\nVector database (backend: {config.VECTOR_BACKEND})") - try: - store = build_vector_store() - print(f" ok {store.describe_backend()}") - print(f" vectors={store.count()}") - except (QdrantUnavailable, StorageUnavailable) as exc: - ready = False - print(f" !! {exc}") - except Exception as exc: - ready = False - print(f" !! {type(exc).__name__}: {exc}") - - if not ready: - print(f"\n{THIN}\nStart the databases with: docker compose up -d\n{THIN}") - return ready - - -# --- Step 1: upload a PDF -------------------------------------------------------------- - -def upload(pdf_path: Path, session_id: str = None, force: bool = False) -> dict: - """ - Ingest one PDF and bind a session to it. - - A fresh session id per upload, because a session holds exactly one PDF -- reusing the - id for a different file is refused by design. Bound `for="both"` so the same upload - serves the chat agent and the resource generator without a second ingest. - """ - pdf_path = Path(pdf_path) - if not pdf_path.exists(): - raise FileNotFoundError(f"No such PDF: {pdf_path}") - - session_id = session_id or f"run-{uuid.uuid4().hex[:10]}" - banner(f"Uploading {pdf_path.name}") - print(f"path {pdf_path}") - print(f"size {pdf_path.stat().st_size / 1_048_576:.2f} MB " - f"(limit {config.MAX_PDF_MB:g} MB)") - print(f"session {session_id} (for chat and resource generation)\n") - - started = time.time() - report = ingest_pdf(pdf_path, session_id=session_id, session_for="both", force=force) - - print(f"\n{'Already ingested' if report['skipped'] else 'Ingested'} in " - f"{report['elapsed_s']}s (wall {time.time() - started:.1f}s)") - print(f" document id {report['doc_id']}") - print(f" pages {report['n_pages']}") - print(f" chunks {report['n_chunks']}") - return report - - -# --- Step 2: chat ---------------------------------------------------------------------- - -def show_chat_result(result: dict) -> None: - """Print one chat turn: the reply, where it came from, and how it was graded.""" - print(f"\n{THIN}") - print(result["reply"] or "(no answer produced)") - print(THIN) - - if result.get("standalone_query") and result["standalone_query"] != result["query"]: - print(f"rewritten as : {result['standalone_query']}") - - if result["mode"] == "pdf": - print(f"mode : PDF (top retrieval score {result['top_score']:.4f})") - for i, (doc, score) in enumerate(zip(result["contexts"], result["scores"]), 1): - preview = " ".join(doc.page_content.split())[:58] - print(f" [{i}] p.{str(doc.metadata.get('page_number')):<4} " - f"{score:.4f} {preview}...") - else: - print(f"mode : general knowledge " - f"(best score {result['top_score']:.4f} < {config.RELEVANCE_THRESHOLD})") - - verdict = result.get("verdict") - if verdict: - status_text = "accepted" if result["accepted"] else "BELOW THRESHOLD (shown anyway)" - print(f"evaluation : {verdict['score']}/100 " - f"(threshold {verdict['threshold']}) - {status_text}") - if len(result["attempts"]) > 1: - trail = " -> ".join(str(a["verdict"]["score"]) for a in result["attempts"] - if a.get("verdict")) - print(f" regenerated once, scores: {trail}") - print(f" {verdict['reasoning']}") - - -def ask_once(agent: ChatAgent, question: str) -> dict: - """One question through the chat graph, timed.""" - print(f"\nYou: {question}") - started = time.time() - result = agent.ask(question) - show_chat_result(result) - print(f"took : {time.time() - started:.1f}s") - return result - - -def chat_loop(agent: ChatAgent) -> None: - """Interactive chat until the user leaves.""" - banner(f"Chat | session {agent.session_id}") - print("Ask anything about the PDF. Commands: 'history', 'reset', 'back'.") - - while True: - try: - question = input("\nYou: ").strip() - except (EOFError, KeyboardInterrupt): - print() - return - - if not question: - continue - low = question.lower() - if low in ("back", "exit", "quit"): - return - if low == "reset": - print(f"Cleared {agent.reset()} turns.") - continue - if low == "history": - turns = agent.history() - if not turns: - print(" (nothing yet)") - for turn in turns: - print(f" {turn['role']:9} {turn['content'][:80]}") - continue - - started = time.time() - try: - show_chat_result(agent.ask(question)) - print(f"took : {time.time() - started:.1f}s") - except Exception as exc: - print(f" [!] {type(exc).__name__}: {exc}") - - -# --- Step 3: generate resources -------------------------------------------------------- - -def make_resource(task: str, doc_id, topic: str = None, count: int = None, - per_page: int = None, evaluate: bool = True, - quiet: bool = False) -> dict: - """ - Generate one resource and print it with its verdict. - - The passage comes from ingestion.build_source_text: with a topic it is the pages that - best match it, otherwise the opening of the document up to the budget. - - A summary of the whole document is the exception. There "the opening pages" is the - wrong answer to "summarise this PDF", so it goes through summarize_document instead, - which reads every page. A summary *of a topic* still takes the normal path -- the - pages matching the topic are the passage, and the rest of the document is not wanted. - """ - label, default_count = TASK_LABELS[task] - # A topic means "just this part of the document", so it keeps the passage path for - # every task. Without one, the whole PDF is read instead -- but only for the two - # question types when a rate was actually asked for, so `all_resources` does not - # silently turn into hundreds of generations. - by_rate = per_page is not None and task in ("keypoints", "practice_qsn") - whole_document = not topic and (task in ("summary", "mcq") or by_rate) - - # No default count for the whole-document summary: it sizes itself to how many pages - # it actually read, and TASK_LABELS' 5 would cap a whole book at five sentences. - if not whole_document or task == "mcq": - count = count or default_count - - banner(f"{label} ({task})") - - started = time.time() - if by_rate and whole_document: - print("source : every page of the document, in groups") - print(f"asking : {per_page} {get_task(task).count_label} per page" - f" evaluation: {'on' if evaluate else 'off'} (per group)\n") - result = generate_document_items(task, doc_id, per_page=per_page, - evaluate=evaluate, verbose=not quiet) - elif whole_document and task == "mcq": - print("source : every page of the document, in groups") - print(f"asking : {count} questions across the whole document" - f" evaluation: {'on' if evaluate else 'off'} (per group)\n") - result = generate_document_items("mcq", doc_id, count=count, evaluate=evaluate, - verbose=not quiet) - elif whole_document: - print("source : every page of the document, summarised then combined") - print(f"asking : {count or 'as many sentences as the document needs'}" - f" evaluation: {'on' if evaluate else 'off'} (on the combined summary)\n") - result = summarize_document(doc_id, count=count, evaluate=evaluate, - verbose=not quiet) - else: - source = build_source_text(doc_id, topic=topic) - print(f"source : {len(source)} chars from the document" - + (f", matching {topic!r}" if topic else " (opening pages)")) - print(f"asking : {count} {'sentences' if task == 'summary' else 'items'}" - f" evaluation: {'on' if evaluate else 'off'}\n") - result = generate_resource(task, source, count=count, doc_id=doc_id, - evaluate=evaluate, verbose=not quiet) - - print(f"\n{THIN}") - print(render(task, result["content"]) or "(nothing generated)") - print(THIN) - - verdict = result.get("verdict") - if result.get("groups"): - # A pooled MCQ set has no single verdict -- each group was judged on its own -- so - # the per-group scores are reported instead of one number that never existed. - scores = [str(a["score"]) for a in result["attempts"] if a.get("score") is not None] - got, asked = len(result["content"] or []), result["requested"] - rate = f" ({result['per_page']} per page)" if result.get("per_page") else "" - print(f"{get_task(task).count_label:9}: {got} of the {asked} asked for{rate}, " - f"from {result['groups']} group(s)" - + ("" if got >= asked else " (the document supported no more)")) - print(f"score : {'per group: ' + ', '.join(scores) if scores else 'not evaluated'}" - f" - {'accepted' if result['accepted'] else 'REJECTED (shown anyway)'}") - elif verdict: - status_text = "accepted" if result["accepted"] else "BELOW THRESHOLD (shown anyway)" - print(f"score : {verdict['score']}/100 " - f"(threshold {verdict['threshold']}) - {status_text}") - print(f"reasoning: {verdict['reasoning']}") - elif result["attempts"]: - # No verdict means a cheaper gate decided it, or no gate ran at all. - last = result["attempts"][-1] - if last.get("stage") in ("validator", "parse"): - print(f"rejected by the {last['stage']} gate: " - f"{'; '.join(last.get('reasons', []))}") - elif last.get("stage") == "skipped": - # `accepted` is True here only because nothing graded it. Saying so matters: - # unevaluated output looks identical to output that passed. - print("score : not evaluated (--no-eval). Nothing checked this content " - "for faithfulness to the passage.") - - print(f"attempts : {len(result['attempts'])} " - f"stored as: {result['resource_id']} took {time.time() - started:.1f}s") - return result - - -def all_resources(doc_id, topic: str = None, evaluate: bool = True, - quiet: bool = False) -> dict: - """Generate all four kinds and summarise how each fared.""" - results = {} - for task in TASK_ORDER: - try: - results[task] = make_resource(task, doc_id, topic=topic, evaluate=evaluate, - quiet=quiet) - except Exception as exc: - print(f" [!] {task} failed: {type(exc).__name__}: {exc}") - results[task] = None - - banner("Resource summary") - print(f"{'task':16} {'verdict':>14} {'score':>6} {'attempts':>9} stored") - print(THIN) - for task, result in results.items(): - if result is None: - print(f"{task:16} {'ERROR':>14}") - continue - verdict = result.get("verdict") or {} - score = verdict.get("score") - # Without a verdict there was no judge, so `accepted` only means "nothing - # objected". Reporting that as "accepted" would overstate it badly. - label = ("accepted" if result["accepted"] else "below threshold") if verdict \ - else "not evaluated" - print(f"{task:16} {label:>14} " - f"{str(score if score is not None else '-'):>6} " - f"{len(result['attempts']):>9} {result['resource_id']}") - - if not evaluate: - print("\nEvaluation was off, so none of this was checked for faithfulness to the") - print("passage. Re-run without --no-eval (or press 'e' in the menu) to grade it.") - return results - - -# --- Status ---------------------------------------------------------------------------- - -def show_status(doc_id=None, session_id: str = None) -> None: - """What is currently stored, across both databases.""" - banner("Stored state") - - documents = pdf_store.list_documents(limit=10) - print(f"Documents ({len(documents)})") - for document in documents: - marker = " <-- this run" if doc_id and document["_id"] == doc_id else "" - print(f" {str(document['_id'])} {str(document.get('n_pages') or '-'):>4}p " - f"{str(document.get('n_chunks') or '-'):>5}c {document['filename']}{marker}") - - if session_id: - bound = content_store.get_session(session_id) - print(f"\nSession {session_id}") - if bound: - print(f" PDF {bound['filename']}") - print(f" for {', '.join(bound.get('kinds') or [])}") - print(f" turns {len(content_store.load_history(session_id, max_turns=999))}") - else: - print(" (no PDF bound yet)") - - if doc_id: - records = list_resources(doc_id=doc_id, limit=10) - print(f"\nGenerated resources for this document ({len(records)})") - for record in records: - print(f" {record['task']:14} score={str(record.get('score') or '-'):>4} " - f"{'PASS' if record['accepted'] else 'fail'} {record['_id']}") - - try: - store = build_vector_store() - print(f"\nVectors: {store.count()} total" - + (f", {store.count(doc_id)} for this document" if doc_id else "")) - except Exception as exc: - print(f"\nVectors: unavailable ({type(exc).__name__})") - - -# --- Interactive menu ------------------------------------------------------------------ - -class Session: - """Everything the menu needs to remember between choices.""" - - def __init__(self): - self.doc_id = None - self.session_id = None - self.filename = None - self.evaluate = True - self.topic = None - - @property - def ready(self) -> bool: - return self.doc_id is not None - - def agent(self) -> ChatAgent: - # `evaluate` has to be passed through, or the menu's toggle would silently apply - # to resource generation only and chat would always pay for the judge. - return ChatAgent(session_id=self.session_id, doc_id=self.doc_id, - evaluate=self.evaluate) - - -def prompt_pdf(state: Session) -> None: - """Ask for a PDF path and ingest it.""" - default = DEFAULT_PDF if DEFAULT_PDF.exists() else None - hint = f" [{default}]" if default else "" - raw = input(f"\nPDF file path{hint}: ").strip().strip('"') - path = Path(raw) if raw else default - - if path is None: - print(" A path is required.") - return - - try: - report = upload(path) - except (FileNotFoundError, ValueError) as exc: - print(f" [!] {exc}") - return - - state.doc_id = report["document"]["_id"] - state.session_id = report["session_id"] - state.filename = report["document"]["filename"] - - -def prompt_count(question: str, choices, default: int) -> int: - """ - Ask how much of something to generate. - - The offered numbers are a menu, not a limit -- any number is accepted and blank takes - the default. An unreadable answer returns the default rather than re-asking: this sits - in front of a generation that takes minutes, and a typo should not cost the run. - """ - offered = "/".join(str(choice) for choice in choices) - raw = input(f"\n{question} [{offered}, default {default}]: ").strip() - if not raw: - return default - try: - chosen = int(raw) - except ValueError: - print(f" Not a number; using {default}.") - return default - if chosen < 1: - print(f" Need at least one; using {default}.") - return default - return chosen - - -# What each whole-document task asks the user for, and how that number is then read. -# "total" is a size for the set; "per_page" is a rate, and the set is however many pages -# the document turns out to have. -WHOLE_DOCUMENT_PROMPTS = { - "mcq": ("How many questions for the whole document?", - MCQ_COUNT_CHOICES, MCQ_DEFAULT_COUNT, "total"), - "keypoints": ("How many key points per page?", - PER_PAGE_CHOICES, DEFAULT_PER_PAGE, "per_page"), - "practice_qsn": ("How many short-answer questions per page?", - PER_PAGE_CHOICES, DEFAULT_PER_PAGE, "per_page"), -} - - -def menu(state: Session) -> None: - """The main loop.""" - while True: - banner("LearnMate") - if state.ready: - print(f"PDF : {state.filename}") - print(f"session : {state.session_id}") - print(f"topic : {state.topic or '(whole document)'}") - else: - print("No PDF uploaded yet.") - print(f"evaluation : {'on' if state.evaluate else 'off'}") - - print(f"\n{THIN}") - print(" 1 Upload a PDF") - print(" 2 Chat about it") - print(" 3 Generate MCQs") - print(" 4 Generate a summary") - print(" 5 Generate key points") - print(" 6 Generate practice questions") - print(" 7 Generate all four") - print(" 8 Show stored state") - print(" 9 Set a topic for generation") - print(" e Toggle evaluation (the judge and its retry)") - print(" 0 Quit") - print(THIN) - - try: - choice = input("Choose: ").strip().lower() - except (EOFError, KeyboardInterrupt): - print("\nBye.") - return - - if choice == "0": - print("Bye.") - return - if choice == "1": - prompt_pdf(state) - continue - if choice == "e": - state.evaluate = not state.evaluate - continue - if choice == "9": - raw = input("Topic (blank for the whole document): ").strip() - state.topic = raw or None - continue - - # Everything below needs a document. - if not state.ready and choice in ("2", "3", "4", "5", "6", "7", "8"): - print("\n Upload a PDF first (option 1).") - continue - - try: - if choice == "2": - chat_loop(state.agent()) - elif choice in ("3", "4", "5", "6"): - task = {"3": "mcq", "4": "summary", "5": "keypoints", - "6": "practice_qsn"}[choice] - count = per_page = None - # Only asked for a whole-document run. With a topic set the passage is a - # few pages, and one call's worth is all it can support. - if not state.topic and task in WHOLE_DOCUMENT_PROMPTS: - question, offered, default, mode = WHOLE_DOCUMENT_PROMPTS[task] - chosen = prompt_count(question, offered, default) - if mode == "per_page": - per_page = chosen - else: - count = chosen - make_resource(task, state.doc_id, topic=state.topic, count=count, - per_page=per_page, evaluate=state.evaluate) - elif choice == "7": - all_resources(state.doc_id, topic=state.topic, evaluate=state.evaluate) - elif choice == "8": - show_status(state.doc_id, state.session_id) - else: - print(" Unknown choice.") - except KeyboardInterrupt: - print("\n Interrupted.") - except Exception as exc: - print(f"\n [!] {type(exc).__name__}: {exc}") - - -# --- Non-interactive modes ------------------------------------------------------------- - -def run_demo(pdf: Path, questions, topic: str, evaluate: bool, quiet: bool) -> int: - """The whole workflow start to finish, with no prompts. Used to smoke-test a change.""" - report = upload(pdf) - doc_id = report["document"]["_id"] - session_id = report["session_id"] - - banner(f"Chat | session {session_id}") - agent = ChatAgent(session_id=session_id, doc_id=doc_id, evaluate=evaluate, - verbose=not quiet) - for question in questions: - ask_once(agent, question) - - results = all_resources(doc_id, topic=topic, evaluate=evaluate, quiet=quiet) - show_status(doc_id, session_id) - - banner("Done") - failed = [t for t, r in results.items() if r is None] - if failed: - print(f"These resources errored: {', '.join(failed)}") - return 1 - print("Every stage ran. Anything marked BELOW THRESHOLD was produced but not accepted") - print("by the evaluator, which is a quality result rather than a failure.") - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser( - description="LearnMate end to end: upload a PDF, chat about it, generate resources.", - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--pdf", type=Path, default=DEFAULT_PDF, - help=("PDF to ingest" - + (f" (default: {DEFAULT_PDF.name})" if DEFAULT_PDF - else "; required, none found under data/"))) - parser.add_argument("--ask", action="append", default=[], - help="ask this question; repeatable") - parser.add_argument("--topic", help="focus resource generation on this topic") - parser.add_argument("--demo", action="store_true", - help="run the whole workflow without prompting") - parser.add_argument("--no-eval", action="store_true", - help="skip the judge and its retry (much faster)") - parser.add_argument("--quiet", action="store_true", help="hide per-node progress") - parser.add_argument("--skip-checks", action="store_true", - help="do not verify the databases and models first") - args = parser.parse_args() - - if not args.skip_checks and not preflight(): - return 2 - - try: - # Non-interactive whenever the caller gave something to do. - if args.demo or args.ask: - if args.pdf is None: - raise SystemExit("No PDF found under data/. Pass one with --pdf .") - questions = args.ask or [ - "What is this document about?", - "What does it say about fundamental rights?", - ] - return run_demo(args.pdf, questions, args.topic, - evaluate=not args.no_eval, quiet=args.quiet) - - state = Session() - state.evaluate = not args.no_eval - state.topic = args.topic - menu(state) - return 0 - except (StorageUnavailable, QdrantUnavailable) as exc: - print(f"\n[!] {exc}", file=sys.stderr) - return 2 - except (FileNotFoundError, ValueError) as exc: - # A missing PDF, one over the size limit, or a session that already holds a - # different PDF. All are the user's to fix, so say so plainly rather than - # printing a traceback at them. - print(f"\n[!] {exc}", file=sys.stderr) - return 1 - except KeyboardInterrupt: - print("\nInterrupted.", file=sys.stderr) - return 130 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/components-Dinura/learnmate/ingestion/__init__.py b/components-Dinura/learnmate/ingestion/__init__.py deleted file mode 100644 index 23ea135..0000000 --- a/components-Dinura/learnmate/ingestion/__init__.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -PDF ingestion: extract, clean, split, embed, store -- for one session. - - read + validate -> store (GridFS) -> extract + clean -> split -> embed - | - bind the session to the document <- - -An upload always belongs to a session, and a session is opened for one purpose: - - chat the PDF will be asked questions about, through the chat agent - resource MCQs, summaries and key points will be generated from it - both one PDF serving the whole workflow - -Both purposes need identical ingestion -- the same chunks for retrieval and the same -stored page text -- so the kind is a statement of intent, not an optimisation. It is -checked when a command runs, so `generate` against a session opened for chat says so -plainly instead of quietly doing something the user did not set up. Opening a second -session on an already-ingested PDF is nearly free: nothing is re-embedded. - -Where things live, in reading order: - - clean.py page extraction and cleaning (running heads, ligatures, hyphenation) - chunking.py cleaned pages -> the overlapping chunks that get embedded - sessions.py session kinds, one-PDF-per-session, and the binding - pipeline.py ingest_pdf() -- the order all of the above happens in - source_text.py build_source_text() -- what a resource session reads back out - -The split between chunking.py and source_text.py is the one worth understanding: chunks -are sized and overlapped for *retrieval*, whole pages are kept for *reading*, and the two -are not interchangeable. Joining chunks back together repeats text at every boundary and -starts mid-sentence. -""" - -from .chunking import build_splitter, pages_to_documents -from .clean import clean_text, extract_pages, is_substantive, preprocess -from .pipeline import ingest_pdf -from .sessions import ( - SESSION_KINDS, - describe_kinds, - kinds_for, - normalise_kinds, - require_kind, -) -from .source_text import build_source_text - -__all__ = [ - "SESSION_KINDS", - "build_source_text", - "build_splitter", - "clean_text", - "describe_kinds", - "extract_pages", - "ingest_pdf", - "is_substantive", - "kinds_for", - "normalise_kinds", - "pages_to_documents", - "preprocess", - "require_kind", -] diff --git a/components-Dinura/learnmate/ingestion/chunking.py b/components-Dinura/learnmate/ingestion/chunking.py deleted file mode 100644 index 8099116..0000000 --- a/components-Dinura/learnmate/ingestion/chunking.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -Turning cleaned pages into the chunks that get embedded. - -Chunking is sub-page, deliberately. One vector per page averages every provision on that -page together, so the specific article a question asks about is diluted by everything -printed beside it -- and a keyword-dense contents page out-scores the article text, -because a list of chapter titles matches almost any topical query. - - cleaned pages -> split -> filter -> Documents carrying their page number -""" - -from typing import Dict, List - -from langchain_core.documents import Document -from langchain_text_splitters import RecursiveCharacterTextSplitter - -from .. import config -from .clean import is_substantive, looks_like_contents - -# Ordered so a split falls at the largest boundary that fits: sentence ends first, then -# the numbered-clause markers that structure legal and academic prose -- "(2)", "(iii)" -- -# then clause punctuation, and only then whitespace. -_SEPARATORS = [ - ". ", # sentence end - "; ", # clause end, very common in legal prose - ": ", - "? ", - "! ", - ") ", # tail of a numbered clause marker - ", ", - " ", - "", -] - - -def build_splitter(chunk_size: int = None, chunk_overlap: int = None - ) -> RecursiveCharacterTextSplitter: - """ - The text splitter used for every document. - - The overlap keeps a provision that straddles a chunk boundary retrievable from either - side, which matters most for exactly the long sentences legal text is made of. - """ - return RecursiveCharacterTextSplitter( - chunk_size=chunk_size or config.CHUNK_SIZE, - chunk_overlap=chunk_overlap or config.CHUNK_OVERLAP, - separators=_SEPARATORS, - keep_separator=True, - length_function=len, - ) - - -def pages_to_documents(pages: List[Dict], doc_id, filename: str, - splitter: RecursiveCharacterTextSplitter = None) -> List[Document]: - """ - Split cleaned pages into LangChain Documents carrying their provenance. - - `chunk_index` is per page and, with doc_id and page_number, gives every chunk a stable - identity -- that triple is the unique key the vector store upserts on, so re-ingesting - a document overwrites its chunks instead of duplicating them. - - The page number riding in the metadata is what lets the chat agent cite "p.52" and - what `build_source_text` uses to read whole pages back. - """ - splitter = splitter or build_splitter() - - documents = [] - for page in pages: - content = page.get("page_content", "") - if not content: - continue - - index = 0 - # Checked here rather than per chunk: the splitter treats ". " as a boundary, so - # by the time a contents page has been split its dot leaders have been broken - # apart and the chunks no longer look like what they are. The page still does. - if looks_like_contents(content, min_leaders=4): - continue - - for piece in splitter.split_text(content): - piece = piece.strip() - # Two cheap filters, in order of cost: too short to carry meaning (a running - # head, a stray caption), then contents-page shapes the page check missed. - if not is_substantive(piece, config.MIN_CHUNK_CHARS): - continue - if looks_like_contents(piece): - continue - - documents.append(Document( - page_content=piece, - metadata={ - "doc_id": doc_id, - "filename": filename, - "page_number": page["page_number"], - "chunk_index": index, - "source": filename, - }, - )) - index += 1 - return documents diff --git a/components-Dinura/learnmate/ingestion/clean.py b/components-Dinura/learnmate/ingestion/clean.py deleted file mode 100644 index 90adfd1..0000000 --- a/components-Dinura/learnmate/ingestion/clean.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Page text extraction and cleaning. - -PDF text arrives with the artefacts of a printed page in it: running heads on every page, -standalone page numbers, curly quotes, bullet glyphs, and words hyphenated across a line -break. All of that ends up in the embedding if it is left alone, and a running head that -repeats on 200 pages is a term the retriever will happily match on. - -Newlines are flattened deliberately. Layout in a PDF reflects the column, not the -sentence, so a line break inside a paragraph is noise; sentence boundaries are recovered -downstream by the splitter. -""" - -import re -import unicodedata -from typing import Dict, List, Union - -# Typographic characters that should not become their own embedding tokens. -_REPLACEMENTS = { - "“": '"', "”": '"', # curly double quotes - "‘": "'", "’": "'", # curly single quotes - "–": "-", "—": "-", # en/em dash - "•": "-", "●": "-", # bullets - "✓": "-", "✔": "-", # check marks used as bullets - " ": " ", # non-breaking space - "fi": "fi", "fl": "fl", # ligatures PyMuPDF leaves intact -} - - -def extract_pages(pdf_bytes: bytes) -> List[Dict]: - """ - Pull text out of a PDF, one entry per page. - - Takes bytes rather than a path so it can read straight from GridFS without staging - the file back onto disk. - """ - import fitz # PyMuPDF - - pages = [] - document = fitz.open(stream=pdf_bytes, filetype="pdf") - try: - for index in range(len(document)): - pages.append({ - "page_index": index, - "page_number": index + 1, - "text": document.load_page(index).get_text("text"), - }) - finally: - document.close() - return pages - - -def clean_text(text: str) -> str: - """Normalise one page of extracted text.""" - text = unicodedata.normalize("NFKC", text or "") - - for old, new in _REPLACEMENTS.items(): - text = text.replace(old, new) - - # Standalone page numbers on their own line. - text = re.sub(r"^\s*\d{1,4}\s*$", "", text, flags=re.MULTILINE) - - # Rejoin words split across a line break: "off-\nshore" -> "offshore". - text = re.sub(r"([A-Za-z])-\s*\n\s*([a-z])", r"\1\2", text) - text = re.sub(r"([a-zA-Z]+)\s*\n\s*-\s*([a-zA-Z]+)", r"\1\2", text) - - # Leading bullet dashes, which carry no meaning once the layout is gone. - text = re.sub(r"(?:^|\n)[ \t]*[-*][ \t]+", " ", text) - - # Flatten remaining line breaks, then collapse the whitespace that leaves behind. - text = text.replace("\n", " ") - text = re.sub(r"\s+", " ", text) - return text.strip() - - -def drop_repeated_lines(pages: List[Dict], min_ratio: float = 0.5) -> List[Dict]: - """ - Remove running heads and footers. - - A line appearing on more than `min_ratio` of the pages is furniture, not content. - Detecting it by frequency generalises across documents, where the previous - hard-coded patterns only matched the two PDFs they were written for. - - Only applied to documents long enough for the ratio to mean something: on a 3-page - handout a genuine heading can easily appear on two pages. - """ - if len(pages) < 6: - return pages - - counts: Dict[str, int] = {} - for page in pages: - for line in {ln.strip() for ln in (page["text"] or "").splitlines() if ln.strip()}: - # Long lines are prose that happens to repeat, not a running head. - if len(line) <= 90: - counts[line] = counts.get(line, 0) + 1 - - threshold = max(3, int(len(pages) * min_ratio)) - furniture = {line for line, count in counts.items() if count >= threshold} - if not furniture: - return pages - - for page in pages: - kept = [ln for ln in (page["text"] or "").splitlines() if ln.strip() not in furniture] - page["text"] = "\n".join(kept) - return pages - - -def preprocess(pdf_bytes: bytes) -> List[Dict]: - """ - Full page pipeline: extract, strip furniture, clean. - - Returns [{page_index, page_number, page_content}, ...] with empty pages dropped. - """ - pages = drop_repeated_lines(extract_pages(pdf_bytes)) - - cleaned = [] - for page in pages: - content = clean_text(page["text"]) - if content: - cleaned.append({ - "page_index": page["page_index"], - "page_number": page["page_number"], - "page_content": content, - }) - return cleaned - - -def is_substantive(text: str, min_chars: int) -> bool: - """ - Reject fragments that are too thin to be worth a vector. - - Measured on alphanumeric characters only, so a line of dot leaders from a contents - page or a row of section numbers does not pass on its punctuation. - """ - return len(re.sub(r"[^A-Za-z0-9]", "", text or "")) >= min_chars - - -# Dot leaders: the run of periods joining an entry to its page number in a table of -# contents or an index. Three or more in one chunk is the signature of a contents page, -# and a single heading followed by an ellipsis never reaches it. -_DOT_LEADER = re.compile(r"(?:\.\s*){3,}") - - -def looks_like_contents(text: str, min_leaders: int = 3) -> bool: - """ - Whether a chunk is table-of-contents or index furniture rather than prose. - - These chunks are actively harmful in both directions. In retrieval a contents page is - a list of every chapter title in the document, so it matches almost any topical query - and out-scores the article the question is actually about. In generation it is worse: - asked for questions about "fundamental rights", the model is handed a page of headings - and page numbers and writes questions about the numbering. - - Detected by dot leaders rather than by page position, because a contents page is not - always at the front -- the index at the back has the same shape. - """ - return len(_DOT_LEADER.findall(text or "")) >= min_leaders diff --git a/components-Dinura/learnmate/ingestion/pipeline.py b/components-Dinura/learnmate/ingestion/pipeline.py deleted file mode 100644 index fc5b433..0000000 --- a/components-Dinura/learnmate/ingestion/pipeline.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -Ingesting one PDF, end to end. - - read + validate -> store (GridFS) -> extract + clean -> split -> embed - | - bind the session to the document <- - -The order is the point of this file. Everything after "store" is the expensive part -- -extraction, splitting, and a few thousand embeddings on CPU -- so every rule that could -refuse the upload is checked before any of it starts: - - empty file pdf_store.read_source - over MAX_PDF_MB pdf_store.read_source - session already used sessions.check_free - -and the session binding is written at the *end*, once the document has proved usable. - -The work itself lives in the modules this one calls: clean.py extracts, chunking.py -splits, the vector store embeds. `ingest_pdf` only decides what happens in what order. -""" - -import hashlib -import time -from pathlib import Path -from typing import Dict, Iterable, Union - -from ..storage import pdf_store -from ..storage.vectors import get_vector_store -from . import sessions -from .chunking import pages_to_documents -from .clean import preprocess - - -def ingest_pdf(source: Union[str, Path, bytes], filename: str = None, - session_id: str = None, session_for: Union[str, Iterable[str]] = "chat", - force: bool = False, verbose: bool = True) -> Dict: - """ - Store and index one PDF, for one session. - - `source` is a path or raw bytes, so this serves a CLI argument and an HTTP upload - equally. Returns a report with the document record and what was indexed. - - `session_id` binds the document to a session, and `session_for` says what that session - is for -- "chat", "resource", or "both". A session holds one PDF: a second, different - PDF ingested into the same session is refused before any work is done. - - A PDF already ingested is skipped unless `force` is set. Identity is the hash of the - file's bytes, so re-uploading the same document under a new name -- or opening a - second session on it for the other purpose -- costs nothing rather than re-embedding - a few thousand chunks. - """ - def log(message): - if verbose: - print(message) - - started = time.time() - kinds = sessions.normalise_kinds(session_for) - - # --- Validate, before anything is written ---------------------------------------- - # read_source raises on an empty file and on one over the size limit; check_free - # raises when this session already has a different PDF. All of it while the session's - # existing document is still untouched. - data, filename = pdf_store.read_source(source, filename) - digest = hashlib.sha256(data).hexdigest() - sessions.check_free(session_id, digest, filename) - - document = pdf_store.store_pdf(data, filename=filename) - doc_id = document["_id"] - - store = get_vector_store() - already_indexed = store.count(doc_id) - - # --- Already done? Bind and stop -------------------------------------------------- - # This is the path a second session on the same PDF takes -- and why opening a - # resource session for a PDF already ingested for chat is nearly free. - if document.get("existing") and already_indexed and not force: - log(f"[=] Already ingested: {document['filename']} " - f"({already_indexed} chunks). Use --force to re-index.") - sessions.bind(session_id, doc_id, document["filename"], digest, kinds, log) - return _report(document, doc_id, session_id, kinds, skipped=True, - n_pages=document.get("n_pages"), n_chunks=already_indexed, - started=started) - - log(f"[*] Stored {document['filename']} " - f"({document['size_bytes'] / 1_048_576:.1f} MB) as {doc_id}") - - # --- Extract ---------------------------------------------------------------------- - pdf_bytes = pdf_store.get_pdf_bytes(doc_id) - pages = preprocess(pdf_bytes) - if not pages: - raise ValueError( - f"No extractable text in {document['filename']}. If it is a scanned PDF it " - "needs OCR before it can be indexed." - ) - log(f"[*] Extracted {len(pages)} pages with text") - - # --- Split ------------------------------------------------------------------------ - documents = pages_to_documents(pages, doc_id, document["filename"]) - if not documents: - raise ValueError(f"No substantive chunks produced from {document['filename']}.") - log(f"[*] {len(pages)} pages -> {len(documents)} chunks") - - if force and already_indexed: - # Clear both collections before rewriting. Upserting alone would leave behind any - # page or chunk the new run no longer produces -- which is exactly what happens - # when a filter is tightened and pages start being dropped. - store.delete(doc_id=doc_id) - pdf_store.delete_pages(doc_id) - - # --- Store the readable text ------------------------------------------------------ - # Chunks are shaped for retrieval; resource generation wants the page as it reads. - # This is what makes a "resource" session possible at all -- see source_text.py. - indexed_pages = {doc.metadata["page_number"] for doc in documents} - pdf_store.store_pages(doc_id, [p for p in pages if p["page_number"] in indexed_pages]) - - # --- Embed ------------------------------------------------------------------------ - log(f"[*] Embedding {len(documents)} chunks into {store.describe_backend()}...") - store.add_documents(documents) - pdf_store.mark_ingested(doc_id, len(pages), len(documents)) - - # --- Bind ------------------------------------------------------------------------- - # Only now that the document is genuinely usable. Binding earlier would tie the - # session to a PDF that turned out to have no extractable text, and the user could - # not then ingest a working one without abandoning the session. - sessions.bind(session_id, doc_id, document["filename"], digest, kinds, log) - - log(f"[+] Ingested {document['filename']} in {round(time.time() - started, 2)}s") - return _report(pdf_store.get_document(doc_id), doc_id, session_id, kinds, - skipped=False, n_pages=len(pages), n_chunks=len(documents), - started=started) - - -def _report(document, doc_id, session_id, kinds, skipped, n_pages, n_chunks, - started) -> Dict: - """The shape every caller gets back, from both exits above.""" - return { - "document": document, - "doc_id": str(doc_id), - "session_id": session_id, - "session_for": list(kinds), - "skipped": skipped, - "n_pages": n_pages, - "n_chunks": n_chunks, - "elapsed_s": round(time.time() - started, 2), - } diff --git a/components-Dinura/learnmate/ingestion/sessions.py b/components-Dinura/learnmate/ingestion/sessions.py deleted file mode 100644 index bcf2d2e..0000000 --- a/components-Dinura/learnmate/ingestion/sessions.py +++ /dev/null @@ -1,161 +0,0 @@ -""" -What a session is: one PDF, opened for one purpose. - -A session records two things at ingest time -- which PDF it is about, and what it is for: - - chat the PDF will be asked questions about, through the chat agent - resource the PDF will have MCQs, summaries and key points generated from it - -Both purposes need exactly the same ingestion work (the same chunks, the same vectors, -the same stored page text), so the kind is not an optimisation. It is a statement of -intent, checked when a command runs so that `generate` against a session opened for chat -tells the user plainly rather than quietly doing something they did not set up. - -Opening a second session for a PDF that is already ingested costs nothing -- the chunks -exist, so `ingest_pdf` takes its "already ingested" path and only writes the new binding. -That is why refusing a mismatch here is cheap advice rather than an obstacle. - -Two rules live in this file: - - check_free() one PDF per session -- a different PDF into a used session is refused - require_kind() a session opened for one purpose is not silently used for the other -""" - -from typing import Iterable, Optional, Sequence, Tuple, Union - -from .. import config -from ..storage import content_store - -# The purposes a session can be opened for. -SESSION_KINDS: Tuple[str, ...] = ("chat", "resource") - -# What a user may type. "both" is there for one PDF that serves the whole workflow -- -# the plural and verb forms because "resources" and "generate" are what people reach for. -_ALIASES = { - "chat": ("chat",), - "resource": ("resource",), - "resources": ("resource",), - "resource_gen": ("resource",), - "generate": ("resource",), - "both": SESSION_KINDS, - "all": SESSION_KINDS, -} - -# How each kind reads in a sentence, for error messages. -_LABELS = {"chat": "chat", "resource": "resource generation"} - - -def normalise_kinds(value: Union[str, Iterable[str], None]) -> Tuple[str, ...]: - """ - Turn what a caller passed into a tuple of valid kinds. - - Accepts a single name, one of the aliases above, or an iterable of names, so an HTTP - handler passing `["chat", "resource"]` and a CLI passing `"both"` both work. - """ - if value is None: - return ("chat",) - - names: Sequence[str] = [value] if isinstance(value, str) else list(value) - - kinds = [] - for name in names: - resolved = _ALIASES.get(str(name).strip().lower()) - if resolved is None: - raise ValueError( - f"Unknown session kind {name!r}; expected one of " - f"{', '.join(sorted(_ALIASES))}." - ) - for kind in resolved: - if kind not in kinds: - kinds.append(kind) - - # Ordered by SESSION_KINDS rather than by what the user typed, so the stored value is - # the same however it was requested and two bindings compare equal. - return tuple(kind for kind in SESSION_KINDS if kind in kinds) - - -def describe_kinds(kinds: Iterable[str]) -> str: - """Render kinds for a message: 'chat', 'resource generation', or both.""" - labels = [_LABELS.get(kind, kind) for kind in kinds] - if len(labels) <= 1: - return labels[0] if labels else "nothing" - return " and ".join(labels) - - -def kinds_for(session_id: str) -> Tuple[str, ...]: - """ - What a session was opened for, or () if it has no PDF bound. - - A binding written before this file existed has no `kinds` field. Those are treated as - permitting everything rather than nothing: an old session is a session that worked, - and breaking it to enforce a rule added later would be the wrong trade. - """ - bound = content_store.get_session(session_id) - if not bound: - return () - return tuple(bound.get("kinds") or SESSION_KINDS) - - -def check_free(session_id: str, digest: str, filename: str) -> None: - """ - Refuse a second, different PDF for a session that already has one. - - Embedding a document is the expensive step -- a few thousand chunks through a CPU - embedding model -- so this runs before anything is stored or embedded, rather than - letting the work start and cleaning up afterwards. - - Re-ingesting the *same* file is allowed: the bound hash is compared, not merely the - presence of a binding, so `--force` can re-index a session's own PDF. - """ - if not session_id or not config.ONE_PDF_PER_SESSION: - return - - bound = content_store.get_session(session_id) - if not bound or bound.get("sha256") == digest: - return - - raise ValueError( - f"Session {session_id!r} is already about {bound['filename']}. One PDF per " - f"session -- ingest {filename} under a new session id instead:\n" - f" ingest_pdf({filename!r}, session_id='')" - ) - - -def require_kind(session_id: str, kind: str) -> None: - """ - Check a session was opened for the purpose it is now being used for. - - A no-op when there is no session or nothing bound to it -- commands run without a - session fall back to the most recent document, and there is no stated intent to - contradict. - """ - if not session_id or kind not in SESSION_KINDS: - return - - bound = content_store.get_session(session_id) - if not bound: - return - - allowed = kinds_for(session_id) - if kind in allowed: - return - - filename = bound.get("filename", "") - raise ValueError( - f"Session {session_id!r} was opened for {describe_kinds(allowed)}, not " - f"{_LABELS[kind]}.\n" - f"Open one for {_LABELS[kind]} on the same PDF -- already ingested, so nothing " - f"is re-embedded:\n" - f" ingest_pdf({filename!r}, session_id='', " - f"session_for={kind!r})" - ) - - -def bind(session_id: str, doc_id, filename: str, digest: str, - kinds: Tuple[str, ...], log=None) -> None: - """Record which PDF a session is about and what it is for. No-op without a session.""" - if not session_id: - return - content_store.bind_session_document(session_id, doc_id, filename, digest, kinds) - if log: - log(f"[*] Session {session_id} -> {filename} (for {describe_kinds(kinds)})") diff --git a/components-Dinura/learnmate/ingestion/source_text.py b/components-Dinura/learnmate/ingestion/source_text.py deleted file mode 100644 index 9c9667c..0000000 --- a/components-Dinura/learnmate/ingestion/source_text.py +++ /dev/null @@ -1,118 +0,0 @@ -""" -Assembling the passage a resource is generated from. - -This is what a "resource" session reads. Where the chat agent retrieves a handful of -chunks per question, resource generation needs one continuous passage to write five MCQs -or a summary from -- and a whole PDF does not fit in a 4k context window, so the -interesting question is which part of it to use. - - topic given -> the pages whose text best matches it. This is what makes - "5 MCQs about directors' duties" work on a 200-page book. - pages given -> exactly those pages - neither -> the opening of the document, up to the budget - -The unit is always a whole page, never the retrieved chunks themselves. Chunks are sized -and overlapped for retrieval: joining them repeats ~150 characters at every boundary and -starts the passage mid-sentence, and a generator handed that writes questions about the -fragments. Retrieval picks *which* pages; the stored page text supplies the prose. -""" - -from typing import List, Optional - -from .. import config -from ..storage import pdf_store -from ..storage.vectors import get_vector_store - - -def _rank_pages_by_topic(doc_id, topic: str) -> List[int]: - """Page numbers ordered by how well their best chunk matches the topic.""" - hits = get_vector_store().similarity_search_with_score(topic, k=12, doc_id=doc_id) - - ranked, seen = [], set() - for document, _ in hits: - number = document.metadata.get("page_number") - # De-duplicated because several chunks of one page can all rank highly, and the - # page is read whole either way. - if number is not None and number not in seen: - seen.add(number) - ranked.append(number) - return ranked - - -def _load_pages(doc_id, selected: Optional[List[int]]) -> List[dict]: - """Stored page text in reading order, falling back to chunks for an older corpus.""" - records = pdf_store.get_pages(doc_id, selected) - if records: - return records - - # Nothing stored by store_pages: either the document predates it or only chunks - # exist. Fall back to the chunks so an older corpus still generates. - documents = get_vector_store().chunks_for(doc_id, pages=selected) - return [{"page_number": d.metadata.get("page_number"), "text": d.page_content} - for d in documents] - - -def _keep_most_relevant(records: List[dict], order: List[int], max_chars: int) -> List[dict]: - """ - Trim to the budget by relevance, then restore reading order. - - Both halves matter: keeping the most relevant pages is what makes a topic query - useful, and re-sorting them by page number is what stops the passage reading as a - sequence of non-sequiturs. - """ - rank = {number: position for position, number in enumerate(order)} - records = sorted(records, key=lambda r: rank.get(r["page_number"], len(rank))) - - kept, total = [], 0 - for record in records: - # `total and` lets the single most relevant page through even if it alone exceeds - # the budget -- it gets truncated below rather than dropped, which beats - # returning nothing. - if total and total + len(record["text"]) > max_chars: - continue - kept.append(record) - total += len(record["text"]) + 2 - - return sorted(kept, key=lambda r: r["page_number"]) - - -def build_source_text(doc_id, topic: str = None, pages: Optional[List[int]] = None, - max_chars: int = None) -> str: - """ - The passage to generate from, as one string of whole pages in reading order. - - Raises ValueError when the document has no indexed text, which the caller surfaces as - "ingest the PDF first" rather than generating from nothing. - """ - max_chars = max_chars or config.MAX_SOURCE_CHARS - - selected = _rank_pages_by_topic(doc_id, topic) if topic else pages - records = _load_pages(doc_id, selected) - - if not records: - raise ValueError( - f"No indexed text for document {doc_id}" - + (f" matching {topic!r}" if topic else "") - + ". Ingest the PDF first." - ) - - if topic: - records = _keep_most_relevant(records, selected, max_chars) - - # Final pass: concatenate up to the budget. Pages arrive in reading order, so this - # only ever truncates the tail. - parts, total = [], 0 - for record in records: - text = (record.get("text") or "").strip() - if not text: - continue - if total + len(text) > max_chars: - remaining = max_chars - total - # Only take a partial page if enough of it fits to be worth reading. - if remaining > 300: - parts.append(text[:remaining]) - break - parts.append(text) - total += len(text) + 2 - - return "\n\n".join(parts).strip() diff --git a/components-Dinura/learnmate/llm/__init__.py b/components-Dinura/learnmate/llm/__init__.py deleted file mode 100644 index 8a4b12e..0000000 --- a/components-Dinura/learnmate/llm/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Model access for LearnMate. - -Three models, reached through three functions: - - get_generator_llm() Qwen2.5-3B-Instruct writes chat replies and study resources - get_judge_llm() Llama-3.2-3B-Instruct scores what the generator wrote - get_embeddings() all-MiniLM-L6-v2 embeds chunks at ingest, queries at retrieval - -Those are the only entry points the agents use, so which model runs and how it is reached -stays a configuration question. All three are cached, so importing them from several -modules still loads each model once per process. - -The two chat models share every line of code between them -- there is no Qwen class and no -Llama class. Which family loads is the GGUF path in config, nothing more. The embedding -model needs its own file because it implements a different LangChain interface -(`Embeddings`, not `BaseChatModel`). - -Where things live, in reading order: - - registry.py get_generator_llm / get_judge_llm -- the entry points, and the - per-role config that decides what they build - llamacpp.py backend 1 a local GGUF, in this process, with JSON grammars - http_api.py backend 2 a served OpenAI-compatible endpoint - messages.py LangChain messages <-> the role/content dicts both backends want - runtime.py the GGUF weight cache, and releasing it cleanly at exit - download.py fetching a missing GGUF from Hugging Face on first use - json_output.py parse_json_reply -- recovering JSON from an unconstrained reply - embeddings.py LearnMateEmbeddings, the retrieval vectors - -Caching happens at three separate layers, which is worth knowing before changing any of -them: `registry._LLM_CACHE` holds wrapper objects keyed by role and sampling settings, -`runtime._LLAMA_CACHE` holds the actual weights keyed by file, and -`embeddings._MODEL_CACHE` holds the sentence-transformers model. Two wrappers can share -one set of weights; that is the point of the split. - -Heavy imports are all function-local -- llama_cpp, sentence_transformers, -huggingface_hub, requests -- so configuring the HTTP backend means llama.cpp never has to -be installed at all. -""" - -from .download import ensure_gguf -from .embeddings import LearnMateEmbeddings, get_embeddings -from .http_api import HttpChatModel -from .json_output import parse_json_reply -from .llamacpp import LlamaCppChatModel -from .registry import get_generator_llm, get_judge_llm - -__all__ = [ - "HttpChatModel", - "LearnMateEmbeddings", - "LlamaCppChatModel", - "ensure_gguf", - "get_embeddings", - "get_generator_llm", - "get_judge_llm", - "parse_json_reply", -] diff --git a/components-Dinura/learnmate/llm/download.py b/components-Dinura/learnmate/llm/download.py deleted file mode 100644 index a011369..0000000 --- a/components-Dinura/learnmate/llm/download.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Getting a GGUF file onto disk. - -Only the llama.cpp backend needs this, and only the first time: the two models this -project uses are ~2 GB each, too large to keep in git, so a missing one is fetched from -Hugging Face rather than being an error the user has to go and fix by hand. -""" - -from pathlib import Path - -from .. import config - - -def ensure_gguf(path: str, repo_id: str, filename: str) -> str: - """ - Return a local path to a GGUF file, downloading it on first use. - - A finetuned model supplied as a plain file is used as-is; the download only happens - when the configured path does not exist and a repo is known. That ordering is what - lets someone drop their own GGUF at the configured path and have it picked up without - touching any config. - """ - target = Path(path) - if target.exists(): - return str(target) - - from huggingface_hub import hf_hub_download # lazy: not needed once models are local - - models_dir = target.parent if target.parent.name else config.MODELS_DIR - models_dir.mkdir(parents=True, exist_ok=True) - print(f"[*] {target.name} not found locally; downloading from {repo_id} (~2 GB, once)...") - - # HF_TOKEN only lifts anonymous rate limits here; both default models are public. - return hf_hub_download( - repo_id=repo_id, - filename=filename, - local_dir=str(models_dir), - token=config.HF_TOKEN, - ) diff --git a/components-Dinura/learnmate/llm/embeddings.py b/components-Dinura/learnmate/llm/embeddings.py deleted file mode 100644 index 2cc6f1c..0000000 --- a/components-Dinura/learnmate/llm/embeddings.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Sentence-Transformers embeddings behind LangChain's Embeddings interface. - -Implemented directly rather than via langchain-huggingface so the project keeps a single -transformers/torch pin: that package tracks its own compatible range and pulling it in -would force a downgrade of the versions already installed here. - -The model loads once per process and is shared, which matters because both ingestion and -every retrieval call ask for it. -""" - -from typing import List - -from langchain_core.embeddings import Embeddings - -from .. import config - -_MODEL_CACHE = {} - - -def _load(model_name: str): - if model_name not in _MODEL_CACHE: - from sentence_transformers import SentenceTransformer - - print(f"[*] Loading embedding model: {model_name} (first load only)...") - _MODEL_CACHE[model_name] = SentenceTransformer(model_name) - return _MODEL_CACHE[model_name] - - -class LearnMateEmbeddings(Embeddings): - """all-MiniLM-L6-v2 by default: 384 dimensions, fast enough to embed a book on CPU.""" - - def __init__(self, model_name: str = None, normalize: bool = True): - self.model_name = model_name or config.EMBEDDING_MODEL - # Normalised vectors make cosine similarity a plain dot product, which is what the - # NumPy fallback in MongoVectorStore relies on to stay fast. - self.normalize = normalize - self._model = None - - @property - def model(self): - if self._model is None: - self._model = _load(self.model_name) - return self._model - - @property - def dimension(self) -> int: - """Vector width, needed when declaring the Atlas vector index.""" - model = self.model - # sentence-transformers renamed this getter in v5 and the old name now warns, so - # the current name is tried first. Falls back to embedding a token and measuring - # the result if neither exists. - for attr in ("get_embedding_dimension", "get_sentence_embedding_dimension"): - getter = getattr(model, attr, None) - if callable(getter): - size = getter() - if size: - return int(size) - return len(self.embed_query("dimension probe")) - - def embed_documents(self, texts: List[str]) -> List[List[float]]: - if not texts: - return [] - vectors = self.model.encode( - texts, - normalize_embeddings=self.normalize, - show_progress_bar=len(texts) > 64, - batch_size=32, - ) - return [vector.tolist() for vector in vectors] - - def embed_query(self, text: str) -> List[float]: - vector = self.model.encode([text], normalize_embeddings=self.normalize)[0] - return vector.tolist() - - -_EMBEDDINGS = None - - -def get_embeddings() -> LearnMateEmbeddings: - """Process-wide embeddings instance.""" - global _EMBEDDINGS - if _EMBEDDINGS is None: - _EMBEDDINGS = LearnMateEmbeddings() - return _EMBEDDINGS diff --git a/components-Dinura/learnmate/llm/http_api.py b/components-Dinura/learnmate/llm/http_api.py deleted file mode 100644 index 7bec303..0000000 --- a/components-Dinura/learnmate/llm/http_api.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Backend 2 of 2: a model served over an OpenAI-compatible HTTP API. - -The path the finetuned model takes if it ends up behind local-model-api/ or -finetuned-model-api/ instead of shipping as a GGUF. Switching to it is a change to -LEARNMATE_GENERATOR_BACKEND and nothing else -- the agents never learn which backend -answered them. -""" - -from typing import Any, Dict, List, Optional - -from langchain_core.callbacks import CallbackManagerForLLMRun -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import BaseMessage -from langchain_core.outputs import ChatResult - -from .messages import _as_result, _to_payload - - -class HttpChatModel(BaseChatModel): - """ - An OpenAI-compatible chat endpoint, for when the model is served rather than loaded. - - `response_schema` is forwarded as OpenAI-style `response_format`; servers that reject - it get a plain retry, so an endpoint without schema support still works. - """ - - base_url: str - model_name: str = "local-model" - api_key: Optional[str] = None - temperature: float = 0.3 - max_tokens: int = 512 - # Generous by design: a 3B model on CPU behind a local server can take minutes for a - # long resource, and a timeout here reads as a failed generation. - timeout: int = 300 - - @property - def _llm_type(self) -> str: - return "learnmate-http" - - @property - def _identifying_params(self) -> Dict[str, Any]: - return {"base_url": self.base_url, "model_name": self.model_name} - - def _post(self, body: Dict[str, Any]) -> str: - """One POST to /chat/completions, returning just the reply text.""" - import requests # lazy, like llama_cpp: neither backend forces the other's deps - - headers = {"Content-Type": "application/json"} - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - - response = requests.post(f"{self.base_url.rstrip('/')}/chat/completions", - json=body, headers=headers, timeout=self.timeout) - response.raise_for_status() - return response.json()["choices"][0]["message"]["content"] - - def _generate( - self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager: Optional[CallbackManagerForLLMRun] = None, - **kwargs: Any, - ) -> ChatResult: - """Same contract as the llama.cpp backend, over the network instead.""" - body: Dict[str, Any] = { - "model": self.model_name, - "messages": _to_payload(messages), - "max_tokens": kwargs.get("max_tokens", self.max_tokens), - "temperature": kwargs.get("temperature", self.temperature), - } - if stop: - body["stop"] = stop - - # The served equivalent of llama.cpp's grammar: a json_schema response_format. - # Not every server implements it, so a rejection falls back to asking plainly - # rather than failing the generation. - schema = kwargs.get("response_schema") - if schema is not None: - try: - constrained = dict(body) - constrained["response_format"] = { - "type": "json_schema", - "json_schema": {"name": "response", "schema": schema, "strict": True}, - } - return _as_result(self._post(constrained)) - except Exception: - pass - - return _as_result(self._post(body)) diff --git a/components-Dinura/learnmate/llm/json_output.py b/components-Dinura/learnmate/llm/json_output.py deleted file mode 100644 index ccf30b8..0000000 --- a/components-Dinura/learnmate/llm/json_output.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Reading JSON back out of a model reply. - -The safety net under the unconstrained path. When grammar-constrained decoding is -available the output is already clean and this is a plain `json.loads`; when it is not -- -an older llama-cpp build, a server without schema support -- the model tends to wrap the -JSON in a code fence or a sentence of preamble, and this is what recovers it. -""" - -import json -import re -from typing import Any - - -def parse_json_reply(raw: str) -> Any: - """ - Parse a model reply that is supposed to be JSON. - - Raises ValueError when there is nothing parseable. That is deliberate: callers treat - it as a failed attempt and retry, which is far better than shipping a placeholder - object downstream and having it surface as an empty quiz three steps later. - """ - # Already-parsed input passes straight through, so a caller need not care whether the - # backend handed back text or a structure. - if isinstance(raw, (dict, list)): - return raw - - text = (raw or "").strip() - try: - return json.loads(text) - except (ValueError, TypeError): - pass - - # Strip a ```json fence if one is present, then take the outermost braces/brackets -- - # which is what survives a model that introduced its answer before giving it. - fenced = re.sub(r"^```(?:json)?|```$", "", text, flags=re.MULTILINE).strip() - match = re.search(r"[\[{].*[\]}]", fenced, re.DOTALL) - if match: - try: - return json.loads(match.group(0)) - except ValueError: - pass - - # Truncated: the full reply can be thousands of tokens, and the opening is enough to - # see what went wrong. - raise ValueError(f"Could not parse model output as JSON: {text[:200]}") diff --git a/components-Dinura/learnmate/llm/llamacpp.py b/components-Dinura/learnmate/llm/llamacpp.py deleted file mode 100644 index 11f95dc..0000000 --- a/components-Dinura/learnmate/llm/llamacpp.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Backend 1 of 2: a local GGUF model, run in this process through llama-cpp-python. - -The default path for both roles. `learnmate/models/` holds the two GGUFs -- Qwen2.5-3B for -the generator, Llama-3.2-3B for the judge -- and which one this class loads is decided -entirely by the `gguf_path` it is constructed with. There is no Qwen class and no Llama -class; the family is a configuration value. - -This is a real BaseChatModel, so it composes with prompts, output parsers and LangGraph -nodes exactly like any stock LangChain chat model. -""" - -from typing import Any, Dict, List, Optional - -from langchain_core.callbacks import CallbackManagerForLLMRun -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import BaseMessage -from langchain_core.outputs import ChatResult - -from .messages import _as_result, _to_payload -from .runtime import _load_llama - - -class LlamaCppChatModel(BaseChatModel): - """A local GGUF model, with optional grammar-constrained JSON output.""" - - gguf_path: str - n_ctx: int = 4096 - n_threads: Optional[int] = None - n_gpu_layers: int = 0 - chat_format: Optional[str] = None - # Defaults only. Every call site in this project passes its own -- see the kwargs - # handling in _generate below. - temperature: float = 0.3 - max_tokens: int = 512 - - @property - def _llm_type(self) -> str: - return "learnmate-llamacpp" - - @property - def _identifying_params(self) -> Dict[str, Any]: - return {"gguf_path": self.gguf_path, "n_ctx": self.n_ctx, - "chat_format": self.chat_format} - - def _generate( - self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager: Optional[CallbackManagerForLLMRun] = None, - **kwargs: Any, - ) -> ChatResult: - """ - The one method BaseChatModel requires. `.invoke(...)` arrives here. - - Weights come from the shared cache, so constructing several of these is cheap -- - only the first one to name a given GGUF actually loads it. - """ - llama = _load_llama(self.gguf_path, self.n_ctx, self.n_threads, - self.n_gpu_layers, self.chat_format) - - # Per-call kwargs win over the constructor defaults. This is how the chat agent - # gets temperature 0.0 for rewriting and 0.3 for generating out of one cached - # model instead of building a new one for each. - params: Dict[str, Any] = { - "messages": _to_payload(messages), - "max_tokens": kwargs.get("max_tokens", self.max_tokens), - "temperature": kwargs.get("temperature", self.temperature), - } - if stop: - params["stop"] = stop - - # --- Grammar-constrained JSON --------------------------------------------------- - # The reason this class exists rather than a stock integration. A 3B model asked - # politely for JSON returns prose about half the time; llama.cpp can instead - # compile the schema into a decoding grammar, so malformed output is impossible. - # Everything structured here -- every resource, every verdict -- depends on it. - schema = kwargs.get("response_schema") - if schema is not None: - try: - constrained = dict(params) - constrained["response_format"] = {"type": "json_object", "schema": schema} - result = llama.create_chat_completion(**constrained) - return _as_result(result["choices"][0]["message"]["content"]) - except Exception: - # Older llama-cpp-python builds have no grammar support. Fall through and - # ask plainly; the callers all parse defensively (see json_output.py). - pass - - result = llama.create_chat_completion(**params) - return _as_result(result["choices"][0]["message"]["content"]) diff --git a/components-Dinura/learnmate/llm/messages.py b/components-Dinura/learnmate/llm/messages.py deleted file mode 100644 index e75b54c..0000000 --- a/components-Dinura/learnmate/llm/messages.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Translation between LangChain's message objects and the wire format both backends use. - -llama.cpp and an OpenAI-compatible HTTP API want the same thing -- a list of -`{"role": ..., "content": ...}` dicts -- so this conversion is shared rather than written -twice. Keeping it here is also what lets the two backend classes stay short enough to read -in one sitting. -""" - -from typing import Dict, List - -from langchain_core.messages import AIMessage, BaseMessage, ChatMessage -from langchain_core.outputs import ChatGeneration, ChatResult - - -def _to_payload(messages: List[BaseMessage]) -> List[Dict[str, str]]: - """ - Convert LangChain messages into the role/content dicts both backends expect. - - LangChain names the roles differently from the chat-completions API ("human" rather - than "user", "ai" rather than "assistant"), so they are mapped rather than passed - through. A ChatMessage carries an explicit role and keeps it. Anything unrecognised - falls back to "user", which is the interpretation that cannot confuse a model into - thinking it wrote the text itself. - """ - role_map = {"human": "user", "ai": "assistant", "system": "system"} - payload = [] - for message in messages: - if isinstance(message, ChatMessage): - role = message.role - else: - role = role_map.get(message.type, "user") - payload.append({"role": role, "content": str(message.content)}) - return payload - - -def _as_result(text: str) -> ChatResult: - """ - Wrap generated text in the ChatResult a BaseChatModel must return. - - Both backends produce a single completion, so the generations list always has exactly - one entry -- there is no n>1 sampling anywhere in this project. - """ - return ChatResult(generations=[ChatGeneration(message=AIMessage(content=text or ""))]) diff --git a/components-Dinura/learnmate/llm/registry.py b/components-Dinura/learnmate/llm/registry.py deleted file mode 100644 index d513540..0000000 --- a/components-Dinura/learnmate/llm/registry.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -The two entry points every agent uses to reach a model. - - get_generator_llm() Qwen2.5-3B by default -- writes chat replies and resources - get_judge_llm() Llama-3.2-3B by default -- scores what the generator wrote - -Both roles are built by the same `_build`, from the same kinds of settings; the only -difference between them is which config block is passed in. That is what makes swapping -Qwen for the finetuned model two lines of .env rather than a code change. - -The two models are deliberately different families. A judge sharing the generator's -weights rates its own output style highly, and the retry loop stops firing. -""" - -from typing import Optional - -from .. import config -from .download import ensure_gguf -from .http_api import HttpChatModel -from .llamacpp import LlamaCppChatModel - -# Wrappers, not weights -- see runtime.py for the layer below this one. Keyed by role and -# sampling settings, so asking for the generator at two temperatures makes two cheap -# wrapper objects that share a single loaded model. -_LLM_CACHE = {} - - -def _build(role: str, backend: str, model: str, repo: str, filename: str, - chat_format: str, n_ctx: int, api_url: str, api_key: str, - temperature: float, max_tokens: int): - """Construct the chat model one role's configuration describes.""" - if backend == "http": - return HttpChatModel( - base_url=api_url, - model_name=model, - api_key=api_key or None, - temperature=temperature, - max_tokens=max_tokens, - ) - - if backend != "llamacpp": - raise ValueError( - f"Unknown {role} backend {backend!r}; expected 'llamacpp' or 'http'." - ) - - # ensure_gguf downloads on first use, so this is where a fresh checkout blocks for a - # few minutes -- not somewhere deep inside a generation. - return LlamaCppChatModel( - gguf_path=ensure_gguf(model, repo, filename), - n_ctx=n_ctx, - n_threads=config.N_THREADS, - n_gpu_layers=config.N_GPU_LAYERS, - chat_format=chat_format or None, - temperature=temperature, - max_tokens=max_tokens, - ) - - -def get_generator_llm(temperature: Optional[float] = None, max_tokens: int = 1024): - """ - The model that writes chat replies and study resources. - - Warmer than the judge on purpose: at temperature 0 a regeneration returns almost - exactly the attempt that was just rejected, which defeats the retry loop. - """ - temp = 0.7 if temperature is None else temperature - key = ("generator", temp, max_tokens) - if key not in _LLM_CACHE: - _LLM_CACHE[key] = _build( - "generator", config.GENERATOR_BACKEND, config.GENERATOR_MODEL, - config.GENERATOR_REPO, config.GENERATOR_FILE, config.GENERATOR_CHAT_FORMAT, - config.GENERATOR_N_CTX, config.GENERATOR_API_URL, config.GENERATOR_API_KEY, - temp, max_tokens, - ) - return _LLM_CACHE[key] - - -def get_judge_llm(temperature: float = 0.0, max_tokens: int = 512): - """The evaluator model. Temperature 0: a grade should not be a dice roll.""" - key = ("judge", temperature, max_tokens) - if key not in _LLM_CACHE: - _LLM_CACHE[key] = _build( - "judge", config.JUDGE_BACKEND, config.JUDGE_MODEL, - config.JUDGE_REPO, config.JUDGE_FILE, config.JUDGE_CHAT_FORMAT, - config.JUDGE_N_CTX, config.JUDGE_API_URL, config.JUDGE_API_KEY, - temperature, max_tokens, - ) - return _LLM_CACHE[key] diff --git a/components-Dinura/learnmate/llm/runtime.py b/components-Dinura/learnmate/llm/runtime.py deleted file mode 100644 index b0cc75a..0000000 --- a/components-Dinura/learnmate/llm/runtime.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -The llama.cpp weight cache: loading GGUF models once, and releasing them cleanly. - -Loading a 2 GB GGUF takes seconds and a copy of the weights in RAM. The generator and the -judge are two different models and both may be asked for from several modules, so what -gets cached -- and what the cache is keyed on -- decides whether this system uses 4 GB or -8 GB of RAM. - -This is the lowest of the three caches in this package: - - runtime.py _LLAMA_CACHE the actual weights, keyed by file - registry.py _LLM_CACHE the chat-model wrappers, keyed by role and sampling - embeddings.py _MODEL_CACHE the sentence-transformers model - -Asking for the generator at two temperatures makes two wrappers that share one set of -weights, because only the wrapper layer has temperature in its key. -""" - -import atexit -from typing import Any, Dict, Optional - -# Keying on the path matters: the generator and the judge are different families on -# purpose -- a judge sharing the generator's weights rates its own output style highly -# and the retry loop never fires -- and a single shared slot would silently hand the -# judge whichever model happened to load first. -_LLAMA_CACHE: Dict[tuple, Any] = {} - - -def _load_llama(gguf_path: str, n_ctx: int, n_threads: Optional[int], - n_gpu_layers: int, chat_format: Optional[str]): - """ - Load (or reuse) a llama.cpp model. - - n_ctx and chat_format join the path in the key because both are baked into the loaded - model: the same GGUF at a different context size is genuinely a different object and - cannot be shared. - """ - key = (gguf_path, n_ctx, chat_format) - if key not in _LLAMA_CACHE: - from llama_cpp import Llama # imported lazily so the HTTP backend needs no llama.cpp - - print(f"[*] Loading model: {gguf_path} (first load only)...") - kwargs: Dict[str, Any] = { - "model_path": gguf_path, - "n_ctx": n_ctx, - "n_threads": n_threads, - "n_gpu_layers": n_gpu_layers, - "verbose": False, - } - # Left unset, llama.cpp reads the prompt template from the GGUF metadata, which is - # correct for anything finetuned from a standard base. - if chat_format: - kwargs["chat_format"] = chat_format - _LLAMA_CACHE[key] = Llama(**kwargs) - return _LLAMA_CACHE[key] - - -@atexit.register -def _release_models() -> None: - """ - Free the llama.cpp contexts while the interpreter is still standing. - - Left to the garbage collector, Llama.__del__ runs during interpreter shutdown and - unwinds an ExitStack whose callbacks have already been torn down, printing a - TypeError traceback after an otherwise clean exit. Closing here means the teardown - happens while those modules still exist. - """ - for model in list(_LLAMA_CACHE.values()): - try: - model.close() - except Exception: - pass - _LLAMA_CACHE.clear() diff --git a/components-Dinura/learnmate/resource_agent/__init__.py b/components-Dinura/learnmate/resource_agent/__init__.py deleted file mode 100644 index 3e4e433..0000000 --- a/components-Dinura/learnmate/resource_agent/__init__.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -The resource-generation agent: MCQs, practice questions, key points and summaries. - -One run is one pass through a LangGraph state machine: - - generate -> check -> decide -+-> persist -> END - ^ | - +------ regenerate ------+ - -`check` runs two gates, cheapest first: the structural validators (plain Python, -microseconds) and then the LLM judge (~25 seconds). Most bad generations fail -mechanically, so the judge is only ever spent on content that is already well-formed. - -The retry budget is 2: one generation plus one regeneration. The last attempt is returned -whether or not it passed, and `accepted` reports which -- a student waiting on a quiz is -better served by flagged output than by nothing. - -Four resource types, one file each. Each owns its prompt, its JSON schema, how to read the -reply and how to render it; everything else is shared, so a fifth type is one new module -plus one line in tasks.py. - - mcq.py multiple-choice questions {question, options[4], correct_answer} - practice_qsn.py short-answer questions {question, answer} - keypoints.py a list of key points ["point", ...] - summary.py one block of connected prose "..." - -Where things live, in reading order: - - task.py the Task contract every resource type implements - mcq.py etc. the four types - tasks.py the registry: TASKS, TASK_NAMES, get_task, render - state.py ResourceState -- what flows between nodes - helpers.py logging, and the revision block used on a retry - generate.py node 1 ask the generator, folding in a critique when retrying - check.py node 2 the two gates - routing.py the accept-or-retry branch - persist.py node 3 store the resource and its attempt trail - graph.py the LangGraph wiring - agent.py generate_resource() -- the public entry point - -Three more modules sit on top, for asks that are about a whole PDF rather than one passage --- neither the 6000-character passage nor the 1024-token reply is enough for those: - - whole_document.py generate_document_items() -- groups of pages generated separately - and pooled, by total (count=40) or by rate (per_page=2) - document_mcqs.py generate_document_mcqs() -- the count-based entry point for mcq - document_summary.py summarize_document() -- every page summarised, then folded into - one comprehensive summary - -The source passage comes from ingestion.build_source_text, which turns a PDF and an -optional topic into the whole pages most relevant to it. -""" - -from .agent import generate_resource -# Re-exported under a qualified name: "COUNT_CHOICES" says nothing at package scope. -from .document_mcqs import COUNT_CHOICES as MCQ_COUNT_CHOICES -from .document_mcqs import DEFAULT_COUNT as MCQ_DEFAULT_COUNT -from .document_mcqs import generate_document_mcqs -from .document_summary import summarize_document -from .whole_document import DEFAULT_PER_PAGE, PER_PAGE_CHOICES, generate_document_items -from .graph import build_resource_graph, get_resource_graph -from .task import Task -from .tasks import TASK_NAMES, TASKS, get_task, render - -__all__ = [ - "DEFAULT_PER_PAGE", - "MCQ_COUNT_CHOICES", - "MCQ_DEFAULT_COUNT", - "PER_PAGE_CHOICES", - "TASKS", - "TASK_NAMES", - "Task", - "build_resource_graph", - "generate_document_items", - "generate_document_mcqs", - "generate_resource", - "get_resource_graph", - "get_task", - "render", - "summarize_document", -] diff --git a/components-Dinura/learnmate/resource_agent/agent.py b/components-Dinura/learnmate/resource_agent/agent.py deleted file mode 100644 index 431f867..0000000 --- a/components-Dinura/learnmate/resource_agent/agent.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -The public entry point: generate one study resource end to end. - - build_source_text(doc_id, topic=...) <- ingestion, picks the passage - | - v - generate_resource("mcq", source, count=5) - | - v - {task, content, accepted, verdict, attempts, resource_id} -""" - -from typing import Dict - -from .. import config -from .graph import get_resource_graph -from .state import ResourceState -from .tasks import get_task - - -def generate_resource(task: str, source: str, count: int = 5, doc_id=None, - threshold: int = None, max_attempts: int = None, - evaluate: bool = True, persist: bool = True, - verbose: bool = True) -> Dict: - """ - Generate one study resource end to end. - - task -- "mcq", "practice_qsn", "keypoints" or "summary" - source -- the PDF passage, from ingestion.build_source_text - count -- how many items; for a summary, roughly how many sentences - - Returns {task, content, accepted, verdict, attempts, resource_id}, where `content` is - the last attempt and `attempts` holds the whole trail in order. `accepted` is False - when the run ended on a rejection, and the content is returned anyway so the caller - can show it with a warning rather than nothing at all. - """ - get_task(task) # fail fast on an unknown task, before loading any model - - if not (source or "").strip(): - raise ValueError("Cannot generate from empty source text.") - - initial: ResourceState = { - "task": task, - "source": source, - "count": count, - "doc_id": doc_id, - # `is not None` rather than `or`: threshold=0 means "accept anything". - "threshold": threshold if threshold is not None else config.EVALUATOR_THRESHOLD, - "max_attempts": max_attempts or config.MAX_ATTEMPTS, - "evaluate": evaluate, - "persist": persist, - "verbose": verbose, - "attempt": 0, - "attempts": [], - } - - # The graph loops, so LangGraph's default recursion budget has to cover - # attempts x (generate + check) plus persist. - limit = 2 * initial["max_attempts"] + 4 - final = get_resource_graph().invoke(initial, {"recursion_limit": limit}) - - return { - "task": task, - "content": final.get("content"), - "accepted": bool(final.get("passed")), - "verdict": final.get("verdict"), - "attempts": final.get("attempts", []), - "resource_id": final.get("resource_id"), - } diff --git a/components-Dinura/learnmate/resource_agent/check.py b/components-Dinura/learnmate/resource_agent/check.py deleted file mode 100644 index 9803cfa..0000000 --- a/components-Dinura/learnmate/resource_agent/check.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Node 2 of 3: check. Both evaluation gates, cheapest first. - - gate 1 validators.validate() plain Python, microseconds - gate 2 the LLM judge ~25 seconds - -Most bad generations fail mechanically -- three options instead of four, a correct_answer -in none of them, an empty summary -- so the judge is only ever spent on content that is -already well-formed. `stage` records which gate decided the attempt, and that is what -content_store.stage_counts() groups by: if the validator is deciding most attempts, the -generation prompt needs work, not the threshold. - - content -> passed, stage, verdict, critique -""" - -import time -from typing import Dict - -from ..evaluator import validators -from ..evaluator.judge import get_judge -from ..storage import content_store -from .helpers import _log -from .state import ResourceState -from .tasks import get_task - - -def check_node(state: ResourceState) -> Dict: - """Structural gate, then the judge. The judge only sees content that passed the gate.""" - # Generation itself failed; there is no content to check and generate_node has - # already recorded the attempt. - if state.get("stage") == "parse": - return {} - - task = get_task(state["task"]) - attempt = state["attempt"] - content = state.get("content") - started = state.get("started", time.time()) - - # --- Gate 1: structural ---------------------------------------------------------- - ok, reasons = validators.validate(task.name, content) - if not ok: - _log(state, f"[*] Structural check FAILED: {'; '.join(reasons)}") - content_store.log_evaluation( - task.name, attempt, None, False, state["threshold"], stage="validator", - elapsed=time.time() - started, doc_id=state.get("doc_id"), - extra={"reasons": reasons}) - return { - "stage": "validator", - "passed": False, - "verdict": None, - # The validator's reasons are already phrased as faults, so they become the - # regeneration instruction directly -- no judge call needed to say what broke. - "critique": "Fix these structural problems and change nothing else: " - + "; ".join(reasons), - "previous": content, - "attempts": [{"attempt": attempt, "stage": "validator", "passed": False, - "score": None, "reasons": reasons, "content": content}], - } - - # --- Evaluation switched off ------------------------------------------------------ - # Generation is intact and both gates are skipped, which is how you measure what - # evaluation is actually costing. - if not state.get("evaluate", True): - return { - "stage": "skipped", "passed": True, "verdict": None, "critique": None, - "attempts": [{"attempt": attempt, "stage": "skipped", "passed": True, - "score": None, "reasons": [], "content": content}], - } - - # --- Gate 2: the judge ------------------------------------------------------------ - _log(state, "[*] Structural check passed. Judging...") - verdict = get_judge().judge( - task.name, task.render(content), source=state["source"], - threshold=state["threshold"]) - - content_store.log_evaluation( - task.name, attempt, verdict["score"], verdict["passed"], state["threshold"], - stage="judge", elapsed=time.time() - started, doc_id=state.get("doc_id")) - _log(state, f"[*] Score {verdict['score']}/100 -> " - f"{'PASS' if verdict['passed'] else 'REGENERATE'}") - - return { - "stage": "judge", - "passed": verdict["passed"], - "verdict": verdict, - "critique": verdict["regeneration_instruction"], - "previous": content, - "attempts": [{"attempt": attempt, "stage": "judge", "passed": verdict["passed"], - "score": verdict["score"], "reasons": [], "content": content, - "verdict": verdict}], - } diff --git a/components-Dinura/learnmate/resource_agent/document_mcqs.py b/components-Dinura/learnmate/resource_agent/document_mcqs.py deleted file mode 100644 index 695a80e..0000000 --- a/components-Dinura/learnmate/resource_agent/document_mcqs.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Multiple-choice questions over a whole document. - -The count-based entry point onto whole_document.generate_document_items, kept separate -because "how many questions do you want" is the question a caller actually has, and -because the counts the CLI offers belong next to the task they size. - - generate_document_mcqs(doc_id, count=40) - -Forty is not something `generate_resource` can deliver: one reply holds about eight MCQs -before it is cut off, and one passage is 6000 characters of the opening pages. See -whole_document.py for how the document is split and the results pooled. -""" - -from typing import Dict - -from .whole_document import generate_document_items - -# What the CLI offers. Not a limit: any count can be passed directly. -COUNT_CHOICES = (10, 20, 40) -DEFAULT_COUNT = 20 - - -def generate_document_mcqs(doc_id, count: int = DEFAULT_COUNT, threshold: int = None, - max_attempts: int = None, evaluate: bool = True, - persist: bool = True, verbose: bool = True, - max_chars: int = None) -> Dict: - """ - Write `count` multiple-choice questions drawn from across the whole document. - - A thin alias for generate_document_items(task="mcq", count=...); see that function for - what comes back and how the count is spread. - """ - return generate_document_items( - "mcq", doc_id, count=count, threshold=threshold, max_attempts=max_attempts, - evaluate=evaluate, persist=persist, verbose=verbose, max_chars=max_chars) diff --git a/components-Dinura/learnmate/resource_agent/document_summary.py b/components-Dinura/learnmate/resource_agent/document_summary.py deleted file mode 100644 index 49faaca..0000000 --- a/components-Dinura/learnmate/resource_agent/document_summary.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -A summary of a whole document, not just the passage that fits the context window. - -`generate_resource("summary", source)` can only see what `build_source_text` handed it -- -MAX_SOURCE_CHARS of the opening pages -- so on anything longer the tail of the PDF is -never read. This walks every stored page instead and folds the results: - - page 1 ---> summary 1 --+ - page 2 ---> summary 2 --+--> one paragraph for the document - ... | - page n ---> summary n --+ - -Two passes, for two different reasons. The per-page pass is what makes the whole document -readable at all: each page fits the window on its own. The fold is what turns n paragraphs -back into one, and it repeats while the page summaries together still overrun the budget, -so a 200-page book reduces in rounds rather than being truncated. - -Only the final round goes through the graph, so the judge grades the paragraph the caller -actually gets -- against the page summaries it was written from, which is the source it was -in fact asked to be faithful to. The per-page pass skips evaluation by default: judging -every page costs ~25 seconds each and buys nothing the final gate does not already check. -""" - -from typing import Dict, List - -from .. import config -from ..storage import pdf_store -from .agent import generate_resource -from .helpers import _log -# The page filter and the grouping are shared with the pooled tasks. The page *loading* is -# not: a summary reports which page each note came from, so it needs the records whole. -from .whole_document import MIN_PAGE_CHARS, batched - -# Sentences asked of a single page. Deliberately tight: these are intermediate notes to be -# folded, and a verbose per-page pass just crowds the fold's own budget. -PAGE_SENTENCES = 2 - -# --- How long the final summary is allowed to be --------------------------------------- -# Scaled to the document rather than fixed: five sentences is a fair summary of a page and -# a useless one of eighty, so the ask grows with how much was actually read. -SENTENCES_PER_PAGE = 1 -MIN_SENTENCES = 5 - -# The ceiling is the generator's output budget, not a stylistic choice. get_generator_llm -# defaults to max_tokens=1024 inside a 4096-token window, and ~20 tokens a sentence puts -# the hard limit near 50 -- asking past that gets a summary cut off mid-word. Raise -# LEARNMATE_GENERATOR_N_CTX and the max_tokens default together before raising this. -MAX_SENTENCES = 30 - -# Rough length of one summary sentence, used to budget an intermediate fold from the -# amount of text it is being handed. Only needs to be the right order of magnitude. -CHARS_PER_SENTENCE = 120 - - -def _summarise(source: str, sentences: int, doc_id) -> str: - """One un-judged, un-stored summary pass. Empty string when the generation failed.""" - result = generate_resource( - "summary", source, count=sentences, doc_id=doc_id, - evaluate=False, persist=False, verbose=False) - return (result.get("content") or "").strip() - - -def _fold_sentences(batch: List[str]) -> int: - """Sentences to ask of one intermediate fold: about half the content it consumes.""" - return max(PAGE_SENTENCES, - min(MAX_SENTENCES, sum(len(text) for text in batch) // CHARS_PER_SENTENCE // 2)) - - -def summarize_document(doc_id, count: int = None, threshold: int = None, - max_attempts: int = None, evaluate: bool = True, - persist: bool = True, verbose: bool = True, - max_chars: int = None) -> Dict: - """ - Summarise every page of a document and combine them into one comprehensive summary. - - count -- roughly how many sentences to write. None sizes it to the document: - about one sentence per page read, floored at MIN_SENTENCES and capped at - MAX_SENTENCES by what the generator can emit in one reply. - Everything else means what it does on `generate_resource`, and applies to the final - fold only -- the per-page pass is never judged or stored. - - Returns generate_resource's dict with one extra key, `pages`: the per-page summaries - as [{"page_number": n, "summary": "..."}], so a caller can show the working rather - than only the conclusion. - - Raises ValueError when the document has no stored page text, which means it was never - ingested (or was ingested before store_pages existed -- re-ingest it). - """ - budget = max_chars or config.MAX_SOURCE_CHARS - - records = pdf_store.get_pages(doc_id) - if not records: - raise ValueError(f"No stored page text for document {doc_id}. Ingest the PDF first.") - - # --- Pass 1: one summary per page -------------------------------------------------- - pages: List[Dict] = [] - for record in records: - text = (record.get("text") or "").strip() - if len(text) < MIN_PAGE_CHARS: - continue - - _log({"verbose": verbose}, f"[*] Summarising page {record['page_number']}...") - # Truncated because one page can still exceed the window on a dense layout. - summary = _summarise(text[:budget], PAGE_SENTENCES, doc_id) - if summary: - pages.append({"page_number": record["page_number"], "summary": summary}) - - if not pages: - raise ValueError(f"Document {doc_id} has no page with enough text to summarise.") - - # How long the answer should be, now that we know how much was actually read. - sentences = count or max(MIN_SENTENCES, - min(MAX_SENTENCES, len(pages) * SENTENCES_PER_PAGE)) - - # --- Pass 2: fold until the notes fit one prompt ------------------------------------ - # Page numbers are kept in the text so the fold can say "the opening chapters" rather - # than treating n paragraphs as n unrelated documents. - notes = [f"Page {page['page_number']}: {page['summary']}" for page in pages] - - while True: - batches = batched(notes, budget) - # One batch means everything fits, so the next pass is the final one. `>=` is the - # no-progress guard: a single note over the budget would otherwise loop forever. - if len(batches) <= 1 or len(batches) >= len(notes): - break - _log({"verbose": verbose}, - f"[*] Folding {len(notes)} page summaries into {len(batches)}...") - # Half the content the batch went in with, measured in characters rather than in - # notes: a batch of four long notes still holds a chapter's worth of material, and - # counting notes would ask it for two sentences. An intermediate fold is not the - # answer and only has to shrink -- halving is what makes the loop converge -- but - # a flat budget here is where the detail was being thrown away, before the final - # pass ever saw it. - notes = [_summarise("\n\n".join(batch), _fold_sentences(batch), doc_id) - for batch in batches] - notes = [note for note in notes if note] - - # --- The one pass that is judged and stored ----------------------------------------- - _log({"verbose": verbose}, - f"[*] Writing the document summary from {len(pages)} pages " - f"(about {sentences} sentences)...") - result = generate_resource( - "summary", "\n\n".join(notes)[:budget], count=sentences, doc_id=doc_id, - threshold=threshold, max_attempts=max_attempts, evaluate=evaluate, - persist=persist, verbose=verbose) - - result["pages"] = pages - return result diff --git a/components-Dinura/learnmate/resource_agent/generate.py b/components-Dinura/learnmate/resource_agent/generate.py deleted file mode 100644 index 3001be4..0000000 --- a/components-Dinura/learnmate/resource_agent/generate.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Node 1 of 3: generate. - -Asks the generator for the resource, folding in a critique when this is a retry. Which -prompt, which schema and how to read the reply all come from the Task -- this node is the -same code for all four resource types. - - source (+ critique) -> content, attempt - -The schema is passed as `response_schema`, so llama.cpp compiles it into a decoding -grammar and malformed JSON becomes impossible on that backend. The HTTP backend forwards -it as `response_format` and falls back to asking plainly, which is why the reply is still -parsed defensively. -""" - -import time -from typing import Dict - -from langchain_core.messages import HumanMessage, SystemMessage - -from ..llm import get_generator_llm, parse_json_reply -from ..storage import content_store -from .helpers import _log, revision_block -from .state import ResourceState -from .tasks import get_task - - -def generate_node(state: ResourceState) -> Dict: - """Ask the generator for the resource, folding in a critique when retrying.""" - task = get_task(state["task"]) - attempt = state.get("attempt", 0) + 1 - - _log(state, f"[*] Generating {task.name} (attempt {attempt}/{state['max_attempts']})...") - - prompt = task.build_prompt(state["source"], state.get("count", 5)) - if state.get("critique"): - prompt += revision_block(task, state["critique"], state.get("previous")) - - messages = [ - SystemMessage(content=task.system_prompt), - HumanMessage(content=prompt), - ] - - started = time.time() - try: - reply = get_generator_llm().invoke(messages, response_schema=task.schema) - content = task.unwrap(parse_json_reply(reply.content)) - return {"attempt": attempt, "content": content, "started": started, - "stage": "generated"} - except Exception as exc: - # Unparseable or failed output is a failed attempt, not a crash: record it and - # let `decide` retry with the parse failure as the critique. Logged with - # stage="parse" so content_store.stage_counts() can show how often the generator, - # rather than the content, was the problem. - _log(state, f"[!] Generation failed: {exc}") - content_store.log_evaluation( - task.name, attempt, None, False, state["threshold"], stage="parse", - elapsed=time.time() - started, doc_id=state.get("doc_id"), - extra={"error": str(exc)[:300]}) - return { - "attempt": attempt, - "content": None, - "started": started, - "stage": "parse", - "passed": False, - "verdict": None, - "critique": "Return valid JSON in exactly the requested shape.", - "previous": None, - "attempts": [{"attempt": attempt, "stage": "parse", "passed": False, - "score": None, "reasons": [str(exc)[:300]], "content": None}], - } diff --git a/components-Dinura/learnmate/resource_agent/graph.py b/components-Dinura/learnmate/resource_agent/graph.py deleted file mode 100644 index 3c8d554..0000000 --- a/components-Dinura/learnmate/resource_agent/graph.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Wiring: the nodes assembled into a LangGraph state machine. - - generate -> check -> decide -+-> persist -> END - ^ | - +------ regenerate ------+ - -No logic of its own -- only how the pieces connect. `decide` is the single place the loop -can end. - -Expressing this as a graph rather than a for-loop is what makes the flow inspectable: -every node's decision lands in the state, so a run that produced bad output can be read -back afterwards instead of guessed at. -""" - -from langgraph.graph import END, StateGraph - -from .check import check_node -from .generate import generate_node -from .persist import persist_node -from .routing import decide -from .state import ResourceState - - -def build_resource_graph(): - """Compile the generate/check/retry graph.""" - graph = StateGraph(ResourceState) - graph.add_node("generate", generate_node) - graph.add_node("check", check_node) - graph.add_node("persist", persist_node) - - graph.set_entry_point("generate") - graph.add_edge("generate", "check") - - # `decide` returns "generate" or "persist"; this mapping turns those strings into the - # actual edges. The "generate" branch is what closes the retry loop. - graph.add_conditional_edges("check", decide, - {"generate": "generate", "persist": "persist"}) - - graph.add_edge("persist", END) - return graph.compile() - - -# Compiled once and reused: the graph is stateless, every run passes its own state in. -_GRAPH = None - - -def get_resource_graph(): - """Process-wide compiled graph, built on first use.""" - global _GRAPH - if _GRAPH is None: - _GRAPH = build_resource_graph() - return _GRAPH diff --git a/components-Dinura/learnmate/resource_agent/helpers.py b/components-Dinura/learnmate/resource_agent/helpers.py deleted file mode 100644 index 26ff07d..0000000 --- a/components-Dinura/learnmate/resource_agent/helpers.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Logging and the revision block, shared by the nodes.""" - -from typing import Any - -from .state import ResourceState -from .task import Task - - -def _log(state: ResourceState, message: str) -> None: - """Print progress unless the caller asked for silence.""" - if state.get("verbose", True): - print(message) - - -def revision_block(task: Task, critique: str, previous: Any) -> str: - """ - Append the rejecting gate's instruction to the next prompt. - - The rejected attempt is included so the model *revises* rather than starting over: - given only the instruction it tends to discard the good items along with the bad one. - Truncated at 2000 chars so a long resource cannot crowd the source passage out of the - context window on the retry. - """ - block = ("\n\n[REVISION REQUIRED] An evaluator rejected your previous attempt.\n" - f"Required fix: {critique}\n") - if previous is not None: - block += f'Your previous attempt:\n"""\n{task.render(previous)[:2000]}\n"""\n' - block += ("Produce the whole resource again with that fix applied, keeping everything " - "the evaluator did not object to. Do not mention this instruction.") - return block diff --git a/components-Dinura/learnmate/resource_agent/keypoints.py b/components-Dinura/learnmate/resource_agent/keypoints.py deleted file mode 100644 index f2f572e..0000000 --- a/components-Dinura/learnmate/resource_agent/keypoints.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Resource type 3 of 4: key points. - - PDF passage -> ["point", "point", ...] - -The only task whose content is a bare list of strings, which is also why it is the only -one whose rendering had to be fixed -- see the note on `render` below. -""" - -from .task import Task - -KEYPOINTS_SCHEMA = { - "type": "object", - "properties": {"keypoints": {"type": "array", "items": {"type": "string"}}}, - "required": ["keypoints"], -} - - -def build_prompt(source: str, count: int) -> str: - return f"""Extract the {count} most important key points from the passage below. - -PASSAGE: -\"\"\" -{source} -\"\"\" - -Rules: -- Every point must be supported by the passage. Add nothing from outside it. -- Each point is one self-contained sentence that makes sense on its own. -- Points must not repeat or paraphrase each other. -- Prefer what the passage treats as significant over incidental detail. - -Return JSON: {{"keypoints": ["...", "..."]}}""" - - -def unwrap(data): - """Pull the list out of the {"keypoints": [...]} envelope, or accept a bare list.""" - if isinstance(data, dict): - return data.get("keypoints", []) - return data if isinstance(data, list) else [] - - -def render(items) -> str: - """ - Numbered, not bulleted -- and that is load-bearing, not cosmetic. - - The judge's prompt puts the rubric and the content in the same message, and every - rubric in evaluator/rubrics.py is a "- " bulleted list. Rendering key points with "- " - too made the two read as one list: the judge counted the five rubric criteria as - points 1-5 and the real key points as 6-10, then rejected a perfectly grounded set for - "invented points (4, 5, 6, 7, 8)" -- items 4-5 being its own criteria. - - Numbering makes the content unmistakably a separate list, which is why the other - list-shaped tasks never showed the fault: mcq and practice_qsn already render as - "Q1.", "Q2.". - """ - return "\n".join(f"{i}. {point}" - for i, point in enumerate(items or [], start=1)) - - -KEYPOINTS = Task( - name="keypoints", - system_prompt=( - "You extract the key points of a passage. Every point you write must be supported " - "by the passage; you never add outside knowledge. Reply with JSON only." - ), - schema=KEYPOINTS_SCHEMA, - build_prompt=build_prompt, - unwrap=unwrap, - render=render, - count_label="points", -) diff --git a/components-Dinura/learnmate/resource_agent/mcq.py b/components-Dinura/learnmate/resource_agent/mcq.py deleted file mode 100644 index bfec5fb..0000000 --- a/components-Dinura/learnmate/resource_agent/mcq.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Resource type 1 of 4: multiple-choice questions. - - PDF passage -> [{question, options[4], correct_answer}, ...] - -The rules in the prompt are chosen to head off exactly what the structural gate would -reject afterwards -- four options, the correct answer copied verbatim from them, no -"all of the above", no position or length giveaway. Asking for it up front is free; -catching it later costs a whole regeneration. -""" - -from .task import Task, unwrap_questions - -# minItems/maxItems 4 is enforced by the decoding grammar itself, so a well-formed reply -# cannot have three options. evaluator/mcq_rules.py checks it again anyway, because the -# HTTP backend may be talking to a server that ignores the schema. -MCQ_SCHEMA = { - "type": "object", - "properties": { - "questions": { - "type": "array", - "items": { - "type": "object", - "properties": { - "question": {"type": "string"}, - "options": { - "type": "array", - "items": {"type": "string"}, - "minItems": 4, - "maxItems": 4, - }, - "correct_answer": {"type": "string"}, - }, - "required": ["question", "options", "correct_answer"], - }, - } - }, - "required": ["questions"], -} - - -def build_prompt(source: str, count: int) -> str: - return f"""Write {count} multiple-choice questions based only on the passage below. - -PASSAGE: -\"\"\" -{source} -\"\"\" - -Rules: -- Every question must be answerable from the passage alone. -- Exactly four options per question. -- "correct_answer" must be copied verbatim from that question's own options. -- The three wrong options must be plausible but clearly wrong according to the passage. -- Do not make the correct option consistently the longest, and do not always put it first. -- Never use "All of the above" or "None of the above". - -Return JSON: {{"questions": [{{"question": "...", "options": ["...","...","...","..."], "correct_answer": "..."}}]}}""" - - -def render(items) -> str: - """Flatten to the exam-paper layout the judge reads and the CLI prints.""" - lines = [] - for i, item in enumerate(items or [], start=1): - lines.append(f"Q{i}. {item.get('question', '')}") - for label, option in zip("ABCD", item.get("options", [])): - lines.append(f" {label}) {option}") - # The answer is shown to the judge on purpose: it cannot check correctness - # against the passage without knowing which option was marked. - lines.append(f" Correct: {item.get('correct_answer', '')}") - return "\n".join(lines) - - -MCQ = Task( - name="mcq", - system_prompt=( - "You are an expert exam writer. You write multiple-choice questions that can be " - "answered from a given passage alone, and you never invent facts the passage does " - "not state. Reply with JSON only." - ), - schema=MCQ_SCHEMA, - build_prompt=build_prompt, - unwrap=unwrap_questions, - render=render, - count_label="questions", -) diff --git a/components-Dinura/learnmate/resource_agent/persist.py b/components-Dinura/learnmate/resource_agent/persist.py deleted file mode 100644 index 39e4616..0000000 --- a/components-Dinura/learnmate/resource_agent/persist.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Node 3 of 3: persist. - -Stores the resource with its whole attempt trail. - -The last attempt is the answer, pass or fail -- once a regeneration has run, its output is -what the caller gets. Earlier attempts stay in the record so a run that needed a retry, -and what the judge objected to the first time, remains answerable afterwards. That trail -is the data behind content_store.evaluation_stats(); keeping only the winner would make -the question "is the threshold set anywhere near right" unanswerable after the fact. -""" - -from typing import Dict - -from ..storage import content_store -from .helpers import _log -from .state import ResourceState - - -def persist_node(state: ResourceState) -> Dict: - """Store the resource with its whole attempt trail.""" - attempts = state.get("attempts", []) - final = attempts[-1] if attempts else {} - - # Off for evaluation runs and tests, which want the content without adding a row. - if not state.get("persist", True): - return {"content": final.get("content"), "resource_id": None} - - resource_id = content_store.save_resource( - doc_id=state.get("doc_id"), - task=state["task"], - content=final.get("content"), - accepted=bool(state.get("passed")), - attempts=attempts, - verdict=state.get("verdict"), - source_preview=state.get("source", ""), - params={"count": state.get("count"), "threshold": state["threshold"], - "evaluated": state.get("evaluate", True)}, - ) - _log(state, f"[+] Stored resource {resource_id}") - - # `content` is republished from the last attempt so the caller always gets the text - # that was actually stored, even when the run ended on a failure. - return {"content": final.get("content"), "resource_id": str(resource_id)} diff --git a/components-Dinura/learnmate/resource_agent/practice_qsn.py b/components-Dinura/learnmate/resource_agent/practice_qsn.py deleted file mode 100644 index cf33187..0000000 --- a/components-Dinura/learnmate/resource_agent/practice_qsn.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Resource type 2 of 4: short-answer practice questions. - - PDF passage -> [{question, answer}, ...] - -The same {"questions": [...]} envelope as mcq -- hence the shared `unwrap_questions` -- -but each item carries a written answer instead of four options. Where an MCQ tests -recognition, these test recall, so the prompt pushes for questions whose answers are -stated in the passage without being a single word lifted from it. -""" - -from .task import Task, unwrap_questions - -PRACTICE_SCHEMA = { - "type": "object", - "properties": { - "questions": { - "type": "array", - "items": { - "type": "object", - "properties": { - "question": {"type": "string"}, - "answer": {"type": "string"}, - }, - "required": ["question", "answer"], - }, - } - }, - "required": ["questions"], -} - - -def build_prompt(source: str, count: int) -> str: - return f"""Write {count} short-answer practice questions based only on the passage below. - -PASSAGE: -\"\"\" -{source} -\"\"\" - -Rules: -- Each answer must be stated in, or directly inferable from, the passage. -- Answers are one or two sentences, not essays. -- Do not ask a question whose answer merely restates the question. -- Vary what the questions test: definitions, conditions, exceptions, consequences. - -Return JSON: {{"questions": [{{"question": "...", "answer": "..."}}]}}""" - - -def render(items) -> str: - """Numbered Q/A pairs -- the same shape as mcq, so neither collides with the rubric.""" - lines = [] - for i, item in enumerate(items or [], start=1): - lines.append(f"Q{i}. {item.get('question', '')}") - lines.append(f" A: {item.get('answer', '')}") - return "\n".join(lines) - - -PRACTICE_QSN = Task( - name="practice_qsn", - system_prompt=( - "You are an expert exam writer. You write short-answer practice questions whose " - "answers appear in the given passage, and you never invent facts the passage does " - "not state. Reply with JSON only." - ), - schema=PRACTICE_SCHEMA, - build_prompt=build_prompt, - unwrap=unwrap_questions, - render=render, - count_label="questions", -) diff --git a/components-Dinura/learnmate/resource_agent/routing.py b/components-Dinura/learnmate/resource_agent/routing.py deleted file mode 100644 index 77d5bc5..0000000 --- a/components-Dinura/learnmate/resource_agent/routing.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -The conditional edge out of check -- the only branch in the graph. - - passed? -> persist - out of attempts? -> persist anyway - otherwise -> back to generate, carrying the critique - -The budget is 2: one generation plus one regeneration. Raising it is not just slower -- a -3B judge tends to oscillate rather than converge over more rounds, and the third attempt -is usually a worse version of the first. -""" - -from .helpers import _log -from .state import ResourceState - - -def decide(state: ResourceState) -> str: - """Stop on a pass or on the attempt budget; otherwise go round again.""" - if state.get("passed"): - return "persist" - if state["attempt"] >= state["max_attempts"]: - return "persist" - _log(state, f"[*] Feedback: {state.get('critique')}") - return "generate" diff --git a/components-Dinura/learnmate/resource_agent/state.py b/components-Dinura/learnmate/resource_agent/state.py deleted file mode 100644 index e984942..0000000 --- a/components-Dinura/learnmate/resource_agent/state.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -The state that flows between nodes across one generation run. - -Same LangGraph rules as the chat agent: a node returns only the keys it wants to change -and the framework merges them, and a field opts out of overwrite-by-default with -`Annotated[type, reducer]`. `attempts` is the only field here that does. -""" - -from typing import Any, Dict, List, Optional - -from typing_extensions import Annotated, TypedDict - - -def _append(left: List, right: List) -> List: - """Reducer: attempts accumulate across loop iterations instead of overwriting.""" - return (left or []) + (right or []) - - -class ResourceState(TypedDict, total=False): - """One generation run's working memory.""" - - # --- Inputs: set by generate_resource, never changed after ------------------------ - task: str # "mcq", "summary", "keypoints", "practice_qsn" - source: str # the PDF passage, from ingestion.build_source_text - count: int # how many items -- or, for summary, roughly how many sentences - doc_id: Any # which document this came from, for the stored record - threshold: int # judge score out of 100 needed to pass - max_attempts: int # total generations allowed, including the first - evaluate: bool # False skips both gates - persist: bool # False returns the resource without storing it - verbose: bool # False silences progress logging - - # --- Written by generate ---------------------------------------------------------- - attempt: int # 1-based counter of generations so far - content: Any # the parsed resource: a list of items, or a summary string - started: float # when this attempt began, for the timing in the evaluation log - - # --- Written by check ------------------------------------------------------------- - stage: str # which gate decided this attempt: parse/validator/judge/skipped - passed: bool # did it clear both gates - verdict: Optional[Dict] # the judge's full result, None when a cheaper gate decided - critique: Optional[str] # what to fix, fed into the next generation - previous: Any # the rejected content, shown to the model on retry - - # --- Written by persist ----------------------------------------------------------- - resource_id: Optional[str] - - # The one accumulating field: the audit trail of the whole run. `content` keeps only - # the newest attempt; this keeps every attempt with the reason it was rejected, which - # is what makes a bad run answerable afterwards instead of guessed at. - attempts: Annotated[List[Dict], _append] diff --git a/components-Dinura/learnmate/resource_agent/summary.py b/components-Dinura/learnmate/resource_agent/summary.py deleted file mode 100644 index 5a8b0ed..0000000 --- a/components-Dinura/learnmate/resource_agent/summary.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Resource type 4 of 4: a summary. - - PDF passage -> "one block of connected prose" - -The odd one out. The other three produce a list of items and `count` means "how many"; -a summary is a single artefact, so `count` is read as a rough sentence budget instead. -That is the only place the shared Task contract has to bend. -""" - -from .task import Task - -SUMMARY_SCHEMA = { - "type": "object", - "properties": {"summary": {"type": "string"}}, - "required": ["summary"], -} - -# Used when the caller passes no count at all. -DEFAULT_SENTENCES = 5 - - -def build_prompt(source: str, count: int) -> str: - sentences = count or DEFAULT_SENTENCES - return f"""Summarise the passage below in about {sentences} sentences. - -PASSAGE: -\"\"\" -{source} -\"\"\" - -Rules: -- Include only what the passage states. Add no outside knowledge and no interpretation. -- Cover the main points rather than only the opening ones. -- Write plain connected prose, not a bulleted list. -- Do not open with a phrase like "This passage discusses"; state the content directly. - -Return JSON: {{"summary": "..."}}""" - - -def unwrap(data): - """Pull the string out of the {"summary": "..."} envelope.""" - if isinstance(data, dict): - return data.get("summary", "") - return data if isinstance(data, str) else str(data) - - -def render(text) -> str: - """Already plain prose; there is nothing to flatten.""" - return text if isinstance(text, str) else str(text) - - -SUMMARY = Task( - name="summary", - system_prompt=( - "You summarise passages faithfully. You never state anything the passage does not " - "support, and you never pad. Reply with JSON only." - ), - schema=SUMMARY_SCHEMA, - build_prompt=build_prompt, - unwrap=unwrap, - render=render, - count_label="sentences", -) diff --git a/components-Dinura/learnmate/resource_agent/task.py b/components-Dinura/learnmate/resource_agent/task.py deleted file mode 100644 index e8798be..0000000 --- a/components-Dinura/learnmate/resource_agent/task.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -The contract every kind of study resource implements. - -A Task owns everything type-specific and nothing else: - - system_prompt who the model is being asked to be - build_prompt the user message, built from the passage and a count - schema the JSON shape, compiled into a decoding grammar by llama.cpp - unwrap parsed JSON -> the content itself - render the content -> plain text, for the judge and for display - -Everything around it -- the model, the two evaluation gates, the retry contract, -persistence -- is shared, so adding a fifth kind of resource means adding one Task and -touching nothing else. The four that exist are one file each: mcq.py, practice_qsn.py, -keypoints.py, summary.py. -""" - -from dataclasses import dataclass -from typing import Any, Callable, Dict - - -@dataclass(frozen=True) -class Task: - """One kind of study resource. Frozen: a task definition is a constant.""" - - name: str - system_prompt: str - schema: Dict[str, Any] - build_prompt: Callable[[str, int], str] - unwrap: Callable[[Any], Any] - render: Callable[[Any], str] - # What `count` means for this task, shown in CLI help. "questions" for the two - # question types, "points" for key points, "sentences" for a summary. - count_label: str = "items" - - -def unwrap_questions(data): - """ - Pull the items out of a {"questions": [...]} envelope. - - Shared by mcq and practice_qsn, which differ in what each question contains but not - in how the list is wrapped. - - Every schema in this package wraps its payload in an object rather than returning a - bare array, because llama.cpp's grammar support is more reliable for a top-level - object. Unwrapping is what hides that from the rest of the system. A model that - ignored the envelope and returned the bare array anyway is still accepted -- there is - no reason to fail an attempt whose content is right. - """ - if isinstance(data, dict): - return data.get("questions", []) - return data if isinstance(data, list) else [] diff --git a/components-Dinura/learnmate/resource_agent/tasks.py b/components-Dinura/learnmate/resource_agent/tasks.py deleted file mode 100644 index 9a0a43d..0000000 --- a/components-Dinura/learnmate/resource_agent/tasks.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -The registry of resource types. - -One entry per kind of study resource that can be generated from a PDF passage. Adding a -fifth means writing one module beside the four below and adding a line here -- the graph, -the gates and the CLI all read this map rather than naming tasks themselves. - - mcq multiple-choice questions -> mcq.py - practice_qsn short-answer questions -> practice_qsn.py - keypoints a list of key points -> keypoints.py - summary one block of connected prose -> summary.py - -Each task name here must also appear in evaluator/validators.py (its structural gate) and -evaluator/rubrics.py (its grading criteria). A task missing from either still runs -- the -validator passes anything unknown through and the judge falls back to the generic -rubric -- but it is then being graded far more loosely than the other four. -""" - -from typing import Dict, List - -from .keypoints import KEYPOINTS -from .mcq import MCQ -from .practice_qsn import PRACTICE_QSN -from .summary import SUMMARY -from .task import Task - -TASKS: Dict[str, Task] = { - "mcq": MCQ, - "practice_qsn": PRACTICE_QSN, - "keypoints": KEYPOINTS, - "summary": SUMMARY, -} - -# Sorted so the CLI's --help and its `choices` list are stable between runs. -TASK_NAMES: List[str] = sorted(TASKS) - -__all__ = ["TASKS", "TASK_NAMES", "Task", "get_task", "render"] - - -def get_task(name: str) -> Task: - """Look up a task, naming the alternatives when it misses.""" - task = TASKS.get(name) - if task is None: - raise ValueError(f"Unknown task {name!r}; expected one of {TASK_NAMES}") - return task - - -def render(task_name: str, content) -> str: - """Flatten generated content into the plain text the judge grades.""" - task = TASKS.get(task_name) - return task.render(content) if task else str(content) diff --git a/components-Dinura/learnmate/resource_agent/whole_document.py b/components-Dinura/learnmate/resource_agent/whole_document.py deleted file mode 100644 index ca27f0b..0000000 --- a/components-Dinura/learnmate/resource_agent/whole_document.py +++ /dev/null @@ -1,274 +0,0 @@ -""" -Generating a list-shaped resource across a whole document. - -`generate_resource` sees one passage: MAX_SOURCE_CHARS of text, answered in one reply of -at most 1024 tokens. Both limits bite as soon as the ask is "for this PDF" rather than -"for this passage" -- nothing past the opening pages is ever read, and a large set comes -back truncated mid-item. So the document is split into groups of whole pages, each group -is asked for its share, and the results are pooled. - -Two ways to decide the share, because the two questions callers actually ask are different: - - count=40 a set of forty questions *about the document*. Forty questions buys five - reads, so five groups are sampled evenly across it. - per_page=2 two key points *for every page*. The rate is a promise about coverage, - so every group is read and the total follows from the page count. - -Each group goes through the normal graph and gets the structural gate, the judge and the -retry it would have got on its own. What this module adds is the part no single call can -do: spreading the work across the document, dropping items two groups both produced, and -running gate 1 over the *pooled* set -- the answer always in slot B is invisible inside a -batch of eight and obvious across forty. - - whole_document.py this: page loading, grouping, the pooled run - document_mcqs.py the count-based entry point for mcq - document_summary.py summaries, which fold rather than pool and so only borrow the - page loading and the grouping -""" - -import math -from typing import Dict, List, Optional, Tuple - -from .. import config -from ..evaluator import validators -from ..evaluator.normalise import norm -from ..storage import content_store, pdf_store -from .agent import generate_resource -from .helpers import _log - -# Pages shorter than this are a title page, a running head or a stray caption -- there is -# no prose on them to work from, and asking anyway spends a generation to be told so. -MIN_PAGE_CHARS = 200 - -# Items one generation call is asked for at most, by task. The generator emits at most -# max_tokens=1024, and these are that budget divided by what one item costs as JSON: an -# MCQ ~90 tokens (stem, four options, answer, keys), a practice question ~70 (question and -# a one-or-two sentence answer), a key point ~35 (one sentence). A call asked for -# materially more than its entry returns a set cut off partway through. -MAX_PER_CALL = {"mcq": 8, "practice_qsn": 10, "keypoints": 16} -DEFAULT_MAX_PER_CALL = 8 - -# Floors, where a task's own gate 1 refuses a set that small. validate_keypoints rejects a -# set of one outright, so a trailing group of a single page at one point per page would -# fail the gate for being exactly what it was asked for. Only tasks listed here get a -# floor: the others would just be over-delivering. -MIN_PER_CALL = {"keypoints": 2} - -# Smallest group worth generating from. Dividing a short document into one group per call -# can leave each with a paragraph, and questions written from a paragraph are trivia. -MIN_GROUP_CHARS = 1200 - -# What the CLI offers for a per-page rate. Not a limit: any rate can be passed directly. -PER_PAGE_CHOICES = (1, 2, 4) -DEFAULT_PER_PAGE = 2 - - -def load_page_texts(doc_id, purpose: str) -> List[str]: - """ - The prose of every page worth reading, in reading order. - - `purpose` is a verb for the error message ("summarise", "question"), so a document with - nothing usable says which thing it could not be used for. - - Raises ValueError when there is no stored page text, which means the document was never - ingested (or was ingested before store_pages existed -- re-ingest it). - """ - records = pdf_store.get_pages(doc_id) - if not records: - raise ValueError(f"No stored page text for document {doc_id}. Ingest the PDF first.") - - texts = [text for text in ((record.get("text") or "").strip() for record in records) - if len(text) >= MIN_PAGE_CHARS] - if not texts: - raise ValueError(f"Document {doc_id} has no page with enough text to {purpose}.") - return texts - - -def batched(texts: List[str], max_chars: int, max_count: int = None) -> List[List[str]]: - """ - Group in reading order into runs bounded by size and, optionally, by how many. - - Order is preserved rather than packed for density: a group of pages 1-8 reads as a - section, a group of pages 1, 40 and 97 reads as three non-sequiturs. - """ - groups: List[List[str]] = [] - group, size = [], 0 - for text in texts: - # `group and` keeps a single oversized entry in a group of its own instead of - # dropping it; the generator truncates it rather than never seeing it. - full = size + len(text) > max_chars or (max_count and len(group) >= max_count) - if group and full: - groups.append(group) - group, size = [], 0 - group.append(text) - size += len(text) + 2 - if group: - groups.append(group) - return groups - - -def _stem(task: str, item) -> str: - """ - What makes two items the same, for deduplication. - - Key points are bare strings and are compared whole; the two question types are - compared on their stem, because a model asked the same thing twice will word the - answer differently and the question identically. - """ - if task == "keypoints": - return norm(item) - return norm((item or {}).get("question") if isinstance(item, dict) else item) - - -def _plan(task: str, texts: List[str], count: Optional[int], per_page: Optional[int], - budget: int) -> Tuple[List[List[str]], Optional[List[int]], int]: - """ - Work out the groups, what to ask each of them, and the total being requested. - - Returns (groups, asks, requested). `asks` is None in count mode, where each group's - share depends on what the groups before it actually produced. - """ - per_call = MAX_PER_CALL.get(task, DEFAULT_MAX_PER_CALL) - - if per_page: - # A rate is a promise that every page is read, so the grouping is driven by how - # many pages' worth of items one call can hold and every group is used. - groups = batched(texts, budget, max_count=max(1, per_call // per_page)) - floor = MIN_PER_CALL.get(task, 1) - asks = [max(floor, per_page * len(group)) for group in groups] - return groups, asks, per_page * len(texts) - - # A total buys a fixed number of calls. Size the groups so there are about that many, - # clamped to what fits the window and to what is worth generating from. - calls = max(1, math.ceil(count / per_call)) - total = sum(len(text) + 2 for text in texts) - groups = batched(texts, min(budget, max(MIN_GROUP_CHARS, math.ceil(total / calls)))) - - # A document bigger than `calls` windows cannot be read whole in the calls a set of - # this size affords -- forty questions buys five reads, and a 400,000-character book is - # sixty-seven. Sample evenly across it rather than taking the first five groups: one - # group per fifth of the book is a set about the book, five groups from chapter one is - # a set about chapter one. The pages between samples are not read; ask for more items - # to read more of the document. - if len(groups) > calls: - step = len(groups) / calls - groups = [groups[int(index * step)] for index in range(calls)] - - return groups, None, count - - -def generate_document_items(task: str, doc_id, count: int = None, per_page: int = None, - threshold: int = None, max_attempts: int = None, - evaluate: bool = True, persist: bool = True, - verbose: bool = True, max_chars: int = None) -> Dict: - """ - Generate a list-shaped resource from across a whole document. - - task -- "mcq", "practice_qsn" or "keypoints". Not "summary": a summary is one - artefact and folds rather than pools, so it has its own module. - count -- how many items in total, sampled across the document, or - per_page -- how many items per page, reading every page. Pass exactly one. - - Everything else means what it does on `generate_resource`. `evaluate` applies per - group; `persist` stores the pooled set once, not each group. - - Returns generate_resource's dict, with `content` the pooled items and `attempts` every - group's trail concatenated, plus `requested`, `generated` and `groups`. `accepted` is - True only when every group was accepted *and* the pooled set passes the structural - gate: a set can be built entirely from accepted batches and still be biased as a whole. - - Fewer items than requested come back when the document is too short to support them or - the generator repeated itself; the shortfall is reported, never padded. - """ - if (count is None) == (per_page is None): - raise ValueError("Pass exactly one of count= (a total) or per_page= (a rate).") - if (count or per_page) < 1: - raise ValueError("count/per_page must be at least 1.") - - budget = max_chars or config.MAX_SOURCE_CHARS - texts = load_page_texts(doc_id, "question" if task != "keypoints" else "summarise") - groups, asks, requested = _plan(task, texts, count, per_page, budget) - - _log({"verbose": verbose}, - f"[*] {requested} {task} item(s) from {len(texts)} page(s), " - + (f"at {per_page} per page, " if per_page else "") - + f"in {len(groups)} group(s)...") - - # --- One call per group ------------------------------------------------------------- - items: List = [] - attempts: List[Dict] = [] - stems = set() - every_group_accepted = True - - for index, group in enumerate(groups, start=1): - if asks is not None: - ask = asks[index - 1] - else: - # Share out what is still missing rather than a fixed slice, so a group that - # came back short is made up for by the ones after it. - remaining = requested - len(items) - if remaining <= 0: - break - per_call = MAX_PER_CALL.get(task, DEFAULT_MAX_PER_CALL) - ask = max(1, min(per_call, math.ceil(remaining / (len(groups) - index + 1)))) - - _log({"verbose": verbose}, - f"[*] Group {index}/{len(groups)}: asking for {ask} item(s)...") - result = generate_resource( - task, "\n\n".join(group)[:budget], count=ask, doc_id=doc_id, - threshold=threshold, max_attempts=max_attempts, evaluate=evaluate, - persist=False, verbose=verbose) - - attempts.extend(result.get("attempts", [])) - every_group_accepted = every_group_accepted and bool(result.get("accepted")) - - for item in result.get("content") or []: - # Neighbouring pages routinely produce the same item twice, and every gate 1 - # checker counts a duplicate as a fault. - stem = _stem(task, item) - if stem and stem not in stems: - stems.add(stem) - items.append(item) - - # A total is a ceiling; a rate is not, since each group was asked for exactly its - # pages' share and going under it is the only way to miss. - if count: - items = items[:count] - - # --- Gate 1 on the pooled set ------------------------------------------------------- - # Cheap, and the only check that can see across groups. - passed, reasons = validators.validate(task, items) - if not passed: - _log({"verbose": verbose}, f"[!] Pooled set: {'; '.join(reasons)}") - if len(items) < requested: - _log({"verbose": verbose}, - f"[!] Got {len(items)} of the {requested} item(s) asked for.") - - accepted = every_group_accepted and passed - - resource_id = None - if persist: - resource_id = content_store.save_resource( - doc_id=doc_id, task=task, content=items, accepted=accepted, - attempts=attempts, verdict=None, - source_preview="\n\n".join(groups[0]) if groups else "", - params={"count": requested, "per_page": per_page, "generated": len(items), - "groups": len(groups), "evaluated": evaluate, "whole_document": True, - "threshold": threshold if threshold is not None - else config.EVALUATOR_THRESHOLD}) - _log({"verbose": verbose}, f"[+] Stored resource {resource_id}") - - return { - "task": task, - "content": items, - "accepted": accepted, - # No single verdict: each group was judged on its own and the trail is in - # `attempts`. Reporting one group's score as the set's would be a lie. - "verdict": None, - "attempts": attempts, - "resource_id": str(resource_id) if resource_id else None, - "requested": requested, - "generated": len(items), - "groups": len(groups), - "per_page": per_page, - } diff --git a/components-Dinura/learnmate/storage/__init__.py b/components-Dinura/learnmate/storage/__init__.py deleted file mode 100644 index 8a07609..0000000 --- a/components-Dinura/learnmate/storage/__init__.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Persistence: two databases, each in its own container with its own named volume. - - MongoDB (learnmate-mongo, :27018) everything that is not a vector - Qdrant (learnmate-qdrant, :6335) the chunk embeddings - -`docker compose up -d` starts both. `docker compose down` keeps the data; only -`down -v` clears it. - -The split is deliberate and asymmetric. MongoDB holds the PDFs (GridFS), the cleaned page -text, session bindings, chat history, generated resources and the evaluation log -- none -of which can be derived from anything else, so losing it loses the corpus. The vectors are -computed *from* that page text, so losing Qdrant only costs a re-ingest. That is why the -vector backend is swappable and MongoDB is not. - -What is stored, per session kind: - - a chat session sessions + chat_turns + the chunks it retrieves from - a resource session sessions + pages (read whole) + resources + evaluations - -Both kinds point at the same document record, so one PDF serves both without being -embedded twice. - -Where things live, in reading order: - - mongo.py the connection, and StorageUnavailable - indexes.py every index the queries depend on, including the three unique ones - ids.py ObjectId coercion, shared by every module that queries by id - - pdf_files.py the PDF bytes, in GridFS - documents.py the document record: store, look up, resolve, delete - pages.py the cleaned page text a resource session reads - pdf_store.py facade over those three - - sessions.py which PDF a session is about, and what it is for - history.py chat turns - resources.py generated resources, with their whole attempt trail - evaluations.py the verdict log and its statistics - content_store.py facade over those four - - vectors.py picks the vector backend - qdrant_vectors.py a Qdrant server: real HNSW, filtering server-side - mongo_vectors.py the same interface over MongoDB, when a second service is not wanted -""" - -from .content_store import ( - bind_session_document, - clear_history, - evaluation_stats, - get_resource, - get_session, - list_resources, - list_sessions, - load_history, - log_evaluation, - save_resource, - save_turn, - session_doc_id, - stage_counts, - unbind_session, -) -from .mongo import StorageUnavailable, close, ensure_indexes, get_db, supports_vector_search -from .mongo_vectors import MongoVectorStore -from .pdf_store import ( - delete_document, - export_pdf, - find_by_hash, - get_active_document, - get_document, - get_pages, - get_pdf_bytes, - list_documents, - mark_ingested, - read_source, - resolve_document, - store_pages, - store_pdf, -) -from .qdrant_vectors import QdrantUnavailable, QdrantVectorStore -from .vectors import build_vector_store, get_vector_store, reset_vector_store - -__all__ = [ - "MongoVectorStore", - "QdrantUnavailable", - "QdrantVectorStore", - "StorageUnavailable", - "bind_session_document", - "build_vector_store", - "clear_history", - "close", - "delete_document", - "ensure_indexes", - "evaluation_stats", - "export_pdf", - "find_by_hash", - "get_active_document", - "get_db", - "get_document", - "get_pages", - "get_pdf_bytes", - "get_resource", - "get_session", - "get_vector_store", - "list_documents", - "list_resources", - "list_sessions", - "load_history", - "log_evaluation", - "mark_ingested", - "read_source", - "reset_vector_store", - "resolve_document", - "save_resource", - "save_turn", - "session_doc_id", - "stage_counts", - "store_pages", - "store_pdf", - "supports_vector_search", - "unbind_session", -] diff --git a/components-Dinura/learnmate/storage/content_store.py b/components-Dinura/learnmate/storage/content_store.py deleted file mode 100644 index 164b801..0000000 --- a/components-Dinura/learnmate/storage/content_store.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Everything the agents produce and read back: the facade over four modules. - - sessions.py which PDF a session is about, and what it is for - history.py chat turns - resources.py generated MCQs, questions, key points and summaries - evaluations.py the verdict log, and the statistics read off it - -Kept as one importable name because every node in both agents already reads -`content_store.save_turn(...)`, `content_store.log_evaluation(...)`. Import the specific -module when you care which collection you are touching. -""" - -from .evaluations import evaluation_stats, log_evaluation, stage_counts -from .history import clear_history, list_sessions, load_history, save_turn -from .resources import get_resource, list_resources, save_resource -from .sessions import ( - bind_session_document, - get_session, - session_doc_id, - unbind_session, -) - -__all__ = [ - "bind_session_document", - "clear_history", - "evaluation_stats", - "get_resource", - "get_session", - "list_resources", - "list_sessions", - "load_history", - "log_evaluation", - "save_resource", - "save_turn", - "session_doc_id", - "stage_counts", - "unbind_session", -] diff --git a/components-Dinura/learnmate/storage/documents.py b/components-Dinura/learnmate/storage/documents.py deleted file mode 100644 index b45c1c8..0000000 --- a/components-Dinura/learnmate/storage/documents.py +++ /dev/null @@ -1,185 +0,0 @@ -""" -The document record: one row per ingested PDF. - - {_id, filename, sha256, size_bytes, gridfs_id, uploaded_at, n_pages, n_chunks, - ingested_at} - -Identity is the SHA-256 of the file bytes, not the filename. Uploading the same PDF twice -under different names is recognised as one document, which keeps the embedding work from -being repeated and gives every stored resource a stable document to point at. - -Retrieval is by indexed _id or sha256, so fetching metadata is a single indexed lookup -and fetching bytes is a GridFS read by id. -""" - -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Optional, Union - -import hashlib - -from bson import ObjectId - -from .. import config -from . import pdf_files -from .ids import coerce_id -from .mongo import get_db - - -def _collection(): - return get_db()[config.COLL_DOCUMENTS] - - -# --- Writing --------------------------------------------------------------------------- - -def store_pdf(source: Union[str, Path, bytes], filename: str = None) -> Dict: - """ - Store a PDF and return its document record. - - `source` is a path or the raw bytes, so this serves both a CLI path argument and an - HTTP upload without the caller staging a temp file. - - A PDF already present is returned untouched with `existing` set, letting the ingestion - pipeline skip re-embedding work that has already been done. That is also what makes - opening a second session on the same PDF nearly free. - """ - data, filename = pdf_files.read_source(source, filename) - digest = hashlib.sha256(data).hexdigest() - - existing = _collection().find_one({"sha256": digest}) - if existing: - existing["existing"] = True - return existing - - # Bytes first: a record pointing at nothing would be worse than orphaned bytes, which - # the unique sha256 index means can only ever happen once per file anyway. - gridfs_id = pdf_files.put(filename, data, digest) - - record = { - "filename": filename, - "sha256": digest, - "size_bytes": len(data), - "gridfs_id": gridfs_id, - "uploaded_at": datetime.now(timezone.utc), - # Filled in by the ingestion pipeline once the text has been processed. - "n_pages": None, - "n_chunks": None, - "ingested_at": None, - } - record["_id"] = _collection().insert_one(record).inserted_id - record["existing"] = False - return record - - -def mark_ingested(doc_id: Union[str, ObjectId], n_pages: int, n_chunks: int) -> None: - """Record the outcome of ingestion on the document record.""" - _collection().update_one( - {"_id": coerce_id(doc_id)}, - {"$set": {"n_pages": n_pages, "n_chunks": n_chunks, - "ingested_at": datetime.now(timezone.utc)}}, - ) - - -def delete_document(doc_id: Union[str, ObjectId]) -> bool: - """ - Remove a document, its stored PDF and the text derived from it. - - Note this clears the `chunks` collection directly, which is where vectors live only - on the MongoDB backend. A caller using Qdrant must also delete through the vector - store -- see ingestion, which does exactly that. - """ - document = get_document(doc_id) - if not document: - return False - - database = get_db() - database[config.COLL_CHUNKS].delete_many({"doc_id": document["_id"]}) - database[config.COLL_PAGES].delete_many({"doc_id": document["_id"]}) - pdf_files.drop(document.get("gridfs_id")) - database[config.COLL_DOCUMENTS].delete_one({"_id": document["_id"]}) - return True - - -# --- Reading --------------------------------------------------------------------------- - -def get_document(doc_id: Union[str, ObjectId]) -> Optional[Dict]: - """Look up one document's metadata by id.""" - oid = coerce_id(doc_id) - if oid is None: - return None - return _collection().find_one({"_id": oid}) - - -def find_by_hash(digest: str) -> Optional[Dict]: - """Look up a document by the SHA-256 of its bytes.""" - return _collection().find_one({"sha256": digest}) - - -def get_pdf_bytes(doc_id: Union[str, ObjectId]) -> Optional[bytes]: - """Read a stored PDF back out of GridFS.""" - document = get_document(doc_id) - if not document: - return None - return pdf_files.get(document.get("gridfs_id")) - - -def export_pdf(doc_id: Union[str, ObjectId], destination: Union[str, Path]) -> Path: - """Write a stored PDF back to disk.""" - data = get_pdf_bytes(doc_id) - if data is None: - raise KeyError(f"No stored PDF for document {doc_id}") - - destination = Path(destination) - if destination.is_dir(): - destination = destination / get_document(doc_id)["filename"] - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(data) - return destination - - -def list_documents(limit: int = 50) -> List[Dict]: - """Most recently uploaded documents first.""" - return list(_collection().find().sort("uploaded_at", -1).limit(limit)) - - -def get_active_document() -> Optional[Dict]: - """ - The most recently ingested PDF, or None if nothing is ingested. - - The fallback for a command given neither a document nor a session. A session's own - PDF -- see sessions.py -- always takes precedence over this. - """ - return _collection().find_one(sort=[("uploaded_at", -1)]) - - -def resolve_document(reference: str) -> Optional[Dict]: - """ - Find a document by id, exact filename, or a unique filename fragment. - - Typing an ObjectId by hand is miserable, so the CLI accepts `--doc constitution` and - this resolves it. A fragment matching several documents is rejected rather than - guessed at, so a command never silently targets the wrong PDF. - """ - if not reference: - return None - - document = get_document(reference) - if document: - return document - - exact = _collection().find_one({"filename": reference}) - if exact: - return exact - - import re - - pattern = re.escape(reference) - matches = list(_collection() - .find({"filename": {"$regex": pattern, "$options": "i"}}) - .limit(5)) - if len(matches) == 1: - return matches[0] - if len(matches) > 1: - names = ", ".join(match["filename"] for match in matches) - raise ValueError(f"{reference!r} matches several documents: {names}") - return None diff --git a/components-Dinura/learnmate/storage/evaluations.py b/components-Dinura/learnmate/storage/evaluations.py deleted file mode 100644 index fbf1171..0000000 --- a/components-Dinura/learnmate/storage/evaluations.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -The evaluation log, and the statistics read back off it. - -Every verdict is recorded -- passes as well as failures -- so score distributions and -timings can be analysed afterwards rather than only failures being visible. - -`stage` says which gate decided an attempt, and that is the more useful of the two -questions this collection answers: - - parse the generator's output could not be read at all - validator structural checks rejected it without the judge running - judge the model scored it - skipped evaluation was switched off - -If the validator is deciding most attempts, the generation prompt needs work, not the -threshold. -""" - -from datetime import datetime, timezone -from typing import Dict, Optional - -from .. import config -from .ids import as_object_id -from .mongo import get_db - - -def _collection(): - return get_db()[config.COLL_EVALUATIONS] - - -def log_evaluation(task: str, attempt: int, score, passed: bool, threshold: int, - stage: str = "judge", elapsed: float = None, - doc_id=None, extra: Optional[Dict] = None) -> None: - """ - Record one verdict. - - Never raises: logging must not be able to take down a generation run. A lost row is a - gap in the statistics; an exception here would be a lost resource. - """ - record = { - "task": task, - "attempt": attempt, - "stage": stage, - "score": score, - "passed": bool(passed), - "threshold": threshold, - "doc_id": as_object_id(doc_id), - "created_at": datetime.now(timezone.utc), - } - if elapsed is not None: - record["elapsed_s"] = round(elapsed, 2) - if extra: - record.update(extra) - - try: - _collection().insert_one(record) - except Exception: - pass - - -def evaluation_stats() -> Dict[str, Dict]: - """ - Score distribution per task, for deciding whether the threshold is meaningful. - - A judge whose scores cluster in a narrow band cannot separate good from bad at any - threshold, however it is set, and `distinct` is what exposes that. Only judge-stage - rows are counted -- a validator rejection has no score to average. - """ - pipeline = [ - {"$match": {"stage": "judge", "score": {"$type": "number"}}}, - {"$group": { - "_id": "$task", - "n": {"$sum": 1}, - "min": {"$min": "$score"}, - "max": {"$max": "$score"}, - "avg": {"$avg": "$score"}, - "scores": {"$push": "$score"}, - "passes": {"$sum": {"$cond": ["$passed", 1, 0]}}, - }}, - {"$sort": {"_id": 1}}, - ] - - out = {} - for row in _collection().aggregate(pipeline): - scores = sorted(row["scores"]) - out[row["_id"]] = { - "n": row["n"], - "min": row["min"], - "median": scores[len(scores) // 2], - "max": row["max"], - "mean": round(row["avg"], 1), - "distinct": len(set(scores)), - "pass_rate": round(row["passes"] / row["n"], 3), - } - return out - - -def stage_counts() -> Dict[str, int]: - """How many evaluations each gate decided -- parse, validator, judge or skipped.""" - pipeline = [{"$group": {"_id": "$stage", "n": {"$sum": 1}}}] - return {row["_id"]: row["n"] for row in _collection().aggregate(pipeline)} diff --git a/components-Dinura/learnmate/storage/history.py b/components-Dinura/learnmate/storage/history.py deleted file mode 100644 index 7929527..0000000 --- a/components-Dinura/learnmate/storage/history.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Chat history: one row per turn. - - {session_id, role, content, doc_id, meta, created_at} - -This is what makes a conversation outlive the process. The chat agent's `rewrite` node -resolves pronouns against exactly what is stored here, so a session resumed tomorrow -behaves like one that never stopped. - -`meta` on an assistant turn carries the audit trail beside the text -- which mode -answered, what it scored, whether it was accepted, how many attempts it took, and which -pages it drew on. -""" - -from datetime import datetime, timezone -from typing import Dict, List, Optional - -from .. import config -from .ids import as_object_id -from .mongo import get_db - - -def _collection(): - return get_db()[config.COLL_CHAT_TURNS] - - -def save_turn(session_id: str, role: str, content: str, doc_id=None, - meta: Optional[Dict] = None) -> None: - """Append one chat turn.""" - _collection().insert_one({ - "session_id": session_id, - "role": role, - "content": content, - "doc_id": as_object_id(doc_id), - "meta": meta or {}, - "created_at": datetime.now(timezone.utc), - }) - - -def load_history(session_id: str, max_turns: int = None) -> List[Dict[str, str]]: - """ - The last N user+assistant pairs, oldest first. - - Fetched newest-first then reversed: with the index on (session_id, created_at) that is - a bounded scan, where sorting the whole session forward and taking the tail is not. - - Bounded rather than complete because history goes into every prompt -- an unbounded - conversation would eventually crowd the retrieved context out of a 4k window. - """ - max_turns = max_turns or config.MAX_HISTORY_TURNS - rows = list(_collection() - .find({"session_id": session_id}, {"role": 1, "content": 1}) - .sort("created_at", -1) - .limit(max_turns * 2)) - rows.reverse() - return [{"role": row["role"], "content": row["content"]} for row in rows] - - -def clear_history(session_id: str) -> int: - """Forget a session's turns. Returns how many were removed.""" - return _collection().delete_many({"session_id": session_id}).deleted_count - - -def list_sessions(limit: int = 20) -> List[Dict]: - """Known chat sessions, most recently active first.""" - pipeline = [ - {"$group": {"_id": "$session_id", "turns": {"$sum": 1}, - "last": {"$max": "$created_at"}}}, - {"$sort": {"last": -1}}, - {"$limit": limit}, - ] - return list(_collection().aggregate(pipeline)) diff --git a/components-Dinura/learnmate/storage/ids.py b/components-Dinura/learnmate/storage/ids.py deleted file mode 100644 index 0902195..0000000 --- a/components-Dinura/learnmate/storage/ids.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Turning whatever a caller passed into a MongoDB ObjectId. - -Document ids cross a lot of boundaries in this system -- a CLI argument, a report dict, a -Qdrant payload, an HTTP request -- and arrive as ObjectId, hex string, or something that -is neither. Every module that queries by id needs the same coercion, so it lives here -once. - -Two variants, because two different failures are wanted: - - as_object_id returns the value unchanged when it is not an id, so a query on it - simply matches nothing instead of raising mid-write - coerce_id returns None, so a lookup can answer "no such document" cleanly -""" - -from typing import Any, Optional, Union - -from bson import ObjectId -from bson.errors import InvalidId - - -def as_object_id(value) -> Union[ObjectId, Any]: - """Coerce to ObjectId, passing the original through when it cannot be one.""" - if isinstance(value, ObjectId) or value is None: - return value - try: - return ObjectId(str(value)) - except (InvalidId, TypeError): - return value - - -def coerce_id(doc_id: Union[str, ObjectId]) -> Optional[ObjectId]: - """Accept an ObjectId or its hex string; return None if it is neither.""" - if isinstance(doc_id, ObjectId): - return doc_id - try: - return ObjectId(str(doc_id)) - except (InvalidId, TypeError): - return None diff --git a/components-Dinura/learnmate/storage/indexes.py b/components-Dinura/learnmate/storage/indexes.py deleted file mode 100644 index 564c4a2..0000000 --- a/components-Dinura/learnmate/storage/indexes.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Every index the queries in this package depend on. - -Created once on first connect. All of them are idempotent, so calling ensure_indexes() -repeatedly is free -- which is what lets get_db() call it unconditionally. - -Three of them are `unique`, and each enforces a rule the rest of the system relies on -rather than merely making a query fast: - - documents.sha256 one document per set of bytes, so re-uploading - the same PDF is recognised instead of duplicated - chunks.(doc_id, page, chunk_index) re-ingesting overwrites a chunk in place - sessions.session_id one PDF per session, structurally -""" - -from pymongo import ASCENDING, DESCENDING -from pymongo.database import Database - -from .. import config - - -def create_indexes(database: Database) -> None: - """Create every index, idempotently.""" - # --- PDFs and their derived text -------------------------------------------------- - # sha256 unique is what makes "store once, retrieve quickly" hold across sessions. - database[config.COLL_DOCUMENTS].create_index([("sha256", ASCENDING)], unique=True) - # get_active_document() and list_documents() both read newest-first. - database[config.COLL_DOCUMENTS].create_index([("uploaded_at", DESCENDING)]) - - # Retrieval always filters by document, and re-ingesting must overwrite a chunk - # rather than append a second copy of it. - database[config.COLL_CHUNKS].create_index([("doc_id", ASCENDING)]) - database[config.COLL_CHUNKS].create_index( - [("doc_id", ASCENDING), ("page_number", ASCENDING), ("chunk_index", ASCENDING)], - unique=True, - ) - - # Whole cleaned page text, kept alongside the chunks. Chunks overlap by design, so - # joining them back together duplicates text at every boundary; resource generation - # needs the page as it actually reads. - database[config.COLL_PAGES].create_index( - [("doc_id", ASCENDING), ("page_number", ASCENDING)], unique=True) - - # --- Generated content and its audit trail ---------------------------------------- - database[config.COLL_RESOURCES].create_index( - [("doc_id", ASCENDING), ("task", ASCENDING), ("created_at", DESCENDING)]) - database[config.COLL_EVALUATIONS].create_index([("created_at", DESCENDING)]) - database[config.COLL_EVALUATIONS].create_index([("task", ASCENDING)]) - - # --- Sessions --------------------------------------------------------------------- - # load_history reads the newest N turns of one session; this index makes that a - # bounded scan rather than a sort of the whole collection. - database[config.COLL_CHAT_TURNS].create_index( - [("session_id", ASCENDING), ("created_at", ASCENDING)]) - - # One record per session. Unique because the binding is what enforces one PDF per - # session -- two records for the same id would mean two answers to "which document - # is this session about". - database[config.COLL_SESSIONS].create_index([("session_id", ASCENDING)], unique=True) diff --git a/components-Dinura/learnmate/storage/mongo.py b/components-Dinura/learnmate/storage/mongo.py deleted file mode 100644 index 4ea68bd..0000000 --- a/components-Dinura/learnmate/storage/mongo.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -The MongoDB connection. - -One external server holds everything the system persists that is not a vector: the -uploaded PDFs (GridFS), their cleaned page text, session bindings, chat history, the -generated resources and the evaluation log. Nothing is written to local disk, so any -process that can reach the URI sees the same corpus. - -It runs as this project's own container -- `docker compose up -d mongo`, published on -27018 with a named volume -- rather than a server shared with other projects. Losing this -database loses the corpus; losing the vector database only costs a re-ingest. - -Indexes live next door in indexes.py and are created on first connect. -""" - -from typing import Optional - -from pymongo import MongoClient -from pymongo.database import Database -from pymongo.errors import PyMongoError, ServerSelectionTimeoutError - -from .. import config -from .indexes import create_indexes - -_CLIENT: Optional[MongoClient] = None -_INDEXES_READY = False - - -class StorageUnavailable(RuntimeError): - """Raised when MongoDB cannot be reached, with the URI that was tried.""" - - -def get_client() -> MongoClient: - """Connect to MongoDB, verifying the server is actually reachable.""" - global _CLIENT - if _CLIENT is None: - client = MongoClient(config.MONGODB_URI, serverSelectionTimeoutMS=5000) - try: - # MongoClient is lazy; ping so a bad URI fails here with a clear message - # rather than deep inside an unrelated query later. - client.admin.command("ping") - except ServerSelectionTimeoutError as exc: - raise StorageUnavailable( - f"Cannot reach MongoDB at {config.MONGODB_URI}. Start it with " - f"`docker compose up -d mongo` (see docker-compose.yml), or set " - f"LEARNMATE_MONGODB_URI. Original error: {exc}" - ) from exc - _CLIENT = client - return _CLIENT - - -def get_db() -> Database: - """The LearnMate database, with indexes ensured on first use.""" - database = get_client()[config.MONGODB_DB] - ensure_indexes(database) - return database - - -def ensure_indexes(database: Database = None) -> None: - """Create the indexes the queries in this package depend on, once per process.""" - global _INDEXES_READY - if _INDEXES_READY: - return - - database = database if database is not None else get_client()[config.MONGODB_DB] - create_indexes(database) - _INDEXES_READY = True - - -def supports_vector_search(database: Database = None) -> bool: - """ - Whether the server can run $vectorSearch. - - Atlas exposes it through Atlas Search; a community server does not have it at any - version. The answer decides whether MongoVectorStore pushes similarity down to the - server or scores in NumPy, so it is checked rather than assumed. - """ - database = database if database is not None else get_db() - try: - list(database[config.COLL_CHUNKS].list_search_indexes()) - return True - except PyMongoError: - return False - - -def close() -> None: - """Drop the connection. Only needed by long-lived callers and tests.""" - global _CLIENT, _INDEXES_READY - if _CLIENT is not None: - _CLIENT.close() - _CLIENT = None - _INDEXES_READY = False diff --git a/components-Dinura/learnmate/storage/mongo_vectors.py b/components-Dinura/learnmate/storage/mongo_vectors.py deleted file mode 100644 index 6f53bf4..0000000 --- a/components-Dinura/learnmate/storage/mongo_vectors.py +++ /dev/null @@ -1,255 +0,0 @@ -""" -A LangChain VectorStore backed by a MongoDB collection. - -The alternative to the Qdrant backend, selected with LEARNMATE_VECTOR_BACKEND=mongodb. -It exists so the system can run without a second service: vectors go in the same external -database that already holds the PDFs, page text and generated content. Qdrant is the -better choice when it is available -- it has a real HNSW index and does its filtering -server-side -- but one fewer moving part is sometimes worth more than that. - -Vectors live in `chunks.embedding`, beside the documents they came from. There is no -embedded database and no local cache directory, so every process reads one shared corpus. - -Similarity is computed one of two ways, chosen by what the server can actually do: - - $vectorSearch Atlas only, an approximate-nearest-neighbour index, scales to millions - NumPy everything else, an exact scan of the candidate chunks - -The fallback is not a degraded mode for a study corpus. A few thousand 384-dimension -vectors score in a couple of milliseconds and the result is exact rather than approximate; -Atlas only starts to win once the corpus outgrows memory. The capability is probed once -and cached, so pointing LEARNMATE_MONGODB_URI at Atlas switches path with no code change. - -Both paths return the same thing: raw cosine similarity in [-1, 1]. Atlas reports cosine -remapped to [0, 1], so it is converted back here -- otherwise RELEVANCE_THRESHOLD would -silently mean two different things on the two backends. -""" - -from datetime import datetime, timezone -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union - -import numpy as np -from bson import ObjectId -from langchain_core.documents import Document -from langchain_core.embeddings import Embeddings -from langchain_core.vectorstores import VectorStore -from pymongo import UpdateOne -from pymongo.errors import PyMongoError - -from .. import config -from ..llm.embeddings import get_embeddings -from .mongo import get_db, supports_vector_search - - -def _as_object_id(value) -> Union[ObjectId, Any]: - if isinstance(value, ObjectId): - return value - try: - return ObjectId(str(value)) - except Exception: - return value - - -class MongoVectorStore(VectorStore): - """Chunk vectors in MongoDB, with Atlas $vectorSearch when the server offers it.""" - - def __init__(self, embedding: Embeddings = None, collection_name: str = None, - index_name: str = None): - self._embedding = embedding or get_embeddings() - self.collection_name = collection_name or config.COLL_CHUNKS - self.index_name = index_name or config.VECTOR_INDEX_NAME - self._vector_search_supported: Optional[bool] = None - - # --- LangChain plumbing ---------------------------------------------------------- - - @property - def embeddings(self) -> Embeddings: - return self._embedding - - @property - def collection(self): - return get_db()[self.collection_name] - - @classmethod - def from_texts(cls, texts: List[str], embedding: Embeddings, - metadatas: Optional[List[dict]] = None, **kwargs: Any) -> "MongoVectorStore": - store = cls(embedding=embedding, **kwargs) - store.add_texts(texts, metadatas) - return store - - # --- Writing --------------------------------------------------------------------- - - def add_texts(self, texts: Iterable[str], metadatas: Optional[List[dict]] = None, - **kwargs: Any) -> List[str]: - """ - Embed and upsert chunks. - - Keyed on (doc_id, page_number, chunk_index), so re-ingesting a document replaces - its chunks instead of piling up a second copy beside them. - """ - texts = list(texts) - if not texts: - return [] - - metadatas = metadatas or [{} for _ in texts] - vectors = self._embedding.embed_documents(texts) - now = datetime.now(timezone.utc) - - operations, keys = [], [] - for text, metadata, vector in zip(texts, metadatas, vectors): - doc_id = _as_object_id(metadata.get("doc_id")) - key = { - "doc_id": doc_id, - "page_number": metadata.get("page_number", 0), - "chunk_index": metadata.get("chunk_index", 0), - } - payload = { - **key, - "text": text, - "embedding": vector, - "filename": metadata.get("filename"), - "source": metadata.get("source"), - "updated_at": now, - } - operations.append(UpdateOne(key, {"$set": payload}, upsert=True)) - keys.append(f"{doc_id}:{key['page_number']}:{key['chunk_index']}") - - self.collection.bulk_write(operations, ordered=False) - return keys - - def delete(self, ids: Optional[List[str]] = None, doc_id=None, **kwargs: Any) -> bool: - """Delete by chunk id, or every chunk of one document.""" - if doc_id is not None: - self.collection.delete_many({"doc_id": _as_object_id(doc_id)}) - return True - if ids: - self.collection.delete_many({"_id": {"$in": [_as_object_id(i) for i in ids]}}) - return True - return False - - # --- Reading --------------------------------------------------------------------- - - def _can_vector_search(self) -> bool: - if self._vector_search_supported is None: - self._vector_search_supported = supports_vector_search(get_db()) - return self._vector_search_supported - - def similarity_search(self, query: str, k: int = None, doc_id=None, - **kwargs: Any) -> List[Document]: - return [doc for doc, _ in - self.similarity_search_with_score(query, k=k, doc_id=doc_id, **kwargs)] - - def similarity_search_with_score(self, query: str, k: int = None, doc_id=None, - **kwargs: Any) -> List[Tuple[Document, float]]: - """Top-k chunks with raw cosine similarity, optionally restricted to one document.""" - k = k or config.TOP_K - vector = self._embedding.embed_query(query) - - query_filter: Dict[str, Any] = {} - if doc_id is not None: - query_filter["doc_id"] = _as_object_id(doc_id) - - if self._can_vector_search(): - try: - return self._atlas_search(vector, k, query_filter) - except PyMongoError: - # An Atlas cluster without the index built yet reaches here. Score exactly - # instead of returning nothing, and stop probing. - self._vector_search_supported = False - - return self._numpy_search(vector, k, query_filter) - - def _atlas_search(self, vector: List[float], k: int, - query_filter: Dict[str, Any]) -> List[Tuple[Document, float]]: - stage: Dict[str, Any] = { - "index": self.index_name, - "path": "embedding", - "queryVector": vector, - # Oversampling the ANN candidate pool is the standard accuracy/latency trade; - # 10x is Atlas's own recommendation for small k. - "numCandidates": max(k * 10, 100), - "limit": k, - } - if query_filter: - stage["filter"] = query_filter - - pipeline = [ - {"$vectorSearch": stage}, - {"$project": {"text": 1, "page_number": 1, "doc_id": 1, "chunk_index": 1, - "filename": 1, "score": {"$meta": "vectorSearchScore"}}}, - ] - - results = [] - for row in self.collection.aggregate(pipeline): - # Atlas reports cosine as (1 + cos) / 2; undo it so both paths agree. - cosine = 2.0 * float(row.get("score", 0.0)) - 1.0 - results.append((self._to_document(row), cosine)) - return results - - def _numpy_search(self, vector: List[float], k: int, - query_filter: Dict[str, Any]) -> List[Tuple[Document, float]]: - """ - Exact cosine similarity over the candidate chunks. - - Both stored and query vectors are L2-normalised by LearnMateEmbeddings, so cosine - similarity is a plain dot product and the whole scan is one matrix multiply. - """ - projection = {"text": 1, "page_number": 1, "doc_id": 1, "chunk_index": 1, - "filename": 1, "embedding": 1} - rows = list(self.collection.find(query_filter, projection)) - if not rows: - return [] - - matrix = np.asarray([row["embedding"] for row in rows], dtype=np.float32) - query_vector = np.asarray(vector, dtype=np.float32) - - # Guard against vectors stored unnormalised (e.g. written by an older ingest). - norms = np.linalg.norm(matrix, axis=1) - norms[norms == 0] = 1.0 - query_norm = np.linalg.norm(query_vector) or 1.0 - - scores = (matrix @ query_vector) / (norms * query_norm) - - top = np.argsort(-scores)[:k] - return [(self._to_document(rows[i]), float(scores[i])) for i in top] - - @staticmethod - def _to_document(row: Dict[str, Any]) -> Document: - return Document( - page_content=row.get("text", ""), - metadata={ - "chunk_id": str(row.get("_id", "")), - "doc_id": str(row.get("doc_id", "")), - "page_number": row.get("page_number"), - "chunk_index": row.get("chunk_index"), - "filename": row.get("filename"), - }, - ) - - # --- Convenience ----------------------------------------------------------------- - - def count(self, doc_id=None) -> int: - query_filter = {"doc_id": _as_object_id(doc_id)} if doc_id is not None else {} - return self.collection.count_documents(query_filter) - - def chunks_for(self, doc_id, limit: int = None, pages: Optional[List[int]] = None - ) -> List[Document]: - """Chunks of one document in reading order, optionally limited to some pages.""" - query_filter: Dict[str, Any] = {"doc_id": _as_object_id(doc_id)} - if pages: - query_filter["page_number"] = {"$in": list(pages)} - - cursor = (self.collection - .find(query_filter, {"text": 1, "page_number": 1, "doc_id": 1, - "chunk_index": 1, "filename": 1}) - .sort([("page_number", 1), ("chunk_index", 1)])) - if limit: - cursor = cursor.limit(limit) - return [self._to_document(row) for row in cursor] - - def describe_backend(self) -> str: - """Which similarity path is in use, for the CLI to report.""" - return ("Atlas $vectorSearch" if self._can_vector_search() - else "exact NumPy cosine (no Atlas vector index on this server)") - - diff --git a/components-Dinura/learnmate/storage/pages.py b/components-Dinura/learnmate/storage/pages.py deleted file mode 100644 index 595107f..0000000 --- a/components-Dinura/learnmate/storage/pages.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -The cleaned text of every page, kept whole. - -Stored separately from the chunks because the two answer different questions. Chunks are -sized and overlapped for *retrieval*, so reassembling a page from them repeats ~150 -characters at every boundary and starts mid-sentence. Resource generation wants the page -as it reads, and a page is also the natural unit for `--pages 12-18`. - -This is what makes a resource-generation session possible: without it, `build_source_text` -would have nothing but chunks to work from. -""" - -from typing import Dict, List, Optional, Union - -from bson import ObjectId -from pymongo import UpdateOne - -from .. import config -from .ids import coerce_id -from .mongo import get_db - - -def _collection(): - return get_db()[config.COLL_PAGES] - - -def store_pages(doc_id: Union[str, ObjectId], pages: List[Dict]) -> int: - """ - Store the cleaned text of every page. Returns how many were written. - - Upserted on (doc_id, page_number), so re-ingesting a document refreshes its pages - rather than piling up a second copy beside them. - """ - oid = coerce_id(doc_id) - operations = [ - UpdateOne( - {"doc_id": oid, "page_number": page["page_number"]}, - {"$set": {"doc_id": oid, - "page_number": page["page_number"], - "page_index": page["page_index"], - "text": page["page_content"]}}, - upsert=True, - ) - for page in pages - ] - if not operations: - return 0 - _collection().bulk_write(operations, ordered=False) - return len(operations) - - -def get_pages(doc_id: Union[str, ObjectId], page_numbers: Optional[List[int]] = None - ) -> List[Dict]: - """Cleaned page text in reading order, optionally only the pages named.""" - query: Dict = {"doc_id": coerce_id(doc_id)} - if page_numbers: - query["page_number"] = {"$in": list(page_numbers)} - return list(_collection() - .find(query, {"page_number": 1, "text": 1}) - .sort("page_number", 1)) - - -def delete_pages(doc_id: Union[str, ObjectId]) -> int: - """Drop a document's stored page text. Returns how many pages were removed.""" - return _collection().delete_many({"doc_id": coerce_id(doc_id)}).deleted_count diff --git a/components-Dinura/learnmate/storage/pdf_files.py b/components-Dinura/learnmate/storage/pdf_files.py deleted file mode 100644 index 4e1547b..0000000 --- a/components-Dinura/learnmate/storage/pdf_files.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -The PDF bytes themselves, in GridFS. - -An uploaded PDF is kept whole, in the database, alongside the chunks derived from it, so a -document can be re-read, re-chunked or handed back to a user without depending on the -uploader's filesystem. GridFS rather than a plain field because a document can exceed -MongoDB's 16 MB per-document limit. - -This module owns bytes in and bytes out. The record describing them lives in -documents.py, and the two are written together by `documents.store_pdf`. -""" - -from pathlib import Path -from typing import Optional, Tuple, Union - -import gridfs -from bson import ObjectId - -from .. import config -from .mongo import get_db - - -def bucket() -> gridfs.GridFSBucket: - """The GridFS bucket PDFs are stored in.""" - return gridfs.GridFSBucket(get_db(), bucket_name=config.GRIDFS_BUCKET) - - -def read_source(source: Union[str, Path, bytes], filename: str = None) -> Tuple[bytes, str]: - """ - Read an upload into bytes and check it is one this system will accept. - - Split out from storing so a caller can validate an upload -- and learn its size and - name -- before anything is written. The ingestion pipeline needs that to refuse a - second PDF for a session without first storing the file it is about to reject. - - Every upload path goes through here, so the size limit holds for a CLI path argument - and an HTTP upload alike; a limit enforced at one of the callers is a limit the other - caller does not have. - """ - if isinstance(source, (str, Path)): - path = Path(source) - if not path.exists(): - raise FileNotFoundError(f"No such PDF: {path}") - data = path.read_bytes() - filename = filename or path.name - else: - data = bytes(source) - filename = filename or "upload.pdf" - - if not data: - raise ValueError("Refusing to store an empty file.") - - if len(data) > config.MAX_PDF_BYTES: - raise ValueError( - f"{filename} is {len(data) / 1_048_576:.1f} MB, over the " - f"{config.MAX_PDF_MB:g} MB upload limit." - ) - - return data, filename - - -def put(filename: str, data: bytes, digest: str) -> ObjectId: - """Write bytes into GridFS and return the id to store on the document record.""" - return bucket().upload_from_stream( - filename, data, metadata={"sha256": digest, "content_type": "application/pdf"}) - - -def get(gridfs_id) -> Optional[bytes]: - """Read a stored PDF back out. Returns None when the id is missing.""" - if not gridfs_id: - return None - stream = bucket().open_download_stream(gridfs_id) - try: - return stream.read() - finally: - stream.close() - - -def drop(gridfs_id) -> None: - """Delete stored bytes, tolerating a record whose bytes are already gone.""" - if not gridfs_id: - return - try: - bucket().delete(gridfs_id) - except gridfs.NoFile: - pass diff --git a/components-Dinura/learnmate/storage/pdf_store.py b/components-Dinura/learnmate/storage/pdf_store.py deleted file mode 100644 index d4a8b65..0000000 --- a/components-Dinura/learnmate/storage/pdf_store.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -PDF storage: the facade over the three modules that make it up. - - pdf_files.py the bytes, in GridFS - documents.py the record describing them, and every way of finding it - pages.py the cleaned page text derived from them - -Kept as one importable name because that is how the rest of the system already reads -- -`pdf_store.store_pdf(...)`, `pdf_store.get_pages(...)` -- and because the three genuinely -are one subject seen from three angles. Import the specific module when you want to know -which layer you are touching; import this when you just want to store or fetch a PDF. -""" - -from .documents import ( - delete_document, - export_pdf, - find_by_hash, - get_active_document, - get_document, - get_pdf_bytes, - list_documents, - mark_ingested, - resolve_document, - store_pdf, -) -from .pages import delete_pages, get_pages, store_pages -from .pdf_files import read_source - -__all__ = [ - "delete_document", - "delete_pages", - "export_pdf", - "find_by_hash", - "get_active_document", - "get_document", - "get_pages", - "get_pdf_bytes", - "list_documents", - "mark_ingested", - "read_source", - "resolve_document", - "store_pages", - "store_pdf", -] diff --git a/components-Dinura/learnmate/storage/qdrant_vectors.py b/components-Dinura/learnmate/storage/qdrant_vectors.py deleted file mode 100644 index b4c8186..0000000 --- a/components-Dinura/learnmate/storage/qdrant_vectors.py +++ /dev/null @@ -1,279 +0,0 @@ -""" -A LangChain VectorStore backed by a Qdrant **server**. - -Server mode, never embedded. `QdrantClient(path=...)` runs Qdrant inside this process -against a local directory and takes an exclusive file lock on it, which means exactly one -process can hold the corpus and nothing else on the network can read it. That is the mode -the previous implementation used and the reason a second script could not run while the -chat agent was open. This module only ever constructs the client with a URL, and refuses -a `path` setting outright rather than silently falling back to it. - -Qdrant is a purpose-built vector database, so unlike the MongoDB fallback the similarity -search is a real HNSW index on the server side: filtering by document and scoring happen -where the vectors live, and neither the vectors nor the payloads are pulled across the -wire to be scored here. - -Scores are raw cosine similarity in [-1, 1], the same scale MongoVectorStore returns, so -RELEVANCE_THRESHOLD means one thing regardless of which backend is configured. -""" - -import uuid -from typing import Any, Dict, Iterable, List, Optional, Tuple - -from langchain_core.documents import Document -from langchain_core.embeddings import Embeddings -from langchain_core.vectorstores import VectorStore - -from .. import config -from ..llm.embeddings import get_embeddings - -# Deterministic point ids. Qdrant accepts only uint64 or UUID ids, so the natural key -# (doc_id, page_number, chunk_index) is hashed into a UUID5. Re-ingesting a document -# therefore overwrites its points in place instead of appending a second copy of them. -_POINT_NAMESPACE = uuid.UUID("6f8a1d3e-2b41-4c9a-9d2f-7e5b0c1a4d88") - - -class QdrantUnavailable(RuntimeError): - """Raised when the Qdrant server cannot be reached, with the URL that was tried.""" - - -def _point_id(doc_id, page_number, chunk_index) -> str: - return str(uuid.uuid5(_POINT_NAMESPACE, f"{doc_id}:{page_number}:{chunk_index}")) - - -class QdrantVectorStore(VectorStore): - """Chunk vectors in a Qdrant server, filtered by document.""" - - def __init__(self, embedding: Embeddings = None, url: str = None, - api_key: str = None, collection_name: str = None): - self._embedding = embedding or get_embeddings() - self.url = url or config.QDRANT_URL - self.api_key = api_key or config.QDRANT_API_KEY or None - self.collection_name = collection_name or config.QDRANT_COLLECTION - self._client = None - self._collection_ready = False - - # --- Connection ------------------------------------------------------------------ - - @property - def client(self): - if self._client is None: - from qdrant_client import QdrantClient - - try: - # url= only. Passing path= here would silently start an embedded instance - # and take a directory lock, which is the mode this project moved off. - client = QdrantClient(url=self.url, api_key=self.api_key, - timeout=config.QDRANT_TIMEOUT) - client.get_collections() # the client is lazy; force a real round trip - except Exception as exc: - raise QdrantUnavailable( - f"Cannot reach the Qdrant server at {self.url}. Start one with " - f"`docker compose up -d qdrant` (see docker-compose.yml), or set " - f"LEARNMATE_QDRANT_URL. Original error: {type(exc).__name__}: {exc}" - ) from exc - self._client = client - return self._client - - def ensure_collection(self) -> None: - """Create the collection and its payload index if they do not exist yet.""" - if self._collection_ready: - return - - from qdrant_client import models - - if not self.client.collection_exists(self.collection_name): - size = self._embedding.dimension - print(f"[*] Creating Qdrant collection {self.collection_name!r} " - f"({size}-d, cosine)...") - self.client.create_collection( - collection_name=self.collection_name, - vectors_config=models.VectorParams( - size=size, distance=models.Distance.COSINE), - ) - - # Every query filters on doc_id. Without a payload index Qdrant falls back to a - # full scan for the filter, which defeats the point of using a vector database. - try: - self.client.create_payload_index( - collection_name=self.collection_name, - field_name="doc_id", - field_schema=models.PayloadSchemaType.KEYWORD, - ) - except Exception: - # Already present; Qdrant has no create-if-missing for payload indexes. - pass - - self._collection_ready = True - - def _doc_filter(self, doc_id=None, pages: Optional[List[int]] = None): - """Build a Qdrant filter, or None when nothing is being narrowed.""" - from qdrant_client import models - - conditions = [] - if doc_id is not None: - conditions.append(models.FieldCondition( - key="doc_id", match=models.MatchValue(value=str(doc_id)))) - if pages: - conditions.append(models.FieldCondition( - key="page_number", match=models.MatchAny(any=list(pages)))) - return models.Filter(must=conditions) if conditions else None - - # --- LangChain plumbing ---------------------------------------------------------- - - @property - def embeddings(self) -> Embeddings: - return self._embedding - - @classmethod - def from_texts(cls, texts: List[str], embedding: Embeddings, - metadatas: Optional[List[dict]] = None, **kwargs: Any - ) -> "QdrantVectorStore": - store = cls(embedding=embedding, **kwargs) - store.add_texts(texts, metadatas) - return store - - # --- Writing --------------------------------------------------------------------- - - def add_texts(self, texts: Iterable[str], metadatas: Optional[List[dict]] = None, - **kwargs: Any) -> List[str]: - """Embed and upsert chunks, in batches so a large document is not one huge request.""" - from qdrant_client import models - - texts = list(texts) - if not texts: - return [] - - self.ensure_collection() - metadatas = metadatas or [{} for _ in texts] - vectors = self._embedding.embed_documents(texts) - - points, ids = [], [] - for text, metadata, vector in zip(texts, metadatas, vectors): - doc_id = str(metadata.get("doc_id", "")) - page_number = metadata.get("page_number", 0) - chunk_index = metadata.get("chunk_index", 0) - identifier = _point_id(doc_id, page_number, chunk_index) - - points.append(models.PointStruct( - id=identifier, - vector=vector, - payload={ - "doc_id": doc_id, - "page_number": page_number, - "chunk_index": chunk_index, - "text": text, - "filename": metadata.get("filename"), - "source": metadata.get("source"), - }, - )) - ids.append(identifier) - - for start in range(0, len(points), config.QDRANT_BATCH_SIZE): - self.client.upsert( - collection_name=self.collection_name, - points=points[start:start + config.QDRANT_BATCH_SIZE], - wait=True, - ) - return ids - - def delete(self, ids: Optional[List[str]] = None, doc_id=None, **kwargs: Any) -> bool: - """Delete by point id, or every point belonging to one document.""" - from qdrant_client import models - - self.ensure_collection() - if doc_id is not None: - self.client.delete( - collection_name=self.collection_name, - points_selector=models.FilterSelector(filter=self._doc_filter(doc_id)), - wait=True, - ) - return True - if ids: - self.client.delete( - collection_name=self.collection_name, - points_selector=models.PointIdsList(points=list(ids)), - wait=True, - ) - return True - return False - - # --- Reading --------------------------------------------------------------------- - - def similarity_search(self, query: str, k: int = None, doc_id=None, - **kwargs: Any) -> List[Document]: - return [doc for doc, _ in - self.similarity_search_with_score(query, k=k, doc_id=doc_id, **kwargs)] - - def similarity_search_with_score(self, query: str, k: int = None, doc_id=None, - **kwargs: Any) -> List[Tuple[Document, float]]: - """Top-k chunks with cosine similarity, optionally restricted to one document.""" - k = k or config.TOP_K - self.ensure_collection() - - response = self.client.query_points( - collection_name=self.collection_name, - query=self._embedding.embed_query(query), - query_filter=self._doc_filter(doc_id), - limit=k, - with_payload=True, - ) - return [(self._to_document(point.payload, point.id), float(point.score)) - for point in response.points] - - @staticmethod - def _to_document(payload: Dict[str, Any], point_id=None) -> Document: - payload = payload or {} - return Document( - page_content=payload.get("text", ""), - metadata={ - "chunk_id": str(point_id or ""), - "doc_id": payload.get("doc_id", ""), - "page_number": payload.get("page_number"), - "chunk_index": payload.get("chunk_index"), - "filename": payload.get("filename"), - }, - ) - - # --- Convenience ----------------------------------------------------------------- - - def count(self, doc_id=None) -> int: - self.ensure_collection() - return self.client.count( - collection_name=self.collection_name, - count_filter=self._doc_filter(doc_id), - exact=True, - ).count - - def chunks_for(self, doc_id, limit: int = None, pages: Optional[List[int]] = None - ) -> List[Document]: - """ - Every chunk of one document in reading order. - - Scrolled in pages rather than fetched in one call: a 300-page PDF is a few - thousand points and Qdrant caps a single scroll response. Ordering is applied - here because scroll returns points in id order, and the ids are content hashes. - """ - self.ensure_collection() - query_filter = self._doc_filter(doc_id, pages) - - documents, offset = [], None - while True: - batch, offset = self.client.scroll( - collection_name=self.collection_name, - scroll_filter=query_filter, - limit=config.QDRANT_BATCH_SIZE, - offset=offset, - with_payload=True, - with_vectors=False, - ) - documents.extend(self._to_document(p.payload, p.id) for p in batch) - if offset is None or (limit and len(documents) >= limit): - break - - documents.sort(key=lambda d: (d.metadata.get("page_number") or 0, - d.metadata.get("chunk_index") or 0)) - return documents[:limit] if limit else documents - - def describe_backend(self) -> str: - return f"Qdrant server at {self.url} (collection {self.collection_name!r})" diff --git a/components-Dinura/learnmate/storage/resources.py b/components-Dinura/learnmate/storage/resources.py deleted file mode 100644 index e24e65f..0000000 --- a/components-Dinura/learnmate/storage/resources.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Generated resources: MCQs, practice questions, key points and summaries. - -Stored with the **full attempt trail**, not just the accepted output. A resource that -needed a retry, and what the judge objected to the first time, is the data that says -whether the threshold is set anywhere near right -- and that question is unanswerable -after the fact if only the winner is kept. -""" - -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - -from bson import ObjectId - -from .. import config -from .ids import as_object_id -from .mongo import get_db - - -def _collection(): - return get_db()[config.COLL_RESOURCES] - - -def save_resource(doc_id, task: str, content: Any, accepted: bool, - attempts: List[Dict], verdict: Optional[Dict] = None, - source_preview: str = "", params: Optional[Dict] = None) -> ObjectId: - """Store one generated resource and return its id.""" - record = { - "doc_id": as_object_id(doc_id), - "task": task, - "content": content, - "accepted": accepted, - "score": (verdict or {}).get("score"), - "threshold": (verdict or {}).get("threshold", config.EVALUATOR_THRESHOLD), - "verdict": verdict, - "attempts": attempts, - "n_attempts": len(attempts), - # Enough of the source to see what the model was working from, without storing a - # second copy of the document inside every resource. - "source_preview": (source_preview or "")[:1000], - "params": params or {}, - "created_at": datetime.now(timezone.utc), - } - return _collection().insert_one(record).inserted_id - - -def get_resource(resource_id) -> Optional[Dict]: - """One resource by id.""" - return _collection().find_one({"_id": as_object_id(resource_id)}) - - -def list_resources(doc_id=None, task: str = None, accepted_only: bool = False, - limit: int = 25) -> List[Dict]: - """ - Most recent resources first, filtered by document and/or task. - - `accepted_only` is off by default on purpose: a rejected resource is still stored and - still worth looking at, and hiding it would make the failure rate invisible. - """ - query: Dict[str, Any] = {} - if doc_id is not None: - query["doc_id"] = as_object_id(doc_id) - if task: - query["task"] = task - if accepted_only: - query["accepted"] = True - - return list(_collection().find(query).sort("created_at", -1).limit(limit)) diff --git a/components-Dinura/learnmate/storage/sessions.py b/components-Dinura/learnmate/storage/sessions.py deleted file mode 100644 index 5114058..0000000 --- a/components-Dinura/learnmate/storage/sessions.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Session bindings: which PDF a session is about, and what it is for. - - {session_id, doc_id, filename, sha256, kinds, bound_at} - -One record per session, with a unique index on session_id -- that uniqueness is what -enforces "one PDF per session" structurally rather than by convention. - -`kinds` is a list of what the session was opened for: ["chat"], ["resource"], or both. -The *rules* about kinds live in ingestion/sessions.py, which owns when a binding may be -written and when a command may use one. This module only reads and writes the record. - -The hash is stored alongside the id so re-ingesting the *same* file into a session can be -told apart from uploading a different one, without a second lookup. -""" - -from datetime import datetime, timezone -from typing import Dict, Optional - -from .. import config -from .ids import as_object_id -from .mongo import get_db - - -def _collection(): - return get_db()[config.COLL_SESSIONS] - - -def get_session(session_id: str) -> Optional[Dict]: - """The session's binding record, or None if it has no PDF yet.""" - if not session_id: - return None - return _collection().find_one({"session_id": session_id}) - - -def session_doc_id(session_id: str): - """The document this session is about, or None if nothing is bound to it yet.""" - return (get_session(session_id) or {}).get("doc_id") - - -def bind_session_document(session_id: str, doc_id, filename: str, sha256: str, - kinds=("chat",)) -> None: - """ - Bind a session to the PDF it will be about, and to what it is for. - - Upsert rather than insert: re-ingesting the same PDF with --force must refresh the - record instead of colliding with the unique index. - """ - _collection().update_one( - {"session_id": session_id}, - {"$set": {"session_id": session_id, - "doc_id": as_object_id(doc_id), - "filename": filename, - "sha256": sha256, - "kinds": list(kinds), - "bound_at": datetime.now(timezone.utc)}}, - upsert=True, - ) - - -def unbind_session(session_id: str) -> bool: - """Release a session's PDF, letting a different one be ingested into it.""" - return _collection().delete_one({"session_id": session_id}).deleted_count > 0 diff --git a/components-Dinura/learnmate/storage/vectors.py b/components-Dinura/learnmate/storage/vectors.py deleted file mode 100644 index c474374..0000000 --- a/components-Dinura/learnmate/storage/vectors.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Vector-store selection. - -Two interchangeable backends implement the same LangChain VectorStore surface, so nothing -upstream -- ingestion, retrieval, the chat graph -- knows or cares which is configured: - - qdrant a Qdrant server. Real HNSW index, filtering and scoring server-side. - mongodb the same MongoDB that holds everything else. Atlas $vectorSearch where - available, exact NumPy scoring otherwise. - -Both return raw cosine similarity in [-1, 1], so RELEVANCE_THRESHOLD carries the same -meaning either way and a corpus can be re-ingested into the other backend without -retuning anything. - -Note that switching backends does not migrate existing vectors. Re-run -`ingest_pdf(, force=True)` to populate the new one; the PDFs and page text are -in MongoDB and are not affected. -""" - -from typing import Optional - -from langchain_core.vectorstores import VectorStore - -from .. import config - -_STORE: Optional[VectorStore] = None - - -def build_vector_store(backend: str = None) -> VectorStore: - """Construct the configured vector store without caching it.""" - backend = (backend or config.VECTOR_BACKEND).lower() - - if backend == "qdrant": - from .qdrant_vectors import QdrantVectorStore - return QdrantVectorStore() - - if backend in ("mongodb", "mongo"): - from .mongo_vectors import MongoVectorStore - return MongoVectorStore() - - raise ValueError( - f"Unknown LEARNMATE_VECTOR_BACKEND {backend!r}; expected 'qdrant' or 'mongodb'." - ) - - -def get_vector_store() -> VectorStore: - """Process-wide vector store.""" - global _STORE - if _STORE is None: - _STORE = build_vector_store() - return _STORE - - -def reset_vector_store() -> None: - """Drop the cached store, so a changed backend takes effect. Used by tests.""" - global _STORE - _STORE = None diff --git a/components-Dinura/requirements.txt b/components-Dinura/requirements.txt deleted file mode 100644 index cd759f5..0000000 --- a/components-Dinura/requirements.txt +++ /dev/null @@ -1,37 +0,0 @@ -# LearnMate dependencies. Versions are the ones this code was built and verified against; -# loosen them if you need to, but the two marked below are the ones that actually matter. - -# --- Agent framework ----------------------------------------------------------------- -langchain>=1.3,<2 -langchain-core>=1.5,<2 -langchain-community>=0.4,<1 -langchain-text-splitters>=1.1,<2 -langgraph>=1.2,<2 - -# --- Local inference ----------------------------------------------------------------- -# llama-cpp-python must be built with grammar support (any 0.2.77+ release is). Every -# structured output in this project depends on JSON-schema-constrained decoding; without -# it a 3B model returns prose where JSON was asked for about half the time. -llama-cpp-python>=0.3,<0.4 - -# --- Embeddings ---------------------------------------------------------------------- -sentence-transformers>=5.0,<6 -torch>=2.6 -transformers>=5.0 - -# --- Storage ------------------------------------------------------------------------- -# dnspython is required for mongodb+srv:// Atlas URIs; pymongo does not pull it in. -pymongo>=4.9,<5 -dnspython>=2.7 - -# Keep this within one minor version of the qdrant/qdrant image tag in -# docker-compose.yml -- the client refuses to talk to a server further away than that. -qdrant-client>=1.15,<2 - -# --- PDF handling -------------------------------------------------------------------- -pymupdf>=1.26 - -# --- Support ------------------------------------------------------------------------- -numpy>=2.0 -python-dotenv>=1.0 -huggingface_hub>=0.20 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c8fdb64 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,125 @@ +services: + mongo: + image: mongo:8 + restart: unless-stopped + networks: + default: null + volumes: + - type: volume + source: mongo_data + target: /data/db + volume: {} + healthcheck: + test: + - CMD + - mongosh + - --quiet + - --eval + - db.adminCommand('ping').ok + retries: 10 + start_period: 20s + + qdrant: + image: qdrant/qdrant:v1.18.1 + restart: unless-stopped + volumes: + - qdrant_data:/qdrant/storage + healthcheck: + test: ["CMD", "/qdrant/qdrant", "--version"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + keycloak: + image: quay.io/keycloak/keycloak:26.0 + restart: unless-stopped + command: start-dev --import-realm + environment: + KC_BOOTSTRAP_ADMIN_USERNAME: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + KC_HEALTH_ENABLED: "true" + KC_HTTP_RELATIVE_PATH: /auth + KC_PROXY_HEADERS: xforwarded + KC_HTTP_ENABLED: "true" + KC_HOSTNAME: ${PUBLIC_HOST} + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + volumes: + - keycloak_data:/opt/keycloak/data + - ./integrated-backend/keycloak/realm:/opt/keycloak/data/import:ro + - ./integrated-backend/keycloak/themes:/opt/keycloak/themes:ro + healthcheck: + test: ["CMD-SHELL", "exec 3<> /dev/tcp/127.0.0.1/8080; echo -e \"GET /auth/health/ready HTTP/1.1\r\nhost: localhost\r\nConnection: close\r\n\r\n\" >&3; head -n 1 <&3 | grep -q '200 OK'"] + interval: 10s + timeout: 5s + retries: 20 + start_period: 60s + + backend: + build: + context: ./integrated-backend + restart: unless-stopped + env_file: + - .env + environment: + JWT_SECRET_KEY: ${JWT_SECRET_KEY} + FRONTEND_ORIGIN: ${PUBLIC_ORIGIN} + KEYCLOAK_ENABLED: "true" + KEYCLOAK_ISSUER: ${PUBLIC_ORIGIN}/auth/realms/learnmate + KEYCLOAK_JWKS_URL: http://keycloak:8080/auth/realms/learnmate/protocol/openid-connect/certs + LEARNMATE_MONGODB_URI: mongodb://mongo:27017 + LEARNMATE_MONGODB_DB: learnmate + LEARNMATE_QDRANT_URL: http://qdrant:6333 + LEARNMATE_QDRANT_COLLECTION: learnmate_chunks + LEARNMATE_MODELS_DIR: /app/models + depends_on: + mongo: + condition: service_healthy + qdrant: + condition: service_healthy + keycloak: + condition: service_healthy + volumes: + - model_data:/app/models + - hf_cache:/app/data/hf_cache + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=5)"] + interval: 15s + timeout: 10s + retries: 20 + start_period: 60s + + frontend: + build: + context: ./integrated-frontend + args: + VITE_API_BASE_URL: "" + VITE_KEYCLOAK_URL: /auth + VITE_KEYCLOAK_REALM: learnmate + VITE_KEYCLOAK_CLIENT_ID: learnmate-frontend + restart: unless-stopped + depends_on: + backend: + condition: service_healthy + + nginx: + image: nginx:1.27-alpine + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + - /etc/letsencrypt:/etc/letsencrypt:ro + depends_on: + - frontend + - backend + - keycloak + +volumes: + mongo_data: + qdrant_data: + keycloak_data: + model_data: + hf_cache: diff --git a/docs/Blank diagram (8).pdf b/docs/Blank diagram (8).pdf deleted file mode 100644 index 233767b..0000000 Binary files a/docs/Blank diagram (8).pdf and /dev/null differ diff --git a/docs/Component Diagram.pdf b/docs/Component Diagram.pdf deleted file mode 100644 index 53226ff..0000000 Binary files a/docs/Component Diagram.pdf and /dev/null differ diff --git a/docs/Project Idea-Group 8.pdf b/docs/Project Idea-Group 8.pdf deleted file mode 100644 index db55df7..0000000 Binary files a/docs/Project Idea-Group 8.pdf and /dev/null differ diff --git a/docs/component-2.drawio b/docs/component-2.drawio deleted file mode 100644 index 98d20a2..0000000 --- a/docs/component-2.drawio +++ /dev/null @@ -1,117 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/component-2.drawio.pdf b/docs/component-2.drawio.pdf deleted file mode 100644 index aa5a68b..0000000 Binary files a/docs/component-2.drawio.pdf and /dev/null differ diff --git a/docs/feature-adders/CHANGELOG.md b/docs/feature-adders/CHANGELOG.md deleted file mode 100644 index 91ecf0d..0000000 --- a/docs/feature-adders/CHANGELOG.md +++ /dev/null @@ -1,46 +0,0 @@ -# Feature adders — changelog - -Logged on `thevindu-feature` (copied from `main`). Do not merge to `main` until the -Tier 3 items have a before/after eval note. - -## Shipped on this branch - -| Feature | Why | Seam | -|---------|-----|------| -| Summary `narrative` / `structured` | Keep connected prose as the default; add a second mode for list-like statutes instead of silently reversing the deck | `summary.py`, Gate 1/2, `GenerateRequest.summary_style` | -| MCQ `easy` / `medium` / `hard` | Medium is today's prompt unchanged; easy/hard are additive | `mcq.py`, item `difficulty`, Analytics | -| MCQ distractor check | Gate 2 judge check that wrong options are actually false; reuses `MAX_ATTEMPTS = 2`; hard tier slightly more lenient | `rubrics.py` | -| UI refresh | Empty states, queued vs running job copy, optional selectors, export buttons, mobile padding | frontend only | -| Export docx/pptx | Format stored judged content; no regenerate | `GET /api/resources/{id}/export` | -| BM25 hybrid retrieve | Additive: ANN top 20, BM25 top 10, merge, **existing** reranker. Logged as `ann` / `bm25` / `both` | ingest sidecar + `retrieve.py` | -| Multi-model | Registry + optional `model_id`; one llama.cpp generator at a time; unload/reload; experimental LoRA labelled, not default | `models_registry.yaml`, `GET /api/models` | -| Upload docx / pptx / tex | Same extract → clean → chunk → embed path as PDF, so MCQ/summary/chat work on lecture slides and LaTeX notes. `.doc`/`.ppt` rejected with a save-as hint. | `extract_office.py`, `validate_upload`, DocumentsCard | - -## Latency, quality, failures (this pass) - -Full analysis: [LATENCY_QUALITY_FAILURES.md](LATENCY_QUALITY_FAILURES.md). Phase 2 code, still on this branch, no default-model change: - -| Item | Why | -|------|-----| -| Stage timings on chat / passage-resource jobs | `rewrite_ms`, `retrieve_ms`, `generate_ms`, `judge_ms`, `model_load_ms` on the result (and `progress.timings`); INFO log per turn | -| Cached BM25Okapi per `doc_id` | Stop rebuilding on every retrieve; word-tokenise chunks; invalidate on ingest | -| `error_code` on failed jobs | `storage` / `model` / `parse` / `timeout` / `interrupted` / `unknown`; frontend `errorMessage` branches | -| `LEARNMATE_JOB_TIMEOUT_S` | Default 0 (off). When set, fail between graph nodes — does not abort llama.cpp mid-token | -| Resource persist of best-scoring attempt | Same policy as chat; full attempt trail kept | - -## Not a backdoor - -`legal-1.5b` is `experimental: true` and `selectable_default: false`. Default remains -`LEARNMATE_GENERATOR_MODEL` (Qwen 2.5 3B). Failed-gate adapters stay out of the silent default. - -## Flags - -- `LEARNMATE_HYBRID_BM25=0` restores ANN-only retrieve. -- `LEARNMATE_JOB_TIMEOUT_S=0` leaves jobs unbounded (whole-document runs). -- Omit `model_id` / `summary_style` / `difficulty` → previous behaviour. - -## Before merging to main - -1. BM25: inspect `retrieval_mix.rerank_kept` on chat turns — if BM25-only chunks are never kept, report that rather than assuming hybrid helped. -2. Any newly selectable default: run `acceptance_thresholds.yaml` the same way as the ML track. - diff --git a/docs/feature-adders/LATENCY_QUALITY_FAILURES.md b/docs/feature-adders/LATENCY_QUALITY_FAILURES.md deleted file mode 100644 index 4cce16c..0000000 --- a/docs/feature-adders/LATENCY_QUALITY_FAILURES.md +++ /dev/null @@ -1,217 +0,0 @@ -# Latency, quality tradeoffs, and failures - -**Branch:** `thevindu-feature` only. Do not merge this to `main` as a substitute for the Tier 3 BM25 / model eval notes in [PLAN.md](PLAN.md). -**Does not promote** `legal-1.5b` / `qwen25-lora-20260815-090709`. That adapter stays `experimental: true`. - -The seven feature adders are already on this branch. They are additive, but several of them spend more wall-clock or create new fail modes. This document maps where time actually goes, why “just update X” fails, which quality–latency levers are safe, and which additions are worth doing. - -Phase 2 of this work (same branch) makes the map measurable: stage timings on jobs, a cached BM25 index, structured `error_code`s, an optional job timeout, and resource persist of the best-scoring attempt. - ---- - -## 1. Bottleneck map - -Every slow call is a **202 job**. One in-process worker runs them. `llama_cpp.Llama` holds a mutable context, so a thread pool would interleave tokens rather than go faster. Chat, ingest, and resource jobs therefore queue behind each other. - -```mermaid -flowchart LR - subgraph request [Request] - POST[POST 202 job] - Q[Single worker queue] - end - subgraph chat [Chat graph sequential] - rewrite[rewrite optional judge] - retrieve[ANN plus BM25 plus rerank] - generate[generator stream] - evaluate[judge or gate skip] - decide[retry or persist] - end - subgraph resource [Resource graph sequential] - genR[generate] - gate1[Gate 1 structural] - gate2[Gate 2 judge] - loopR[retry max 2] - end - POST --> Q - Q --> chat - Q --> resource -``` - -### Chat turn (local GGUFs, CPU) - -| Stage | Typical cost | Notes | -|-------|----------------|-------| -| Rewrite | 0 or a small judge call | Heuristic `_needs_rewrite` skips most standalone questions. Follow-ups call the **judge** LLM (`max_tokens=100`). | -| Retrieve | tens–hundreds of ms | Embed query, ANN, optional BM25 merge, cross-encoder rerank (~100 ms / 20 pairs once loaded). | -| Generate | seconds to tens of seconds | Streamed; first tokens ~2 s after a warm GGUF. `max_tokens=320`. | -| Judge | **~25–36 s** | Dominant cost in **pdf** mode. **Skipped** in `general` mode (`LEARNMATE_JUDGE_GATE_MODES=general`). | -| Retry | up to another generate + judge | `MAX_ATTEMPTS=2`. Chat skips retry when there is no critique, or `score < threshold - 25`. | -| Persist | Mongo writes | Chat keeps the **best-scoring** attempt, not necessarily the last. | - -Documented wall-clock: **30–60 s per pdf turn** on the local backend. - -### Resource (passage) - -Gate 1 (structural) is microseconds and runs first. Gate 2 is the same ~25 s judge. Easy-MCQ distractor text and structured-summary rubric lines make the judge prompt slightly longer and can raise reject rate → a second generate + judge. - -### Whole-document MCQ / keypoints / practice - -Groups run **one after another**. Each group pays generate + Gate 1 + Gate 2 + optional retry. Minutes, not seconds. Document **summary** is cheaper on the judge: per-page folds with `evaluate=False`, one judged pass at the end. - -### Ingest - -Extract, chunk, embed, BM25 sidecar write. First process after boot also pays ~16 s of embedding-model + import warm-up unless `API_WARM_UP=1` (default on). - -### Model switch (`model_id`) - -Unload previous **generator** GGUF, load the next. Several seconds, reported on `progress.message`. The judge stays loaded. Two generators in RAM at once is **rejected**. - -### Export (docx / pptx) - -Not a bottleneck. `GET /api/resources/{id}/export` reads stored content and formats it. No generate, no judge. - -### Frontend - -`waitForJob` polls 300 ms while streaming, then 1500 ms. **No client timeout** (whole-document runs are allowed to take minutes). A 401 on any API call clears the session and hard-redirects to `/login`, which immediately sends Keycloak `login()` — that is the “localhost buffers and checks again” loop when the backend is down or restarting. - ---- - -## 2. Measured vs estimated - -Until Phase 2 timers land on the job record, numbers above are from code comments and the ML eval log, not from live `thevindu-feature` jobs. - -**Already measured (offline ML track, Colab T4, not this server):** - -- LoRA candidate p95 **16.4 s / 14.9 s** vs **≤ 8 s** bar (`acceptance_thresholds.yaml`). That hardware is **not** production serving; do not treat it as the live GGUF p95. - -**Estimated from comments / config (live path):** - -- Judge ~25 s (resources) / median ~36 s (chat pdf). -- Boot embedding warm-up ~16 s. -- Rerank ~100 ms / 20 pairs. -- Chat turn 30–60 s. - -**What Phase 2 timers prove:** `rewrite_ms`, `retrieve_ms`, `generate_ms`, `judge_ms`, `model_load_ms` on the chat (and passage-resource) job result, plus one INFO log line per turn. Use those before claiming BM25 or a model switch “made it slower.” - ---- - -## 3. Why updates fail - -Three failure classes. Mixing them produces the wrong fix (lowering an accuracy bar, or adding retries, or making the failed LoRA the default). - -### 3.1 ML track update failed - -Candidate `qwen25-lora-20260815-090709` on `lm-legal-v0.1` failed [acceptance_thresholds.yaml](../../model-Thevindu/03_testing_and_versioning/acceptance_thresholds.yaml). Authoritative numbers are in [model_card.md](../../model-Thevindu/04_docs/model_card.md) and `version_registry.csv`. - -| Gate | `test` | `test_strict` | Threshold | Result | -|------|--------|---------------|-----------|--------| -| Accuracy (LLM-judge) | 0.557 | 0.621 | ≥ 0.70 | **FAIL** | -| Groundedness | 0.877 | 0.921 | ≥ 0.85 | pass (after `validate_pairs.py`) | -| Hallucination | 0.123 | 0.079 | ≤ 0.15 | pass | -| Latency p95 | 16368 ms | 14879 ms | ≤ 8000 ms | **FAIL** on eval hardware | -| Beat API fallback | 0.557 vs 0.871 | 0.621 vs 0.918 | slack 0.05 | **FAIL** | - -The first groundedness fail was an **eval bug** (naive regex over-flagged hallucinations). Rescoring is not a model win. - -Making this the silent generator default would fail the live product the same way: weaker general answers, extra unload/reload if anyone switches back, and a false “we improved the model” story. It stays `experimental: true` in `learnmate/models_registry.yaml`. A second training run will not beat `gpt-4o-mini` / Gemini on this corpus size; keep an API backend as the high-quality option. - -### 3.2 Feature-adder updates that look like quality wins - -- **Stricter Gate 2** (easy MCQ distractors, structured summary rubric) → more rejects → second generate + second judge (~2× 25 s). Do **not** raise `MAX_ATTEMPTS`. The 3B judge oscillates rather than converges. -- **Hybrid BM25** without inspecting `retrieval_mix.rerank_kept` → cannot tell if BM25-only chunks ever survive the reranker. Shipping it as “better retrieve” without that note is the failure [CHANGELOG.md](CHANGELOG.md) already warns about. Rebuild-every-turn of `BM25Okapi` from raw strings was also wasted work (and treated documents as character lists). Phase 2 caches a word-tokenised index per `doc_id`. -- **Switching `model_id` every request** → multi-second unload/reload. Two concurrent generators are unsafe (mutable llama.cpp context). -- **Skipping Gate 1** to “go faster” → the judge spends 25 s on malformed JSON. Gate 1 is the cheap filter; keep it. -- **Raising retries / parallel llama.cpp threads** → known-wrong answers or interleaved tokens, not speed. - -### 3.3 Runtime failures the new system handled poorly (before Phase 2) - -- Worker stored one error string. The UI could not tell Mongo down vs bad `model_id` vs timeout vs restart. -- Empty chat reply on llama.cpp error still walked evaluate/retry. -- `--reload` watching `venv` restarts the process; Keycloak `check-sso` then looks like a hang. -- Client `waitForJob` had no timeout. Optional `LEARNMATE_JOB_TIMEOUT_S` (default **0 = off**) fails the job between stages so whole-document runs stay legal unless you set it. - ---- - -## 4. Quality vs latency tradeoffs - -Levers ranked by **time saved vs quality risk**. Defaults on this branch stay as they are unless a flag is set. - -**Already correct — do not reverse** - -- Skip the judge in `general` mode (`LEARNMATE_JUDGE_GATE_MODES`). -- `reply_ready`: the student can read while the judge runs. -- Document-summary judges only the final fold. -- Gate 1 in front of Gate 2. -- Chat persist of the **best** attempt (resources did not, until Phase 2). -- One worker / one generator GGUF. - -**Opt-in, high savings** - -- `LEARNMATE_GENERATOR_BACKEND=gemini` (and/or judge): seconds instead of ~30 s; text leaves the machine. Keep local GGUF as default. -- `API_WARM_MODELS=1` on a demo box: first question is faster; `--reload` becomes painful. -- GPU / Metal: `LEARNMATE_N_GPU_LAYERS=-1` and a CUDA/Vulkan/Metal build of llama-cpp-python. Comments cite ~40 s vs ~6 s depending on the machine. - -**Opt-in, medium savings** - -- `evaluate=False` for drafts. -- Skip rewrite LLM entirely (follow-ups retrieve worse). -- Narrower `LEARNMATE_JUDGE_GATE_MODES` (quality risk on pdf). -- `LEARNMATE_HYBRID_BM25=0` if mix logs show BM25-only chunks never kept. - -**Unsafe “speed”** - -- Raise `MAX_ATTEMPTS`. -- Skip Gate 1. -- Make the experimental LoRA the default. -- Two generator contexts / a worker pool over llama.cpp. - ---- - -## 5. Failure handling — as-is vs gaps - -| Layer | As-is | Gap / Phase 2 | -|-------|--------|----------------| -| Jobs | `queued → running → done/failed`; stale jobs failed on restart | `error_code`: `storage`, `model`, `parse`, `timeout`, `interrupted`, `unknown` | -| Timeout | none | `LEARNMATE_JOB_TIMEOUT_S=0` off; when set, raise between graph nodes (cannot abort a llama.cpp call mid-token) | -| Gate 1 | fail fast, critique for retry | keep | -| Gate 2 | fail closed on unparseable verdict (score 1) | keep | -| Chat retry | skip if no critique or score far below threshold | keep | -| Chat generate exception | empty reply, graph continues | still recoverable via retry; job-level `model` if it raises out | -| Storage | 503 `StorageUnavailable` / `QdrantUnavailable` | mapped to `error_code=storage` on jobs | -| Rerank / BM25 miss | fall back to vector order / ANN-only | keep | -| Frontend 401 | wipe token, `/login` | start the venv server on 8010; do not use `--reload` against `venv` | -| Frontend job fail | generic `Error` message | `errorMessage` branches on `error_code` | - ---- - -## 6. Additions worth doing - -### Done in Phase 2 (this change set) - -1. **Stage timings** on chat and passage-resource jobs (`rewrite_ms`, `retrieve_ms`, `generate_ms`, `judge_ms`, `model_load_ms`). -2. **Cache `BM25Okapi` per `doc_id`**, word-tokenised, invalidate on ingest. `LEARNMATE_HYBRID_BM25=0` still disables hybrid. -3. **Structured job failures** (`error` stays human-readable; `error_code` for the UI). -4. **Optional job timeout** (`LEARNMATE_JOB_TIMEOUT_S`, default 0). -5. **Resources persist the best-scoring attempt**, same policy as chat. Trail of all attempts is kept. - -### Ranked, not in this change set - -| Rank | Addition | Why wait | -|------|----------|----------| -| 1 | Inspect `retrieval_mix.rerank_kept` on real chat turns | Tier 3 gate before `main`; needs traffic, not more code | -| 2 | Gemini / HTTP backend as a documented “fast demo” profile | Env-only; do not flip the default | -| 3 | GPU llama-cpp-python wheel | Machine-specific; `.env` already has the knobs | -| 4 | Fairer job queue (ingest vs chat) | Needs a real broker or leases; one-process assumption | -| 5 | Parallel whole-document groups | Unsafe with one llama.cpp context | -| 6 | Keycloak silent-SSO UX | Separate from generation latency | - ---- - -## 7. What we will not do - -- Promote `qwen25-lora-20260815-090709` / `legal-1.5b` to `selectable_default`. -- Load two generator GGUFs at once. -- Raise `MAX_ATTEMPTS`. -- Reverse narrative-summary or MCQ-medium defaults. -- Merge this branch to `main` until BM25 mix eval and any new default model pass the same discipline as the ML track. -- Commit adapters, `.env`, or `venv`. diff --git a/docs/feature-adders/PLAN.md b/docs/feature-adders/PLAN.md deleted file mode 100644 index 0c7c618..0000000 --- a/docs/feature-adders/PLAN.md +++ /dev/null @@ -1,79 +0,0 @@ -# Feature adders — plan, feasibility, and ship rules - -**Branch:** `thevindu-feature` (copied from `main`). Do not merge to `main` until each item’s gate below is signed. -**Pattern:** slot into existing seams (task prompts, Gate 1/2, `GenerateRequest`, retrieve node, `.env`). No new architecture. - -Bottlenecks, quality–latency levers, and why “just update the model / add retries” fails: [LATENCY_QUALITY_FAILURES.md](LATENCY_QUALITY_FAILURES.md). - ---- - -## Decision already taken (do not reverse silently) - -Summaries stay **narrative connected prose by default**. “Bolder / point-by-point” is a **second mode** (`structured`), not a replacement. That is option (b) from the spec: students can override; a list-like statute can default to structured via a heuristic. - -MCQ **medium** = today’s generator, unchanged. Easy and hard are additive. - ---- - -## Feasibility - -| # | Feature | Feasible? | Seam | Risk | Re-eval before `main`? | -|---|---------|-----------|------|------|------------------------| -| 1 | Summary `narrative` / `structured` | **Yes** | `summary.py` prompt + Gate 1 length/points + Gate 2 rubric line | Low | Light spot-check | -| 2 | MCQ `easy` / `medium` / `hard` | **Yes** | `mcq.py` prompt + stamp `difficulty` on items + params | Low | Light spot-check | -| 3 | UI/UX refresh | **Yes** | empty states, JobProgress copy, optional selectors, mobile | Low | No | -| 4 | MCQ distractor checker | **Yes** | Gate **2** extra rubric (not a new retry budget) | Moderate — false rejects | Yes — watch reject rate | -| 5 | Export docx/pptx | **Yes** | `GET /api/resources/{id}/export` reads stored content only | Moderate (new deps) | No | -| 6 | BM25 hybrid | **Yes** | ingest sidecar + retrieve merge **before** existing reranker | High — live answers | Yes — ANN vs BM25 vs both | -| 7 | Multi-model | **Yes** | registry YAML + optional `model_id`; **one** llama.cpp load; unload/reload | High — live generation | Yes — per-model gate; experimental LoRA stays labelled | - -**Not feasible / not done:** keeping two GGUFs in RAM for concurrent roles is already how generator+judge work (two files). Keeping **two generators** loaded at once is **rejected** — same mutable-context rule as the single worker. - -**Promotion backdoor:** a failed LoRA may appear as `experimental: true` in the registry. It must **not** become `selectable_default`. Default remains the live Qwen 2.5-3B GGUF. - ---- - -## Suggested git layout (this branch) - -Work lands on `thevindu-feature` in **tier order**, one commit family per feature. Optional pointers (created if isolation is needed later): - -- `feature/summary-style` -- `feature/mcq-difficulty` -- `feature/ui-refresh` -- `feature/mcq-distractor-checker` -- `feature/export-office` -- `feature/bm25-hybrid` -- `feature/multi-model` - -Tier 1 can merge internally as soon as spot-checked. Tier 3 stays on this branch until eval notes exist. - ---- - -## Env / flags (live path safety) - -| Flag | Default | Meaning | -|------|---------|---------| -| `LEARNMATE_HYBRID_BM25` | `1` on this branch | Hybrid retrieve; set `0` to restore ANN-only | -| `LEARNMATE_GENERATOR_MODEL` | unchanged | Fallback when `model_id` omitted | -| `LEARNMATE_JOB_TIMEOUT_S` | `0` (off) | Cooperative job ceiling; `timeout` error_code when set | -| Registry file | `learnmate/models_registry.yaml` | Selectable generators | - ---- - -## Acceptance (Tier 3) - -Before merging BM25 or a new default model to `main`: - -1. BM25: log `retrieval_mix` (`ann` / `bm25` / `both`) and report how often the reranker keeps BM25-only chunks. -2. Models: run the same `acceptance_thresholds.yaml` discipline as the ML track; experimental ids stay experimental. - ---- - -## Demo order - -1. Generate a **narrative** summary (looks as today). -2. Toggle **structured** on a numbered-section statute. -3. Generate **easy** vs **hard** MCQs; difficulty badge on the quiz. -4. Empty Documents/Resources/Chat states. -5. Export a resource to Word. -6. (If enabled) job message “Loading generator …” on model switch; analytics filter by model. diff --git a/docs/plan.txt b/docs/plan.txt deleted file mode 100644 index 7d01d55..0000000 --- a/docs/plan.txt +++ /dev/null @@ -1,39 +0,0 @@ -E-Learning Platform for Self-Learning Using Large Language Models - -An e-learning platform that transforms uploaded lecture materials into an interactive self-learning experience. -Students can upload course materials in PDF format, after which the system automatically extracts the content and uses one or more Large Language Models (LLMs) to generate educational resources that support independent learning. - -1. Select suitable LLMs, design and implement a pipeline to generate educational resources. -2. Develop a web application that meets the identified functional and non-functional requirements to demonstrate the workflow. -3. The developed system should perform with reasonable accuracy and operational performance. -4. Documentation covering each stage of the implementation project. - ----------------------------------------------------------------------------------------------------------- - -Title : E-Learning Platform for Self-Learning Using Large Language Models - - -Introduction : - -Proposed Solution (Briefly): - -1.Any Subject - only possible types of supporting resources for that subject will be generated. -2.PDF Document Processing Pipeline to be implemented.(Architecture needs to be decided, LLM Usage needs to be decided.) -3.Resource generation - Several types such as Lecture summaries, Concept explanations, Flashcards, Multiple-choice questions, Short-answer questions, Practice exercises, AI-based question answering chat -4.Web Application with React Frontend, Python FASTAPI backend, MongoDB nosql database. - -web application will be developed with features such as: - -User authentication and User login/Dashboard -PDF upload and management -Generated learning resources accessing through dashboard (all resources types can be accessed seperately) -Progress tracking Dashboard page -Hosting and Deployment, Monitoring - -Datasets: -PDF of educational materials - - -Similar Projects: - - diff --git a/docs/planv1.txt b/docs/planv1.txt deleted file mode 100644 index 6290661..0000000 --- a/docs/planv1.txt +++ /dev/null @@ -1,213 +0,0 @@ -E-Learning Platform for Self-Learning Using Large Language Models - -An e-learning platform that transforms uploaded lecture materials into an interactive self-learning experience. -Students can upload course materials in PDF format, after which the system automatically extracts the content and uses one or more Large Language Models (LLMs) to generate educational resources that support independent learning. - -1. Select suitable LLMs, design and implement a pipeline to generate educational resources. -2. Develop a web application that meets the identified functional and non-functional requirements to demonstrate the workflow. -3. The developed system should perform with reasonable accuracy and operational performance. -4. Documentation covering each stage of the implementation project. - ----------------------------------------------------------------------------------------------------------- - -Title : E-Learning Platform for Self-Learning Using Large Language Models - - -Introduction : - -Proposed Solution (Briefly): - -1.Any Subject - only possible types of supporting resources for that subject will be generated. -2.PDF Document Processing Pipeline to be implemented.(Architecture needs to be decided, LLM Usage needs to be decided.) -3.Resource generation - Several types such as Lecture summaries, Concept explanations, Flashcards, Multiple-choice questions, Short-answer questions, Practice exercises, AI-based question answering chat -4.Web Application with React Frontend, Python FASTAPI backend, MongoDB nosql database. - -web application will be developed with features such as: - -User authentication and User login/Dashboard -PDF upload and management -Generated learning resources accessing through dashboard (all resources types can be accessed seperately) -Progress tracking Dashboard page -Hosting and Deployment, Monitoring - -Datasets: -PDF of educational materials - - -Similar Projects: - - ------------------------------------------------------------------------------------------------ - -#Currently flowchart order for the process is , - - - -PDF upload - -PDF validation - -PDF Metadata(title, pages, size, author) - -PDF storage(original) + PDF classification(Initially we are implementing only for standard lecture material, not targeting for qsn papers, tutes, research papers, etc. therefore classification is not needed) - -Layout-aware PDF Parsing Engine (Text, Images, Tables, Captions, Fonts) -> Structured JSON Document - -Document Cleaning & Normalization (Strips headers/footers/artifacts) - -Document Structure Reconstruction (Sections, Headings, Subheadings) - -Content Relationship Detection(May need Knowledge graphs eg-Neo4J but no need if it adds complexity) - -Intelligent Semantic Chunking (Chunk in meaningful way, not in fixed size blocks) - - - -Embedding Generation + Metadata (For PDF based AI chat, we need Embeddings. Without embeddings, the chatbot would need to read the entire PDF every time, which is inefficient and scales poorly. Embeddings let you retrieve only the most relevant chunks before asking the LLM to answer.) - -MongoDB + FAISS(vector embeddings database) Storage - - -Learning Resource Engine + Chat Engine - - - - - - ------------------------------------------Diagram ---------------------------------------------------- - - - - - - - USER - │ - ▼ - Upload PDF - │ - ▼ - PDF Validation - (format, size, readability, corruption) - │ - ▼ - Store Original PDF + store metadata - (MongoDB/GridFS/File System) - │ - ▼ - Layout-aware PDF Parsing Engine - (Text, Images, Tables, Captions, Metadata Extraction) - │ - ▼ - Intermediate Document Representation (JSON) - │ - ▼ - Document Cleaning & Normalization - (remove headers, footers, page numbers, fix text) - │ - ▼ - Document Structure Reconstruction + Content Relationship Detection/ knowledge graph of document - (Title → Sections → Subsections → Paragraph hierarchy) - │ - ▼ - Intelligent Semantic Chunking - (semantic chunks instead of fixed-size chunks) - │ - ┌───────────────────────┴────────────────────────┐ - │ │ - ▼ ▼ - Learning Resource Generation by LLM Embedding Generation - │ │ - ┌───────────┼───────────────┐ - │ │ │ │ - ▼ ▼ ▼ and more ▼ - Summaries practice.qsn MCQs Vector Database (FAISS) - │ │ │ │ - └───────────┴───────────────┘ - │ │ - ▼ ▼ - Store Generated Resources RAG Chat Pipeline - (MongoDB) │ - ▼ - Student Question - │ - ▼ - Query Embedding - │ - ▼ - Similarity Search - (Top-K Chunks) - │ - ▼ - Prompt Construction - │ - ▼ - LLM - │ - ▼ - Contextual Answer - - - - -how many LLMs? - multi-agent approach - 3 LLMs - - -1. Embedding Generation ------------------------ -Objective: -Fast, high-dimensional vectorization for FAISS. - -Recommended Models: -API: -- text-embedding-004 (Google) -- text-embedding-3-small (OpenAI) - -Local: -- bge-large-en-v1.5 -- all-MiniLM-L6-v2 (via HuggingFace) - - -2. Resource Generation ----------------------- -Objective: -High reasoning capability, excellent JSON formatting (for parsing into MongoDB), and large context understanding. - -Recommended Models: -API: -- Gemini 1.5 Pro (excellent for large document reasoning) -- GPT-4o - -Local: -- Llama-3-70B (if hardware permits) - - -3. RAG Chat Engine ------------------- -Objective: -Low latency, conversational tone, and strong instruction following. - -Recommended Models: -API: -- Gemini 1.5 Flash (extremely fast) -- Claude 3.5 Haiku - -Local: -- Llama-3-8B-Instruct -- Mistral-7B - - - - - - - -The Application Layer -│ -│ ├── Document DB (MongoDB for storing users & outputs) -│ ├── Backend API (FastAPI) -│ └── Client Dashboard (React) - - - diff --git a/frontend/.env b/frontend/.env deleted file mode 100644 index b53dc6b..0000000 --- a/frontend/.env +++ /dev/null @@ -1 +0,0 @@ -VITE_API_BASE_URL=http://localhost:8000 \ No newline at end of file diff --git a/frontend/.env.example b/frontend/.env.example deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/.gitignore b/frontend/.gitignore deleted file mode 100644 index a547bf3..0000000 --- a/frontend/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-ssr -*.local - -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? diff --git a/frontend/README.md b/frontend/README.md deleted file mode 100644 index a36934d..0000000 --- a/frontend/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# React + Vite - -This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. - -Currently, two official plugins are available: - -- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs) -- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) - -## React Compiler - -The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). - -## Expanding the ESLint configuration - -If you are developing a production application, we recommend using TypeScript with type-aware lint rules enabled. Check out the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for information on how to integrate TypeScript and [`typescript-eslint`](https://typescript-eslint.io) in your project. diff --git a/frontend/api/routes.jsx b/frontend/api/routes.jsx deleted file mode 100644 index 00a60a8..0000000 --- a/frontend/api/routes.jsx +++ /dev/null @@ -1,53 +0,0 @@ -import axios from "axios"; - -const API_BASE_URL = import.meta.env.VITE_API_BASE_URL; - -const api = axios.create({ - baseURL: API_BASE_URL, -}); - -// Attach the JWT (if one exists) to every outgoing request automatically. -api.interceptors.request.use((config) => { - const token = localStorage.getItem("token"); - if (token) { - config.headers.Authorization = `Bearer ${token}`; - } - return config; -}); - -export function registerUser({ name, email, password }) { - return api.post("/api/auth/register", { name, email, password }); -} - -export function loginUser({ email, password }) { - return api.post("/api/auth/login", { email, password }); -} - -export function uploadDocument({ file, subject, onUploadProgress }) { - const formData = new FormData(); - formData.append("file", file); - formData.append("subject", subject); - - return api.post("/api/documents/upload", formData, { - headers: { "Content-Type": "multipart/form-data" }, - onUploadProgress, - }); -} - -export function listDocuments() { - return api.get("/api/documents"); -} - -export function getDocumentFile(documentId) { - return api.get(`/api/documents/${documentId}/file`, { responseType: "blob" }); -} - -export function generateResource({ documentId, resourceType }) { - return api.post("/api/resources/generate", { document_id: documentId, resource_type: resourceType }); -} - -export function listResources(documentId) { - return api.get("/api/resources", { params: { document_id: documentId } }); -} - -export default api; \ No newline at end of file diff --git a/frontend/app.jsx b/frontend/app.jsx deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js deleted file mode 100644 index ea36dd3..0000000 --- a/frontend/eslint.config.js +++ /dev/null @@ -1,21 +0,0 @@ -import js from '@eslint/js' -import globals from 'globals' -import reactHooks from 'eslint-plugin-react-hooks' -import reactRefresh from 'eslint-plugin-react-refresh' -import { defineConfig, globalIgnores } from 'eslint/config' - -export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{js,jsx}'], - extends: [ - js.configs.recommended, - reactHooks.configs.flat.recommended, - reactRefresh.configs.vite, - ], - languageOptions: { - globals: globals.browser, - parserOptions: { ecmaFeatures: { jsx: true } }, - }, - }, -]) diff --git a/frontend/index.html b/frontend/index.html deleted file mode 100644 index f94d687..0000000 --- a/frontend/index.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - frontend - - -
- - - diff --git a/frontend/main.jsx b/frontend/main.jsx deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/package-lock.json b/frontend/package-lock.json deleted file mode 100644 index 21171c7..0000000 --- a/frontend/package-lock.json +++ /dev/null @@ -1,3359 +0,0 @@ -{ - "name": "frontend", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "frontend", - "version": "0.0.0", - "dependencies": { - "@tailwindcss/vite": "^4.3.3", - "axios": "^1.19.0", - "react": "^19.2.8", - "react-dom": "^19.2.8", - "react-router-dom": "^7.18.2", - "tailwindcss": "^4.3.3" - }, - "devDependencies": { - "@eslint/js": "^10.0.1", - "@types/react": "^19.2.17", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.4", - "eslint": "^10.8.0", - "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-react-refresh": "^0.5.3", - "globals": "^17.7.0", - "vite": "^8.2.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", - "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.29.7", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", - "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", - "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.7", - "@babel/helper-compilation-targets": "^7.29.7", - "@babel/helper-module-transforms": "^7.29.7", - "@babel/helpers": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/template": "^7.29.7", - "@babel/traverse": "^7.29.7", - "@babel/types": "^7.29.7", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", - "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.8", - "@babel/types": "^7.29.8", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", - "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.29.7", - "@babel/helper-validator-option": "^7.29.7", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", - "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", - "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", - "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.29.7", - "@babel/helper-validator-identifier": "^7.29.7", - "@babel/traverse": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", - "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", - "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", - "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", - "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", - "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.8" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/template": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", - "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", - "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.8", - "@babel/helper-globals": "^7.29.7", - "@babel/parser": "^7.29.8", - "@babel/template": "^7.29.7", - "@babel/types": "^7.29.8", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", - "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.29.7", - "@babel/helper-validator-identifier": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@emnapi/core": { - "version": "2.0.0-alpha.3", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.3.tgz", - "integrity": "sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==", - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "2.0.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "2.0.0-alpha.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.3.tgz", - "integrity": "sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.0.1.tgz", - "integrity": "sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", - "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^3.4.3" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.12.2", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", - "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/config-array": { - "version": "0.23.5", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", - "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/object-schema": "^3.0.5", - "debug": "^4.3.1", - "minimatch": "^10.2.4" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - } - }, - "node_modules/@eslint/config-helpers": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", - "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^1.2.1" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - } - }, - "node_modules/@eslint/core": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", - "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@types/json-schema": "^7.0.15" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - } - }, - "node_modules/@eslint/js": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", - "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://eslint.org/donate" - }, - "peerDependencies": { - "eslint": "^10.0.0" - }, - "peerDependenciesMeta": { - "eslint": { - "optional": true - } - } - }, - "node_modules/@eslint/object-schema": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", - "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - } - }, - "node_modules/@eslint/plugin-kit": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz", - "integrity": "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^1.2.1", - "levn": "^0.4.1" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - } - }, - "node_modules/@humanfs/core": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", - "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@humanfs/types": "^0.15.0" - }, - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanfs/node": { - "version": "0.16.8", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", - "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@humanfs/core": "^0.19.2", - "@humanfs/types": "^0.15.0", - "@humanwhocodes/retry": "^0.4.0" - }, - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanfs/types": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", - "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@humanwhocodes/retry": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", - "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.2.tgz", - "integrity": "sha512-JfB4kuJQjaoHuCTseIINHtHWeJnvgEcxjwA5t/Y00ZgaOO1Crz3fjT/p8kT28zA/Caz7oiUMn3d6H2yOVCVwuw==", - "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.3" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=23.5.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.3", - "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" - } - }, - "node_modules/@oxc-project/types": { - "version": "0.142.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz", - "integrity": "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.1.tgz", - "integrity": "sha512-02hOeOSryYxVrOIphmLAsqnCJWxwlzFk+pEt/N/i6OgT3lShHO7xGCU5cpgchRDHboAEbSjzgGh+O/u1GswQmA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.1.tgz", - "integrity": "sha512-fMsTOnN0OjFm3CyppWPitKnc8UlliVARUULW6cfU6AIqjdtgmSFWSk9vecHzZduv/yMWIHDlRhM1e8Iff9uAfA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.1.tgz", - "integrity": "sha512-1wjKdz/XLGKHaTNHjQveQ/B23TKx4ItAqm1JbyVuvNPc4Ze0Fb48s49TAd/2zcplPl8okE/UbTgmlVfwT7eFeQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.1.tgz", - "integrity": "sha512-Fa0jHR07E7YBN4vOEsbVf2briYNsuOowfLJaXULZM0ldMlaCaj2LJgLMbMe4iacRyZmvR8efFhgR9wKuGclQUg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.1.tgz", - "integrity": "sha512-pzkgu1SSHGgRRyRZ4fbmSgmajbVt+epaLP99NDjFft69v/ypfTi6swBMiVdh2EkQ0OSnHE1lZDM7DRGkyAzUpA==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.1.tgz", - "integrity": "sha512-QI5SEDY8cbiYWHx0VO4vIc3UlS6a32vXHjU8Qy/17adEmZIPuByJg13UEvo9c/UCiUkdcVWY83C+b+JrwnNyUg==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.1.tgz", - "integrity": "sha512-Sm41FyCeXqmYcERoYOCbGIL5hNfd8w9LQ7Y61Bev48HkcjaJqV/iiVOaiDxjVTRMS+QKrZmD8cfPt4uMVnvM+A==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.1.tgz", - "integrity": "sha512-2x+WhXTGl9yJYPbltW/BSEPTVz9OIWQyER4N+gJEDWkkn904eRcBzELqh/Hf7K0w/ubGbKNMv0ZC+94QK/IFEg==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.1.tgz", - "integrity": "sha512-eEjmQpuRQayHPWWnywaWHkFT3ToPbP3RYy42VVd/B9aBGDA+Ol25EIWHxKQST3IiWJjikCWUF7KtbfqwZrzVwQ==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.1.tgz", - "integrity": "sha512-/Orga1fZYkLc/56jBICcHrKchl8Z2UKdDSr3LG9ToWO1lQ6a4Livk9Xz+9WN91zsz5QR3XQz2NNoSDEvP6qadw==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.1.tgz", - "integrity": "sha512-xxBJRL+0q0Kce7orznGWLuylHDY65vuARXZRpX+hPdv+DqK2c3NlCsVA98tlWzWNEE7yPqA/1NQ5nnCrj49Y5A==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.1.tgz", - "integrity": "sha512-M6AdXIXw3s+/8XpKMzdGDEXGS1S7kwUsy+rcTIUIOx5Ge4nXKCtAFHFV9YKkXvGcC5WMoTjAteLzlsQROVI0Yw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.2.1.tgz", - "integrity": "sha512-/TX0SoRGojHzSAHpfVBbavRVSazg5U3h3Y3VXfcc0cdugq6kxdqw8LPGFiPr+/7gE/60zRcsOY2Vi9b9eT0jww==", - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "2.0.0-alpha.3", - "@emnapi/runtime": "2.0.0-alpha.3", - "@napi-rs/wasm-runtime": "^1.2.0" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=23.5.0" - } - }, - "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.1.tgz", - "integrity": "sha512-EvRrivJieyHG+AO9lleZWgq+g0+S7oV2C51yuqlcyU/R9net+sI4Pj0F+lUoP2bEr6TWX3SqFaaS0SzfLxSzkw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.1.tgz", - "integrity": "sha512-Z4eCmn5QJ/5+azF9knpLWKfVd9aidn0mAe9TpJgvBLId9Ax3t0+JVxBmT25Bv7NBbVW1TZyKjQjQReouMeH5UQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "license": "MIT" - }, - "node_modules/@tailwindcss/node": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz", - "integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==", - "license": "MIT", - "dependencies": { - "@jridgewell/remapping": "^2.3.5", - "enhanced-resolve": "^5.24.1", - "jiti": "^2.7.0", - "lightningcss": "1.32.0", - "magic-string": "^0.30.21", - "source-map-js": "^1.2.1", - "tailwindcss": "4.3.3" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", - "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-android-arm64": "1.32.0", - "lightningcss-darwin-arm64": "1.32.0", - "lightningcss-darwin-x64": "1.32.0", - "lightningcss-freebsd-x64": "1.32.0", - "lightningcss-linux-arm-gnueabihf": "1.32.0", - "lightningcss-linux-arm64-gnu": "1.32.0", - "lightningcss-linux-arm64-musl": "1.32.0", - "lightningcss-linux-x64-gnu": "1.32.0", - "lightningcss-linux-x64-musl": "1.32.0", - "lightningcss-win32-arm64-msvc": "1.32.0", - "lightningcss-win32-x64-msvc": "1.32.0" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-android-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", - "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-darwin-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", - "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-darwin-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", - "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-freebsd-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", - "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", - "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", - "cpu": [ - "arm" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", - "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-linux-arm64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", - "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-linux-x64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", - "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-linux-x64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", - "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", - "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/node/node_modules/lightningcss-win32-x64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", - "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@tailwindcss/oxide": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz", - "integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==", - "license": "MIT", - "engines": { - "node": ">= 20" - }, - "optionalDependencies": { - "@tailwindcss/oxide-android-arm64": "4.3.3", - "@tailwindcss/oxide-darwin-arm64": "4.3.3", - "@tailwindcss/oxide-darwin-x64": "4.3.3", - "@tailwindcss/oxide-freebsd-x64": "4.3.3", - "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", - "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", - "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", - "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", - "@tailwindcss/oxide-linux-x64-musl": "4.3.3", - "@tailwindcss/oxide-wasm32-wasi": "4.3.3", - "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", - "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" - } - }, - "node_modules/@tailwindcss/oxide-android-arm64": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.3.tgz", - "integrity": "sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-darwin-arm64": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.3.tgz", - "integrity": "sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-darwin-x64": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.3.tgz", - "integrity": "sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-freebsd-x64": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.3.tgz", - "integrity": "sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.3.tgz", - "integrity": "sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.3.tgz", - "integrity": "sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-linux-arm64-musl": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.3.tgz", - "integrity": "sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-linux-x64-gnu": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz", - "integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-linux-x64-musl": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz", - "integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.3.tgz", - "integrity": "sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==", - "bundleDependencies": [ - "@napi-rs/wasm-runtime", - "@emnapi/core", - "@emnapi/runtime", - "@tybys/wasm-util", - "@emnapi/wasi-threads", - "tslib" - ], - "cpu": [ - "wasm32" - ], - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.11.1", - "@emnapi/runtime": "^1.11.1", - "@emnapi/wasi-threads": "^1.2.2", - "@napi-rs/wasm-runtime": "^1.1.4", - "@tybys/wasm-util": "^0.10.2", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", - "integrity": "sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/oxide-win32-x64-msvc": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.3.tgz", - "integrity": "sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 20" - } - }, - "node_modules/@tailwindcss/vite": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.3.tgz", - "integrity": "sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==", - "license": "MIT", - "dependencies": { - "@tailwindcss/node": "4.3.3", - "@tailwindcss/oxide": "4.3.3", - "tailwindcss": "4.3.3" - }, - "peerDependencies": { - "vite": "^5.2.0 || ^6 || ^7 || ^8" - } - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", - "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@types/esrecurse": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", - "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/estree": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", - "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/react": { - "version": "19.2.18", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", - "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", - "dev": true, - "license": "MIT", - "dependencies": { - "csstype": "^3.2.2" - } - }, - "node_modules/@types/react-dom": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz", - "integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "@types/react": "^19.2.0" - } - }, - "node_modules/@vitejs/plugin-react": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.5.tgz", - "integrity": "sha512-BOVzne/NL162sMdResB25mUv+vWMF5NoAjNf09TeGlE7ZpszZWSD3winycicLJw72yeVsoCn/2kOhEuCvEShMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rolldown/pluginutils": "^1.0.1" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "peerDependencies": { - "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", - "babel-plugin-react-compiler": "^1.0.0", - "vite": "^8.0.0" - }, - "peerDependenciesMeta": { - "@rolldown/plugin-babel": { - "optional": true - }, - "babel-plugin-react-compiler": { - "optional": true - } - } - }, - "node_modules/acorn": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", - "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/ajv": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", - "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.19.0.tgz", - "integrity": "sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw==", - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.16.0", - "form-data": "^4.0.6", - "https-proxy-agent": "^5.0.1", - "proxy-from-env": "^2.1.0" - } - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/baseline-browser-mapping": { - "version": "2.11.9", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.9.tgz", - "integrity": "sha512-cp447VUsGS07+n1Dqf7YSQ8maeJrjEhaDxTm1ZefbqDtypHBC5GzGMQbklR6IPR13Y8OAJRHZWEMtZipJLCttg==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/browserslist": { - "version": "4.28.7", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz", - "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.10.44", - "caniuse-lite": "^1.0.30001806", - "electron-to-chromium": "^1.5.393", - "node-releases": "^2.0.51", - "update-browserslist-db": "^1.2.3" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001806", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz", - "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/csstype": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", - "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/electron-to-chromium": { - "version": "1.5.399", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.399.tgz", - "integrity": "sha512-lEcqhErbHjXRvd41rnWLpzbyU/IXfIYo7QwaFWmxGeLiLyY2TBCdHnWY88vB+p3ubnihRypDm66panXl7TylLA==", - "dev": true, - "license": "ISC" - }, - "node_modules/enhanced-resolve": { - "version": "5.24.5", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", - "integrity": "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==", - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.4", - "tapable": "^2.3.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", - "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint": { - "version": "10.8.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.8.0.tgz", - "integrity": "sha512-nuKKvN+oIBO0koN7Tm7dlkmnkc21mtt0QJLwAKzjLq14y6lRTdVG36MZHJ8eQHwdJMwZbQNMlPOYedMq/oVJvQ==", - "dev": true, - "license": "MIT", - "workspaces": [ - "packages/*" - ], - "dependencies": { - "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.5", - "@eslint/config-helpers": "^0.7.0", - "@eslint/core": "^1.2.1", - "@eslint/plugin-kit": "^0.7.2", - "@humanfs/node": "^0.16.6", - "@humanwhocodes/module-importer": "^1.0.1", - "@humanwhocodes/retry": "^0.4.2", - "@types/estree": "^1.0.6", - "ajv": "^6.14.0", - "cross-spawn": "^7.0.6", - "debug": "^4.3.2", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^9.1.2", - "eslint-visitor-keys": "^5.0.1", - "espree": "^11.2.0", - "esquery": "^1.7.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^8.0.0", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "minimatch": "^10.2.5", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://eslint.org/donate" - }, - "peerDependencies": { - "jiti": "*" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - } - } - }, - "node_modules/eslint-plugin-react-hooks": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.1.1.tgz", - "integrity": "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.24.4", - "@babel/parser": "^7.24.4", - "hermes-parser": "^0.25.1", - "zod": "^3.25.0 || ^4.0.0", - "zod-validation-error": "^3.5.0 || ^4.0.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" - } - }, - "node_modules/eslint-plugin-react-refresh": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.5.3.tgz", - "integrity": "sha512-5EMmLCV98Pi4o/f/3DP/v/tNqLHMIc9I8LKClNDWhZ9JTho89/kQcitCXQBMG7sAfVRK0Ie3T2EDOzp1YXYiVA==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "eslint": "^9 || ^10" - } - }, - "node_modules/eslint-scope": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", - "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "@types/esrecurse": "^4.3.1", - "@types/estree": "^1.0.8", - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/espree": { - "version": "11.2.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", - "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "acorn": "^8.16.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^5.0.1" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/esquery": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", - "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "estraverse": "^5.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/file-entry-cache": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", - "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "flat-cache": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/flat-cache": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", - "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", - "dev": true, - "license": "MIT", - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.4" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/flatted": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", - "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", - "dev": true, - "license": "ISC" - }, - "node_modules/follow-redirects": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", - "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/globals": { - "version": "17.8.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.8.0.tgz", - "integrity": "sha512-Zz/LMDZScFmkakeL2cTHzf+PbWKdpU3uclqkZT7TjDG58j5WPt0PpA+n9uPI24fZtlw07q0OtEi84K+umsRzqQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "license": "ISC" - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hermes-estree": { - "version": "0.25.1", - "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", - "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", - "dev": true, - "license": "MIT" - }, - "node_modules/hermes-parser": { - "version": "0.25.1", - "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", - "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hermes-estree": "0.25.1" - } - }, - "node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/jiti": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", - "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", - "license": "MIT", - "bin": { - "jiti": "lib/jiti-cli.mjs" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "json-buffer": "3.0.1" - } - }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/lightningcss": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", - "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-android-arm64": "1.33.0", - "lightningcss-darwin-arm64": "1.33.0", - "lightningcss-darwin-x64": "1.33.0", - "lightningcss-freebsd-x64": "1.33.0", - "lightningcss-linux-arm-gnueabihf": "1.33.0", - "lightningcss-linux-arm64-gnu": "1.33.0", - "lightningcss-linux-arm64-musl": "1.33.0", - "lightningcss-linux-x64-gnu": "1.33.0", - "lightningcss-linux-x64-musl": "1.33.0", - "lightningcss-win32-arm64-msvc": "1.33.0", - "lightningcss-win32-x64-msvc": "1.33.0" - } - }, - "node_modules/lightningcss-android-arm64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", - "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-arm64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", - "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-x64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", - "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-freebsd-x64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", - "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", - "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", - "cpu": [ - "arm" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", - "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", - "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", - "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-musl": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", - "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", - "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", - "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/magic-string": { - "version": "0.30.21", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", - "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.8" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.51", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", - "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/postcss": { - "version": "8.5.25", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", - "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.16", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/proxy-from-env": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", - "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/react": { - "version": "19.2.8", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", - "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/react-dom": { - "version": "19.2.8", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", - "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", - "license": "MIT", - "dependencies": { - "scheduler": "^0.27.0" - }, - "peerDependencies": { - "react": "^19.2.8" - } - }, - "node_modules/react-router": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", - "integrity": "sha512-aUVMjFm3GAPTTZL7oYr5E7ETiqfQCHRLH+B+5afnICvf0r7kkK4eR6SMuwbSTJw/7t+12khT/Kahij49fqOCIg==", - "license": "MIT", - "dependencies": { - "cookie": "^1.0.1", - "set-cookie-parser": "^2.6.0" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "react": ">=18", - "react-dom": ">=18" - }, - "peerDependenciesMeta": { - "react-dom": { - "optional": true - } - } - }, - "node_modules/react-router-dom": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.2.tgz", - "integrity": "sha512-AIKJ/jgGlFb3EbfCXk5Gzshiwt+l3mqbCrNjmEWMMjqQxNJ3svBa6bgzFyCC2Sw3RA0VWF1kg3uQf2OFhxb8hw==", - "license": "MIT", - "dependencies": { - "react-router": "7.18.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "react": ">=18", - "react-dom": ">=18" - } - }, - "node_modules/rolldown": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.1.tgz", - "integrity": "sha512-4FKJhg8d3OiyQOA6Q1Q0hoFFpW9/OoX+VsHzpECsdsIZoOArrAK90gl59YK/Z+gnDel45bgJZK03ozH/9bCqEw==", - "license": "MIT", - "dependencies": { - "@oxc-project/types": "=0.142.0", - "@rolldown/pluginutils": "^1.0.0" - }, - "bin": { - "rolldown": "bin/cli.mjs" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.2.1", - "@rolldown/binding-darwin-arm64": "1.2.1", - "@rolldown/binding-darwin-x64": "1.2.1", - "@rolldown/binding-freebsd-x64": "1.2.1", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.1", - "@rolldown/binding-linux-arm64-gnu": "1.2.1", - "@rolldown/binding-linux-arm64-musl": "1.2.1", - "@rolldown/binding-linux-ppc64-gnu": "1.2.1", - "@rolldown/binding-linux-s390x-gnu": "1.2.1", - "@rolldown/binding-linux-x64-gnu": "1.2.1", - "@rolldown/binding-linux-x64-musl": "1.2.1", - "@rolldown/binding-openharmony-arm64": "1.2.1", - "@rolldown/binding-wasm32-wasi": "1.2.1", - "@rolldown/binding-win32-arm64-msvc": "1.2.1", - "@rolldown/binding-win32-x64-msvc": "1.2.1" - } - }, - "node_modules/scheduler": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", - "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/set-cookie-parser": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", - "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", - "license": "MIT" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/tailwindcss": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", - "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", - "license": "MIT" - }, - "node_modules/tapable": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", - "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", - "license": "MIT", - "engines": { - "node": ">=6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, - "node_modules/tinyglobby": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", - "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.4" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true - }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/vite": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.0.tgz", - "integrity": "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==", - "license": "MIT", - "dependencies": { - "lightningcss": "^1.33.0", - "picomatch": "^4.0.5", - "postcss": "^8.5.23", - "rolldown": "~1.2.0", - "tinyglobby": "^0.2.17" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.4.0", - "esbuild": "^0.27.0 || ^0.28.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "@vitejs/devtools": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/zod": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", - "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/zod-validation-error": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-4.0.2.tgz", - "integrity": "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.0.0" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - } - } - } -} diff --git a/frontend/package.json b/frontend/package.json deleted file mode 100644 index b70905a..0000000 --- a/frontend/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "frontend", - "private": true, - "version": "0.0.0", - "type": "module", - "scripts": { - "dev": "vite", - "build": "vite build", - "lint": "eslint .", - "preview": "vite preview" - }, - "dependencies": { - "@tailwindcss/vite": "^4.3.3", - "axios": "^1.19.0", - "react": "^19.2.8", - "react-dom": "^19.2.8", - "react-router-dom": "^7.18.2", - "tailwindcss": "^4.3.3" - }, - "devDependencies": { - "@eslint/js": "^10.0.1", - "@types/react": "^19.2.17", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.4", - "eslint": "^10.8.0", - "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-react-refresh": "^0.5.3", - "globals": "^17.7.0", - "vite": "^8.2.0" - } -} diff --git a/frontend/pages/chat-home.jsx b/frontend/pages/chat-home.jsx deleted file mode 100644 index de6174d..0000000 --- a/frontend/pages/chat-home.jsx +++ /dev/null @@ -1,12 +0,0 @@ -function ChatHome() { - return ( -
-

Chat

-

- The document-grounded chat assistant will be built here starting Day 11. -

-
- ); -} - -export default ChatHome; \ No newline at end of file diff --git a/frontend/pages/chat.jsx b/frontend/pages/chat.jsx deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/pages/dashboard.jsx b/frontend/pages/dashboard.jsx deleted file mode 100644 index 4c2ad53..0000000 --- a/frontend/pages/dashboard.jsx +++ /dev/null @@ -1,30 +0,0 @@ -import DocumentsCard from "../src/components/DocumentsCard.jsx"; - -function Dashboard() { - return ( -
-

Your Workspace

- -
- - -
-

Generated Resources

-

- Once you upload a document, generated summaries, key points, and quizzes will appear here. -

-
- -
-

Your Analytics

-

Track your study progress over time.

- - View analytics → - -
-
-
- ); -} - -export default Dashboard; \ No newline at end of file diff --git a/frontend/pages/documents.jsx b/frontend/pages/documents.jsx deleted file mode 100644 index 89ba3d0..0000000 --- a/frontend/pages/documents.jsx +++ /dev/null @@ -1,132 +0,0 @@ -import { useState, useEffect } from "react"; -import { listDocuments, getDocumentFile } from "../api/routes.jsx"; -import ResourcesPanel from "../src/components/ResourcesPanel.jsx"; - - -function Documents() { - const [documents, setDocuments] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(""); - const [selectedDoc, setSelectedDoc] = useState(null); - const [pdfUrl, setPdfUrl] = useState(null); - const [viewerLoading, setViewerLoading] = useState(false); - - async function fetchDocuments() { - setLoading(true); - setError(""); - try { - const res = await listDocuments(); - setDocuments(res.data); - } catch (err) { - setError(err.response?.data?.detail || "Could not load documents."); - } finally { - setLoading(false); - } - } - - async function handleView(doc) { - setSelectedDoc(doc); - setViewerLoading(true); - setError(""); - - if (pdfUrl) URL.revokeObjectURL(pdfUrl); - setPdfUrl(null); - - try { - const res = await getDocumentFile(doc.id); - const blobUrl = URL.createObjectURL(res.data); - setPdfUrl(blobUrl); - } catch { - setError("Could not load this document. Please try again."); - } finally { - setViewerLoading(false); - } - } - - useEffect(() => { - // eslint-disable-next-line react-hooks/set-state-in-effect -- fetch-on-mount pattern, see https://react.dev/learn/you-might-not-need-an-effect - fetchDocuments(); - return () => { - if (pdfUrl) URL.revokeObjectURL(pdfUrl); - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - return ( -
-
-

Your Documents

- -
- {error &&

{error}

} - -
-
- {loading ? ( -

Loading documents...

- ) : documents.length === 0 ? ( -

- No documents yet — upload one from the Dashboard to get started. -

- ) : ( - - - - - - - - - - - - {documents.map((doc) => ( - handleView(doc)} - className={`cursor-pointer border-b hover:bg-gray-50 ${ - selectedDoc?.id === doc.id ? "bg-blue-50" : "" - }`} - > - - - - - - - ))} - -
FilenameSubjectPagesStatusChunks
{doc.filename}{doc.subject}{doc.page_count}{doc.chunk_count} - - {doc.processing_status} - -
- )} -
- -
-
- {!selectedDoc ? ( -

Select a document to view it here.

- ) : viewerLoading ? ( -

Loading preview...

- ) : pdfUrl ? ( -