From b565f0b87dad137e4cfd9f8a0083fa2adbdda966 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 14:51:32 +0000 Subject: [PATCH 1/2] feat(storage): optional S3/object-storage driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Put storage behind a backend-agnostic driver interface selected by the STORAGE_DRIVER env var: `local` (default, unchanged on-disk behaviour) or `s3` for S3 / S3-compatible object storage (AWS S3, MinIO, Cloudflare R2, Hetzner). Additive and opt-in — without STORAGE_DRIVER the local driver is used and behaviour is byte-for-byte the same as before. - src/lib/storage/ replaces the single storage.ts: - driver.ts: StorageDriver interface, shared pure helpers (safeInlineMime, sanitizeName, StorageLimitError), the canonical rel-path key builder, and the shared size+sha256+maxBytes upload meter. - local.ts: LocalStorageDriver (the previous fs implementation, verbatim). - s3.ts: S3StorageDriver — AWS SDK imported lazily so the local path never loads it; PutObject/GetObject/HeadObject/DeleteObject + lib-storage multipart Upload for streams; ranged GetObject for HTTP range requests; credentials via the standard AWS chain. - index.ts: facade keeping every existing named export + new readFileBuffer, caches the selected driver. - fileStream is now async (S3 GetObject is async); the file route and the zip-unpacker await it. - extract.ts and media.ts now read stored binaries through readFileBuffer instead of reading local disk directly, so OCR/transcription/PDF/DOCX extraction work against S3 too. - No schema change: the stored key is backend-agnostic. - Tests: mocked S3 driver (put/get/head/delete/range/key-prefix/safe-key) and a real LocalStorageDriver round-trip (save/read/size/delete, streaming with size+hash+maxBytes, byte-range). - Docs: STORAGE_DRIVER / S3_* documented in .env.example, docker-compose.yml and docs/self-hosting.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0191JikNRN8Q2HBtf6fpLXmH --- .env.example | 14 + docker-compose.yml | 12 + docs/self-hosting.md | 25 + package-lock.json | 620 +++++++++++++++++++++++ package.json | 3 + src/app/api/materials/[id]/file/route.ts | 13 +- src/lib/ai/extract.ts | 22 +- src/lib/ai/media.ts | 14 +- src/lib/jobs/unpack-zip.ts | 2 +- src/lib/storage.ts | 129 ----- src/lib/storage/driver.ts | 93 ++++ src/lib/storage/index.ts | 80 +++ src/lib/storage/local.ts | 78 +++ src/lib/storage/s3.ts | 162 ++++++ src/lib/storage/storage.test.ts | 187 +++++++ 15 files changed, 1288 insertions(+), 166 deletions(-) delete mode 100644 src/lib/storage.ts create mode 100644 src/lib/storage/driver.ts create mode 100644 src/lib/storage/index.ts create mode 100644 src/lib/storage/local.ts create mode 100644 src/lib/storage/s3.ts create mode 100644 src/lib/storage/storage.test.ts diff --git a/.env.example b/.env.example index a687046..4bdc9a3 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,20 @@ ENCRYPTION_KEY=change-me # TZ=Europe/Berlin +# Object storage for uploaded files. Default is local disk (the uploads volume +# above). For large multi-GB modules or multi-node setups, use S3 or any +# S3-compatible service (AWS S3, MinIO, Cloudflare R2). See docs/self-hosting.md. +# STORAGE_DRIVER=s3 +# S3_BUCKET=study-helper +# S3_REGION=eu-central-1 +# S3_ENDPOINT=https://s3.example.com # only for S3-compatible services (MinIO/R2) +# S3_FORCE_PATH_STYLE=true # required by MinIO / some endpoints +# S3_KEY_PREFIX=study-helper # optional object key prefix +# Credentials via the standard AWS chain: +# AWS_ACCESS_KEY_ID=... +# AWS_SECRET_ACCESS_KEY=... + + # Demo data on startup: creates admin@example.com / admin-test-1234 (admin) # and user@example.com / user-test-1234, each with sample study content. # Runs once (skipped if the accounts exist). Never enable in production. diff --git a/docker-compose.yml b/docker-compose.yml index 3753bff..85551a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,8 +30,20 @@ services: # the web tier, set WORKERS_IN_PROCESS: "false" here and run the worker # service below. By default (unset) the app runs workers in-process — no # extra service needed. + # + # To store uploaded files in S3 / S3-compatible object storage instead of the + # local uploads volume (recommended for large multi-GB modules or multi-node + # deployments), set STORAGE_DRIVER: "s3" and the S3_* / AWS_* vars below. + # See docs/self-hosting.md → "Object storage (S3)". # environment: # WORKERS_IN_PROCESS: "false" + # STORAGE_DRIVER: "s3" + # S3_BUCKET: "study-helper" + # S3_REGION: "eu-central-1" + # S3_ENDPOINT: "https://s3.example.com" # only for S3-compatible services + # S3_FORCE_PATH_STYLE: "true" # required by MinIO / some endpoints + # AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" + # AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" # Optional dedicated job worker. Runs `npm run worker`, which uses tsx — so it # needs an image that includes the TS sources + devDependencies (a source diff --git a/docs/self-hosting.md b/docs/self-hosting.md index b8a3739..36a2c46 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -88,10 +88,35 @@ and links to the release. Installing it is a manual | `STUDYHELPER_VERSION` | no | Image tag to run (default `latest`); pin e.g. `1.0.0` for reproducible deploys | | `DATA_DIR` | no | Host directory for the database + uploads volumes (default `./data`, next to `docker-compose.yml`) — see [Where data is stored](#where-data-is-stored) | | `UPLOAD_DIR` | no | Upload path **inside the container** (default `/data/uploads`) — only relevant for non-Docker deployments; Docker users should set `DATA_DIR` instead | +| `STORAGE_DRIVER` | no | Where uploaded files live: `local` (default, disk under `UPLOAD_DIR`) or `s3` (S3 / S3-compatible object storage) — see [Object storage (S3)](#object-storage-s3) | +| `WORKERS_IN_PROCESS` | no | `false` runs background jobs only in a separate worker process (`npm run worker`) instead of the web tier (default `true` — in-process) | | `SEED_TEST_DATA` | no | `true` seeds demo accounts (admin@example.com / admin-test-1234, user@example.com / user-test-1234) with sample study content on startup — for evaluation only, never in production | **Do not lose `ENCRYPTION_KEY`** — encrypted settings (AI keys, SMTP, OIDC secrets) become unreadable without it. +## Object storage (S3) + +By default uploaded files are stored on disk (`STORAGE_DRIVER=local`, under +`UPLOAD_DIR` / the `uploads` volume). For large multi-GB modules or multi-node +deployments you can store them in S3 or any S3-compatible service (AWS S3, +MinIO, Cloudflare R2, Hetzner Object Storage) instead: + +| Variable | Required | Description | +| --------------------- | --------------- | ----------------------------------------------------------------------------------- | +| `STORAGE_DRIVER` | set to `s3` | Enables the S3 driver | +| `S3_BUCKET` | yes (with `s3`) | Target bucket name | +| `S3_REGION` | no | Bucket region (falls back to `AWS_REGION`, then `us-east-1`) | +| `S3_ENDPOINT` | no | Custom endpoint URL for S3-compatible services (e.g. MinIO/R2); omit for AWS S3 | +| `S3_FORCE_PATH_STYLE` | no | `true` for path-style addressing (needed by MinIO and some S3-compatible endpoints) | +| `S3_KEY_PREFIX` | no | Optional key prefix (folder) for all objects, e.g. `study-helper` | + +Credentials come from the standard AWS credential chain — set +`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` (and `AWS_SESSION_TOKEN` if +applicable), or run on infrastructure with an attached IAM role. The value +stored in the database is a backend-agnostic key, so nothing schema-wise +changes; switching drivers does **not** migrate existing files, so pick a +driver before uploading (or copy the objects yourself when migrating). + ## Reverse proxy example (Caddy) ``` diff --git a/package-lock.json b/package-lock.json index 1eb8984..e65a357 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,8 @@ "@ai-sdk/openai": "^4.0.10", "@ai-sdk/openai-compatible": "^3.0.7", "@ai-sdk/react": "^4.0.20", + "@aws-sdk/client-s3": "^3.1088.0", + "@aws-sdk/lib-storage": "^3.1088.0", "@base-ui/react": "^1.6.0", "@better-auth/passkey": "^1.6.23", "@dnd-kit/core": "^6.3.1", @@ -72,6 +74,7 @@ "@types/sql.js": "^1.4.11", "@types/web-push": "^3.6.4", "@vitejs/plugin-react": "^6.0.3", + "aws-sdk-client-mock": "^4.1.0", "drizzle-kit": "^0.31.10", "eslint": "^9", "eslint-config-next": "16.2.10", @@ -327,6 +330,334 @@ "dev": true, "license": "MIT" }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.18.tgz", + "integrity": "sha512-IImkbEyXdV6/uaF5r6Wkk+8718mQw1ll83j0a4a30R3JM/rHVFdWAiT4jtJpFjJiIwM/oJ6SxIxr0z2TaQUGqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1088.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1088.0.tgz", + "integrity": "sha512-9ryKKxnjtJRKLDI24P+2gQZki7k28BeV6gz1AySvFDWayhjAZuh4QZjCyx55tqR8Oz0IJxWutgJRO43dDOcXoQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.18", + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.69", + "@aws-sdk/middleware-sdk-s3": "^3.972.64", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.975.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.3.tgz", + "integrity": "sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@aws-sdk/xml-builder": "^3.972.36", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.4", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.59.tgz", + "integrity": "sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.61", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.61.tgz", + "integrity": "sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.3.tgz", + "integrity": "sha512-WpuqYX4gGkx++fCTSWE8+41JzkZVcrI50SH48Ml4CsG1pyuHKyMmpw/FixBHDrmjoQ553PmeCLa/fZIcst+WyA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-login": "^3.972.65", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.65", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.65.tgz", + "integrity": "sha512-xr9rgjYEdmC2Tpg2lwt9o+nOEaK9Qpd+dBjzrVCuWWyQfvhO91Ezu0Hh9ts2VUxOZxmS/k5T9msa34e4R1bnrQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.69", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.69.tgz", + "integrity": "sha512-wbJGGesd0Tl18bmUcbj1xJ+e7CpuRJ6PIpMywLFuUttGy615lua87cJ0EA8pFpY/QgPuUXbnupWBtSPJ9tyZhg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-ini": "^3.973.3", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.59.tgz", + "integrity": "sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.3.tgz", + "integrity": "sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/token-providers": "3.1088.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.65", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.65.tgz", + "integrity": "sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.1088.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.1088.0.tgz", + "integrity": "sha512-OElyotfOuTLAkWeYfWvgFQ/ABVs5xi9+UzlnKveFnbyfzV9um0guDWGanG/xGlw9ofAiplscHlNauChtJUBNWA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.1088.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.64", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.64.tgz", + "integrity": "sha512-RBi43anhDBUv+HCfxCOXwGOE7GmT4n7ChV04Mwr22RhXTNcamW/iWnJlOotDPCZSrJ4dEvhZSiWWQMwLX+ZhFA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.33", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.33.tgz", + "integrity": "sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.41.tgz", + "integrity": "sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1088.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1088.0.tgz", + "integrity": "sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", + "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.36", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.36.tgz", + "integrity": "sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -4646,6 +4977,128 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", + "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@sinonjs/samsam": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.3.tgz", + "integrity": "sha512-hw6HbX+GyVZzmaYNh82Ecj1vdGZrqVIn/keDTg63IgAwiQPO+xCz99uG6Woqgb4tM0mUiFENKZ4cqd7IX94AXQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "type-detect": "^4.1.0" + } + }, + "node_modules/@sinonjs/samsam/node_modules/type-detect": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", + "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/@smithy/core": { + "version": "3.29.4", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.4.tgz", + "integrity": "sha512-G1GRglAabzEhqghJMBAd54FkRS7SAFGHEwbhcI9r+O+LIMuFsLyXkLZkCoFSgAglRu8s/URVXJB0hglq3ZipIg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.9", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.9.tgz", + "integrity": "sha512-2nfV4qRKiYeXU4zD2vvSCfg5dfp/BuhrM73vt7q9gzBhxs4rbPxXY21wo+kyI3bRmXcEGRnCLTaW8O437jzHIg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.6", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.6.tgz", + "integrity": "sha512-NHLgAlORUFZjn5ZfhYuyyKMlXA1WLYOdGxEhyNxrPpbJzoacGbl0chn1lN2KiZ8mpNVk0tV5607CSYlYs/OFgw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.9.6", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.6.tgz", + "integrity": "sha512-odd+HYx3OLcXRSEz0ZeF3JQdSYdK8QnRgA2N87cPW7coWIbKfRk7a9VQjfeWQLqnzrDLk23KMEn46p8N7M/JFg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.5", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.5.tgz", + "integrity": "sha512-MO5VEhwVl0BN7xVoVeNrZfiUFoQtqxUbgl6/RwOTlMMxCSjblG8twSrVTwz3J4w9WZxd2rBfBAUXjH77agspBg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -5433,6 +5886,23 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/sinon": { + "version": "17.0.4", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.4.tgz", + "integrity": "sha512-RHnIrhfPO3+tJT0s7cFaXGZvsL4bbR3/k7z3P312qMS4JaS2Tk+KiwiLx1S0rQ56ERj00u1/BtdyVd0FY+Pdew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/sinonjs__fake-timers": "*" + } + }, + "node_modules/@types/sinonjs__fake-timers": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-15.0.1.tgz", + "integrity": "sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/sql.js": { "version": "1.4.11", "resolved": "https://registry.npmjs.org/@types/sql.js/-/sql.js-1.4.11.tgz", @@ -6683,6 +7153,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-sdk-client-mock": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/aws-sdk-client-mock/-/aws-sdk-client-mock-4.1.0.tgz", + "integrity": "sha512-h/tOYTkXEsAcV3//6C1/7U4ifSpKyJvb6auveAepqqNJl6TdZaPFEtKjBQNf8UxQdDP850knB2i/whq4zlsxJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/sinon": "^17.0.3", + "sinon": "^18.0.1", + "tslib": "^2.1.0" + } + }, "node_modules/axe-core": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.12.1.tgz", @@ -6936,6 +7418,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.16", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", @@ -6992,6 +7480,16 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -9402,6 +9900,15 @@ "node": ">= 0.6" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -10439,6 +10946,26 @@ "@formatjs/icu-messageformat-parser": "^3.4.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -11396,6 +11923,13 @@ "setimmediate": "^1.0.5" } }, + "node_modules/just-extend": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", + "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -13254,6 +13788,29 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/nise": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/nise/-/nise-6.1.5.tgz", + "integrity": "sha512-SnRDPDBjxZZoU2n0+gzzLtSvo1OZo7j6jnbXsoh3AFxEGhaFU7ZF0TmefuKERq79wxR2U+MPn7ArW+Tl+clC3A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "^15.1.1", + "just-extend": "^6.2.0", + "path-to-regexp": "^8.3.0" + } + }, + "node_modules/nise/node_modules/@sinonjs/fake-timers": { + "version": "15.4.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.4.0.tgz", + "integrity": "sha512-DsG+8/LscQIQg68J6Ef3dv10u6nVyetYn923s3/sus5eaGfTo1of5WMZSLf0UJc9KDuKPilPH0UDJCjvNbDNCA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1" + } + }, "node_modules/node-addon-api": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", @@ -15728,6 +16285,35 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/sinon": { + "version": "18.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-18.0.1.tgz", + "integrity": "sha512-a2N2TDY1uGviajJ6r4D1CyRAkzE9NNVlYOV1wX5xQDuAk0ONgzgRl0EjCQuRCPxOwp13ghsMwt9Gdldujs39qw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "11.2.2", + "@sinonjs/samsam": "^8.0.0", + "diff": "^5.2.0", + "nise": "^6.0.0", + "supports-color": "^7" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/sinon" + } + }, + "node_modules/sinon/node_modules/diff": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", + "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -15860,6 +16446,30 @@ "node": ">= 0.4" } }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "node_modules/stream-browserify/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -16587,6 +17197,16 @@ "node": ">= 0.8.0" } }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/type-fest": { "version": "5.8.0", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.8.0.tgz", diff --git a/package.json b/package.json index 9c74ac0..b7a0506 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "@ai-sdk/openai": "^4.0.10", "@ai-sdk/openai-compatible": "^3.0.7", "@ai-sdk/react": "^4.0.20", + "@aws-sdk/client-s3": "^3.1088.0", + "@aws-sdk/lib-storage": "^3.1088.0", "@base-ui/react": "^1.6.0", "@better-auth/passkey": "^1.6.23", "@dnd-kit/core": "^6.3.1", @@ -82,6 +84,7 @@ "@types/sql.js": "^1.4.11", "@types/web-push": "^3.6.4", "@vitejs/plugin-react": "^6.0.3", + "aws-sdk-client-mock": "^4.1.0", "drizzle-kit": "^0.31.10", "eslint": "^9", "eslint-config-next": "16.2.10", diff --git a/src/app/api/materials/[id]/file/route.ts b/src/app/api/materials/[id]/file/route.ts index bc519a6..61dc025 100644 --- a/src/app/api/materials/[id]/file/route.ts +++ b/src/app/api/materials/[id]/file/route.ts @@ -4,10 +4,7 @@ import { material } from "@/db/schema" import { getSession } from "@/lib/auth/session" import { fileSize, fileStream, safeInlineMime } from "@/lib/storage" -export async function GET( - request: Request, - { params }: { params: Promise<{ id: string }> } -) { +export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) { const session = await getSession() if (!session) return new Response("Unauthorized", { status: 401 }) @@ -38,12 +35,10 @@ export async function GET( const match = /bytes=(\d*)-(\d*)/.exec(range) if (match && (match[1] || match[2])) { // "bytes=-N" is a suffix range: the last N bytes of the file - const start = match[1] - ? parseInt(match[1], 10) - : Math.max(size - parseInt(match[2], 10), 0) + const start = match[1] ? parseInt(match[1], 10) : Math.max(size - parseInt(match[2], 10), 0) const end = match[1] && match[2] ? Math.min(parseInt(match[2], 10), size - 1) : size - 1 if (start <= end && start < size) { - return new Response(fileStream(row.storagePath, start, end), { + return new Response(await fileStream(row.storagePath, start, end), { status: 206, headers: { "Content-Type": mime, @@ -56,7 +51,7 @@ export async function GET( } } - return new Response(fileStream(row.storagePath), { + return new Response(await fileStream(row.storagePath), { headers: { "Content-Type": mime, "Content-Length": String(size), diff --git a/src/lib/ai/extract.ts b/src/lib/ai/extract.ts index 0a363e3..da74045 100644 --- a/src/lib/ai/extract.ts +++ b/src/lib/ai/extract.ts @@ -1,11 +1,8 @@ import "server-only" -import { readFile } from "node:fs/promises" -import path from "node:path" +import { readFileBuffer } from "@/lib/storage" import { classifyFile } from "./filetypes" import { extractPptxText, extractXlsxText } from "./office" -const UPLOAD_DIR = process.env.UPLOAD_DIR ?? path.join(process.cwd(), "data", "uploads") - /** * Cap on plain-text/code files read verbatim. Raised well above the old 2 MB so * large lecture transcripts / CSV exports are covered; still bounded so a single @@ -14,12 +11,6 @@ const UPLOAD_DIR = process.env.UPLOAD_DIR ?? path.join(process.cwd(), "data", "u */ const MAX_TEXT_BYTES = (Number(process.env.MAX_TEXT_EXTRACT_MB) || 25) * 1024 * 1024 -function absolute(storagePath: string): string { - const abs = path.resolve(UPLOAD_DIR, storagePath) - if (!abs.startsWith(path.resolve(UPLOAD_DIR))) throw new Error("Invalid path") - return abs -} - /** * Extracts plain text from a stored material file. Routing is by extension + * MIME (`classifyFile`), covering PDF, DOCX, PPTX, XLSX and all text/code @@ -30,32 +21,31 @@ export async function extractText( storagePath: string, mimeType: string | null ): Promise { - const abs = absolute(storagePath) const strategy = classifyFile(storagePath, mimeType) switch (strategy) { case "pdf": { const { extractText: extractPdf, getDocumentProxy } = await import("unpdf") - const buffer = await readFile(abs) + const buffer = await readFileBuffer(storagePath) const pdf = await getDocumentProxy(new Uint8Array(buffer)) const { text } = await extractPdf(pdf, { mergePages: true }) return text || null } case "docx": { const mammoth = await import("mammoth") - const { value } = await mammoth.extractRawText({ buffer: await readFile(abs) }) + const { value } = await mammoth.extractRawText({ buffer: await readFileBuffer(storagePath) }) return value || null } case "pptx": { - const text = extractPptxText(new Uint8Array(await readFile(abs))) + const text = extractPptxText(new Uint8Array(await readFileBuffer(storagePath))) return text || null } case "xlsx": { - const text = extractXlsxText(new Uint8Array(await readFile(abs))) + const text = extractXlsxText(new Uint8Array(await readFileBuffer(storagePath))) return text || null } case "text": { - const buffer = await readFile(abs) + const buffer = await readFileBuffer(storagePath) return buffer.subarray(0, MAX_TEXT_BYTES).toString("utf8") || null } default: diff --git a/src/lib/ai/media.ts b/src/lib/ai/media.ts index cc98e56..b06dde3 100644 --- a/src/lib/ai/media.ts +++ b/src/lib/ai/media.ts @@ -1,18 +1,10 @@ import "server-only" -import { readFile } from "node:fs/promises" import path from "node:path" import { experimental_transcribe as transcribe, generateText, type TranscriptionModel } from "ai" +import { readFileBuffer } from "@/lib/storage" import { getLanguageModel, getTranscriptionModel, resolveModelForUser } from "./registry" import { runAi } from "./run" -const UPLOAD_DIR = process.env.UPLOAD_DIR ?? path.join(process.cwd(), "data", "uploads") - -function absolute(storagePath: string): string { - const abs = path.resolve(UPLOAD_DIR, storagePath) - if (!abs.startsWith(path.resolve(UPLOAD_DIR))) throw new Error("Invalid path") - return abs -} - const IMAGE_PROMPT = "Extract all readable text from this image verbatim (OCR). Then add a short " + "description of any diagrams, charts or notable visual content. Respond in the " + @@ -32,7 +24,7 @@ export async function extractImageText( const ref = await resolveModelForUser(userId) if (!ref) return null const model = await getLanguageModel(ref, userId) - const buffer = await readFile(absolute(storagePath)) + const buffer = await readFileBuffer(storagePath) const { text } = await runAi( { userId, @@ -75,7 +67,7 @@ export async function transcribeMedia(storagePath: string, userId: string): Prom try { const transcription = await getTranscriptionModel(userId) if (!transcription) return null - const buffer = await readFile(absolute(storagePath)) + const buffer = await readFileBuffer(storagePath) const { text } = await runAi( { userId, diff --git a/src/lib/jobs/unpack-zip.ts b/src/lib/jobs/unpack-zip.ts index 08994d7..45b7ee6 100644 --- a/src/lib/jobs/unpack-zip.ts +++ b/src/lib/jobs/unpack-zip.ts @@ -20,7 +20,7 @@ export type UnpackZipPayload = { /** Reads a stored file fully into a Buffer. */ async function readStored(storagePath: string): Promise { const chunks: Uint8Array[] = [] - const reader = fileStream(storagePath).getReader() + const reader = (await fileStream(storagePath)).getReader() for (;;) { const { done, value } = await reader.read() if (done) break diff --git a/src/lib/storage.ts b/src/lib/storage.ts deleted file mode 100644 index 3f7a2dd..0000000 --- a/src/lib/storage.ts +++ /dev/null @@ -1,129 +0,0 @@ -import "server-only" -import { createHash } from "node:crypto" -import { createReadStream, createWriteStream } from "node:fs" -import { mkdir, readFile, stat, unlink, writeFile } from "node:fs/promises" -import path from "node:path" -import { Readable, Transform } from "node:stream" -import { pipeline } from "node:stream/promises" - -/** Thrown by saveStream when the incoming stream exceeds the byte limit. */ -export class StorageLimitError extends Error { - constructor(message = "File exceeds the maximum allowed size") { - super(message) - this.name = "StorageLimitError" - } -} - -/** - * Local-disk storage for uploaded materials. All paths are relative to - * UPLOAD_DIR so the database stays portable across hosts. - */ -const UPLOAD_DIR = process.env.UPLOAD_DIR ?? path.join(process.cwd(), "data", "uploads") - -function absolute(relPath: string): string { - const abs = path.resolve(UPLOAD_DIR, relPath) - if (!abs.startsWith(path.resolve(UPLOAD_DIR))) throw new Error("Invalid path") - return abs -} - -// MIME types a browser would execute in the app's origin when served inline. -const ACTIVE_CONTENT_MIMES = new Set([ - "text/html", - "application/xhtml+xml", - "image/svg+xml", - "text/xml", - "application/xml", -]) - -/** - * Returns a MIME type that is safe to serve inline from the app origin. - * Active content (HTML, SVG, XML) is downgraded to octet-stream so a stored - * file can never run scripts in the app's origin. - */ -export function safeInlineMime(mime: string | null | undefined): string { - const normalized = (mime ?? "").split(";")[0].trim().toLowerCase() - if (!normalized || ACTIVE_CONTENT_MIMES.has(normalized)) return "application/octet-stream" - return normalized -} - -function sanitizeName(filename: string): string { - return filename.replace(/[^\w.\-()\[\] ]/g, "_").slice(-150) || "file" -} - -/** Stores a buffer and returns the relative storage path. */ -export async function saveFile(userId: string, filename: string, data: Buffer): Promise { - const relPath = path.join(userId, `${crypto.randomUUID()}-${sanitizeName(filename)}`) - const abs = absolute(relPath) - await mkdir(path.dirname(abs), { recursive: true }) - await writeFile(abs, data) - return relPath.replaceAll(path.sep, "/") -} - -/** - * Streams an upload straight to disk without ever holding the whole file in - * memory (unlike `formData()` + `arrayBuffer()`), computing the byte size and a - * sha256 content hash as it goes. Enforces `maxBytes` mid-stream and removes the - * partial file if the limit is exceeded or the stream errors. - */ -export async function saveStream( - userId: string, - filename: string, - body: ReadableStream, - opts: { maxBytes?: number } = {} -): Promise<{ storagePath: string; size: number; hash: string }> { - const relPath = path.join(userId, `${crypto.randomUUID()}-${sanitizeName(filename)}`) - const abs = absolute(relPath) - await mkdir(path.dirname(abs), { recursive: true }) - - const hash = createHash("sha256") - let size = 0 - const meter = new Transform({ - transform(chunk: Buffer, _enc, cb) { - size += chunk.length - if (opts.maxBytes != null && size > opts.maxBytes) { - cb(new StorageLimitError()) - return - } - hash.update(chunk) - cb(null, chunk) - }, - }) - - try { - const source = Readable.fromWeb(body as Parameters[0]) - await pipeline(source, meter, createWriteStream(abs)) - } catch (error) { - await unlink(abs).catch(() => {}) - throw error - } - - return { storagePath: relPath.replaceAll(path.sep, "/"), size, hash: hash.digest("hex") } -} - -/** Persists extracted plain text to disk (out of the DB) and returns its path. */ -export async function saveText(userId: string, filename: string, text: string): Promise { - return saveFile(userId, filename, Buffer.from(text, "utf8")) -} - -/** Reads a stored UTF-8 text file (e.g. extracted material text). */ -export async function readStoredText(relPath: string): Promise { - return readFile(absolute(relPath), "utf8") -} - -export async function deleteFile(relPath: string): Promise { - try { - await unlink(absolute(relPath)) - } catch { - // already gone — fine - } -} - -export async function fileSize(relPath: string): Promise { - return (await stat(absolute(relPath))).size -} - -/** Returns a Web ReadableStream for a stored file, optionally a byte range. */ -export function fileStream(relPath: string, start?: number, end?: number): ReadableStream { - const nodeStream = createReadStream(absolute(relPath), start != null ? { start, end } : {}) - return Readable.toWeb(nodeStream) as ReadableStream -} diff --git a/src/lib/storage/driver.ts b/src/lib/storage/driver.ts new file mode 100644 index 0000000..bbc066e --- /dev/null +++ b/src/lib/storage/driver.ts @@ -0,0 +1,93 @@ +import { createHash, type Hash } from "node:crypto" +import { Transform } from "node:stream" + +/** Thrown by saveStream when the incoming stream exceeds the byte limit. */ +export class StorageLimitError extends Error { + constructor(message = "File exceeds the maximum allowed size") { + super(message) + this.name = "StorageLimitError" + } +} + +// MIME types a browser would execute in the app's origin when served inline. +const ACTIVE_CONTENT_MIMES = new Set([ + "text/html", + "application/xhtml+xml", + "image/svg+xml", + "text/xml", + "application/xml", +]) + +/** + * Returns a MIME type that is safe to serve inline from the app origin. + * Active content (HTML, SVG, XML) is downgraded to octet-stream so a stored + * file can never run scripts in the app's origin. + */ +export function safeInlineMime(mime: string | null | undefined): string { + const normalized = (mime ?? "").split(";")[0].trim().toLowerCase() + if (!normalized || ACTIVE_CONTENT_MIMES.has(normalized)) return "application/octet-stream" + return normalized +} + +function sanitizeName(filename: string): string { + return filename.replace(/[^\w.\-()\[\] ]/g, "_").slice(-150) || "file" +} + +/** + * Builds the canonical, forward-slash storage key for a new upload: + * `/-`. This string is the value persisted in the + * database and the identifier every driver maps onto its backend (a filesystem + * path for local, an object key for S3), so it must stay backend-agnostic. + */ +export function buildRelPath(userId: string, filename: string): string { + return `${userId}/${crypto.randomUUID()}-${sanitizeName(filename)}` +} + +/** Result of streaming an upload to storage. */ +export type SaveStreamResult = { storagePath: string; size: number; hash: string } + +/** + * A pass-through transform that counts bytes, computes a sha256 hash, and aborts + * with StorageLimitError once `maxBytes` is exceeded. Shared by every driver so + * size/hash/limit semantics are identical regardless of the storage backend. + */ +export function createUploadMeter(opts: { maxBytes?: number }): { + meter: Transform + hash: Hash + size: () => number +} { + const hash = createHash("sha256") + let size = 0 + const meter = new Transform({ + transform(chunk: Buffer, _enc, cb) { + size += chunk.length + if (opts.maxBytes != null && size > opts.maxBytes) { + cb(new StorageLimitError()) + return + } + hash.update(chunk) + cb(null, chunk) + }, + }) + return { meter, hash, size: () => size } +} + +/** + * Backend-agnostic storage operations. Two implementations exist — a local-disk + * driver (default) and an S3/object-storage driver — selected by STORAGE_DRIVER. + * Every method takes/returns the backend-agnostic relative path from + * `buildRelPath`, so switching drivers never changes what is stored in the DB. + */ +export interface StorageDriver { + saveBuffer(userId: string, filename: string, data: Buffer): Promise + saveStream( + userId: string, + filename: string, + body: ReadableStream, + opts?: { maxBytes?: number } + ): Promise + readBuffer(relPath: string): Promise + deleteFile(relPath: string): Promise + fileSize(relPath: string): Promise + fileStream(relPath: string, start?: number, end?: number): Promise +} diff --git a/src/lib/storage/index.ts b/src/lib/storage/index.ts new file mode 100644 index 0000000..72492da --- /dev/null +++ b/src/lib/storage/index.ts @@ -0,0 +1,80 @@ +import "server-only" +import type { SaveStreamResult, StorageDriver } from "./driver" +import { LocalStorageDriver } from "./local" + +export { safeInlineMime, StorageLimitError } from "./driver" +export type { SaveStreamResult } from "./driver" + +/** + * Storage backend selection. `local` (default) writes to UPLOAD_DIR on disk; + * `s3` uses S3 / S3-compatible object storage (see ./s3). The driver is created + * once and cached, and the AWS SDK is only loaded when `s3` is selected, so the + * default local path is byte-for-byte unchanged and pays nothing for S3. + */ +function driverName(): "local" | "s3" { + return (process.env.STORAGE_DRIVER ?? "local").toLowerCase() === "s3" ? "s3" : "local" +} + +let driverPromise: Promise | undefined + +function getDriver(): Promise { + if (!driverPromise) { + driverPromise = + driverName() === "s3" + ? import("./s3").then(({ S3StorageDriver }) => new S3StorageDriver()) + : Promise.resolve(new LocalStorageDriver()) + } + return driverPromise +} + +/** Stores a buffer and returns the relative storage path. */ +export async function saveFile(userId: string, filename: string, data: Buffer): Promise { + return (await getDriver()).saveBuffer(userId, filename, data) +} + +/** + * Streams an upload straight to storage without ever holding the whole file in + * memory, computing the byte size and a sha256 content hash as it goes. Enforces + * `maxBytes` mid-stream and removes the partial object if the limit is exceeded + * or the stream errors. + */ +export async function saveStream( + userId: string, + filename: string, + body: ReadableStream, + opts: { maxBytes?: number } = {} +): Promise { + return (await getDriver()).saveStream(userId, filename, body, opts) +} + +/** Persists extracted plain text out of the DB and returns its path. */ +export async function saveText(userId: string, filename: string, text: string): Promise { + return (await getDriver()).saveBuffer(userId, filename, Buffer.from(text, "utf8")) +} + +/** Reads a stored UTF-8 text file (e.g. extracted material text). */ +export async function readStoredText(relPath: string): Promise { + return (await (await getDriver()).readBuffer(relPath)).toString("utf8") +} + +/** Reads a stored file fully into a Buffer (binary-safe). */ +export async function readFileBuffer(relPath: string): Promise { + return (await getDriver()).readBuffer(relPath) +} + +export async function deleteFile(relPath: string): Promise { + return (await getDriver()).deleteFile(relPath) +} + +export async function fileSize(relPath: string): Promise { + return (await getDriver()).fileSize(relPath) +} + +/** Returns a Web ReadableStream for a stored file, optionally a byte range. */ +export async function fileStream( + relPath: string, + start?: number, + end?: number +): Promise { + return (await getDriver()).fileStream(relPath, start, end) +} diff --git a/src/lib/storage/local.ts b/src/lib/storage/local.ts new file mode 100644 index 0000000..f41018b --- /dev/null +++ b/src/lib/storage/local.ts @@ -0,0 +1,78 @@ +import "server-only" +import { createReadStream, createWriteStream } from "node:fs" +import { mkdir, readFile, stat, unlink, writeFile } from "node:fs/promises" +import path from "node:path" +import { Readable } from "node:stream" +import { pipeline } from "node:stream/promises" +import { + buildRelPath, + createUploadMeter, + type SaveStreamResult, + type StorageDriver, +} from "./driver" + +/** + * Local-disk storage for uploaded materials. All paths are relative to + * UPLOAD_DIR so the database stays portable across hosts. This is the default + * driver; behaviour is unchanged from before the driver abstraction existed. + */ +const UPLOAD_DIR = process.env.UPLOAD_DIR ?? path.join(process.cwd(), "data", "uploads") + +function absolute(relPath: string): string { + const abs = path.resolve(UPLOAD_DIR, relPath) + if (!abs.startsWith(path.resolve(UPLOAD_DIR))) throw new Error("Invalid path") + return abs +} + +export class LocalStorageDriver implements StorageDriver { + async saveBuffer(userId: string, filename: string, data: Buffer): Promise { + const relPath = buildRelPath(userId, filename) + const abs = absolute(relPath) + await mkdir(path.dirname(abs), { recursive: true }) + await writeFile(abs, data) + return relPath + } + + async saveStream( + userId: string, + filename: string, + body: ReadableStream, + opts: { maxBytes?: number } = {} + ): Promise { + const relPath = buildRelPath(userId, filename) + const abs = absolute(relPath) + await mkdir(path.dirname(abs), { recursive: true }) + + const { meter, hash, size } = createUploadMeter(opts) + try { + const source = Readable.fromWeb(body as Parameters[0]) + await pipeline(source, meter, createWriteStream(abs)) + } catch (error) { + await unlink(abs).catch(() => {}) + throw error + } + + return { storagePath: relPath, size: size(), hash: hash.digest("hex") } + } + + async readBuffer(relPath: string): Promise { + return readFile(absolute(relPath)) + } + + async deleteFile(relPath: string): Promise { + try { + await unlink(absolute(relPath)) + } catch { + // already gone — fine + } + } + + async fileSize(relPath: string): Promise { + return (await stat(absolute(relPath))).size + } + + async fileStream(relPath: string, start?: number, end?: number): Promise { + const nodeStream = createReadStream(absolute(relPath), start != null ? { start, end } : {}) + return Readable.toWeb(nodeStream) as ReadableStream + } +} diff --git a/src/lib/storage/s3.ts b/src/lib/storage/s3.ts new file mode 100644 index 0000000..1064d66 --- /dev/null +++ b/src/lib/storage/s3.ts @@ -0,0 +1,162 @@ +import "server-only" +import { Readable } from "node:stream" +import type { S3Client } from "@aws-sdk/client-s3" +import { + buildRelPath, + createUploadMeter, + type SaveStreamResult, + type StorageDriver, +} from "./driver" + +type S3Config = { + bucket: string + region: string + endpoint?: string + forcePathStyle: boolean + keyPrefix: string +} + +function readS3Config(): S3Config { + const bucket = process.env.S3_BUCKET + if (!bucket) { + throw new Error("S3_BUCKET must be set when STORAGE_DRIVER=s3") + } + return { + bucket, + region: process.env.S3_REGION ?? process.env.AWS_REGION ?? "us-east-1", + endpoint: process.env.S3_ENDPOINT || undefined, + forcePathStyle: /^(1|true|yes)$/i.test(process.env.S3_FORCE_PATH_STYLE ?? ""), + keyPrefix: (process.env.S3_KEY_PREFIX ?? "").replace(/^\/+|\/+$/g, ""), + } +} + +/** Rejects keys that could escape the bucket / configured-prefix namespace. */ +function assertSafeKey(relPath: string): void { + if (!relPath || relPath.startsWith("/") || relPath.split("/").includes("..")) { + throw new Error("Invalid path") + } +} + +/** + * S3 / S3-compatible object-storage driver (AWS S3, MinIO, Cloudflare R2, …). + * The AWS SDK is imported lazily so the default local driver never loads it and + * local deploys pay nothing for it. Credentials come from the standard AWS + * credential chain (env vars or an IAM role); only region/endpoint/path-style + * are read from our own env. The relative path from `buildRelPath` is the + * canonical object key (optionally under S3_KEY_PREFIX), matching the local + * driver so the value stored in the DB is identical across backends. + */ +export class S3StorageDriver implements StorageDriver { + private readonly cfg: S3Config + private clientPromise?: Promise + + constructor() { + this.cfg = readS3Config() + } + + private async client(): Promise { + if (!this.clientPromise) { + this.clientPromise = import("@aws-sdk/client-s3").then( + ({ S3Client: Client }) => + new Client({ + region: this.cfg.region, + forcePathStyle: this.cfg.forcePathStyle, + ...(this.cfg.endpoint ? { endpoint: this.cfg.endpoint } : {}), + }) + ) + } + return this.clientPromise + } + + private key(relPath: string): string { + assertSafeKey(relPath) + return this.cfg.keyPrefix ? `${this.cfg.keyPrefix}/${relPath}` : relPath + } + + async saveBuffer(userId: string, filename: string, data: Buffer): Promise { + const relPath = buildRelPath(userId, filename) + const { PutObjectCommand } = await import("@aws-sdk/client-s3") + const client = await this.client() + await client.send( + new PutObjectCommand({ Bucket: this.cfg.bucket, Key: this.key(relPath), Body: data }) + ) + return relPath + } + + async saveStream( + userId: string, + filename: string, + body: ReadableStream, + opts: { maxBytes?: number } = {} + ): Promise { + const relPath = buildRelPath(userId, filename) + const { Upload } = await import("@aws-sdk/lib-storage") + const client = await this.client() + + const { meter, hash, size } = createUploadMeter(opts) + const source = Readable.fromWeb(body as Parameters[0]) + // Capture the meter's error (e.g. StorageLimitError) so it can be rethrown + // unwrapped — lib-storage would otherwise surface a generic stream error. + let meterError: Error | undefined + meter.once("error", (error: Error) => { + meterError = error + }) + source.pipe(meter) + + const upload = new Upload({ + client, + params: { Bucket: this.cfg.bucket, Key: this.key(relPath), Body: meter }, + }) + try { + await upload.done() + } catch (error) { + await upload.abort().catch(() => {}) + source.destroy() + throw meterError ?? error + } + + return { storagePath: relPath, size: size(), hash: hash.digest("hex") } + } + + async readBuffer(relPath: string): Promise { + const { GetObjectCommand } = await import("@aws-sdk/client-s3") + const client = await this.client() + const res = await client.send( + new GetObjectCommand({ Bucket: this.cfg.bucket, Key: this.key(relPath) }) + ) + const bytes = await res.Body!.transformToByteArray() + return Buffer.from(bytes) + } + + async deleteFile(relPath: string): Promise { + try { + const { DeleteObjectCommand } = await import("@aws-sdk/client-s3") + const client = await this.client() + await client.send( + new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: this.key(relPath) }) + ) + } catch (error) { + // Delete is best-effort / idempotent, mirroring the local driver. + console.warn("[storage:s3] delete failed", relPath, error) + } + } + + async fileSize(relPath: string): Promise { + const { HeadObjectCommand } = await import("@aws-sdk/client-s3") + const client = await this.client() + const res = await client.send( + new HeadObjectCommand({ Bucket: this.cfg.bucket, Key: this.key(relPath) }) + ) + return res.ContentLength ?? 0 + } + + async fileStream(relPath: string, start?: number, end?: number): Promise { + const { GetObjectCommand } = await import("@aws-sdk/client-s3") + const client = await this.client() + const range = start != null ? `bytes=${start}-${end ?? ""}` : undefined + const res = await client.send( + new GetObjectCommand({ Bucket: this.cfg.bucket, Key: this.key(relPath), Range: range }) + ) + return Readable.toWeb(res.Body as Readable) as ReadableStream + } +} diff --git a/src/lib/storage/storage.test.ts b/src/lib/storage/storage.test.ts new file mode 100644 index 0000000..9d484d7 --- /dev/null +++ b/src/lib/storage/storage.test.ts @@ -0,0 +1,187 @@ +import { createHash } from "node:crypto" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { Readable } from "node:stream" +import { mockClient } from "aws-sdk-client-mock" +import { + DeleteObjectCommand, + GetObjectCommand, + type GetObjectCommandOutput, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} from "@aws-sdk/client-s3" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +// The storage modules import "server-only", which throws outside a React Server +// Component bundle (i.e. under vitest). Neutralise it for this unit test. +vi.mock("server-only", () => ({})) + +const s3Mock = mockClient(S3Client) + +/** Wraps a fake body as a GetObject output without leaking `any`. */ +function getOutput(body: unknown): GetObjectCommandOutput { + return { Body: body } as unknown as GetObjectCommandOutput +} + +async function readWebStream(stream: ReadableStream): Promise { + const chunks: Uint8Array[] = [] + const reader = stream.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) break + if (value) chunks.push(value) + } + return Buffer.concat(chunks) +} + +describe("S3StorageDriver", () => { + const OPTIONAL_ENV = ["S3_REGION", "S3_ENDPOINT", "S3_FORCE_PATH_STYLE", "S3_KEY_PREFIX"] + + beforeEach(() => { + s3Mock.reset() + process.env.S3_BUCKET = "test-bucket" + for (const key of OPTIONAL_ENV) delete process.env[key] + }) + + async function makeDriver(env: Record = {}) { + for (const [key, value] of Object.entries(env)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + const { S3StorageDriver } = await import("./s3") + return new S3StorageDriver() + } + + it("requires S3_BUCKET", async () => { + await expect(makeDriver({ S3_BUCKET: undefined })).rejects.toThrow(/S3_BUCKET/) + }) + + it("saveBuffer PUTs the object under the canonical key and returns the rel path", async () => { + s3Mock.on(PutObjectCommand).resolves({}) + const driver = await makeDriver() + const relPath = await driver.saveBuffer("user-1", "notes.pdf", Buffer.from("data")) + + expect(relPath).toMatch(/^user-1\/[0-9a-f-]+-notes\.pdf$/) + const calls = s3Mock.commandCalls(PutObjectCommand) + expect(calls).toHaveLength(1) + expect(calls[0].args[0].input.Bucket).toBe("test-bucket") + expect(calls[0].args[0].input.Key).toBe(relPath) + }) + + it("prepends S3_KEY_PREFIX to the object key but keeps the rel path prefix-free", async () => { + s3Mock.on(PutObjectCommand).resolves({}) + const driver = await makeDriver({ S3_KEY_PREFIX: "media" }) + const relPath = await driver.saveBuffer("user-1", "a.txt", Buffer.from("x")) + + expect(relPath.startsWith("user-1/")).toBe(true) + expect(s3Mock.commandCalls(PutObjectCommand)[0].args[0].input.Key).toBe(`media/${relPath}`) + }) + + it("readBuffer returns the object bytes", async () => { + s3Mock + .on(GetObjectCommand) + .resolves(getOutput({ transformToByteArray: async () => new Uint8Array([104, 105]) })) + const driver = await makeDriver() + const buf = await driver.readBuffer("user-1/x-a.txt") + expect(buf.toString("utf8")).toBe("hi") + }) + + it("fileSize returns ContentLength from a HEAD request", async () => { + s3Mock.on(HeadObjectCommand).resolves({ ContentLength: 42 }) + const driver = await makeDriver() + expect(await driver.fileSize("user-1/x-a.txt")).toBe(42) + }) + + it("fileStream requests a byte range and streams the body", async () => { + s3Mock.on(GetObjectCommand).resolves(getOutput(Readable.from([Buffer.from("hello")]))) + const driver = await makeDriver() + const stream = await driver.fileStream("user-1/x-a.txt", 0, 4) + + expect(await readWebStream(stream)).toEqual(Buffer.from("hello")) + expect(s3Mock.commandCalls(GetObjectCommand)[0].args[0].input.Range).toBe("bytes=0-4") + }) + + it("fileStream omits the Range header when no range is given", async () => { + s3Mock.on(GetObjectCommand).resolves(getOutput(Readable.from([Buffer.from("full")]))) + const driver = await makeDriver() + await driver.fileStream("user-1/x-a.txt") + expect(s3Mock.commandCalls(GetObjectCommand)[0].args[0].input.Range).toBeUndefined() + }) + + it("deleteFile issues a DELETE and swallows errors (idempotent)", async () => { + s3Mock.on(DeleteObjectCommand).rejects(new Error("boom")) + const driver = await makeDriver() + await expect(driver.deleteFile("user-1/x-a.txt")).resolves.toBeUndefined() + expect(s3Mock.commandCalls(DeleteObjectCommand)).toHaveLength(1) + }) + + it("rejects keys that try to escape the namespace", async () => { + const driver = await makeDriver() + await expect(driver.readBuffer("../secret")).rejects.toThrow(/Invalid path/) + }) +}) + +describe("LocalStorageDriver", () => { + let dir: string + + beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "storage-test-")) + process.env.UPLOAD_DIR = dir + vi.resetModules() // re-read UPLOAD_DIR at module load + }) + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }) + }) + + it("round-trips a buffer through save/read/size and deletes it", async () => { + const { LocalStorageDriver } = await import("./local") + const driver = new LocalStorageDriver() + + const relPath = await driver.saveBuffer("user-1", "a.txt", Buffer.from("hello world")) + expect(relPath).toMatch(/^user-1\/[0-9a-f-]+-a\.txt$/) + expect((await driver.readBuffer(relPath)).toString("utf8")).toBe("hello world") + expect(await driver.fileSize(relPath)).toBe(11) + + await driver.deleteFile(relPath) + await expect(driver.fileSize(relPath)).rejects.toBeTruthy() + await expect(driver.deleteFile(relPath)).resolves.toBeUndefined() // idempotent + }) + + it("streams to disk while computing size + sha256 and honours maxBytes", async () => { + const { LocalStorageDriver } = await import("./local") + const { StorageLimitError } = await import("./driver") + const driver = new LocalStorageDriver() + + const payload = Buffer.from("the quick brown fox") + const body = Readable.toWeb(Readable.from([payload])) as ReadableStream + const result = await driver.saveStream("user-1", "f.bin", body) + + expect(result.size).toBe(payload.length) + expect(result.hash).toBe(createHash("sha256").update(payload).digest("hex")) + expect((await driver.readBuffer(result.storagePath)).equals(payload)).toBe(true) + + const big = Readable.toWeb(Readable.from([Buffer.alloc(100)])) as ReadableStream + await expect( + driver.saveStream("user-1", "big.bin", big, { maxBytes: 10 }) + ).rejects.toBeInstanceOf(StorageLimitError) + }) + + it("serves a byte range via fileStream", async () => { + const { LocalStorageDriver } = await import("./local") + const driver = new LocalStorageDriver() + const relPath = await driver.saveBuffer("user-1", "r.txt", Buffer.from("0123456789")) + + const stream = await driver.fileStream(relPath, 2, 5) + const chunks: Uint8Array[] = [] + const reader = stream.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) break + if (value) chunks.push(value) + } + expect(Buffer.concat(chunks).toString("utf8")).toBe("2345") + }) +}) From e493c17e1183615e550d452d3a5f4e821c02de75 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 16:06:11 +0000 Subject: [PATCH 2/2] feat(generation): optional Batch API for complete generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the coverage-generation MAP step through the provider's async Batch API (Anthropic Message Batches / OpenAI Batch) — ~50% cheaper for large modules. Additive and opt-in via the `ai.useBatchApi` admin flag (default off); with the flag off, or for non-Anthropic/OpenAI providers, the synchronous live path is unchanged and is also the fallback if a batch submit fails. - src/lib/ai/generation/batch-adapter.ts (new): pure, unit-tested builders and parsers (Zod->JSON schema, Anthropic tool-use requests, OpenAI json_schema tasks, result parsing with usage) plus thin network calls; the vendor SDKs (@anthropic-ai/sdk, openai) are imported lazily so non-batch deploys never load them. resolveBatchProvider reads credentials from settings + BYOK. - generate.ts: extract the two MAP prompts into shared buildCardPrompt/ buildQuestionPrompt and the dedup+insert into persistCards/persistQuestions, so live and batch paths produce identical output. runCoverageGeneration submits one batch for all uncovered topics when the flag is on and returns; grounding/retrieval still runs live. - src/lib/ai/generation/batch-poll.ts (new): cron-driven poller that, once a vendor batch ends, fetches results, de-dupes + inserts, records token usage per topic (requirement A), and completes the job. Failed items are marked in coverage so a re-run regenerates only them. - jobs: new poll-batches queue on a */5 schedule (matches send-reminders). - schema: generation_job gains nullable batch_ref + batch_model (migration 0034); no other data-model change. - settings: ai.useBatchApi flag + admin toggle (Switch) with i18n. The batch path is asynchronous: results arrive over minutes to hours and the coverage view fills gradually. Submit/poll/fetch against the real vendor APIs is verified on the production stand; CI covers the pure builder/parser logic. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0191JikNRN8Q2HBtf6fpLXmH --- docs/self-hosting.md | 12 +- drizzle/0034_old_ben_parker.sql | 2 + drizzle/meta/0034_snapshot.json | 5775 +++++++++++++++++++ drizzle/meta/_journal.json | 7 + messages/de.json | 2 + messages/en.json | 2 + package-lock.json | 94 + package.json | 2 + src/app/[locale]/(app)/admin/ai/page.tsx | 4 +- src/components/admin/ai-settings-form.tsx | 10 + src/db/schema/generation.ts | 6 + src/lib/ai/generation/batch-adapter.test.ts | 173 + src/lib/ai/generation/batch-adapter.ts | 314 + src/lib/ai/generation/batch-poll.ts | 165 + src/lib/ai/generation/generate.ts | 282 +- src/lib/jobs/index.ts | 8 + src/lib/settings.ts | 9 +- 17 files changed, 6791 insertions(+), 76 deletions(-) create mode 100644 drizzle/0034_old_ben_parker.sql create mode 100644 drizzle/meta/0034_snapshot.json create mode 100644 src/lib/ai/generation/batch-adapter.test.ts create mode 100644 src/lib/ai/generation/batch-adapter.ts create mode 100644 src/lib/ai/generation/batch-poll.ts diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 36a2c46..c57b958 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -65,12 +65,12 @@ first (see Backup below), or set it correctly before the first Everything else is configured in **Admin → Settings**: -| Area | What | -| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Sign-in & SSO | open/closed registration, GitHub/Google login, generic OIDC (Keycloak, Authentik, Zitadel, Authelia, …) | -| AI | providers (Anthropic, OpenAI, Google, Mistral, Groq, Ollama, OpenAI-compatible), models, default + embedding model (enables RAG), monthly token limits | -| Email | SMTP for password resets and reminders, test email | -| Branding | app name, max upload size | +| Area | What | +| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Sign-in & SSO | open/closed registration, GitHub/Google login, generic OIDC (Keycloak, Authentik, Zitadel, Authelia, …) | +| AI | providers (Anthropic, OpenAI, Google, Mistral, Groq, Ollama, OpenAI-compatible), models, default + embedding model (enables RAG), monthly token limits, optional Batch API for cheaper async complete-generation (Anthropic/OpenAI) | +| Email | SMTP for password resets and reminders, test email | +| Branding | app name, max upload size | A small dot next to the version number in the sidebar (visible to admins) shows when a newer release is available on GitHub — checked once a day — diff --git a/drizzle/0034_old_ben_parker.sql b/drizzle/0034_old_ben_parker.sql new file mode 100644 index 0000000..7760e97 --- /dev/null +++ b/drizzle/0034_old_ben_parker.sql @@ -0,0 +1,2 @@ +ALTER TABLE "generation_job" ADD COLUMN "batch_ref" text;--> statement-breakpoint +ALTER TABLE "generation_job" ADD COLUMN "batch_model" text; \ No newline at end of file diff --git a/drizzle/meta/0034_snapshot.json b/drizzle/meta/0034_snapshot.json new file mode 100644 index 0000000..9efcfe3 --- /dev/null +++ b/drizzle/meta/0034_snapshot.json @@ -0,0 +1,5775 @@ +{ + "id": "00c7232f-2a8d-478c-a065-3437869b6e3f", + "prevId": "6e7cab55-c19e-4e42-83bc-e4d08963a8ff", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_conversation": { + "name": "ai_conversation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'New conversation'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'general'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ai_conversation_userId_idx": { + "name": "ai_conversation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "ai_conversation_user_id_user_id_fk": { + "name": "ai_conversation_user_id_user_id_fk", + "tableFrom": "ai_conversation", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ai_conversation_module_id_module_id_fk": { + "name": "ai_conversation_module_id_module_id_fk", + "tableFrom": "ai_conversation", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ai_message": { + "name": "ai_message", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parts": { + "name": "parts", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ai_message_conversationId_idx": { + "name": "ai_message_conversationId_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "ai_message_conversation_id_ai_conversation_id_fk": { + "name": "ai_message_conversation_id_ai_conversation_id_fk", + "tableFrom": "ai_message", + "tableTo": "ai_conversation", + "columnsFrom": [ + "conversation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ai_usage_log": { + "name": "ai_usage_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "feature": { + "name": "feature", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'chat'" + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ai_usage_userId_idx": { + "name": "ai_usage_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ai_usage_createdAt_idx": { + "name": "ai_usage_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "ai_usage_log_user_id_user_id_fk": { + "name": "ai_usage_log_user_id_user_id_fk", + "tableFrom": "ai_usage_log", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_ai_key": { + "name": "user_ai_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_key": { + "name": "encrypted_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_ai_key_user_id_user_id_fk": { + "name": "user_ai_key_user_id_user_id_fk", + "tableFrom": "user_ai_key", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_config": { + "name": "app_config", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.assignment": { + "name": "assignment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "due_date": { + "name": "due_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'graded'" + }, + "ai_generated": { + "name": "ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "points_achieved": { + "name": "points_achieved", + "type": "numeric(7, 2)", + "primaryKey": false, + "notNull": false + }, + "points_max": { + "name": "points_max", + "type": "numeric(7, 2)", + "primaryKey": false, + "notNull": false + }, + "subtasks": { + "name": "subtasks", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "assignment_module_idx": { + "name": "assignment_module_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "assignment_user_idx": { + "name": "assignment_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "assignment_user_id_user_id_fk": { + "name": "assignment_user_id_user_id_fk", + "tableFrom": "assignment", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assignment_module_id_module_id_fk": { + "name": "assignment_module_id_module_id_fk", + "tableFrom": "assignment", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.assignment_material": { + "name": "assignment_material", + "schema": "", + "columns": { + "assignment_id": { + "name": "assignment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "material_id": { + "name": "material_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "assignment_material_assignment_id_assignment_id_fk": { + "name": "assignment_material_assignment_id_assignment_id_fk", + "tableFrom": "assignment_material", + "tableTo": "assignment", + "columnsFrom": [ + "assignment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assignment_material_material_id_material_id_fk": { + "name": "assignment_material_material_id_material_id_fk", + "tableFrom": "assignment_material", + "tableTo": "material", + "columnsFrom": [ + "material_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "assignment_material_assignment_id_material_id_pk": { + "name": "assignment_material_assignment_id_material_id_pk", + "columns": [ + "assignment_id", + "material_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor": { + "name": "actor", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user'" + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_label": { + "name": "entity_label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "before": { + "name": "before", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "after": { + "name": "after", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "undone": { + "name": "undone", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_log_user_created_idx": { + "name": "audit_log_user_created_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "audit_log_user_id_user_id_fk": { + "name": "audit_log_user_id_user_id_fk", + "tableFrom": "audit_log", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.generation_coverage": { + "name": "generation_coverage", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "topic_id": { + "name": "topic_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "produced_count": { + "name": "produced_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "generation_coverage_target_topic_idx": { + "name": "generation_coverage_target_topic_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "topic_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "generation_coverage_job_idx": { + "name": "generation_coverage_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "generation_coverage_topic_id_outline_topic_id_fk": { + "name": "generation_coverage_topic_id_outline_topic_id_fk", + "tableFrom": "generation_coverage", + "tableTo": "outline_topic", + "columnsFrom": [ + "topic_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "generation_coverage_job_id_generation_job_id_fk": { + "name": "generation_coverage_job_id_generation_job_id_fk", + "tableFrom": "generation_coverage", + "tableTo": "generation_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.generation_job": { + "name": "generation_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "outline_version": { + "name": "outline_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "topics_total": { + "name": "topics_total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "topics_done": { + "name": "topics_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "produced_count": { + "name": "produced_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "params": { + "name": "params", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "batch_ref": { + "name": "batch_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "batch_model": { + "name": "batch_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "generation_job_user_idx": { + "name": "generation_job_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "generation_job_target_idx": { + "name": "generation_job_target_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "generation_job_user_id_user_id_fk": { + "name": "generation_job_user_id_user_id_fk", + "tableFrom": "generation_job", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "generation_job_module_id_module_id_fk": { + "name": "generation_job_module_id_module_id_fk", + "tableFrom": "generation_job", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.assessment_attempt": { + "name": "assessment_attempt", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "assessment_id": { + "name": "assessment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "result_percent": { + "name": "result_percent", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "passed": { + "name": "passed", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "assessment_attempt_assessmentId_idx": { + "name": "assessment_attempt_assessmentId_idx", + "columns": [ + { + "expression": "assessment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "assessment_attempt_assessment_id_module_assessment_id_fk": { + "name": "assessment_attempt_assessment_id_module_assessment_id_fk", + "tableFrom": "assessment_attempt", + "tableTo": "module_assessment", + "columnsFrom": [ + "assessment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.degree_program": { + "name": "degree_program", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "degree_type": { + "name": "degree_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "institution": { + "name": "institution", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_ects": { + "name": "target_ects", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "grading_system": { + "name": "grading_system", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'german'" + }, + "grade_goal": { + "name": "grade_goal", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grade_scale": { + "name": "grade_scale", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "thesis_max_attempts": { + "name": "thesis_max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "degree_program_userId_idx": { + "name": "degree_program_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "degree_program_user_id_user_id_fk": { + "name": "degree_program_user_id_user_id_fk", + "tableFrom": "degree_program", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.external_resource": { + "name": "external_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "program_id": { + "name": "program_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'website'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_note": { + "name": "encrypted_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "external_resource_userId_idx": { + "name": "external_resource_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "external_resource_moduleId_idx": { + "name": "external_resource_moduleId_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "external_resource_user_id_user_id_fk": { + "name": "external_resource_user_id_user_id_fk", + "tableFrom": "external_resource", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "external_resource_program_id_degree_program_id_fk": { + "name": "external_resource_program_id_degree_program_id_fk", + "tableFrom": "external_resource", + "tableTo": "degree_program", + "columnsFrom": [ + "program_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "external_resource_module_id_module_id_fk": { + "name": "external_resource_module_id_module_id_fk", + "tableFrom": "external_resource", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grade": { + "name": "grade", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": true + }, + "weight": { + "name": "weight", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": true, + "default": "'1'" + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "graded_at": { + "name": "graded_at", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "grade_moduleId_idx": { + "name": "grade_moduleId_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "grade_module_id_module_id_fk": { + "name": "grade_module_id_module_id_fk", + "tableFrom": "grade", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.module_assessment": { + "name": "module_assessment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'exam'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "module_assessment_moduleId_idx": { + "name": "module_assessment_moduleId_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "module_assessment_module_id_module_id_fk": { + "name": "module_assessment_module_id_module_id_fk", + "tableFrom": "module_assessment", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "module_assessment_module_id_unique": { + "name": "module_assessment_module_id_unique", + "nullsNotDistinct": false, + "columns": [ + "module_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.module_contact": { + "name": "module_contact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "module_contact_moduleId_idx": { + "name": "module_contact_moduleId_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "module_contact_module_id_module_id_fk": { + "name": "module_contact_module_id_module_id_fk", + "tableFrom": "module_contact", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.semester": { + "name": "semester", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "program_id": { + "name": "program_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "start_date": { + "name": "start_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "end_date": { + "name": "end_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "semester_programId_idx": { + "name": "semester_programId_idx", + "columns": [ + { + "expression": "program_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "semester_program_id_degree_program_id_fk": { + "name": "semester_program_id_degree_program_id_fk", + "tableFrom": "semester", + "tableTo": "degree_program", + "columnsFrom": [ + "program_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.event": { + "name": "event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'other'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starts_at": { + "name": "starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ends_at": { + "name": "ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "all_day": { + "name": "all_day", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "ai_generated": { + "name": "ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "reminder_offsets": { + "name": "reminder_offsets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "recurrence": { + "name": "recurrence", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "recurrence_until": { + "name": "recurrence_until", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "recurrence_weekdays": { + "name": "recurrence_weekdays", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "recurrence_interval": { + "name": "recurrence_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "event_userId_idx": { + "name": "event_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "event_startsAt_idx": { + "name": "event_startsAt_idx", + "columns": [ + { + "expression": "starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "event_user_id_user_id_fk": { + "name": "event_user_id_user_id_fk", + "tableFrom": "event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "event_module_id_module_id_fk": { + "name": "event_module_id_module_id_fk", + "tableFrom": "event", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.module": { + "name": "module", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "semester_id": { + "name": "semester_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ects": { + "name": "ects", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "instructor": { + "name": "instructor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "exam_type": { + "name": "exam_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'planned'" + }, + "is_thesis": { + "name": "is_thesis", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "pass_fail": { + "name": "pass_fail", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "bonus_type": { + "name": "bonus_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "bonus_value": { + "name": "bonus_value", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false + }, + "bonus_min_avg_percent": { + "name": "bonus_min_avg_percent", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false + }, + "bonus_min_completed_share": { + "name": "bonus_min_completed_share", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "module_semesterId_idx": { + "name": "module_semesterId_idx", + "columns": [ + { + "expression": "semester_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "module_semester_id_semester_id_fk": { + "name": "module_semester_id_semester_id_fk", + "tableFrom": "module", + "tableTo": "semester", + "columnsFrom": [ + "semester_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_prefs": { + "name": "user_prefs", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "ics_token": { + "name": "ics_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "active_program_id": { + "name": "active_program_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "active_semester_id": { + "name": "active_semester_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preferred_model": { + "name": "preferred_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "weekly_goal_minutes": { + "name": "weekly_goal_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_prefs_user_id_user_id_fk": { + "name": "user_prefs_user_id_user_id_fk", + "tableFrom": "user_prefs", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_prefs_active_program_id_degree_program_id_fk": { + "name": "user_prefs_active_program_id_degree_program_id_fk", + "tableFrom": "user_prefs", + "tableTo": "degree_program", + "columnsFrom": [ + "active_program_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "user_prefs_active_semester_id_semester_id_fk": { + "name": "user_prefs_active_semester_id_semester_id_fk", + "tableFrom": "user_prefs", + "tableTo": "semester", + "columnsFrom": [ + "active_semester_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_prefs_ics_token_unique": { + "name": "user_prefs_ics_token_unique", + "nullsNotDistinct": false, + "columns": [ + "ics_token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.material": { + "name": "material", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "folder": { + "name": "folder", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "text_content": { + "name": "text_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "text_storage_path": { + "name": "text_storage_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "extraction_status": { + "name": "extraction_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "extraction_error": { + "name": "extraction_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "chunks_total": { + "name": "chunks_total", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "chunks_embedded": { + "name": "chunks_embedded", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "material_userId_idx": { + "name": "material_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_moduleId_idx": { + "name": "material_moduleId_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_folderId_idx": { + "name": "material_folderId_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_contentHash_idx": { + "name": "material_contentHash_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "content_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "material_user_id_user_id_fk": { + "name": "material_user_id_user_id_fk", + "tableFrom": "material", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "material_module_id_module_id_fk": { + "name": "material_module_id_module_id_fk", + "tableFrom": "material", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "material_folder_id_material_folder_id_fk": { + "name": "material_folder_id_material_folder_id_fk", + "tableFrom": "material", + "tableTo": "material_folder", + "columnsFrom": [ + "folder_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.material_annotation": { + "name": "material_annotation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "material_id": { + "name": "material_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "page": { + "name": "page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "rect": { + "name": "rect", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yellow'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "material_annotation_material_idx": { + "name": "material_annotation_material_idx", + "columns": [ + { + "expression": "material_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "material_annotation_material_id_material_id_fk": { + "name": "material_annotation_material_id_material_id_fk", + "tableFrom": "material_annotation", + "tableTo": "material", + "columnsFrom": [ + "material_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "material_annotation_user_id_user_id_fk": { + "name": "material_annotation_user_id_user_id_fk", + "tableFrom": "material_annotation", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.material_folder": { + "name": "material_folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "material_folder_userId_idx": { + "name": "material_folder_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_folder_moduleId_idx": { + "name": "material_folder_moduleId_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_folder_parentId_idx": { + "name": "material_folder_parentId_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "material_folder_user_id_user_id_fk": { + "name": "material_folder_user_id_user_id_fk", + "tableFrom": "material_folder", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "material_folder_module_id_module_id_fk": { + "name": "material_folder_module_id_module_id_fk", + "tableFrom": "material_folder", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "material_folder_parent_id_material_folder_id_fk": { + "name": "material_folder_parent_id_material_folder_id_fk", + "tableFrom": "material_folder", + "tableTo": "material_folder", + "columnsFrom": [ + "parent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.material_chunk": { + "name": "material_chunk", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "material_id": { + "name": "material_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector", + "primaryKey": false, + "notNull": false + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "parent_chunk_id": { + "name": "parent_chunk_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contextual_header": { + "name": "contextual_header", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('simple', \"material_chunk\".\"content\")", + "type": "stored" + } + } + }, + "indexes": { + "material_chunk_materialId_idx": { + "name": "material_chunk_materialId_idx", + "columns": [ + { + "expression": "material_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_chunk_material_level_idx": { + "name": "material_chunk_material_level_idx", + "columns": [ + { + "expression": "material_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "material_chunk_tsv_idx": { + "name": "material_chunk_tsv_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "material_chunk_material_id_material_id_fk": { + "name": "material_chunk_material_id_material_id_fk", + "tableFrom": "material_chunk", + "tableTo": "material", + "columnsFrom": [ + "material_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "material_chunk_parent_chunk_id_material_chunk_id_fk": { + "name": "material_chunk_parent_chunk_id_material_chunk_id_fk", + "tableFrom": "material_chunk", + "tableTo": "material_chunk", + "columnsFrom": [ + "parent_chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.answer_log": { + "name": "answer_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "attempt_id": { + "name": "attempt_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "question_id": { + "name": "question_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "answer": { + "name": "answer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "correct": { + "name": "correct", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "feedback": { + "name": "feedback", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "answer_log_attemptId_idx": { + "name": "answer_log_attemptId_idx", + "columns": [ + { + "expression": "attempt_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "answer_log_attempt_id_quiz_attempt_id_fk": { + "name": "answer_log_attempt_id_quiz_attempt_id_fk", + "tableFrom": "answer_log", + "tableTo": "quiz_attempt", + "columnsFrom": [ + "attempt_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "answer_log_question_id_question_id_fk": { + "name": "answer_log_question_id_question_id_fk", + "tableFrom": "answer_log", + "tableTo": "question", + "columnsFrom": [ + "question_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deck": { + "name": "deck", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_generated": { + "name": "ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'normal'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "deck_userId_idx": { + "name": "deck_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deck_user_id_user_id_fk": { + "name": "deck_user_id_user_id_fk", + "tableFrom": "deck", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deck_module_id_module_id_fk": { + "name": "deck_module_id_module_id_fk", + "tableFrom": "deck", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.flashcard": { + "name": "flashcard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "deck_id": { + "name": "deck_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "front": { + "name": "front", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "back": { + "name": "back", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "due": { + "name": "due", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "stability": { + "name": "stability", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "difficulty": { + "name": "difficulty", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "elapsed_days": { + "name": "elapsed_days", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "scheduled_days": { + "name": "scheduled_days", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "learning_steps": { + "name": "learning_steps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reps": { + "name": "reps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "lapses": { + "name": "lapses", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "state": { + "name": "state", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_review": { + "name": "last_review", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "flashcard_deckId_idx": { + "name": "flashcard_deckId_idx", + "columns": [ + { + "expression": "deck_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "flashcard_due_idx": { + "name": "flashcard_due_idx", + "columns": [ + { + "expression": "due", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "flashcard_deck_id_deck_id_fk": { + "name": "flashcard_deck_id_deck_id_fk", + "tableFrom": "flashcard", + "tableTo": "deck", + "columnsFrom": [ + "deck_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.question": { + "name": "question", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "quiz_id": { + "name": "quiz_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "correct_index": { + "name": "correct_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "reference_answer": { + "name": "reference_answer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "explanation": { + "name": "explanation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "question_quizId_idx": { + "name": "question_quizId_idx", + "columns": [ + { + "expression": "quiz_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "question_quiz_id_quiz_id_fk": { + "name": "question_quiz_id_quiz_id_fk", + "tableFrom": "question", + "tableTo": "quiz", + "columnsFrom": [ + "quiz_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.quiz": { + "name": "quiz", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_generated": { + "name": "ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "quiz_userId_idx": { + "name": "quiz_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "quiz_user_id_user_id_fk": { + "name": "quiz_user_id_user_id_fk", + "tableFrom": "quiz", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "quiz_module_id_module_id_fk": { + "name": "quiz_module_id_module_id_fk", + "tableFrom": "quiz", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.quiz_attempt": { + "name": "quiz_attempt", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "quiz_id": { + "name": "quiz_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "score": { + "name": "score", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "quiz_attempt_quizId_idx": { + "name": "quiz_attempt_quizId_idx", + "columns": [ + { + "expression": "quiz_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "quiz_attempt_userId_idx": { + "name": "quiz_attempt_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "quiz_attempt_quiz_id_quiz_id_fk": { + "name": "quiz_attempt_quiz_id_quiz_id_fk", + "tableFrom": "quiz_attempt", + "tableTo": "quiz", + "columnsFrom": [ + "quiz_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "quiz_attempt_user_id_user_id_fk": { + "name": "quiz_attempt_user_id_user_id_fk", + "tableFrom": "quiz_attempt", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_log": { + "name": "review_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_log_cardId_idx": { + "name": "review_log_cardId_idx", + "columns": [ + { + "expression": "card_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_log_userId_idx": { + "name": "review_log_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "review_log_card_id_flashcard_id_fk": { + "name": "review_log_card_id_flashcard_id_fk", + "tableFrom": "review_log", + "tableTo": "flashcard", + "columnsFrom": [ + "card_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "review_log_user_id_user_id_fk": { + "name": "review_log_user_id_user_id_fk", + "tableFrom": "review_log", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.study_plan": { + "name": "study_plan", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_generated": { + "name": "ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "study_plan_userId_idx": { + "name": "study_plan_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "study_plan_user_id_user_id_fk": { + "name": "study_plan_user_id_user_id_fk", + "tableFrom": "study_plan", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "study_plan_module_id_module_id_fk": { + "name": "study_plan_module_id_module_id_fk", + "tableFrom": "study_plan", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.study_plan_item": { + "name": "study_plan_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scheduled_date": { + "name": "scheduled_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "done": { + "name": "done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "study_plan_item_planId_idx": { + "name": "study_plan_item_planId_idx", + "columns": [ + { + "expression": "plan_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "study_plan_item_plan_id_study_plan_id_fk": { + "name": "study_plan_item_plan_id_study_plan_id_fk", + "tableFrom": "study_plan_item", + "tableTo": "study_plan", + "columnsFrom": [ + "plan_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.thesis_milestone": { + "name": "thesis_milestone", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "thesis_id": { + "name": "thesis_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "due_date": { + "name": "due_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "done": { + "name": "done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "thesis_milestone_thesisId_idx": { + "name": "thesis_milestone_thesisId_idx", + "columns": [ + { + "expression": "thesis_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "thesis_milestone_thesis_id_thesis_project_id_fk": { + "name": "thesis_milestone_thesis_id_thesis_project_id_fk", + "tableFrom": "thesis_milestone", + "tableTo": "thesis_project", + "columnsFrom": [ + "thesis_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.thesis_project": { + "name": "thesis_project", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "program_id": { + "name": "program_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "semester_id": { + "name": "semester_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thesis_type": { + "name": "thesis_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'topic'" + }, + "research_question": { + "name": "research_question", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outline": { + "name": "outline", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "due_date": { + "name": "due_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "superseded_by_id": { + "name": "superseded_by_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "thesis_project_userId_idx": { + "name": "thesis_project_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "thesis_active_per_program_uq": { + "name": "thesis_active_per_program_uq", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "program_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"thesis_project\".\"superseded_by_id\" is null and \"thesis_project\".\"program_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "thesis_project_user_id_user_id_fk": { + "name": "thesis_project_user_id_user_id_fk", + "tableFrom": "thesis_project", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thesis_project_program_id_degree_program_id_fk": { + "name": "thesis_project_program_id_degree_program_id_fk", + "tableFrom": "thesis_project", + "tableTo": "degree_program", + "columnsFrom": [ + "program_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thesis_project_semester_id_semester_id_fk": { + "name": "thesis_project_semester_id_semester_id_fk", + "tableFrom": "thesis_project", + "tableTo": "semester", + "columnsFrom": [ + "semester_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_prefs": { + "name": "notification_prefs", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email_reminders": { + "name": "email_reminders", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "push_reminders": { + "name": "push_reminders", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "channels": { + "name": "channels", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "notification_prefs_user_id_user_id_fk": { + "name": "notification_prefs_user_id_user_id_fk", + "tableFrom": "notification_prefs", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_sent": { + "name": "notification_sent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "notification_sent_user_id_user_id_fk": { + "name": "notification_sent_user_id_user_id_fk", + "tableFrom": "notification_sent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "notification_sent_user_key": { + "name": "notification_sent_user_key", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.push_subscription": { + "name": "push_subscription", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "p256dh": { + "name": "p256dh", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "auth": { + "name": "auth", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "push_subscription_userId_idx": { + "name": "push_subscription_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "push_subscription_user_id_user_id_fk": { + "name": "push_subscription_user_id_user_id_fk", + "tableFrom": "push_subscription", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "push_subscription_endpoint_unique": { + "name": "push_subscription_endpoint_unique", + "nullsNotDistinct": false, + "columns": [ + "endpoint" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.reminder_sent": { + "name": "reminder_sent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "offset_minutes": { + "name": "offset_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "occurrence_date": { + "name": "occurrence_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "reminder_sent_event_id_event_id_fk": { + "name": "reminder_sent_event_id_event_id_fk", + "tableFrom": "reminder_sent", + "tableTo": "event", + "columnsFrom": [ + "event_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "reminder_sent_event_offset_occurrence": { + "name": "reminder_sent_event_offset_occurrence", + "nullsNotDistinct": false, + "columns": [ + "event_id", + "offset_minutes", + "occurrence_date" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invite": { + "name": "invite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "max_uses": { + "name": "max_uses", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "used_count": { + "name": "used_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "invite_created_by_user_id_fk": { + "name": "invite_created_by_user_id_fk", + "tableFrom": "invite", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invite_token_unique": { + "name": "invite_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.study_session": { + "name": "study_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pomodoro'" + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "study_session_userId_idx": { + "name": "study_session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "study_session_startedAt_idx": { + "name": "study_session_startedAt_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "study_session_user_id_user_id_fk": { + "name": "study_session_user_id_user_id_fk", + "tableFrom": "study_session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "study_session_module_id_module_id_fk": { + "name": "study_session_module_id_module_id_fk", + "tableFrom": "study_session", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.semester_plan": { + "name": "semester_plan", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "semester_id": { + "name": "semester_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "availability": { + "name": "availability", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "generated_at": { + "name": "generated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "semester_plan_user_idx": { + "name": "semester_plan_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "semester_plan_user_id_user_id_fk": { + "name": "semester_plan_user_id_user_id_fk", + "tableFrom": "semester_plan", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "semester_plan_semester_id_semester_id_fk": { + "name": "semester_plan_semester_id_semester_id_fk", + "tableFrom": "semester_plan", + "tableTo": "semester", + "columnsFrom": [ + "semester_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "semester_plan_semester_id_unique": { + "name": "semester_plan_semester_id_unique", + "nullsNotDistinct": false, + "columns": [ + "semester_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.semester_plan_item": { + "name": "semester_plan_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assignment_id": { + "name": "assignment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'study'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "start_time": { + "name": "start_time", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_minutes": { + "name": "duration_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 60 + }, + "done": { + "name": "done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "semester_plan_item_plan_idx": { + "name": "semester_plan_item_plan_idx", + "columns": [ + { + "expression": "plan_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "semester_plan_item_plan_id_semester_plan_id_fk": { + "name": "semester_plan_item_plan_id_semester_plan_id_fk", + "tableFrom": "semester_plan_item", + "tableTo": "semester_plan", + "columnsFrom": [ + "plan_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "semester_plan_item_module_id_module_id_fk": { + "name": "semester_plan_item_module_id_module_id_fk", + "tableFrom": "semester_plan_item", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "semester_plan_item_assignment_id_assignment_id_fk": { + "name": "semester_plan_item_assignment_id_assignment_id_fk", + "tableFrom": "semester_plan_item", + "tableTo": "assignment", + "columnsFrom": [ + "assignment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.module_outline": { + "name": "module_outline", + "schema": "", + "columns": { + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'idle'" + }, + "topic_count": { + "name": "topic_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "module_outline_module_id_module_id_fk": { + "name": "module_outline_module_id_module_id_fk", + "tableFrom": "module_outline", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "module_outline_user_id_user_id_fk": { + "name": "module_outline_user_id_user_id_fk", + "tableFrom": "module_outline", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outline_topic": { + "name": "outline_topic", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "module_id": { + "name": "module_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title_key": { + "name": "title_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_material_ids": { + "name": "source_material_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "outline_topic_module_idx": { + "name": "outline_topic_module_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outline_topic_module_version_idx": { + "name": "outline_topic_module_version_idx", + "columns": [ + { + "expression": "module_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "outline_topic_module_id_module_id_fk": { + "name": "outline_topic_module_id_module_id_fk", + "tableFrom": "outline_topic", + "tableTo": "module", + "columnsFrom": [ + "module_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "outline_topic_user_id_user_id_fk": { + "name": "outline_topic_user_id_user_id_fk", + "tableFrom": "outline_topic", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "outline_topic_parent_id_outline_topic_id_fk": { + "name": "outline_topic_parent_id_outline_topic_id_fk", + "tableFrom": "outline_topic", + "tableTo": "outline_topic", + "columnsFrom": [ + "parent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 7bf0bcb..6bdf894 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -239,6 +239,13 @@ "when": 1784141433362, "tag": "0033_phase2_hybrid_search", "breakpoints": true + }, + { + "idx": 34, + "version": "7", + "when": 1784217381713, + "tag": "0034_old_ben_parker", + "breakpoints": true } ] } \ No newline at end of file diff --git a/messages/de.json b/messages/de.json index da24a55..71d0932 100644 --- a/messages/de.json +++ b/messages/de.json @@ -171,6 +171,8 @@ "defaultEmbeddingModel": "Standard-Embedding-Modell (RAG)", "noModels": "Noch keine Modelle konfiguriert", "monthlyLimit": "Monatliches Token-Limit pro User (0 = unbegrenzt)", + "useBatchApi": "Batch-API für vollständige Generierung nutzen", + "useBatchApiHint": "Führt die „Complete\"-Generierung über die Batch-API von Anthropic/OpenAI aus (~50 % günstiger). Ergebnisse treffen asynchron ein; nur für Anthropic-/OpenAI-Provider, sonst weiter live.", "saved": "KI-Einstellungen gespeichert.", "usage": "Token-Verbrauch (30 Tage)", "usageUser": "Benutzer", diff --git a/messages/en.json b/messages/en.json index 4de09f8..8de75d4 100644 --- a/messages/en.json +++ b/messages/en.json @@ -171,6 +171,8 @@ "defaultEmbeddingModel": "Default embedding model (RAG)", "noModels": "No models configured yet", "monthlyLimit": "Monthly token limit per user (0 = unlimited)", + "useBatchApi": "Use the Batch API for complete generation", + "useBatchApiHint": "Runs \"complete\" generation through Anthropic/OpenAI's Batch API (~50% cheaper). Results arrive asynchronously; only for Anthropic/OpenAI providers, otherwise it stays live.", "saved": "AI settings saved.", "usage": "Token usage (30 days)", "usageUser": "User", diff --git a/package-lock.json b/package-lock.json index e65a357..794d00a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@ai-sdk/openai": "^4.0.10", "@ai-sdk/openai-compatible": "^3.0.7", "@ai-sdk/react": "^4.0.20", + "@anthropic-ai/sdk": "^0.111.0", "@aws-sdk/client-s3": "^3.1088.0", "@aws-sdk/lib-storage": "^3.1088.0", "@base-ui/react": "^1.6.0", @@ -43,6 +44,7 @@ "next-intl": "^4.13.1", "next-themes": "^0.4.6", "nodemailer": "^9.0.3", + "openai": "^6.47.0", "pdfjs-dist": "^6.1.200", "pg-boss": "^12.25.1", "postgres": "^3.4.9", @@ -279,6 +281,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.111.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.111.0.tgz", + "integrity": "sha512-1hUqKi+uJQoS5X90+InwHbFAXMvgq0DnsC5hVLEeSRaODiU5WvmqDAcVCmGS2wC0pN9Z8jtWCbWw7JLzeDdm/Q==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@asamuzakjp/css-color": { "version": "5.1.11", "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", @@ -5099,6 +5122,12 @@ "node": ">=18.0.0" } }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -10083,6 +10112,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, "node_modules/fast-uri": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", @@ -11851,6 +11886,19 @@ "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", "license": "(AFL-2.1 OR BSD-3-Clause)" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -14103,6 +14151,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "6.47.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.47.0.tgz", + "integrity": "sha512-xYr+R9woSzWxVxeiqkkNbHhv89tZDEI6eBMbrdPnv3poh+mijHvbhS35a+3o6xHa411/ns8j5ENY3So9DCXWYw==", + "license": "Apache-2.0", + "peerDependencies": { + "@aws-sdk/credential-provider-node": ">=3.972.0 <4", + "@smithy/hash-node": ">=4.3.0 <5", + "@smithy/signature-v4": ">=5.4.0 <6", + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@smithy/hash-node": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/option": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", @@ -16404,6 +16482,16 @@ "devOptional": true, "license": "MIT" }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -17074,6 +17162,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", diff --git a/package.json b/package.json index b7a0506..73a9fab 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@ai-sdk/openai": "^4.0.10", "@ai-sdk/openai-compatible": "^3.0.7", "@ai-sdk/react": "^4.0.20", + "@anthropic-ai/sdk": "^0.111.0", "@aws-sdk/client-s3": "^3.1088.0", "@aws-sdk/lib-storage": "^3.1088.0", "@base-ui/react": "^1.6.0", @@ -53,6 +54,7 @@ "next-intl": "^4.13.1", "next-themes": "^0.4.6", "nodemailer": "^9.0.3", + "openai": "^6.47.0", "pdfjs-dist": "^6.1.200", "pg-boss": "^12.25.1", "postgres": "^3.4.9", diff --git a/src/app/[locale]/(app)/admin/ai/page.tsx b/src/app/[locale]/(app)/admin/ai/page.tsx index 31c7a0c..66fcb41 100644 --- a/src/app/[locale]/(app)/admin/ai/page.tsx +++ b/src/app/[locale]/(app)/admin/ai/page.tsx @@ -30,7 +30,9 @@ export default async function AdminAiPage() { return (
- + p.modelsText @@ -102,6 +104,7 @@ export function AiSettingsForm({ initial }: { initial: AiSettings }) { defaultModel: defaultModel || undefined, defaultEmbeddingModel: defaultEmbeddingModel || undefined, monthlyTokenLimitPerUser: monthlyLimit, + useBatchApi, }) toast.success(t("saved")) } catch (error) { @@ -273,6 +276,13 @@ export function AiSettingsForm({ initial }: { initial: AiSettings }) { onChange={(e) => setMonthlyLimit(Number(e.target.value))} />
+
+
+ + +
+

{t("useBatchApiHint")}

+
diff --git a/src/db/schema/generation.ts b/src/db/schema/generation.ts index 587b8d2..713b6c2 100644 --- a/src/db/schema/generation.ts +++ b/src/db/schema/generation.ts @@ -34,6 +34,12 @@ export const generationJob = pgTable( topicsDone: integer("topics_done").notNull().default(0), producedCount: integer("produced_count").notNull().default(0), params: jsonb("params"), + /** When the MAP step ran via a provider Batch API: the vendor batch id + * (while awaiting results) and the "providerId:modelId" it was submitted + * with (so a poller can resolve credentials and record usage). Both null + * for the default synchronous live path. */ + batchRef: text("batch_ref"), + batchModel: text("batch_model"), error: text("error"), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }) diff --git a/src/lib/ai/generation/batch-adapter.test.ts b/src/lib/ai/generation/batch-adapter.test.ts new file mode 100644 index 0000000..3b277b1 --- /dev/null +++ b/src/lib/ai/generation/batch-adapter.test.ts @@ -0,0 +1,173 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import { z } from "zod" + +// Neutralise server-only and mock the heavy server deps so the module imports in +// a plain unit-test context. The pure builders/parsers don't touch these; only +// resolveBatchProvider does (via the getSetting / userAiKey mocks below). +vi.mock("server-only", () => ({})) + +const getSettingMock = vi.fn() +const findFirstMock = vi.fn().mockResolvedValue(undefined) +vi.mock("@/lib/settings", () => ({ getSetting: (key: string) => getSettingMock(key) })) +vi.mock("@/db", () => ({ + db: { query: { userAiKey: { findFirst: (...args: unknown[]) => findFirstMock(...args) } } }, +})) +vi.mock("@/db/schema", () => ({ userAiKey: { userId: {}, providerId: {} } })) +vi.mock("@/lib/crypto", () => ({ decrypt: (s: string) => `decrypted:${s}` })) +vi.mock("@/lib/ai/registry", () => ({ + parseModelRef: (ref: string) => { + const i = ref.indexOf(":") + return { providerId: ref.slice(0, i), modelId: ref.slice(i + 1) } + }, +})) + +import { + buildAnthropicRequests, + buildOpenAiTasks, + parseAnthropicResults, + parseOpenAiResults, + resolveBatchProvider, + toJsonSchema, + type BatchItem, +} from "./batch-adapter" + +const items: BatchItem[] = [ + { customId: "topic-1", prompt: "PROMPT", jsonSchema: { type: "object" }, maxTokens: 2000 }, +] + +describe("toJsonSchema", () => { + it("converts a Zod object schema to JSON Schema", () => { + const schema = z.object({ cards: z.array(z.object({ front: z.string(), back: z.string() })) }) + const json = toJsonSchema(schema) + expect(json.type).toBe("object") + expect(json).toHaveProperty("properties.cards.type", "array") + }) +}) + +describe("buildAnthropicRequests", () => { + it("forces a single tool call carrying the JSON schema", () => { + const reqs = buildAnthropicRequests(items, "claude-x") + expect(reqs).toHaveLength(1) + expect(reqs[0].custom_id).toBe("topic-1") + expect(reqs[0].params.model).toBe("claude-x") + expect(reqs[0].params.max_tokens).toBe(2000) + expect(reqs[0].params.tool_choice).toEqual({ type: "tool", name: "emit_result" }) + expect(reqs[0].params.tools[0].input_schema).toEqual({ type: "object" }) + expect(reqs[0].params.messages).toEqual([{ role: "user", content: "PROMPT" }]) + }) +}) + +describe("buildOpenAiTasks", () => { + it("builds json_schema chat-completion tasks", () => { + const tasks = buildOpenAiTasks(items, "gpt-x") + expect(tasks[0].custom_id).toBe("topic-1") + expect(tasks[0].method).toBe("POST") + expect(tasks[0].url).toBe("/v1/chat/completions") + expect(tasks[0].body.model).toBe("gpt-x") + expect(tasks[0].body.response_format.json_schema.schema).toEqual({ type: "object" }) + expect(tasks[0].body.messages).toEqual([{ role: "user", content: "PROMPT" }]) + }) +}) + +describe("parseAnthropicResults", () => { + it("extracts the tool_use input and usage for succeeded items, flags errors", () => { + const parsed = parseAnthropicResults([ + { + custom_id: "topic-1", + result: { + type: "succeeded", + message: { + content: [ + { type: "text" }, + { type: "tool_use", input: { cards: [{ front: "a", back: "b" }] } }, + ], + usage: { input_tokens: 10, output_tokens: 20 }, + }, + }, + }, + { custom_id: "topic-2", result: { type: "errored", error: { message: "bad request" } } }, + ]) + expect(parsed[0]).toEqual({ + customId: "topic-1", + object: { cards: [{ front: "a", back: "b" }] }, + usage: { inputTokens: 10, outputTokens: 20 }, + error: undefined, + }) + expect(parsed[1].object).toBeNull() + expect(parsed[1].error).toBe("bad request") + expect(parsed[1].usage).toEqual({ inputTokens: 0, outputTokens: 0 }) + }) +}) + +describe("parseOpenAiResults", () => { + it("parses JSONL output lines, tolerating errored/empty lines", () => { + const jsonl = [ + JSON.stringify({ + custom_id: "topic-1", + response: { + body: { + choices: [ + { message: { content: JSON.stringify({ cards: [{ front: "a", back: "b" }] }) } }, + ], + usage: { prompt_tokens: 5, completion_tokens: 7 }, + }, + }, + }), + "", + JSON.stringify({ custom_id: "topic-2", error: { message: "boom" } }), + ].join("\n") + const parsed = parseOpenAiResults(jsonl) + expect(parsed).toHaveLength(2) + expect(parsed[0]).toEqual({ + customId: "topic-1", + object: { cards: [{ front: "a", back: "b" }] }, + usage: { inputTokens: 5, outputTokens: 7 }, + }) + expect(parsed[1].object).toBeNull() + expect(parsed[1].error).toBe("batch item error") + }) +}) + +describe("resolveBatchProvider", () => { + beforeEach(() => { + getSettingMock.mockReset() + findFirstMock.mockReset() + findFirstMock.mockResolvedValue(undefined) + }) + + it("resolves an Anthropic provider with its admin key", async () => { + getSettingMock.mockResolvedValue({ + providers: [{ id: "anthropic", type: "anthropic", apiKey: "sk-admin", models: [] }], + }) + const provider = await resolveBatchProvider("anthropic:claude-x", "user-1") + expect(provider).toEqual({ + type: "anthropic", + apiKey: "sk-admin", + baseUrl: undefined, + modelId: "claude-x", + }) + }) + + it("prefers a per-user BYOK key over the admin key", async () => { + getSettingMock.mockResolvedValue({ + providers: [{ id: "openai", type: "openai", apiKey: "sk-admin", models: [] }], + }) + findFirstMock.mockResolvedValue({ encryptedKey: "enc" }) + const provider = await resolveBatchProvider("openai:gpt-x", "user-1") + expect(provider?.apiKey).toBe("decrypted:enc") + }) + + it("returns null for a non-batch provider type", async () => { + getSettingMock.mockResolvedValue({ + providers: [{ id: "google", type: "google", apiKey: "k", models: [] }], + }) + expect(await resolveBatchProvider("google:gemini", "user-1")).toBeNull() + }) + + it("returns null when the provider has no usable key", async () => { + getSettingMock.mockResolvedValue({ + providers: [{ id: "anthropic", type: "anthropic", models: [] }], + }) + expect(await resolveBatchProvider("anthropic:claude-x", "user-1")).toBeNull() + }) +}) diff --git a/src/lib/ai/generation/batch-adapter.ts b/src/lib/ai/generation/batch-adapter.ts new file mode 100644 index 0000000..3c45883 --- /dev/null +++ b/src/lib/ai/generation/batch-adapter.ts @@ -0,0 +1,314 @@ +import "server-only" +import { and, eq } from "drizzle-orm" +import type { z } from "zod" +import { z as zod } from "zod" +import { db } from "@/db" +import { userAiKey } from "@/db/schema" +import { decrypt } from "@/lib/crypto" +import { getSetting } from "@/lib/settings" +import { parseModelRef } from "@/lib/ai/registry" + +/** + * Optional Batch-API adapter for the coverage-generation MAP step. Anthropic + * Message Batches and OpenAI Batch run the same per-topic requests + * asynchronously at ~50% of the cost. This module is split into pure, testable + * builders/parsers and thin network calls; the vendor SDKs are imported lazily + * so deployments that don't use batching never load them. Everything here is + * opt-in (settings flag `ai.useBatchApi`) with the synchronous live path as the + * default and the fallback. + */ + +export type BatchProviderType = "anthropic" | "openai" + +export type BatchProvider = { + type: BatchProviderType + apiKey: string + baseUrl?: string + modelId: string +} + +export type BatchItem = { + /** Correlates the request with an outline topic id. */ + customId: string + prompt: string + jsonSchema: Record + maxTokens: number +} + +export type BatchUsage = { inputTokens: number; outputTokens: number } + +export type BatchResult = { + customId: string + /** The parsed structured object, or null when the item errored. */ + object: unknown | null + usage: BatchUsage + error?: string +} + +export type BatchPollStatus = "processing" | "completed" | "failed" + +const TOOL_NAME = "emit_result" + +// ---- Provider resolution -------------------------------------------------------- + +/** + * Resolves a "providerId:modelId" ref to batch credentials, or null when the + * provider isn't batch-capable (only anthropic/openai) or has no usable key. + * Mirrors the BYOK precedence of registry.resolveApiKey. A null result means + * the caller must fall back to the live path. + */ +export async function resolveBatchProvider( + modelRef: string, + userId: string +): Promise { + const { providerId, modelId } = parseModelRef(modelRef) + const ai = await getSetting("ai") + const provider = ai?.providers.find((p) => p.id === providerId) + if (!provider || (provider.type !== "anthropic" && provider.type !== "openai")) return null + const byok = await db.query.userAiKey.findFirst({ + where: and(eq(userAiKey.userId, userId), eq(userAiKey.providerId, provider.id)), + }) + const apiKey = byok ? decrypt(byok.encryptedKey) : provider.apiKey + if (!apiKey) return null + return { type: provider.type, apiKey, baseUrl: provider.baseUrl, modelId } +} + +/** Converts a Zod schema to a JSON Schema for the batch request body. */ +export function toJsonSchema(schema: z.ZodType): Record { + return zod.toJSONSchema(schema) as Record +} + +// ---- Anthropic Message Batches -------------------------------------------------- + +type AnthropicRequest = { + custom_id: string + params: { + model: string + max_tokens: number + tools: { name: string; description: string; input_schema: Record }[] + tool_choice: { type: "tool"; name: string } + messages: { role: "user"; content: string }[] + } +} + +/** Builds Anthropic batch requests that force a single tool call whose input is + * the structured result (the batch equivalent of `generateObject`). */ +export function buildAnthropicRequests(items: BatchItem[], modelId: string): AnthropicRequest[] { + return items.map((item) => ({ + custom_id: item.customId, + params: { + model: modelId, + max_tokens: item.maxTokens, + tools: [ + { + name: TOOL_NAME, + description: "Return the requested structured result.", + input_schema: item.jsonSchema, + }, + ], + tool_choice: { type: "tool", name: TOOL_NAME }, + messages: [{ role: "user", content: item.prompt }], + }, + })) +} + +type AnthropicResultEntry = { + custom_id: string + result?: { + type: string + message?: { + content?: { type: string; input?: unknown }[] + usage?: { input_tokens?: number; output_tokens?: number } + } + error?: { type?: string; message?: string } + } +} + +/** Parses Anthropic batch result entries into the shared BatchResult shape. */ +export function parseAnthropicResults(entries: AnthropicResultEntry[]): BatchResult[] { + return entries.map((entry) => { + const result = entry.result + if (!result || result.type !== "succeeded" || !result.message) { + return { + customId: entry.custom_id, + object: null, + usage: { inputTokens: 0, outputTokens: 0 }, + error: result?.error?.message ?? result?.type ?? "no result", + } + } + const toolUse = (result.message.content ?? []).find((b) => b.type === "tool_use") + const usage: BatchUsage = { + inputTokens: result.message.usage?.input_tokens ?? 0, + outputTokens: result.message.usage?.output_tokens ?? 0, + } + return { + customId: entry.custom_id, + object: toolUse?.input ?? null, + usage, + error: toolUse ? undefined : "no tool_use block in result", + } + }) +} + +// ---- OpenAI Batch --------------------------------------------------------------- + +type OpenAiTask = { + custom_id: string + method: "POST" + url: "/v1/chat/completions" + body: { + model: string + max_tokens: number + messages: { role: "user"; content: string }[] + response_format: { + type: "json_schema" + json_schema: { name: string; schema: Record; strict: boolean } + } + } +} + +/** Builds OpenAI batch task lines (one JSONL entry per topic) that request a + * json_schema-shaped response (the batch equivalent of `generateObject`). */ +export function buildOpenAiTasks(items: BatchItem[], modelId: string): OpenAiTask[] { + return items.map((item) => ({ + custom_id: item.customId, + method: "POST", + url: "/v1/chat/completions", + body: { + model: modelId, + max_tokens: item.maxTokens, + messages: [{ role: "user", content: item.prompt }], + response_format: { + type: "json_schema", + json_schema: { name: "result", schema: item.jsonSchema, strict: false }, + }, + }, + })) +} + +type OpenAiResultLine = { + custom_id: string + response?: { + body?: { + choices?: { message?: { content?: string } }[] + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + } + error?: unknown +} + +/** Parses an OpenAI batch output file (JSONL) into the shared BatchResult shape. */ +export function parseOpenAiResults(jsonl: string): BatchResult[] { + const lines = jsonl + .split("\n") + .map((l) => l.trim()) + .filter(Boolean) + return lines.map((line) => { + const parsed = JSON.parse(line) as OpenAiResultLine + const body = parsed.response?.body + const usage: BatchUsage = { + inputTokens: body?.usage?.prompt_tokens ?? 0, + outputTokens: body?.usage?.completion_tokens ?? 0, + } + const content = body?.choices?.[0]?.message?.content + if (parsed.error || !content) { + return { + customId: parsed.custom_id, + object: null, + usage, + error: parsed.error ? "batch item error" : "no content in response", + } + } + try { + return { customId: parsed.custom_id, object: JSON.parse(content), usage } + } catch { + return { customId: parsed.custom_id, object: null, usage, error: "invalid JSON content" } + } + }) +} + +// ---- Network calls (lazy vendor SDKs) ------------------------------------------- + +async function anthropicClient(provider: BatchProvider) { + const { default: Anthropic } = await import("@anthropic-ai/sdk") + return new Anthropic({ + apiKey: provider.apiKey, + ...(provider.baseUrl ? { baseURL: provider.baseUrl } : {}), + }) +} + +async function openaiClient(provider: BatchProvider) { + const { default: OpenAI } = await import("openai") + return new OpenAI({ + apiKey: provider.apiKey, + ...(provider.baseUrl ? { baseURL: provider.baseUrl } : {}), + }) +} + +/** Submits all items as one vendor batch and returns the vendor batch id. */ +export async function submitBatch(provider: BatchProvider, items: BatchItem[]): Promise { + if (provider.type === "anthropic") { + const client = await anthropicClient(provider) + const requests = buildAnthropicRequests(items, provider.modelId) + const batch = await client.messages.batches.create({ + requests: requests as Parameters[0]["requests"], + }) + return batch.id + } + const { default: OpenAI, toFile } = await import("openai") + const client = new OpenAI({ + apiKey: provider.apiKey, + ...(provider.baseUrl ? { baseURL: provider.baseUrl } : {}), + }) + const jsonl = buildOpenAiTasks(items, provider.modelId) + .map((t) => JSON.stringify(t)) + .join("\n") + const file = await client.files.create({ + file: await toFile(Buffer.from(jsonl), "batch.jsonl"), + purpose: "batch", + }) + const batch = await client.batches.create({ + input_file_id: file.id, + endpoint: "/v1/chat/completions", + completion_window: "24h", + }) + return batch.id +} + +/** Polls a vendor batch's lifecycle state. */ +export async function pollBatch( + provider: BatchProvider, + batchRef: string +): Promise { + if (provider.type === "anthropic") { + const client = await anthropicClient(provider) + const batch = await client.messages.batches.retrieve(batchRef) + // "ended" means all requests finished (individual items may still have errored). + return batch.processing_status === "ended" ? "completed" : "processing" + } + const client = await openaiClient(provider) + const batch = await client.batches.retrieve(batchRef) + if (batch.status === "completed") return "completed" + if (["failed", "expired", "cancelled", "cancelling"].includes(batch.status)) return "failed" + return "processing" +} + +/** Fetches and parses a completed vendor batch's results. */ +export async function fetchBatchResults( + provider: BatchProvider, + batchRef: string +): Promise { + if (provider.type === "anthropic") { + const client = await anthropicClient(provider) + const entries: AnthropicResultEntry[] = [] + for await (const entry of await client.messages.batches.results(batchRef)) { + entries.push(entry as unknown as AnthropicResultEntry) + } + return parseAnthropicResults(entries) + } + const client = await openaiClient(provider) + const batch = await client.batches.retrieve(batchRef) + if (!batch.output_file_id) return [] + const content = await client.files.content(batch.output_file_id) + return parseOpenAiResults(await content.text()) +} diff --git a/src/lib/ai/generation/batch-poll.ts b/src/lib/ai/generation/batch-poll.ts new file mode 100644 index 0000000..5c7dd98 --- /dev/null +++ b/src/lib/ai/generation/batch-poll.ts @@ -0,0 +1,165 @@ +import "server-only" +import { and, eq, inArray, isNotNull } from "drizzle-orm" +import { db } from "@/db" +import { generationJob, outlineTopic } from "@/db/schema" +import { getSetting } from "@/lib/settings" +import { recordAiUsage, type AiCallContext } from "@/lib/ai/run" +import { + fetchBatchResults, + pollBatch, + resolveBatchProvider, + type BatchResult, +} from "./batch-adapter" +import { Deduper, embedTexts } from "./dedup" +import { + cardSchema, + loadExistingKeys, + persistCards, + persistQuestions, + questionSchema, + upsertCoverage, +} from "./generate" + +type JobRow = typeof generationJob.$inferSelect + +/** + * Cron-driven poller for in-flight generation batches. For every job that + * submitted a batch (`batchRef` set, still `running`), it checks the vendor + * batch status; once the batch has finished, it fetches the results, + * de-duplicates + inserts the items, records token usage per topic + * (requirement A — even though the tokens only arrive now), and completes the + * job. Individual failed items are marked in coverage so a re-run regenerates + * only them. Each job is isolated so one bad batch can't block the others. + */ +export async function pollPendingBatches(): Promise { + const jobs = await db.query.generationJob.findMany({ + where: and(isNotNull(generationJob.batchRef), eq(generationJob.status, "running")), + }) + for (const job of jobs) { + try { + await processBatchJob(job) + } catch (error) { + console.error("[batch-poll] job failed", job.id, error) + } + } +} + +async function failBatch(job: JobRow, message: string): Promise { + await db + .update(generationJob) + .set({ status: "failed", error: message.slice(0, 500), batchRef: null }) + .where(eq(generationJob.id, job.id)) +} + +async function processBatchJob(job: JobRow): Promise { + if (!job.batchRef || !job.batchModel) return + const provider = await resolveBatchProvider(job.batchModel, job.userId) + if (!provider) { + await failBatch(job, "Batch provider is no longer available") + return + } + + const status = await pollBatch(provider, job.batchRef) + if (status === "processing") return + if (status === "failed") { + await failBatch(job, "Provider batch failed") + return + } + + const results = await fetchBatchResults(provider, job.batchRef) + await applyBatchResults(job, results) +} + +async function applyBatchResults(job: JobRow, results: BatchResult[]): Promise { + const embeddingRef = (await getSetting("ai"))?.defaultEmbeddingModel ?? null + + // Re-seed the de-duplicator from the target's existing items; the whole batch + // is de-duped at once now that all topics arrive together. + const deduper = new Deduper(0.9) + const existingKeys = await loadExistingKeys(job.kind, job.targetId) + deduper.seedKeys(existingKeys) + if (embeddingRef && existingKeys.length > 0) { + deduper.seedVectors(await embedTexts(job.userId, embeddingRef, existingKeys)) + } + + const topicRows = results.length + ? await db.query.outlineTopic.findMany({ + where: inArray( + outlineTopic.id, + results.map((r) => r.customId) + ), + columns: { id: true, title: true }, + }) + : [] + const titleById = new Map(topicRows.map((t) => [t.id, t.title])) + + let topicsDone = job.topicsDone + let producedTotal = job.producedCount + + for (const res of results) { + let produced = 0 + let ok = false + if (res.object != null) { + try { + if (job.kind === "deck") { + const parsed = cardSchema.safeParse(res.object) + if (!parsed.success) throw new Error("schema mismatch") + produced = await persistCards( + job.userId, + job.targetId, + deduper, + embeddingRef, + parsed.data.cards + ) + } else { + const parsed = questionSchema.safeParse(res.object) + if (!parsed.success) throw new Error("schema mismatch") + produced = await persistQuestions( + job.userId, + job.targetId, + deduper, + embeddingRef, + parsed.data.questions + ) + } + ok = true + } catch (error) { + console.error("[batch-poll] item failed", res.customId, error) + } + } + + await upsertCoverage(job.targetId, res.customId, job.id, ok ? "done" : "failed", produced) + + // Requirement A: the token ledger records every AI action's usage — matching + // the live path (which also logs usage per topic without a per-topic audit + // entry), just with the tokens that only became available now. + const ctx: AiCallContext = { + userId: job.userId, + model: job.batchModel ?? "batch", + feature: job.kind === "deck" ? "flashcards" : "quiz", + operation: "ai_generate", + jobId: job.id, + entityType: job.kind, + entityId: job.targetId, + entityLabel: titleById.get(res.customId) ?? res.customId, + itemCount: produced, + } + try { + await recordAiUsage(ctx, { + inputTokens: res.usage.inputTokens, + outputTokens: res.usage.outputTokens, + totalTokens: res.usage.inputTokens + res.usage.outputTokens, + }) + } catch (error) { + console.error("[batch-poll] usage logging failed", error) + } + + topicsDone++ + producedTotal += produced + } + + await db + .update(generationJob) + .set({ status: "completed", batchRef: null, topicsDone, producedCount: producedTotal }) + .where(eq(generationJob.id, job.id)) +} diff --git a/src/lib/ai/generation/generate.ts b/src/lib/ai/generation/generate.ts index 5e3fa36..4685a29 100644 --- a/src/lib/ai/generation/generate.ts +++ b/src/lib/ai/generation/generate.ts @@ -20,6 +20,7 @@ import { assertWithinLimit } from "@/lib/ai/usage" import { getModuleMaterialSample, searchChunks, searchChunksInMaterials } from "@/lib/ai/rag" import { buildModuleOutline, type OutlineTopic } from "./outline" import { Deduper, embedTexts } from "./dedup" +import { resolveBatchProvider, submitBatch, toJsonSchema, type BatchItem } from "./batch-adapter" const MAX_GROUNDING_CHARS = 12000 const DEFAULT_CARDS_PER_TOPIC = 6 @@ -31,11 +32,11 @@ export type GenParams = { mixed?: boolean } -const cardSchema = z.object({ +export const cardSchema = z.object({ cards: z.array(z.object({ front: z.string(), back: z.string() })).max(60), }) -const questionSchema = z.object({ +export const questionSchema = z.object({ questions: z .array( z.object({ @@ -50,6 +51,41 @@ const questionSchema = z.object({ .max(30), }) +/** The flashcard MAP prompt. Shared by the live path and the batch adapter so + * both produce identical output. */ +export function buildCardPrompt( + topic: OutlineTopic, + grounding: string, + count: number, + language: string +): string { + return `Create up to ${count} high-quality spaced-repetition flashcards about the topic "${topic.title}".${ + topic.summary ? ` Topic scope: ${topic.summary}` : "" + } +Base the cards strictly on the excerpts below — do not invent facts not present. Each card: a concise question/term on the front, a precise answer/definition on the back. Write all cards in ${language}. + +Excerpts: +${grounding || "(no excerpts available — use the topic title/scope)"}` +} + +/** The quiz MAP prompt. Shared by the live path and the batch adapter. */ +export function buildQuestionPrompt( + topic: OutlineTopic, + grounding: string, + count: number, + mixed: boolean, + language: string +): string { + return `Create up to ${count} exam-style quiz questions about the topic "${topic.title}".${ + topic.summary ? ` Topic scope: ${topic.summary}` : "" + } +${mixed ? "Mix multiple_choice (exactly 4 plausible options, correctIndex set) and free_text (referenceAnswer set), about 70/30." : "Use only multiple_choice questions with exactly 4 plausible options and correctIndex set."} +Give each question a short explanation of the correct answer. Base questions strictly on the excerpts. Write everything in ${language}. + +Excerpts: +${grounding || "(no excerpts available — use the topic title/scope)"}` +} + // ---- Job creation --------------------------------------------------------------- async function createJob( @@ -149,13 +185,7 @@ async function generateTopicCards( generateObject({ model, schema: cardSchema, - prompt: `Create up to ${count} high-quality spaced-repetition flashcards about the topic "${topic.title}".${ - topic.summary ? ` Topic scope: ${topic.summary}` : "" - } -Base the cards strictly on the excerpts below — do not invent facts not present. Each card: a concise question/term on the front, a precise answer/definition on the back. Write all cards in ${language}. - -Excerpts: -${grounding || "(no excerpts available — use the topic title/scope)"}`, + prompt: buildCardPrompt(topic, grounding, count, language), }) ) return object.cards.slice(0, count) @@ -187,14 +217,7 @@ async function generateTopicQuestions( generateObject({ model, schema: questionSchema, - prompt: `Create up to ${count} exam-style quiz questions about the topic "${topic.title}".${ - topic.summary ? ` Topic scope: ${topic.summary}` : "" - } -${mixed ? "Mix multiple_choice (exactly 4 plausible options, correctIndex set) and free_text (referenceAnswer set), about 70/30." : "Use only multiple_choice questions with exactly 4 plausible options and correctIndex set."} -Give each question a short explanation of the correct answer. Base questions strictly on the excerpts. Write everything in ${language}. - -Excerpts: -${grounding || "(no excerpts available — use the topic title/scope)"}`, + prompt: buildQuestionPrompt(topic, grounding, count, mixed, language), }) ) return object.questions.slice(0, count) @@ -202,7 +225,7 @@ ${grounding || "(no excerpts available — use the topic title/scope)"}`, // ---- Coverage orchestration ----------------------------------------------------- -async function upsertCoverage( +export async function upsertCoverage( targetId: string, topicId: string, jobId: string, @@ -218,7 +241,7 @@ async function upsertCoverage( }) } -async function loadExistingKeys(kind: GenerationKind, targetId: string): Promise { +export async function loadExistingKeys(kind: GenerationKind, targetId: string): Promise { if (kind === "deck") { const rows = await db .select({ front: flashcard.front }) @@ -243,6 +266,68 @@ async function nextQuizSortOrder(quizId: string): Promise { return (row?.sortOrder ?? -1) + 1 } +type Card = z.infer["cards"][number] +type Question = z.infer["questions"][number] + +/** De-duplicates generated cards against `deduper` and inserts the fresh ones. + * Shared by the live loop and the batch-result path. Returns items inserted. */ +export async function persistCards( + userId: string, + targetId: string, + deduper: Deduper, + embeddingRef: string | null, + cards: Card[] +): Promise { + const keys = cards.map((c) => c.front) + const vecs = + embeddingRef && keys.length > 0 ? await embedTexts(userId, embeddingRef, keys) : undefined + const fresh = deduper.filter( + cards.map((c) => ({ key: c.front, item: c })), + vecs + ) + if (fresh.length > 0) { + await db + .insert(flashcard) + .values(fresh.map((c) => ({ deckId: targetId, front: c.front, back: c.back }))) + } + return fresh.length +} + +/** De-duplicates generated questions against `deduper` and inserts the fresh + * ones. Shared by the live loop and the batch-result path. Returns items + * inserted. */ +export async function persistQuestions( + userId: string, + targetId: string, + deduper: Deduper, + embeddingRef: string | null, + questions: Question[] +): Promise { + const keys = questions.map((q) => q.prompt) + const vecs = + embeddingRef && keys.length > 0 ? await embedTexts(userId, embeddingRef, keys) : undefined + const fresh = deduper.filter( + questions.map((q) => ({ key: q.prompt, item: q })), + vecs + ) + if (fresh.length > 0) { + const base = await nextQuizSortOrder(targetId) + await db.insert(question).values( + fresh.map((q, i) => ({ + quizId: targetId, + kind: q.kind, + prompt: q.prompt, + options: q.kind === "multiple_choice" ? q.options : null, + correctIndex: q.kind === "multiple_choice" && q.correctIndex >= 0 ? q.correctIndex : null, + referenceAnswer: q.kind === "free_text" ? q.referenceAnswer : null, + explanation: q.explanation || null, + sortOrder: base + i, + })) + ) + } + return fresh.length +} + async function fail(jobId: string, message: string): Promise { await db .update(generationJob) @@ -250,6 +335,85 @@ async function fail(jobId: string, message: string): Promise { .where(eq(generationJob.id, jobId)) } +/** Batch-request token budget per topic (Anthropic requires max_tokens). */ +function batchMaxTokens(count: number): number { + return Math.min(8000, Math.max(1500, count * 400)) +} + +/** + * Submits all uncovered topics as ONE provider batch for the MAP step and marks + * the job as awaiting the batch (status stays `running`, `batchRef` set). Returns + * true when a batch was submitted (or there was nothing left to do) — the caller + * then returns and the batch poller finishes the job once results arrive. Returns + * false to fall back to the live path (provider not batch-capable, or submit + * failed). Grounding/retrieval still runs live here; only the LLM calls batch. + */ +async function trySubmitBatchGeneration( + job: typeof generationJob.$inferSelect, + topics: OutlineTopic[], + modelRef: string, + covByTopic: Map, + opts: { perTopic: number; language: string; mixed: boolean } +): Promise { + const provider = await resolveBatchProvider(modelRef, job.userId) + if (!provider) return false + + const pending = topics.filter((t) => covByTopic.get(t.id)?.status !== "done") + const doneRows = topics.filter((t) => covByTopic.get(t.id)?.status === "done") + const alreadyProduced = doneRows.reduce( + (sum, t) => sum + (covByTopic.get(t.id)?.producedCount ?? 0), + 0 + ) + + if (pending.length === 0) { + await db + .update(generationJob) + .set({ status: "completed", topicsDone: doneRows.length, producedCount: alreadyProduced }) + .where(eq(generationJob.id, job.id)) + return true + } + + try { + await assertWithinLimit(job.userId) + } catch { + await fail(job.id, "Monthly token limit reached") + return true + } + + const jsonSchema = toJsonSchema(job.kind === "deck" ? cardSchema : questionSchema) + const maxTokens = batchMaxTokens(opts.perTopic) + + const items: BatchItem[] = [] + for (const topic of pending) { + const grounding = await groundTopic(job.userId, job.moduleId, topic) + const prompt = + job.kind === "deck" + ? buildCardPrompt(topic, grounding, opts.perTopic, opts.language) + : buildQuestionPrompt(topic, grounding, opts.perTopic, opts.mixed, opts.language) + items.push({ customId: topic.id, prompt, jsonSchema, maxTokens }) + await upsertCoverage(job.targetId, topic.id, job.id, "generating") + } + + let batchRef: string + try { + batchRef = await submitBatch(provider, items) + } catch (error) { + console.error("[generation] batch submit failed, falling back to live path", error) + return false + } + + await db + .update(generationJob) + .set({ + batchRef, + batchModel: modelRef, + topicsDone: doneRows.length, + producedCount: alreadyProduced, + }) + .where(eq(generationJob.id, job.id)) + return true +} + /** * Runs a coverage-driven generation job: (re)builds the module outline, then for * every topic that isn't already covered for this target, generates items from @@ -284,7 +448,8 @@ export async function runCoverageGeneration(jobId: string): Promise { return } const model = await getLanguageModel(modelRef, job.userId) - const embeddingRef = (await getSetting("ai"))?.defaultEmbeddingModel ?? null + const ai = await getSetting("ai") + const embeddingRef = ai?.defaultEmbeddingModel ?? null const params = (job.params ?? {}) as GenParams const perTopic = @@ -292,14 +457,6 @@ export async function runCoverageGeneration(jobId: string): Promise { const language = params.language ?? "the language of the source materials" const mixed = params.mixed ?? true - // Seed de-dup with what the target already contains. - const deduper = new Deduper(0.9) - const existingKeys = await loadExistingKeys(job.kind, job.targetId) - deduper.seedKeys(existingKeys) - if (embeddingRef && existingKeys.length > 0) { - deduper.seedVectors(await embedTexts(job.userId, embeddingRef, existingKeys)) - } - const coverage = await db.query.generationCoverage.findMany({ where: eq(generationCoverage.targetId, job.targetId), }) @@ -310,6 +467,28 @@ export async function runCoverageGeneration(jobId: string): Promise { .set({ topicsTotal: topics.length }) .where(eq(generationJob.id, jobId)) + // Optional: run the per-topic MAP step through the provider's async Batch API + // (~50% cheaper). Submits one batch for all uncovered topics and returns; the + // batch poller records the results (with tokens + audit) and completes the job + // when the batch finishes. Falls through to the live path when the flag is off + // or the active provider isn't batch-capable. + if (ai?.useBatchApi) { + const submitted = await trySubmitBatchGeneration(job, topics, modelRef, covByTopic, { + perTopic, + language, + mixed, + }) + if (submitted) return + } + + // Live path: seed de-dup with what the target already contains, then iterate. + const deduper = new Deduper(0.9) + const existingKeys = await loadExistingKeys(job.kind, job.targetId) + deduper.seedKeys(existingKeys) + if (embeddingRef && existingKeys.length > 0) { + deduper.seedVectors(await embedTexts(job.userId, embeddingRef, existingKeys)) + } + let topicsDone = 0 let producedTotal = 0 @@ -348,21 +527,7 @@ export async function runCoverageGeneration(jobId: string): Promise { perTopic, language ) - const keys = cards.map((c) => c.front) - const vecs = - embeddingRef && keys.length > 0 - ? await embedTexts(job.userId, embeddingRef, keys) - : undefined - const fresh = deduper.filter( - cards.map((c) => ({ key: c.front, item: c })), - vecs - ) - if (fresh.length > 0) { - await db - .insert(flashcard) - .values(fresh.map((c) => ({ deckId: job.targetId, front: c.front, back: c.back }))) - } - produced = fresh.length + produced = await persistCards(job.userId, job.targetId, deduper, embeddingRef, cards) } else { const questions = await generateTopicQuestions( job.userId, @@ -375,32 +540,13 @@ export async function runCoverageGeneration(jobId: string): Promise { mixed, language ) - const keys = questions.map((q) => q.prompt) - const vecs = - embeddingRef && keys.length > 0 - ? await embedTexts(job.userId, embeddingRef, keys) - : undefined - const fresh = deduper.filter( - questions.map((q) => ({ key: q.prompt, item: q })), - vecs + produced = await persistQuestions( + job.userId, + job.targetId, + deduper, + embeddingRef, + questions ) - if (fresh.length > 0) { - const base = await nextQuizSortOrder(job.targetId) - await db.insert(question).values( - fresh.map((q, i) => ({ - quizId: job.targetId, - kind: q.kind, - prompt: q.prompt, - options: q.kind === "multiple_choice" ? q.options : null, - correctIndex: - q.kind === "multiple_choice" && q.correctIndex >= 0 ? q.correctIndex : null, - referenceAnswer: q.kind === "free_text" ? q.referenceAnswer : null, - explanation: q.explanation || null, - sortOrder: base + i, - })) - ) - } - produced = fresh.length } await upsertCoverage(job.targetId, topic.id, jobId, "done", produced) } catch (error) { diff --git a/src/lib/jobs/index.ts b/src/lib/jobs/index.ts index 5e62164..de9d6bb 100644 --- a/src/lib/jobs/index.ts +++ b/src/lib/jobs/index.ts @@ -7,6 +7,7 @@ const globalForBoss = globalThis as unknown as { boss?: Promise } export const QUEUE_EMBED_MATERIAL = "embed-material" export const QUEUE_SUMMARIZE_MATERIAL = "summarize-material" export const QUEUE_GENERATE_COVERAGE = "generate-coverage" +export const QUEUE_POLL_BATCHES = "poll-batches" export const QUEUE_REINDEX_VECTORS = "reindex-vectors" export const QUEUE_UNPACK_ZIP = "unpack-zip" export const QUEUE_SEND_REMINDERS = "send-reminders" @@ -17,6 +18,7 @@ const ALL_QUEUES = [ QUEUE_EMBED_MATERIAL, QUEUE_SUMMARIZE_MATERIAL, QUEUE_GENERATE_COVERAGE, + QUEUE_POLL_BATCHES, QUEUE_REINDEX_VECTORS, QUEUE_UNPACK_ZIP, QUEUE_SEND_REMINDERS, @@ -40,6 +42,7 @@ export async function startClient(): Promise { await boss.start() for (const queue of ALL_QUEUES) await boss.createQueue(queue) await boss.schedule(QUEUE_SEND_REMINDERS, "*/5 * * * *") + await boss.schedule(QUEUE_POLL_BATCHES, "*/5 * * * *") await boss.schedule(QUEUE_DAILY_PLAN, "0 7 * * *") await boss.schedule(QUEUE_CHECK_UPDATES, "0 6 * * *") return boss @@ -74,6 +77,11 @@ export async function registerWorkers(boss: PgBoss): Promise { } }) + await boss.work(QUEUE_POLL_BATCHES, async () => { + const { pollPendingBatches } = await import("@/lib/ai/generation/batch-poll") + await pollPendingBatches() + }) + await boss.work(QUEUE_REINDEX_VECTORS, async () => { const { reindexVectors } = await import("@/lib/ai/ann") await reindexVectors() diff --git a/src/lib/settings.ts b/src/lib/settings.ts index 2d98d58..c7856e7 100644 --- a/src/lib/settings.ts +++ b/src/lib/settings.ts @@ -95,6 +95,13 @@ export const aiSettingsSchema = z.object({ defaultEmbeddingModel: z.string().optional(), /** Monthly token limit per user (input+output). 0 = unlimited */ monthlyTokenLimitPerUser: z.number().int().min(0).default(0), + /** + * Run the coverage-generation MAP step through the provider's async Batch API + * (Anthropic Message Batches / OpenAI Batch) — ~50% cheaper, but results + * arrive asynchronously. Only applies when the active model is an Anthropic or + * OpenAI provider; otherwise the synchronous live path is used. Default off. + */ + useBatchApi: z.boolean().default(false), }) export type AiSettings = z.infer @@ -146,7 +153,7 @@ const defaults: { [K in SettingKey]: SettingValue } = { smtp: undefined as never, // no default — unset means email disabled branding: { appName: "StudyHelper" }, uploads: { maxUploadMb: 200, storageQuotaMbPerUser: 0 }, - ai: { providers: [], monthlyTokenLimitPerUser: 0 }, + ai: { providers: [], monthlyTokenLimitPerUser: 0, useBatchApi: false }, "ai.ann": { status: "idle" }, "push.vapid": undefined as never, // generated on first use "system.updateCheck": undefined as never, // set once the first check has run