diff --git a/.env.example b/.env.example index 4bdc9a3..4fff336 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,12 @@ ENCRYPTION_KEY=change-me # TZ=Europe/Berlin +# Staging directory for resumable (tus) uploads of very large files. Files live +# here only until they are moved into storage, then are deleted. Defaults to +# ./data/tus-incoming; put it on a persistent volume so an interrupted upload can +# resume after a restart. +# TUS_DIR=/data/tus-incoming + # Object storage for uploaded files. Default is local disk (the uploads volume # above). For large multi-GB modules or multi-node setups, use S3 or any # S3-compatible service (AWS S3, MinIO, Cloudflare R2). See docs/self-hosting.md. diff --git a/docker-compose.yml b/docker-compose.yml index 85551a6..c7d3502 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,13 @@ services: # Server timezone — event times, reminders and daily plans use it. # Set to your users' timezone; containers default to UTC otherwise. TZ: ${TZ:-Europe/Berlin} + # Staging directory for resumable (tus) uploads (see volume below). + TUS_DIR: /data/tus-incoming volumes: - ${DATA_DIR:-./data}/uploads:/data/uploads + # Staging area for resumable (tus) uploads of very large files. A + # persistent volume lets an interrupted upload resume after a restart. + - ${DATA_DIR:-./data}/tus-incoming:/data/tus-incoming depends_on: db: condition: service_healthy @@ -55,9 +60,12 @@ services: # environment: # DATABASE_URL: postgres://study:${POSTGRES_PASSWORD:-study}@db:5432/study # UPLOAD_DIR: /data/uploads + # # The worker finalizes resumable (tus) uploads, so it needs the staging dir. + # TUS_DIR: /data/tus-incoming # TZ: ${TZ:-Europe/Berlin} # volumes: # - ${DATA_DIR:-./data}/uploads:/data/uploads + # - ${DATA_DIR:-./data}/tus-incoming:/data/tus-incoming # depends_on: # db: # condition: service_healthy diff --git a/docs/self-hosting.md b/docs/self-hosting.md index c57b958..0f06756 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -44,10 +44,11 @@ Update with `docker compose pull && docker compose up -d`. Images are built for `docker compose` mounts two directories from the **host** into the containers, so your data survives container restarts/rebuilds: -| Host path (default) | Container path | Contents | -| ------------------- | ----------------- | ------------------------------------------------------------------------------------------- | -| `./data/db` | Postgres data dir | Everything except files: users, modules, flashcards, grades, events, chat history, settings | -| `./data/uploads` | `/data/uploads` | Uploaded files (PDFs, images, …) | +| Host path (default) | Container path | Contents | +| --------------------- | -------------------- | ------------------------------------------------------------------------------------------- | +| `./data/db` | Postgres data dir | Everything except files: users, modules, flashcards, grades, events, chat history, settings | +| `./data/uploads` | `/data/uploads` | Uploaded files (PDFs, images, …) | +| `./data/tus-incoming` | `/data/tus-incoming` | Staging for in-flight resumable (tus) uploads — transient; cleared once finalized | By default both live under `./data`, next to `docker-compose.yml`. To store them elsewhere — a separate disk, a NAS mount, outside the git checkout — @@ -88,6 +89,7 @@ and links to the release. Installing it is a manual | `STUDYHELPER_VERSION` | no | Image tag to run (default `latest`); pin e.g. `1.0.0` for reproducible deploys | | `DATA_DIR` | no | Host directory for the database + uploads volumes (default `./data`, next to `docker-compose.yml`) — see [Where data is stored](#where-data-is-stored) | | `UPLOAD_DIR` | no | Upload path **inside the container** (default `/data/uploads`) — only relevant for non-Docker deployments; Docker users should set `DATA_DIR` instead | +| `TUS_DIR` | no | Staging dir for resumable (tus) uploads of very large files (default `/data/tus-incoming`); use a persistent volume so interrupted uploads resume after a restart | | `STORAGE_DRIVER` | no | Where uploaded files live: `local` (default, disk under `UPLOAD_DIR`) or `s3` (S3 / S3-compatible object storage) — see [Object storage (S3)](#object-storage-s3) | | `WORKERS_IN_PROCESS` | no | `false` runs background jobs only in a separate worker process (`npm run worker`) instead of the web tier (default `true` — in-process) | | `SEED_TEST_DATA` | no | `true` seeds demo accounts (admin@example.com / admin-test-1234, user@example.com / user-test-1234) with sample study content on startup — for evaluation only, never in production | diff --git a/package-lock.json b/package-lock.json index d39c77e..e14c3f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,8 @@ "@fullcalendar/interaction": "^6.1.21", "@fullcalendar/react": "^6.1.21", "@fullcalendar/timegrid": "^6.1.21", + "@tus/file-store": "^2.1.0", + "@tus/server": "^2.4.1", "ai": "^7.0.19", "better-auth": "^1.6.23", "class-variance-authority": "^0.7.1", @@ -61,6 +63,7 @@ "sql.js": "^1.14.1", "tailwind-merge": "^3.6.0", "ts-fsrs": "^5.4.1", + "tus-js-client": "^4.3.1", "tw-animate-css": "^1.4.0", "unpdf": "^1.6.2", "web-push": "^3.6.7", @@ -3495,6 +3498,13 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT", + "optional": true + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -4646,6 +4656,31 @@ } } }, + "node_modules/@redis/client": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.12.1.tgz", + "integrity": "sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==", + "license": "MIT", + "optional": true, + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", @@ -5762,6 +5797,57 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@tus/file-store": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@tus/file-store/-/file-store-2.1.0.tgz", + "integrity": "sha512-RW/aYO2pp0L4AVG6NebPL+Q9FKKtDXF2+skBq3NbOMOEoxWFHSCgtzIX1pBPezFdZDwqdZMIlx+NQGQYYp6/3Q==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.7.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^5.0.0" + } + }, + "node_modules/@tus/server": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@tus/server/-/server-2.4.1.tgz", + "integrity": "sha512-WShIwCqtthAL0x++f/VCF6ZpbULjRIYLPumQoOYNHa4NVMxW1pDBAKsNE3Y+rnVTzOttEumIXD1WDNRsIf7x7Q==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.7.0", + "debug": "^4.3.4", + "lodash.throttle": "^4.1.1", + "set-cookie-parser": "^2.7.1", + "srvx": "~0.11.15" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^5.0.0", + "ioredis": "^5.4.1" + } + }, + "node_modules/@tus/server/node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/@tus/utils": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.7.0.tgz", + "integrity": "sha512-BPom8lVwR4fkk1EynPeQwuN0e5bH2gBphwt69uF5rS4wRD2N7IyHDPqdjcAY0I7gNEWfUitPyDodywbD4r7dEQ==", + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/@tybys/wasm-util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", @@ -7529,7 +7615,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "devOptional": true, "license": "MIT" }, "node_modules/bundle-name": { @@ -7804,6 +7889,16 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cmdk": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.1.1.tgz", @@ -7844,6 +7939,15 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "dependencies": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -8104,6 +8208,12 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, + "node_modules/custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "license": "ISC" + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -8352,6 +8462,16 @@ "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", "license": "MIT" }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -11079,6 +11199,39 @@ "@formatjs/icu-messageformat-parser": "3.5.13" } }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "optional": true, + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ip-address": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", @@ -11776,6 +11929,12 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-base64": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.1.tgz", + "integrity": "sha512-U73qptcvf/HIOauFOmqT3a0mDUp0MYlfd15oqoe9kqZt5XhiXVb+HG09sLvI9PQ9tZIBFS4nlErai8zbWazP0g==", + "license": "BSD-3-Clause" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -12378,6 +12537,52 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "license": "MIT", + "dependencies": { + "lodash._stringtopath": "~4.8.0" + } + }, + "node_modules/lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "license": "MIT" + }, + "node_modules/lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "license": "MIT", + "dependencies": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "node_modules/lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "license": "MIT" + }, + "node_modules/lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "license": "MIT" + }, + "node_modules/lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "license": "MIT", + "dependencies": { + "lodash._basetostring": "~4.12.0" + } + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -12385,6 +12590,22 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "license": "MIT" + }, + "node_modules/lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "license": "MIT", + "dependencies": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, "node_modules/log-symbols": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", @@ -15167,6 +15388,23 @@ "react-is": "^16.13.1" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, "node_modules/property-information": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", @@ -15251,6 +15489,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -15466,6 +15710,29 @@ "node": ">= 4" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "optional": true, + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -15666,6 +15933,12 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "license": "ISC" }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, "node_modules/reselect": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.2.0.tgz", @@ -15746,6 +16019,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -16468,6 +16750,18 @@ "integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==", "license": "MIT" }, + "node_modules/srvx": { + "version": "0.11.22", + "resolved": "https://registry.npmjs.org/srvx/-/srvx-0.11.22.tgz", + "integrity": "sha512-LqZxxBDMKuMAZzFzJnDCkFOrs9MZQZr0LvHiO/SuSZVdQaXD7xQ5UWTUxheJrQPve1qk9MG2B/yttUvJxw8egQ==", + "license": "MIT", + "bin": { + "srvx": "bin/srvx.mjs" + }, + "engines": { + "node": ">=20.16.0" + } + }, "node_modules/stable-hash": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stable-hash/-/stable-hash-0.0.5.tgz", @@ -16482,6 +16776,13 @@ "devOptional": true, "license": "MIT" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT", + "optional": true + }, "node_modules/standardwebhooks": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", @@ -17269,6 +17570,36 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "license": "0BSD" }, + "node_modules/tus-js-client": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-4.3.1.tgz", + "integrity": "sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.1.2", + "combine-errors": "^3.0.3", + "is-stream": "^2.0.0", + "js-base64": "^3.7.2", + "lodash.throttle": "^4.1.1", + "proper-lockfile": "^4.1.2", + "url-parse": "^1.5.7" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tus-js-client/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tw-animate-css": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.4.0.tgz", @@ -17741,6 +18072,16 @@ "punycode": "^2.1.0" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/use-callback-ref": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", diff --git a/package.json b/package.json index 38c1be2..a49c90e 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,8 @@ "@fullcalendar/interaction": "^6.1.21", "@fullcalendar/react": "^6.1.21", "@fullcalendar/timegrid": "^6.1.21", + "@tus/file-store": "^2.1.0", + "@tus/server": "^2.4.1", "ai": "^7.0.19", "better-auth": "^1.6.23", "class-variance-authority": "^0.7.1", @@ -71,6 +73,7 @@ "sql.js": "^1.14.1", "tailwind-merge": "^3.6.0", "ts-fsrs": "^5.4.1", + "tus-js-client": "^4.3.1", "tw-animate-css": "^1.4.0", "unpdf": "^1.6.2", "web-push": "^3.6.7", diff --git a/src/app/api/materials/tus/[[...slug]]/route.ts b/src/app/api/materials/tus/[[...slug]]/route.ts new file mode 100644 index 0000000..2ec993c --- /dev/null +++ b/src/app/api/materials/tus/[[...slug]]/route.ts @@ -0,0 +1,107 @@ +import "server-only" +import { FileStore } from "@tus/file-store" +import { MemoryLocker, Server, type Upload } from "@tus/server" +import { getSession } from "@/lib/auth/session" +import { getSetting } from "@/lib/settings" +import { ownModule } from "@/lib/studies/access" +import { ownFolder } from "@/lib/materials/folders" +import { assertStorageWithinLimit } from "@/lib/materials/usage" +import { TUS_DIR, TUS_PATH } from "@/lib/materials/tus-config" + +// Resumable (tus protocol) upload endpoint for very large files: an interrupted +// upload resumes at the last byte offset instead of restarting. Files are staged +// under TUS_DIR and, on completion, a background job streams them into the +// configured storage backend (local or S3) and creates the material — reusing +// the same ingest path as the direct upload route. The small direct-upload route +// (/api/materials/upload) remains the default for small files. + +const globalForTus = globalThis as unknown as { tusServer?: Server } + +/** Aborts a tus request with a specific HTTP status (tus reads status_code/body). */ +function httpError(statusCode: number, message: string): Error { + return Object.assign(new Error(message), { status_code: statusCode, body: message }) +} + +function metaString(value: string | null | undefined): string { + return (value ?? "").trim() +} + +function getServer(): Server { + if (globalForTus.tusServer) return globalForTus.tusServer + + const server = new Server({ + path: TUS_PATH, + datastore: new FileStore({ directory: TUS_DIR }), + // Process-local lock: fine because all PATCH/HEAD requests for one upload + // hit the web tier, and finalization happens later in a background job. + locker: new MemoryLocker(), + respectForwardedHeaders: true, + async onIncomingRequest() { + const session = await getSession() + if (!session) throw httpError(401, "Unauthorized") + }, + async onUploadCreate(_req, upload) { + const session = await getSession() + if (!session) throw httpError(401, "Unauthorized") + const userId = session.user.id + + const meta = upload.metadata ?? {} + const moduleId = metaString(meta.moduleId) + const folderId = metaString(meta.folderId) || null + if (!moduleId) throw httpError(400, "moduleId required") + + try { + await ownModule(moduleId, userId) + if (folderId) { + const folder = await ownFolder(folderId, userId) + if (folder.moduleId !== moduleId) throw new Error("mismatch") + } + } catch { + throw httpError(404, "Not found") + } + + const uploads = await getSetting("uploads") + const maxUploadMb = uploads?.maxUploadMb ?? 200 + const maxBytes = maxUploadMb * 1024 * 1024 + if (upload.size != null && upload.size > maxBytes) { + throw httpError(413, `File too large (max ${maxUploadMb} MB)`) + } + if (upload.size != null && upload.size > 0) { + try { + await assertStorageWithinLimit(userId, upload.size) + } catch { + throw httpError(413, "Storage quota exceeded") + } + } + + // Stamp the authenticated userId into the stored metadata so the finalize + // job trusts it rather than any client-supplied value. + return { metadata: { ...meta, userId } } + }, + async onUploadFinish(_req, upload) { + await enqueueFinalize(upload) + return {} + }, + }) + + globalForTus.tusServer = server + return server +} + +async function enqueueFinalize(upload: Upload): Promise { + const meta = upload.metadata ?? {} + const { enqueueFinalizeUpload } = await import("@/lib/jobs") + await enqueueFinalizeUpload({ + tusId: upload.id, + userId: metaString(meta.userId), + moduleId: metaString(meta.moduleId), + folderId: metaString(meta.folderId) || null, + relativePath: metaString(meta.relativePath) || undefined, + fileName: metaString(meta.filename) || metaString(meta.name) || upload.id, + mimeType: meta.filetype ?? null, + }) +} + +const handler = (request: Request): Promise => getServer().handleWeb(request) + +export { handler as POST, handler as PATCH, handler as HEAD, handler as OPTIONS, handler as DELETE } diff --git a/src/app/api/materials/upload/route.ts b/src/app/api/materials/upload/route.ts index abf37fd..6111f06 100644 --- a/src/app/api/materials/upload/route.ts +++ b/src/app/api/materials/upload/route.ts @@ -1,15 +1,11 @@ import { NextResponse } from "next/server" -import path from "node:path" -import { and, eq } from "drizzle-orm" -import { db } from "@/db" -import { material } from "@/db/schema" import { getSession } from "@/lib/auth/session" import { getSetting } from "@/lib/settings" -import { deleteFile, safeInlineMime, saveStream, StorageLimitError } from "@/lib/storage" +import { saveStream, StorageLimitError } from "@/lib/storage" import { ownModule } from "@/lib/studies/access" -import { findOrCreateFolderPath, ownFolder, splitPath } from "@/lib/materials/folders" +import { ownFolder } from "@/lib/materials/folders" import { assertStorageWithinLimit } from "@/lib/materials/usage" -import { isZip } from "@/lib/materials/paths" +import { QuotaExceededError, registerUploadedFile } from "@/lib/materials/ingest" // Uploads stream the raw request body straight to disk (see saveStream) instead // of buffering the whole file in memory via formData()/arrayBuffer(). Metadata @@ -62,14 +58,6 @@ export async function POST(request: Request) { } } - // Resolve the destination folder: any leading directories of relativePath are - // created (nested) under the current folder. - let targetFolderId = folderId - if (relativePath) { - const segments = splitPath(path.posix.dirname(relativePath)) - targetFolderId = await findOrCreateFolderPath(userId, moduleId, segments, folderId) - } - let saved: { storagePath: string; size: number; hash: string } try { saved = await saveStream(userId, fileName, request.body, { maxBytes }) @@ -81,82 +69,27 @@ export async function POST(request: Request) { return NextResponse.json({ error: "Upload failed" }, { status: 500 }) } - // Authoritative storage-quota check now that the real size is known. + // Folder resolution, quota, zip, dedup, insert + embedding are shared with the + // tus resumable-upload finalizer (see @/lib/materials/ingest). try { - await assertStorageWithinLimit(userId, saved.size) - } catch { - await deleteFile(saved.storagePath) - return NextResponse.json({ error: "Storage quota exceeded" }, { status: 413 }) - } - - // Zip archives are unpacked in the background into a same-named folder — the - // archive itself is not kept as a material. - if (isZip(fileName, mimeType)) { - try { - const { enqueueUnpackZip } = await import("@/lib/jobs") - await enqueueUnpackZip({ - userId, - moduleId, - parentFolderId: targetFolderId, - zipStoragePath: saved.storagePath, - zipName: fileName, - }) - } catch (error) { - console.error("[upload] failed to enqueue unpack job", error) - await deleteFile(saved.storagePath) - return NextResponse.json({ error: "Failed to queue unpack" }, { status: 500 }) - } - return NextResponse.json({ ok: true, queued: true }) - } - - // Incremental reuse: an identical file (same content hash) already in this - // module is not re-stored or re-processed. - const duplicate = await db.query.material.findFirst({ - where: and( - eq(material.userId, userId), - eq(material.moduleId, moduleId), - eq(material.contentHash, saved.hash) - ), - columns: { id: true }, - }) - if (duplicate) { - await deleteFile(saved.storagePath) - return NextResponse.json({ ok: true, deduped: true, id: duplicate.id }) - } - - const [created] = await db - .insert(material) - .values({ + const result = await registerUploadedFile({ userId, moduleId, - kind: "file", - name: fileName, - storagePath: saved.storagePath, - mimeType: safeInlineMime(mimeType), - sizeBytes: saved.size, - contentHash: saved.hash, - folderId: targetFolderId, - extractionStatus: "pending", + folderId, + relativePath: relativePath || undefined, + fileName, + mimeType, + saved, }) - .returning() - - const { logAudit } = await import("@/lib/audit") - await logAudit({ - userId, - operation: "create", - entityType: "material", - entityId: created.id, - entityLabel: fileName, - after: created, - }) - - // Kick off text extraction + embedding in the background - try { - const { enqueueEmbedMaterial } = await import("@/lib/jobs") - await enqueueEmbedMaterial(created.id) + if (result.kind === "queued") return NextResponse.json({ ok: true, queued: true }) + if (result.kind === "deduped") + return NextResponse.json({ ok: true, deduped: true, id: result.id }) + return NextResponse.json({ ok: true, id: result.id }) } catch (error) { - console.error("[upload] failed to enqueue embedding job", error) + if (error instanceof QuotaExceededError) { + return NextResponse.json({ error: "Storage quota exceeded" }, { status: 413 }) + } + console.error("[upload] register failed", error) + return NextResponse.json({ error: "Upload failed" }, { status: 500 }) } - - return NextResponse.json({ ok: true, id: created.id }) } diff --git a/src/components/materials/upload-client.ts b/src/components/materials/upload-client.ts index 7b28c85..1aa611a 100644 --- a/src/components/materials/upload-client.ts +++ b/src/components/materials/upload-client.ts @@ -1,9 +1,53 @@ /** Shared client-side upload helpers for the materials feature. */ +import { Upload as TusUpload, isSupported as tusSupported } from "tus-js-client" +import { shouldUseTus } from "./upload-transport" + export type UploadItem = { file: File; relativePath?: string } export type UploadProgress = { done: number; total: number; percent: number } +/** + * Uploads one large file via the resumable tus protocol. Survives interruptions: + * a dropped connection is retried from the server's last byte offset, and a + * previous (unfinished) upload of the same file is resumed. The material is + * created by a background job once the upload completes, so this resolves with + * `queued: true`. + */ +function tusUpload( + file: File, + params: { moduleId: string; folderId: string | null; relativePath?: string }, + onPercent: (percent: number) => void +): Promise<{ queued?: boolean }> { + return new Promise((resolve, reject) => { + const upload = new TusUpload(file, { + endpoint: "/api/materials/tus", + chunkSize: 50 * 1024 * 1024, + retryDelays: [0, 1000, 3000, 5000, 10000], + removeFingerprintOnSuccess: true, + metadata: { + filename: file.name, + filetype: file.type || "application/octet-stream", + moduleId: params.moduleId, + ...(params.folderId ? { folderId: params.folderId } : {}), + ...(params.relativePath ? { relativePath: params.relativePath } : {}), + }, + onError: (error) => reject(error), + onProgress: (sent, total) => { + if (total) onPercent(Math.round((sent / total) * 100)) + }, + onSuccess: () => resolve({ queued: true }), + }) + upload + .findPreviousUploads() + .then((previous) => { + if (previous.length > 0) upload.resumeFromPreviousUpload(previous[0]) + upload.start() + }) + .catch(() => upload.start()) + }) +} + /** * POSTs one file (with optional relative path) to the upload endpoint. The file * is sent as the raw request body (not multipart) so the server can stream it @@ -59,11 +103,14 @@ export async function uploadFiles( let queued = 0 for (let i = 0; i < items.length; i++) { const { file, relativePath } = items[i] - const res = await xhrUpload( - file, - { moduleId: opts.moduleId, folderId: opts.folderId, relativePath }, - (percent) => opts.onProgress?.({ done: i, total: items.length, percent }) - ) + const params = { moduleId: opts.moduleId, folderId: opts.folderId, relativePath } + const onPercent = (percent: number) => + opts.onProgress?.({ done: i, total: items.length, percent }) + // Large files go through the resumable tus endpoint; small ones keep the + // simpler direct upload. tus is skipped where the browser can't support it. + const res = shouldUseTus(file.size, tusSupported) + ? await tusUpload(file, params, onPercent) + : await xhrUpload(file, params, onPercent) if (res.queued) queued++ } opts.onProgress?.({ done: items.length, total: items.length, percent: 100 }) diff --git a/src/components/materials/upload-transport.test.ts b/src/components/materials/upload-transport.test.ts new file mode 100644 index 0000000..211f3cd --- /dev/null +++ b/src/components/materials/upload-transport.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest" +import { shouldUseTus, TUS_THRESHOLD } from "./upload-transport" + +describe("shouldUseTus", () => { + it("uses tus for files at or above the threshold when supported", () => { + expect(shouldUseTus(TUS_THRESHOLD, true)).toBe(true) + expect(shouldUseTus(TUS_THRESHOLD + 1, true)).toBe(true) + }) + + it("keeps small files on the direct path", () => { + expect(shouldUseTus(TUS_THRESHOLD - 1, true)).toBe(false) + expect(shouldUseTus(0, true)).toBe(false) + }) + + it("never uses tus when the browser can't support it", () => { + expect(shouldUseTus(TUS_THRESHOLD * 4, false)).toBe(false) + }) +}) diff --git a/src/components/materials/upload-transport.ts b/src/components/materials/upload-transport.ts new file mode 100644 index 0000000..7682b12 --- /dev/null +++ b/src/components/materials/upload-transport.ts @@ -0,0 +1,12 @@ +/** Transport selection for material uploads (pure — safe to unit-test). */ + +// Files at or above this size use the resumable tus endpoint (survives +// connection drops / page reloads); smaller files take the simpler direct path. +export const TUS_THRESHOLD = 50 * 1024 * 1024 + +/** Whether a file should be uploaded via the resumable tus protocol rather than + * the direct upload route. Large files use tus; it is skipped when the browser + * can't support it (then the direct path handles all sizes). */ +export function shouldUseTus(fileSize: number, tusAvailable: boolean): boolean { + return fileSize >= TUS_THRESHOLD && tusAvailable +} diff --git a/src/lib/jobs/index.ts b/src/lib/jobs/index.ts index de9d6bb..81835d1 100644 --- a/src/lib/jobs/index.ts +++ b/src/lib/jobs/index.ts @@ -10,6 +10,7 @@ export const QUEUE_GENERATE_COVERAGE = "generate-coverage" export const QUEUE_POLL_BATCHES = "poll-batches" export const QUEUE_REINDEX_VECTORS = "reindex-vectors" export const QUEUE_UNPACK_ZIP = "unpack-zip" +export const QUEUE_FINALIZE_UPLOAD = "finalize-upload" export const QUEUE_SEND_REMINDERS = "send-reminders" export const QUEUE_DAILY_PLAN = "daily-plan-reminder" export const QUEUE_CHECK_UPDATES = "check-updates" @@ -21,6 +22,7 @@ const ALL_QUEUES = [ QUEUE_POLL_BATCHES, QUEUE_REINDEX_VECTORS, QUEUE_UNPACK_ZIP, + QUEUE_FINALIZE_UPLOAD, QUEUE_SEND_REMINDERS, QUEUE_DAILY_PLAN, QUEUE_CHECK_UPDATES, @@ -94,6 +96,16 @@ export async function registerWorkers(boss: PgBoss): Promise { } }) + await boss.work( + QUEUE_FINALIZE_UPLOAD, + async (jobs) => { + const { finalizeUpload } = await import("@/lib/materials/tus-finalize") + for (const job of jobs) { + await finalizeUpload(job.data) + } + } + ) + await boss.work(QUEUE_SEND_REMINDERS, async () => { const { sendDueReminders } = await import("./reminders") await sendDueReminders() @@ -197,3 +209,16 @@ export async function enqueueUnpackZip( // Unpacking isn't idempotent (would duplicate files), so don't retry. await boss.send(QUEUE_UNPACK_ZIP, payload, { retryLimit: 0 }) } + +export async function enqueueFinalizeUpload( + payload: import("@/lib/materials/tus-finalize").FinalizeUploadPayload +): Promise { + const boss = await getBoss() + // singletonKey on the tus id coalesces a double onUploadFinish; the finalizer + // removes the staging file when done, so a retry that finds it gone is a no-op. + await boss.send(QUEUE_FINALIZE_UPLOAD, payload, { + retryLimit: 3, + retryDelay: 30, + singletonKey: payload.tusId, + }) +} diff --git a/src/lib/materials/ingest.ts b/src/lib/materials/ingest.ts new file mode 100644 index 0000000..8c617fc --- /dev/null +++ b/src/lib/materials/ingest.ts @@ -0,0 +1,131 @@ +import "server-only" +import path from "node:path" +import { and, eq } from "drizzle-orm" +import { db } from "@/db" +import { material } from "@/db/schema" +import { deleteFile, safeInlineMime } from "@/lib/storage" +import { findOrCreateFolderPath, splitPath } from "@/lib/materials/folders" +import { assertStorageWithinLimit } from "@/lib/materials/usage" +import { isZip } from "@/lib/materials/paths" + +/** Thrown when the finished file would exceed the user's storage quota. The + * partial storage object has already been removed when this is thrown. */ +export class QuotaExceededError extends Error { + constructor() { + super("Storage quota exceeded") + this.name = "QuotaExceededError" + } +} + +export type SavedFile = { storagePath: string; size: number; hash: string } + +export type RegisterInput = { + userId: string + moduleId: string + folderId: string | null + relativePath?: string + fileName: string + mimeType: string | null + saved: SavedFile +} + +export type RegisterResult = + { kind: "queued" } | { kind: "deduped"; id: string } | { kind: "created"; id: string } + +/** + * Turns a file already streamed into storage (`saved`) into a material: + * resolves the destination folder, enforces the storage quota, unpacks zips in + * the background, de-duplicates by content hash, inserts the material row and + * enqueues embedding. Shared by the direct upload route and the tus + * resumable-upload finalizer so both paths behave identically. + */ +export async function registerUploadedFile(input: RegisterInput): Promise { + const { userId, moduleId, folderId, relativePath, fileName, mimeType, saved } = input + + // Resolve destination folder: leading directories of relativePath are created + // (nested) under the current folder. + let targetFolderId = folderId + if (relativePath) { + const segments = splitPath(path.posix.dirname(relativePath)) + targetFolderId = await findOrCreateFolderPath(userId, moduleId, segments, folderId) + } + + // Authoritative storage-quota check now that the real size is known. + try { + await assertStorageWithinLimit(userId, saved.size) + } catch { + await deleteFile(saved.storagePath) + throw new QuotaExceededError() + } + + // Zip archives are unpacked in the background into a same-named folder — the + // archive itself is not kept as a material. + if (isZip(fileName, mimeType)) { + try { + const { enqueueUnpackZip } = await import("@/lib/jobs") + await enqueueUnpackZip({ + userId, + moduleId, + parentFolderId: targetFolderId, + zipStoragePath: saved.storagePath, + zipName: fileName, + }) + } catch (error) { + console.error("[ingest] failed to enqueue unpack job", error) + await deleteFile(saved.storagePath) + throw error + } + return { kind: "queued" } + } + + // Incremental reuse: an identical file (same content hash) already in this + // module is not re-stored or re-processed. + const duplicate = await db.query.material.findFirst({ + where: and( + eq(material.userId, userId), + eq(material.moduleId, moduleId), + eq(material.contentHash, saved.hash) + ), + columns: { id: true }, + }) + if (duplicate) { + await deleteFile(saved.storagePath) + return { kind: "deduped", id: duplicate.id } + } + + const [created] = await db + .insert(material) + .values({ + userId, + moduleId, + kind: "file", + name: fileName, + storagePath: saved.storagePath, + mimeType: safeInlineMime(mimeType), + sizeBytes: saved.size, + contentHash: saved.hash, + folderId: targetFolderId, + extractionStatus: "pending", + }) + .returning() + + const { logAudit } = await import("@/lib/audit") + await logAudit({ + userId, + operation: "create", + entityType: "material", + entityId: created.id, + entityLabel: fileName, + after: created, + }) + + // Kick off text extraction + embedding in the background. + try { + const { enqueueEmbedMaterial } = await import("@/lib/jobs") + await enqueueEmbedMaterial(created.id) + } catch (error) { + console.error("[ingest] failed to enqueue embedding job", error) + } + + return { kind: "created", id: created.id } +} diff --git a/src/lib/materials/tus-config.ts b/src/lib/materials/tus-config.ts new file mode 100644 index 0000000..b40007e --- /dev/null +++ b/src/lib/materials/tus-config.ts @@ -0,0 +1,11 @@ +import "server-only" +import path from "node:path" + +/** Staging directory for in-flight tus (resumable) uploads. Files live here + * only until the finalize job streams them into the configured storage backend + * and removes them. Put it on a persistent volume so interrupted uploads can + * resume across restarts. */ +export const TUS_DIR = process.env.TUS_DIR ?? path.join(process.cwd(), "data", "tus-incoming") + +/** The route the tus server is mounted at. */ +export const TUS_PATH = "/api/materials/tus" diff --git a/src/lib/materials/tus-finalize.ts b/src/lib/materials/tus-finalize.ts new file mode 100644 index 0000000..ef764d5 --- /dev/null +++ b/src/lib/materials/tus-finalize.ts @@ -0,0 +1,67 @@ +import "server-only" +import { Readable } from "node:stream" +import { getSetting } from "@/lib/settings" +import { saveStream } from "@/lib/storage" +import { QuotaExceededError, registerUploadedFile } from "@/lib/materials/ingest" +import { TUS_DIR } from "@/lib/materials/tus-config" + +export type FinalizeUploadPayload = { + tusId: string + userId: string + moduleId: string + folderId: string | null + relativePath?: string + fileName: string + mimeType: string | null +} + +/** + * Finalizes a completed tus upload: streams the staged file into the configured + * storage backend via `saveStream` (which computes size + sha256 for dedup), + * registers the material through the shared ingest path, then removes the + * staging file. Driver-agnostic — the same code path works for local disk and + * S3 because it goes through `saveStream`. + */ +export async function finalizeUpload(payload: FinalizeUploadPayload): Promise { + const { tusId, userId, moduleId, folderId, relativePath, fileName, mimeType } = payload + if (!userId || !moduleId) { + console.error("[tus-finalize] missing userId/moduleId for", tusId) + return + } + + const { FileStore } = await import("@tus/file-store") + const store = new FileStore({ directory: TUS_DIR }) + + const uploads = await getSetting("uploads") + const maxBytes = (uploads?.maxUploadMb ?? 200) * 1024 * 1024 + + let saved: { storagePath: string; size: number; hash: string } + try { + const nodeStream = store.read(tusId) + const webStream = Readable.toWeb(nodeStream) as ReadableStream + saved = await saveStream(userId, fileName, webStream, { maxBytes }) + } catch (error) { + console.error("[tus-finalize] failed to move staged upload into storage", tusId, error) + await store.remove(tusId).catch(() => {}) + return + } + + try { + await registerUploadedFile({ + userId, + moduleId, + folderId, + relativePath, + fileName, + mimeType, + saved, + }) + } catch (error) { + // registerUploadedFile already removed the stored object on quota failure. + if (!(error instanceof QuotaExceededError)) { + console.error("[tus-finalize] register failed", tusId, error) + } + } finally { + await store.remove(tusId).catch(() => {}) + } +}