Skalierungs-Add-ons: dedizierter Worker + optionaler HNSW-ANN-Index

docker-compose.yml

@@ -26,6 +26,30 @@ services:
       db:
         condition: service_healthy
     restart: unless-stopped
+    # To move background jobs (embedding, summarization, coverage generation) off
+    # the web tier, set WORKERS_IN_PROCESS: "false" here and run the worker
+    # service below. By default (unset) the app runs workers in-process — no
+    # extra service needed.
+    # environment:
+    #   WORKERS_IN_PROCESS: "false"
+
+  # Optional dedicated job worker. Runs `npm run worker`, which uses tsx — so it
+  # needs an image that includes the TS sources + devDependencies (a source
+  # checkout, or a custom image built from the repo). The default prebuilt
+  # standalone image runs jobs in-process instead, so this service is opt-in.
+  # worker:
+  #   image: ghcr.io/veniplex/study-helper:${STUDYHELPER_VERSION:-latest}
+  #   command: ["npm", "run", "worker"]
+  #   environment:
+  #     DATABASE_URL: postgres://study:${POSTGRES_PASSWORD:-study}@db:5432/study
+  #     UPLOAD_DIR: /data/uploads
+  #     TZ: ${TZ:-Europe/Berlin}
+  #   volumes:
+  #     - ${DATA_DIR:-./data}/uploads:/data/uploads
+  #   depends_on:
+  #     db:
+  #       condition: service_healthy
+  #   restart: unless-stopped
 
   db:
     image: pgvector/pgvector:pg17

messages/de.json

@@ -176,7 +176,16 @@
       "usageUser": "Benutzer",
       "usageInput": "Input-Tokens",
       "usageOutput": "Output-Tokens",
-      "usageEmpty": "Noch kein Verbrauch."
+      "usageEmpty": "Noch kein Verbrauch.",
+      "annTitle": "Vektor-Index (ANN, schnelle Suche)",
+      "annDescription": "Optionaler HNSW-Index für schnelle Vektorsuche bei sehr großen Materialmengen. Baut einen typisierten Index für das aktuelle Embedding-Modell auf. Standardmäßig aus; ohne Index läuft die Suche als (langsamerer) sequentieller Scan.",
+      "annStatusIdle": "Nicht aufgebaut",
+      "annStatusBuilding": "Wird aufgebaut …",
+      "annStatusReady": "Aktiv",
+      "annStatusFailed": "Fehlgeschlagen",
+      "annRebuild": "Index (neu) aufbauen",
+      "annRebuilding": "Index-Aufbau gestartet – läuft im Hintergrund.",
+      "annNeedEmbedding": "Zuerst ein Standard-Embedding-Modell konfigurieren."
     },
     "email": {
       "title": "E-Mail (SMTP)",

messages/en.json

@@ -176,7 +176,16 @@
       "usageUser": "User",
       "usageInput": "Input tokens",
       "usageOutput": "Output tokens",
-      "usageEmpty": "No usage yet."
+      "usageEmpty": "No usage yet.",
+      "annTitle": "Vector index (ANN, fast search)",
+      "annDescription": "Optional HNSW index for fast vector search over very large material sets. Builds a typed index for the current embedding model. Off by default; without it, search runs as a (slower) sequential scan.",
+      "annStatusIdle": "Not built",
+      "annStatusBuilding": "Building …",
+      "annStatusReady": "Active",
+      "annStatusFailed": "Failed",
+      "annRebuild": "Build / rebuild index",
+      "annRebuilding": "Index build started — running in the background.",
+      "annNeedEmbedding": "Configure a default embedding model first."
     },
     "email": {
       "title": "Email (SMTP)",

package.json

@@ -6,6 +6,7 @@
     "dev": "next dev",
     "build": "next build",
     "start": "next start",
+    "worker": "NODE_OPTIONS=--conditions=react-server tsx src/worker.ts",
     "lint": "eslint",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
@@ -87,6 +88,7 @@
     "jsdom": "^29.1.1",
     "prettier": "^3.9.4",
     "tailwindcss": "^4",
+    "tsx": "^4.23.0",
     "typescript": "^5",
     "vitest": "^4.1.10"
   }

src/app/[locale]/(app)/admin/actions.ts

@@ -114,3 +114,18 @@ export async function saveAiSettings(value: unknown) {
   revalidatePath("/", "layout")
   return { ok: true as const }
 }
+
+/** Kicks off a background rebuild of the pgvector HNSW ANN index. */
+export async function startVectorReindex() {
+  await requireAdmin()
+  const { enqueueReindexVectors } = await import("@/lib/jobs")
+  await enqueueReindexVectors()
+  return { ok: true as const }
+}
+
+/** Current ANN index state (status/model/dimensions), for the admin UI. */
+export async function getAnnStatus() {
+  await requireAdmin()
+  const { getSetting } = await import("@/lib/settings")
+  return (await getSetting("ai.ann")) ?? { status: "idle" as const }
+}

src/app/[locale]/(app)/admin/ai/page.tsx

@@ -6,17 +6,14 @@ import { requireAdmin } from "@/lib/auth/session"
 import { getSetting } from "@/lib/settings"
 import { daysAgo } from "@/lib/utils"
 import { AiSettingsForm } from "@/components/admin/ai-settings-form"
-import {
-  Card,
-  CardContent,
-  CardHeader,
-  CardTitle,
-} from "@/components/ui/card"
+import { AnnIndexCard } from "@/components/admin/ann-index-card"
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
 
 export default async function AdminAiPage() {
   await requireAdmin()
   const t = await getTranslations("admin.ai")
   const ai = await getSetting("ai")
+  const ann = await getSetting("ai.ann")
 
   const thirtyDaysAgo = daysAgo(30)
   const usage = await db
@@ -33,8 +30,10 @@ export default async function AdminAiPage() {
 
   return (
     <div className="space-y-6">
-      <AiSettingsForm
-        initial={ai ?? { providers: [], monthlyTokenLimitPerUser: 0 }}
+      <AiSettingsForm initial={ai ?? { providers: [], monthlyTokenLimitPerUser: 0 }} />
+      <AnnIndexCard
+        initial={ann ?? { status: "idle" }}
+        embeddingConfigured={Boolean(ai?.defaultEmbeddingModel)}
       />
       <Card>
         <CardHeader>

src/components/admin/ann-index-card.tsx

@@ -0,0 +1,95 @@
+"use client"
+
+import * as React from "react"
+import { Loader2 } from "lucide-react"
+import { useTranslations } from "next-intl"
+import { toast } from "sonner"
+import { Button } from "@/components/ui/button"
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
+import { getAnnStatus, startVectorReindex } from "@/app/[locale]/(app)/admin/actions"
+
+type AnnStatus = {
+  status: "idle" | "building" | "ready" | "failed"
+  embeddingModel?: string
+  dimensions?: number
+  error?: string
+}
+
+/**
+ * Admin control for the optional pgvector HNSW ANN index: shows its state and
+ * triggers a background rebuild. While building it polls for status.
+ */
+export function AnnIndexCard({
+  initial,
+  embeddingConfigured,
+}: {
+  initial: AnnStatus
+  embeddingConfigured: boolean
+}) {
+  const t = useTranslations("admin.ai")
+  const [status, setStatus] = React.useState<AnnStatus>(initial)
+  const [pending, setPending] = React.useState(false)
+
+  React.useEffect(() => {
+    if (status.status !== "building") return
+    let active = true
+    const timer = setInterval(async () => {
+      try {
+        const s = (await getAnnStatus()) as AnnStatus
+        if (active) setStatus(s)
+      } catch {
+        // transient — keep polling
+      }
+    }, 3000)
+    return () => {
+      active = false
+      clearInterval(timer)
+    }
+  }, [status.status])
+
+  async function onRebuild() {
+    setPending(true)
+    try {
+      await startVectorReindex()
+      setStatus((s) => ({ ...s, status: "building" }))
+      toast.success(t("annRebuilding"))
+    } catch (error) {
+      toast.error(error instanceof Error ? error.message : String(error))
+    } finally {
+      setPending(false)
+    }
+  }
+
+  const label: Record<AnnStatus["status"], string> = {
+    idle: t("annStatusIdle"),
+    building: t("annStatusBuilding"),
+    ready: t("annStatusReady"),
+    failed: t("annStatusFailed"),
+  }
+  const busy = pending || status.status === "building"
+
+  return (
+    <Card>
+      <CardHeader>
+        <CardTitle className="text-base">{t("annTitle")}</CardTitle>
+      </CardHeader>
+      <CardContent className="space-y-3">
+        <p className="text-muted-foreground text-sm">{t("annDescription")}</p>
+        <div className="text-sm">
+          <span className="font-medium">{label[status.status]}</span>
+          {status.status === "ready" && status.dimensions
+            ? ` · ${status.embeddingModel} · dim ${status.dimensions}`
+            : ""}
+          {status.status === "failed" && status.error ? ` — ${status.error}` : ""}
+        </div>
+        <Button onClick={onRebuild} disabled={busy || !embeddingConfigured}>
+          {busy && <Loader2 className="size-4 animate-spin" />}
+          {t("annRebuild")}
+        </Button>
+        {!embeddingConfigured && (
+          <p className="text-muted-foreground text-xs">{t("annNeedEmbedding")}</p>
+        )}
+      </CardContent>
+    </Card>
+  )
+}