GB-scale module processing: complete, incremental AI generation with full audit/token logging

drizzle/0031_phase0_incremental_ingestion.sql

@@ -0,0 +1,15 @@
+ALTER TABLE "material" ADD COLUMN "text_storage_path" text;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "char_count" integer;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "content_hash" text;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "summary" text;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "extraction_status" text DEFAULT 'pending' NOT NULL;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "extraction_error" text;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "chunks_total" integer;--> statement-breakpoint
+ALTER TABLE "material" ADD COLUMN "chunks_embedded" integer;--> statement-breakpoint
+ALTER TABLE "material_chunk" ADD COLUMN "level" integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "material_chunk" ADD COLUMN "parent_chunk_id" text;--> statement-breakpoint
+ALTER TABLE "material_chunk" ADD CONSTRAINT "material_chunk_parent_chunk_id_material_chunk_id_fk" FOREIGN KEY ("parent_chunk_id") REFERENCES "public"."material_chunk"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "material_contentHash_idx" ON "material" USING btree ("user_id","content_hash");--> statement-breakpoint
+CREATE INDEX "material_chunk_material_level_idx" ON "material_chunk" USING btree ("material_id","level");--> statement-breakpoint
+-- Backfill: materials that already have extracted text are ready, not pending.
+UPDATE "material" SET "extraction_status" = 'ready' WHERE "text_content" IS NOT NULL;

drizzle/0032_phase1_outline_generation.sql

@@ -0,0 +1,68 @@
+CREATE TABLE "generation_coverage" (
+	"id" text PRIMARY KEY NOT NULL,
+	"target_id" text NOT NULL,
+	"topic_id" text NOT NULL,
+	"job_id" text,
+	"status" text DEFAULT 'pending' NOT NULL,
+	"produced_count" integer DEFAULT 0 NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "generation_job" (
+	"id" text PRIMARY KEY NOT NULL,
+	"user_id" text NOT NULL,
+	"module_id" text NOT NULL,
+	"kind" text NOT NULL,
+	"target_id" text NOT NULL,
+	"status" text DEFAULT 'pending' NOT NULL,
+	"outline_version" integer,
+	"topics_total" integer DEFAULT 0 NOT NULL,
+	"topics_done" integer DEFAULT 0 NOT NULL,
+	"produced_count" integer DEFAULT 0 NOT NULL,
+	"params" jsonb,
+	"error" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "module_outline" (
+	"module_id" text PRIMARY KEY NOT NULL,
+	"user_id" text NOT NULL,
+	"version" integer DEFAULT 0 NOT NULL,
+	"fingerprint" text,
+	"status" text DEFAULT 'idle' NOT NULL,
+	"topic_count" integer DEFAULT 0 NOT NULL,
+	"error" text,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "outline_topic" (
+	"id" text PRIMARY KEY NOT NULL,
+	"module_id" text NOT NULL,
+	"user_id" text NOT NULL,
+	"version" integer NOT NULL,
+	"parent_id" text,
+	"title" text NOT NULL,
+	"title_key" text NOT NULL,
+	"summary" text,
+	"source_material_ids" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"weight" integer DEFAULT 5 NOT NULL,
+	"sort_order" integer DEFAULT 0 NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "generation_coverage" ADD CONSTRAINT "generation_coverage_topic_id_outline_topic_id_fk" FOREIGN KEY ("topic_id") REFERENCES "public"."outline_topic"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "generation_coverage" ADD CONSTRAINT "generation_coverage_job_id_generation_job_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."generation_job"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "generation_job" ADD CONSTRAINT "generation_job_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "generation_job" ADD CONSTRAINT "generation_job_module_id_module_id_fk" FOREIGN KEY ("module_id") REFERENCES "public"."module"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "module_outline" ADD CONSTRAINT "module_outline_module_id_module_id_fk" FOREIGN KEY ("module_id") REFERENCES "public"."module"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "module_outline" ADD CONSTRAINT "module_outline_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "outline_topic" ADD CONSTRAINT "outline_topic_module_id_module_id_fk" FOREIGN KEY ("module_id") REFERENCES "public"."module"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "outline_topic" ADD CONSTRAINT "outline_topic_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "outline_topic" ADD CONSTRAINT "outline_topic_parent_id_outline_topic_id_fk" FOREIGN KEY ("parent_id") REFERENCES "public"."outline_topic"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE UNIQUE INDEX "generation_coverage_target_topic_idx" ON "generation_coverage" USING btree ("target_id","topic_id");--> statement-breakpoint
+CREATE INDEX "generation_coverage_job_idx" ON "generation_coverage" USING btree ("job_id");--> statement-breakpoint
+CREATE INDEX "generation_job_user_idx" ON "generation_job" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX "generation_job_target_idx" ON "generation_job" USING btree ("target_id");--> statement-breakpoint
+CREATE INDEX "outline_topic_module_idx" ON "outline_topic" USING btree ("module_id");--> statement-breakpoint
+CREATE INDEX "outline_topic_module_version_idx" ON "outline_topic" USING btree ("module_id","version");

drizzle/0033_phase2_hybrid_search.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "material_chunk" ADD COLUMN "contextual_header" text;--> statement-breakpoint
+ALTER TABLE "material_chunk" ADD COLUMN "content_tsv" "tsvector" GENERATED ALWAYS AS (to_tsvector('simple', "material_chunk"."content")) STORED;--> statement-breakpoint
+CREATE INDEX "material_chunk_tsv_idx" ON "material_chunk" USING gin ("content_tsv");