From 07291e2226ea56d201aa8bbf6d707567b7a84ac7 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sun, 19 Jul 2026 10:57:11 +0800 Subject: [PATCH 01/29] feat: add founder weekly review persistence --- .../lifecycle.integration.test.ts | 445 ++++++++++++++ .../founderWeeklyReview/migration.test.ts | 85 +++ .../__tests__/founderWeeklyReview/testDb.ts | 174 ++++++ .../0016_founder_weekly_review_runs.sql | 74 +++ apps/web/scripts/migrate.mjs | 12 +- packages/core/src/db/schema.ts | 1 + .../src/db/schema/founder-weekly-review.ts | 136 +++++ packages/features/package.json | 4 + .../src/founder-weekly-review/README.md | 64 ++ .../src/founder-weekly-review/contracts.ts | 248 ++++++++ .../src/founder-weekly-review/errors.ts | 57 ++ .../src/founder-weekly-review/index.ts | 5 + .../src/founder-weekly-review/repository.ts | 564 ++++++++++++++++++ .../src/founder-weekly-review/user-service.ts | 183 ++++++ .../founder-weekly-review/worker-service.ts | 128 ++++ packages/features/src/index.ts | 2 +- 16 files changed, 2179 insertions(+), 3 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/migration.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/testDb.ts create mode 100644 apps/web/drizzle/0016_founder_weekly_review_runs.sql create mode 100644 packages/core/src/db/schema/founder-weekly-review.ts create mode 100644 packages/features/src/founder-weekly-review/README.md create mode 100644 packages/features/src/founder-weekly-review/contracts.ts create mode 100644 packages/features/src/founder-weekly-review/errors.ts create mode 100644 packages/features/src/founder-weekly-review/index.ts create mode 100644 packages/features/src/founder-weekly-review/repository.ts create mode 100644 packages/features/src/founder-weekly-review/user-service.ts create mode 100644 packages/features/src/founder-weekly-review/worker-service.ts diff --git a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts new file mode 100644 index 000000000..e3410b511 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts @@ -0,0 +1,445 @@ +import { randomUUID } from "node:crypto"; + +import { sql } from "drizzle-orm"; +import { company } from "@launchstack/core/db/schema"; +import { + FounderWeeklyReviewConflictError, + FounderWeeklyReviewInvalidPayloadError, + FounderWeeklyReviewInvalidTransitionError, + FounderWeeklyReviewNotFoundError, + FounderWeeklyReviewRepository, + FounderWeeklyReviewUserService, + FounderWeeklyReviewWorkerService, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewPayload, +} from "@launchstack/features/founder-weekly-review"; + +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeIfDatabase = + process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL + ? describe + : describe.skip; + +function createEvidenceSnapshot(): FounderWeeklyReviewEvidenceSnapshot { + return { + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-18T10:00:00.000Z", + reportingPeriod: { + start: "2026-07-07", + end: "2026-07-13", + }, + workspaceTimezone: "America/Los_Angeles", + items: [ + { + sourceType: "workspace_document", + sourceId: "doc-1", + title: "Weekly Product Notes", + sourceTimestamp: "2026-07-10T18:15:00.000Z", + excerpt: "Shipped billing exports and fixed onboarding activation delay.", + workspaceDeepLink: "launchstack://workspace/documents/doc-1", + metadata: { + documentType: "note", + tags: ["product", "ops"], + }, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback-1", + title: "Customer call summary", + excerpt: "Two enterprise prospects asked for stronger audit logging.", + canonicalUrl: "https://example.com/customer-feedback/1", + metadata: { + sentiment: "mixed", + }, + }, + ], + sourceWarnings: [ + { + code: "missing_github_activity", + message: "GitHub activity collection is not part of LAU-5.", + sourceType: "github_activity", + }, + ], + }; +} + +function createPayload(seed: string): FounderWeeklyReviewPayload { + const observed = { + kind: "observed_fact" as const, + text: `Observed fact ${seed}`, + sourceIds: ["doc-1"], + confidence: "high" as const, + }; + const recommended = { + kind: "recommended_item" as const, + text: `Recommended item ${seed}`, + sourceIds: ["feedback-1"], + confidence: "medium" as const, + }; + + return { + schemaVersion: "founder-weekly-review/v1", + sections: { + whatChanged: { heading: "What changed", items: [observed] }, + whatShipped: { heading: "What shipped", items: [observed] }, + whatCustomersSaid: { heading: "What customers said", items: [observed] }, + currentBlockers: { heading: "Current blockers", items: [recommended] }, + nextPriorities: { heading: "Next priorities", items: [recommended] }, + }, + }; +} + +async function insertCompany( + db: Awaited>["db"], + name: string +): Promise { + const rows = await db.execute(sql` + INSERT INTO "pdr_ai_v2_company" ("name", "numberOfEmployees") + VALUES (${name}, '5') + RETURNING "id" + `); + + const [result] = rows; + + if (!result || typeof result !== "object" || !("id" in result)) { + throw new Error(`Failed to insert company fixture for ${name}`); + } + + const id = result.id; + + if ( + typeof id !== "number" && + typeof id !== "string" && + typeof id !== "bigint" + ) { + throw new Error(`Invalid company ID returned for ${name}`); + } + + return BigInt(id); +} + +describeIfDatabase("Founder Weekly Review lifecycle integration", () => { + it("covers create idempotency, isolation, lifecycle transitions, claims, retries, and immutability", async () => { + const testDb = await createFounderWeeklyReviewTestDatabase(); + const extraSession = await testDb.createSession(); + const thirdSession = await testDb.createSession(); + + try { + const companyAId = await insertCompany(testDb.db, "Alpha"); + const companyBId = await insertCompany(testDb.db, "Beta"); + const repositoryA = new FounderWeeklyReviewRepository(testDb.db); + const repositoryB = new FounderWeeklyReviewRepository(extraSession.db); + const userServiceA = new FounderWeeklyReviewUserService(repositoryA); + const userServiceB = new FounderWeeklyReviewUserService(repositoryB); + const workerA = new FounderWeeklyReviewWorkerService(repositoryA); + const workerB = new FounderWeeklyReviewWorkerService( + new FounderWeeklyReviewRepository(thirdSession.db) + ); + + const actorA = { + externalUserId: "clerk_alpha", + internalUserId: 101n, + companyId: companyAId, + role: "owner", + }; + const actorB = { + externalUserId: "clerk_beta", + internalUserId: 202n, + companyId: companyBId, + role: "owner", + }; + + const snapshot = createEvidenceSnapshot(); + expect(JSON.stringify(snapshot)).not.toMatch( + /token|api[_-]?key|authorization|signedUrl|presigned/i + ); + + const created = await userServiceA.createOrGetRun(actorA, { + requestKey: "create-1", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + expect(created.status).toBe("queued"); + expect(created.evidenceSnapshot).toEqual(snapshot); + + const createdAgain = await userServiceA.createOrGetRun(actorA, { + requestKey: "create-1", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: { + ...snapshot, + items: [], + }, + }); + expect(createdAgain.id).toBe(created.id); + expect(createdAgain.evidenceSnapshot).toEqual(snapshot); + + const companyBRun = await userServiceB.createOrGetRun(actorB, { + requestKey: "create-1", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + expect(companyBRun.id).not.toBe(created.id); + + await expect(userServiceB.getRun(actorB, created.id)).rejects.toBeInstanceOf( + FounderWeeklyReviewNotFoundError + ); + await expect( + userServiceB.updateDraft(actorB, created.id, createPayload("wrong-company")) + ).rejects.toBeInstanceOf(FounderWeeklyReviewNotFoundError); + await expect(userServiceB.publishDraft(actorB, created.id)).rejects.toBeInstanceOf( + FounderWeeklyReviewNotFoundError + ); + await expect( + userServiceB.retryFailedRun(actorB, created.id, "retry-wrong-company") + ).rejects.toBeInstanceOf(FounderWeeklyReviewNotFoundError); + await expect( + workerB.claimQueuedRun({ + companyId: companyBId, + runId: created.id, + generationClaimId: "claim-wrong-company", + }) + ).rejects.toBeInstanceOf(FounderWeeklyReviewNotFoundError); + + const claimOne = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-1", + generationJobId: "job-1", + }); + expect(claimOne.status).toBe("generating"); + expect(claimOne.generationAttempt).toBe(1); + + const claimOneAgain = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-1", + generationJobId: "job-1", + }); + expect(claimOneAgain.generationAttempt).toBe(1); + + await expect( + workerB.saveGeneratedDraft( + { + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-2", + }, + createPayload("claim-mismatch"), + null + ) + ).rejects.toMatchObject({ code: "claim_ownership_mismatch" }); + + const generatedDraft = await workerA.saveGeneratedDraft( + { + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-1", + generationJobId: "job-1", + }, + createPayload("draft-1"), + { + provider: "openai", + model: "gpt-5-mini", + attributes: { latencyMs: 1300 }, + } + ); + expect(generatedDraft.status).toBe("draft"); + expect(generatedDraft.generatedAt).not.toBeNull(); + + const editedDraft = await userServiceA.updateDraft( + actorA, + created.id, + createPayload("draft-2") + ); + expect(editedDraft.status).toBe("draft"); + expect(editedDraft.reviewPayload?.sections.whatChanged.items[0]).toMatchObject({ + text: "Observed fact draft-2", + }); + + const published = await userServiceA.publishDraft(actorA, created.id); + expect(published.status).toBe("published"); + expect(published.publishedAt).not.toBeNull(); + expect(published.evidenceSnapshot).toEqual(snapshot); + + const republished = await userServiceA.publishDraft(actorA, created.id); + expect(republished.status).toBe("published"); + expect(republished.publishedAt?.toISOString()).toBe( + published.publishedAt?.toISOString() + ); + + await expect( + userServiceA.updateDraft(actorA, created.id, createPayload("after-publish")) + ).rejects.toBeInstanceOf(FounderWeeklyReviewInvalidTransitionError); + await expect( + userServiceA.retryFailedRun(actorA, created.id, "retry-published") + ).rejects.toBeInstanceOf(FounderWeeklyReviewInvalidTransitionError); + + const failedBeforeClaim = await userServiceA.createOrGetRun(actorA, { + requestKey: "queued-fail", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + const queuedFailure = await workerA.markQueuedRunFailed( + companyAId, + failedBeforeClaim.id, + { + errorCode: "EVIDENCE_INVALID", + errorMessage: "e".repeat(2000), + } + ); + expect(queuedFailure.status).toBe("failed"); + expect(queuedFailure.failureSequence).toBe(1); + expect(queuedFailure.errorMessage?.length).toBeLessThanOrEqual(1024); + + const retried = await userServiceA.retryFailedRun( + actorA, + failedBeforeClaim.id, + "retry-1" + ); + expect(retried.status).toBe("queued"); + expect(retried.retryCount).toBe(1); + expect(retried.errorCode).toBeNull(); + expect(retried.generationClaimId).toBeNull(); + expect(retried.evidenceSnapshot).toEqual(snapshot); + + const retriedAgain = await userServiceA.retryFailedRun( + actorA, + failedBeforeClaim.id, + "retry-1" + ); + expect(retriedAgain.retryCount).toBe(1); + + const claimRaceRun = await userServiceA.createOrGetRun(actorA, { + requestKey: "claim-race", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + const raceResults = await Promise.allSettled([ + workerA.claimQueuedRun({ + companyId: companyAId, + runId: claimRaceRun.id, + generationClaimId: "race-1", + }), + workerB.claimQueuedRun({ + companyId: companyAId, + runId: claimRaceRun.id, + generationClaimId: "race-2", + }), + ]); + const fulfilledClaims = raceResults.filter( + (result): result is PromiseFulfilledResult> => + result.status === "fulfilled" + ); + const rejectedClaims = raceResults.filter( + (result): result is PromiseRejectedResult => result.status === "rejected" + ); + expect(fulfilledClaims).toHaveLength(1); + expect(rejectedClaims).toHaveLength(1); + expect(rejectedClaims[0]?.reason).toBeInstanceOf(FounderWeeklyReviewConflictError); + + const generatingFailureRun = await userServiceA.createOrGetRun(actorA, { + requestKey: "generation-fail", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + const generatingFailureClaim = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-1", + }); + expect(generatingFailureClaim.status).toBe("generating"); + + const failed = await workerA.markGenerationFailed( + { + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-1", + }, + { + errorCode: "MODEL_TIMEOUT", + errorMessage: "provider timed out", + } + ); + expect(failed.status).toBe("failed"); + expect(failed.failureSequence).toBe(1); + + const failedAgain = await workerA.markGenerationFailed( + { + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-1", + }, + { + errorCode: "MODEL_TIMEOUT", + errorMessage: "provider timed out", + } + ); + expect(failedAgain.failureSequence).toBe(1); + + const retryRaceRun = await userServiceA.retryFailedRun( + actorA, + generatingFailureRun.id, + "retry-generation-1" + ); + expect(retryRaceRun.retryCount).toBe(1); + + const secondClaim = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-2", + }); + expect(secondClaim.generationAttempt).toBe(2); + + const secondFailure = await workerA.markGenerationFailed( + { + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-2", + }, + { + errorCode: "MODEL_TIMEOUT", + errorMessage: "provider timed out again", + } + ); + expect(secondFailure.failureSequence).toBe(2); + + await expect( + userServiceA.retryFailedRun(actorA, generatingFailureRun.id, "retry-generation-1") + ).rejects.toBeInstanceOf(FounderWeeklyReviewConflictError); + + const concurrentRetryRun = await userServiceA.createOrGetRun(actorA, { + requestKey: "retry-race", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + await workerA.markQueuedRunFailed(companyAId, concurrentRetryRun.id, { + errorCode: "DISPATCH_FAILED", + errorMessage: "queue dispatch failed", + }); + const concurrentRetryResults = await Promise.all([ + userServiceA.retryFailedRun(actorA, concurrentRetryRun.id, "same-retry-key"), + userServiceA.retryFailedRun(actorA, concurrentRetryRun.id, "same-retry-key"), + ]); + expect(concurrentRetryResults[0].retryCount).toBe(1); + expect(concurrentRetryResults[1].retryCount).toBe(1); + + const listed = await userServiceA.listRuns(actorA); + expect(listed.length).toBeGreaterThanOrEqual(5); + + await expect( + userServiceA.updateDraft( + actorA, + claimRaceRun.id, + { + schemaVersion: "founder-weekly-review/v1", + } as FounderWeeklyReviewPayload + ) + ).rejects.toBeInstanceOf(FounderWeeklyReviewInvalidPayloadError); + } finally { + await thirdSession.close(); + await extraSession.close(); + await testDb.close(); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/migration.test.ts b/apps/web/__tests__/founderWeeklyReview/migration.test.ts new file mode 100644 index 000000000..e7f1f8c7f --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/migration.test.ts @@ -0,0 +1,85 @@ +import { sql } from "drizzle-orm"; + +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeIfDatabase = + process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL + ? describe + : describe.skip; + +describeIfDatabase("Founder Weekly Review migrations", () => { + it("replays all migrations into a clean schema and creates the new tables and constraints", async () => { + const testDb = await createFounderWeeklyReviewTestDatabase(); + try { + const runsTable = await testDb.db.execute(sql` + SELECT to_regclass('pdr_ai_v2_founder_weekly_review_runs') AS name + `); + const opsTable = await testDb.db.execute(sql` + SELECT to_regclass('pdr_ai_v2_founder_weekly_review_operations') AS name + `); + const indexes = await testDb.db.execute(sql` + SELECT indexname + FROM pg_indexes + WHERE tablename IN ( + 'pdr_ai_v2_founder_weekly_review_runs', + 'pdr_ai_v2_founder_weekly_review_operations' + ) + ORDER BY indexname + `); + + expect(runsTable[0]?.name).toBe("pdr_ai_v2_founder_weekly_review_runs"); + expect(opsTable[0]?.name).toBe("pdr_ai_v2_founder_weekly_review_operations"); + expect(indexes.map((row) => row.indexname)).toEqual( + expect.arrayContaining([ + "founder_weekly_review_runs_company_request_key_unique", + "founder_weekly_review_operations_run_type_request_key_unique", + "founder_weekly_review_runs_company_status_created_at_idx", + ]) + ); + + await testDb.db.execute(sql` + INSERT INTO "pdr_ai_v2_company" + ("name", "numberOfEmployees") + VALUES + ('Migration Test Co', '3') + `); + await testDb.db.execute(sql` + INSERT INTO "pdr_ai_v2_founder_weekly_review_runs" + ( + "id", + "company_id", + "request_key", + "reporting_period_start", + "reporting_period_end", + "status", + "review_schema_version", + "evidence_snapshot", + "evidence_schema_version", + "created_by_actor_id" + ) + VALUES + ( + 'fwr_migration_test', + 1, + 'request-key', + DATE '2026-07-07', + DATE '2026-07-13', + 'queued', + 'founder-weekly-review/v1', + '{"schemaVersion":"founder-weekly-review-evidence/v1","capturedAt":"2026-07-18T00:00:00.000Z","reportingPeriod":{"start":"2026-07-07","end":"2026-07-13"},"workspaceTimezone":"UTC","items":[],"sourceWarnings":[]}'::jsonb, + 'founder-weekly-review-evidence/v1', + 'user:test' + ) + `); + + const inserted = await testDb.db.execute(sql` + SELECT "status" + FROM "pdr_ai_v2_founder_weekly_review_runs" + WHERE "id" = 'fwr_migration_test' + `); + expect(inserted[0]?.status).toBe("queued"); + } finally { + await testDb.close(); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/testDb.ts b/apps/web/__tests__/founderWeeklyReview/testDb.ts new file mode 100644 index 000000000..b8ad11c5d --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/testDb.ts @@ -0,0 +1,174 @@ +import { randomUUID } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import postgres, { type Sql } from "postgres"; +import { drizzle } from "drizzle-orm/postgres-js"; + +import * as coreSchema from "@launchstack/core/db/schema"; +import type { DbClient } from "@launchstack/core/db"; + +const rootDir = join(__dirname, "..", ".."); +const migrationsDir = join(rootDir, "drizzle"); + +function buildSearchPathSql(schemaName: string): string { + return `SET search_path TO "${schemaName}", public`; +} + +async function listMigrationFiles(): Promise { + const entries = await readdir(migrationsDir, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".sql")) + .map((entry) => entry.name) + .sort(); +} + +function normalizeSqlMigrationBody(body: string): string { + return body.replace(/^\uFEFF/, ""); +} + +async function bootstrapIsolatedSchema( + client: Sql, + schemaName: string +): Promise { + await client.unsafe(buildSearchPathSql(schemaName)); + await client.unsafe(` + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_company" ( + "id" serial PRIMARY KEY, + "name" varchar(256) NOT NULL, + "slug" varchar(64), + "description" text, + "industry" varchar(256), + "swatch" integer NOT NULL DEFAULT 1, + "embedding_index_key" varchar(128), + "active_embedding_index_key" varchar(128), + "pending_embedding_index_key" varchar(128), + "reindex_status" varchar(16) NOT NULL DEFAULT 'STABLE', + "reindex_job_id" text, + "reindex_started_at" timestamptz, + "reindex_completed_at" timestamptz, + "reindex_error" text, + "embedding_openai_api_key" text, + "embedding_huggingface_api_key" text, + "embedding_ollama_base_url" varchar(1024), + "embedding_ollama_model" varchar(256), + "employerPasskey" varchar(256) NOT NULL DEFAULT '', + "employeePasskey" varchar(256) NOT NULL DEFAULT '', + "numberOfEmployees" varchar(256) NOT NULL, + "use_uploadthing" boolean NOT NULL DEFAULT true, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_users" ( + "id" serial PRIMARY KEY, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "role" varchar(256) NOT NULL, + "last_active_at" timestamptz, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document" ( + "id" serial PRIMARY KEY, + "company_id" bigint REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "url" varchar(256), + "category" varchar(256), + "title" varchar(256), + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_retrieval_chunks" ( + "id" bigint PRIMARY KEY + ); + + CREATE TABLE IF NOT EXISTS "company" ( + "id" serial PRIMARY KEY, + "name" varchar(256) NOT NULL + ); + + CREATE TABLE IF NOT EXISTS "document" ( + "id" serial PRIMARY KEY + ); + + CREATE TABLE IF NOT EXISTS "ocr_jobs" ( + "id" varchar(256) PRIMARY KEY + ); + + CREATE TABLE IF NOT EXISTS "file_uploads" ( + "id" serial PRIMARY KEY, + "user_id" varchar(256) NOT NULL, + "filename" varchar(256) NOT NULL, + "mime_type" varchar(128) NOT NULL, + "file_data" text NOT NULL, + "file_size" integer NOT NULL, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + `); +} + +export interface FounderWeeklyReviewTestSession { + db: DbClient; + close(): Promise; +} + +export interface FounderWeeklyReviewTestDatabase { + schemaName: string; + db: DbClient; + createSession(): Promise; + close(): Promise; +} + +async function createSession( + connectionString: string, + schemaName: string +): Promise { + const client = postgres(connectionString, { max: 1 }); + await client.unsafe(buildSearchPathSql(schemaName)); + return { + db: drizzle(client, { schema: coreSchema }), + close: () => client.end({ timeout: 5 }), + }; +} + +export async function createFounderWeeklyReviewTestDatabase(): Promise { + const connectionString = + process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL; + + if (!connectionString) { + throw new Error( + "LAUNCHSTACK_TEST_DATABASE_URL or DATABASE_URL is required for founder weekly review integration tests." + ); + } + + const adminClient = postgres(connectionString, { max: 1 }); + const schemaName = `lau5_${randomUUID().replace(/-/g, "")}`; + await adminClient.unsafe(`CREATE SCHEMA "${schemaName}"`); + await adminClient.unsafe(`CREATE EXTENSION IF NOT EXISTS vector`); + await adminClient.unsafe(buildSearchPathSql(schemaName)); + await bootstrapIsolatedSchema(adminClient, schemaName); + + const migrationFiles = await listMigrationFiles(); + for (const fileName of migrationFiles) { + const body = normalizeSqlMigrationBody( + await readFile(join(migrationsDir, fileName), "utf8") + ); + await adminClient.begin(async (tx) => { + await tx.unsafe(buildSearchPathSql(schemaName)); + await tx.unsafe(body); + }); + } + + const primary = await createSession(connectionString, schemaName); + + return { + schemaName, + db: primary.db, + createSession: () => createSession(connectionString, schemaName), + async close() { + await primary.close(); + await adminClient.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`); + await adminClient.end({ timeout: 5 }); + }, + }; +} diff --git a/apps/web/drizzle/0016_founder_weekly_review_runs.sql b/apps/web/drizzle/0016_founder_weekly_review_runs.sql new file mode 100644 index 000000000..b8115e02e --- /dev/null +++ b/apps/web/drizzle/0016_founder_weekly_review_runs.sql @@ -0,0 +1,74 @@ +-- Founder Weekly Review persistence and lifecycle foundation (LAU-5). +-- +-- Creates: +-- 1. pdr_ai_v2_founder_weekly_review_runs +-- 2. pdr_ai_v2_founder_weekly_review_operations +-- +-- V1 decision: multiple runs are allowed per company and reporting period. +-- Idempotency is enforced by: +-- - create: UNIQUE(company_id, request_key) +-- - retry: UNIQUE(run_id, operation_type, request_key) + +CREATE TABLE IF NOT EXISTS "pdr_ai_v2_founder_weekly_review_runs" ( + "id" varchar(64) PRIMARY KEY, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "request_key" varchar(128) NOT NULL, + "reporting_period_start" date NOT NULL, + "reporting_period_end" date NOT NULL, + "status" varchar(32) NOT NULL DEFAULT 'queued', + "review_payload" jsonb, + "review_schema_version" varchar(64) NOT NULL, + "evidence_snapshot" jsonb NOT NULL, + "evidence_schema_version" varchar(64) NOT NULL, + "model_metadata" jsonb, + "created_by_actor_id" varchar(256) NOT NULL, + "retry_count" integer NOT NULL DEFAULT 0, + "failure_sequence" integer NOT NULL DEFAULT 0, + "generation_attempt" integer NOT NULL DEFAULT 0, + "generation_claim_id" varchar(128), + "generation_job_id" varchar(256), + "queued_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "claimed_at" timestamptz, + "generation_started_at" timestamptz, + "generated_at" timestamptz, + "published_at" timestamptz, + "error_code" varchar(128), + "error_message" varchar(1024), + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS "pdr_ai_v2_founder_weekly_review_operations" ( + "id" varchar(64) PRIMARY KEY, + "run_id" varchar(64) NOT NULL REFERENCES "pdr_ai_v2_founder_weekly_review_runs"("id") ON DELETE CASCADE, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "operation_type" varchar(32) NOT NULL, + "request_key" varchar(128) NOT NULL, + "source_failure_sequence" integer NOT NULL, + "actor_id" varchar(256) NOT NULL, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_request_key_unique" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "request_key"); + +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_operations_run_type_request_key_unique" + ON "pdr_ai_v2_founder_weekly_review_operations" ("run_id", "operation_type", "request_key"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "created_at"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_status_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "status", "created_at"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_period_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "reporting_period_start", "reporting_period_end"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_claim_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "id", "status", "generation_claim_id"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_operations_company_run_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_operations" ("company_id", "run_id", "created_at"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_operations_run_type_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_operations" ("run_id", "operation_type", "created_at"); diff --git a/apps/web/scripts/migrate.mjs b/apps/web/scripts/migrate.mjs index 6d1a6b883..a86c5f3f0 100644 --- a/apps/web/scripts/migrate.mjs +++ b/apps/web/scripts/migrate.mjs @@ -35,6 +35,10 @@ const migrationsDir = join(__dirname, "..", "drizzle"); const sql = postgres(url, { max: 1 }); +function normalizeSqlMigrationBody(body) { + return body.replace(/^\uFEFF/, ""); +} + async function ensureMigrationsTable() { await sql` CREATE TABLE IF NOT EXISTS _launchstack_migrations ( @@ -59,7 +63,9 @@ async function listMigrationFiles() { } async function applyMigration(name) { - const body = await readFile(join(migrationsDir, name), "utf8"); + const body = normalizeSqlMigrationBody( + await readFile(join(migrationsDir, name), "utf8"), + ); const checksum = createHash("sha256").update(body).digest("hex"); console.log(`[migrate] applying ${name}`); @@ -83,7 +89,9 @@ async function main() { let drift = 0; for (const name of files) { - const body = await readFile(join(migrationsDir, name), "utf8"); + const body = normalizeSqlMigrationBody( + await readFile(join(migrationsDir, name), "utf8"), + ); const checksum = createHash("sha256").update(body).digest("hex"); const recorded = applied.get(name); diff --git a/packages/core/src/db/schema.ts b/packages/core/src/db/schema.ts index 02e3eb78e..c8e3bd36e 100644 --- a/packages/core/src/db/schema.ts +++ b/packages/core/src/db/schema.ts @@ -16,3 +16,4 @@ export * from "./schema/client-prospector"; export * from "./schema/company-metadata"; export * from "./schema/marketing-history"; export * from "./schema/credits"; +export * from "./schema/founder-weekly-review"; diff --git a/packages/core/src/db/schema/founder-weekly-review.ts b/packages/core/src/db/schema/founder-weekly-review.ts new file mode 100644 index 000000000..21fed15ea --- /dev/null +++ b/packages/core/src/db/schema/founder-weekly-review.ts @@ -0,0 +1,136 @@ +import { sql } from "drizzle-orm"; +import type { InferSelectModel } from "drizzle-orm"; +import { + bigint, + date, + index, + integer, + jsonb, + uniqueIndex, + varchar, + timestamp, +} from "drizzle-orm/pg-core"; + +import { company } from "./base"; +import { pgTable } from "./helpers"; + +export const founderWeeklyReviewRunStatusEnum = [ + "queued", + "generating", + "draft", + "published", + "failed", +] as const; + +export const founderWeeklyReviewOperationTypeEnum = ["retry"] as const; + +export const founderWeeklyReviewRuns = pgTable( + "founder_weekly_review_runs", + { + id: varchar("id", { length: 64 }).primaryKey(), + companyId: bigint("company_id", { mode: "bigint" }) + .notNull() + .references(() => company.id, { onDelete: "cascade" }), + requestKey: varchar("request_key", { length: 128 }).notNull(), + reportingPeriodStart: date("reporting_period_start").notNull(), + reportingPeriodEnd: date("reporting_period_end").notNull(), + status: varchar("status", { + length: 32, + enum: founderWeeklyReviewRunStatusEnum, + }) + .notNull() + .default("queued"), + reviewPayload: jsonb("review_payload").$type | null>(), + reviewSchemaVersion: varchar("review_schema_version", { length: 64 }).notNull(), + evidenceSnapshot: jsonb("evidence_snapshot") + .$type>() + .notNull(), + evidenceSchemaVersion: varchar("evidence_schema_version", { length: 64 }).notNull(), + modelMetadata: jsonb("model_metadata").$type | null>(), + createdByActorId: varchar("created_by_actor_id", { length: 256 }).notNull(), + retryCount: integer("retry_count").notNull().default(0), + failureSequence: integer("failure_sequence").notNull().default(0), + generationAttempt: integer("generation_attempt").notNull().default(0), + generationClaimId: varchar("generation_claim_id", { length: 128 }), + generationJobId: varchar("generation_job_id", { length: 256 }), + queuedAt: timestamp("queued_at", { withTimezone: true }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + claimedAt: timestamp("claimed_at", { withTimezone: true }), + generationStartedAt: timestamp("generation_started_at", { withTimezone: true }), + generatedAt: timestamp("generated_at", { withTimezone: true }), + publishedAt: timestamp("published_at", { withTimezone: true }), + errorCode: varchar("error_code", { length: 128 }), + errorMessage: varchar("error_message", { length: 1024 }), + createdAt: timestamp("created_at", { withTimezone: true }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).$onUpdate( + () => new Date() + ), + }, + (table) => ({ + requestKeyUnique: uniqueIndex( + "founder_weekly_review_runs_company_request_key_unique" + ).on(table.companyId, table.requestKey), + companyCreatedIdx: index("founder_weekly_review_runs_company_created_at_idx").on( + table.companyId, + table.createdAt + ), + companyStatusCreatedIdx: index( + "founder_weekly_review_runs_company_status_created_at_idx" + ).on(table.companyId, table.status, table.createdAt), + companyPeriodIdx: index("founder_weekly_review_runs_company_period_idx").on( + table.companyId, + table.reportingPeriodStart, + table.reportingPeriodEnd + ), + claimIdx: index("founder_weekly_review_runs_claim_idx").on( + table.companyId, + table.id, + table.status, + table.generationClaimId + ), + }) +); + +export const founderWeeklyReviewOperations = pgTable( + "founder_weekly_review_operations", + { + id: varchar("id", { length: 64 }).primaryKey(), + runId: varchar("run_id", { length: 64 }) + .notNull() + .references(() => founderWeeklyReviewRuns.id, { onDelete: "cascade" }), + companyId: bigint("company_id", { mode: "bigint" }) + .notNull() + .references(() => company.id, { onDelete: "cascade" }), + operationType: varchar("operation_type", { + length: 32, + enum: founderWeeklyReviewOperationTypeEnum, + }).notNull(), + requestKey: varchar("request_key", { length: 128 }).notNull(), + sourceFailureSequence: integer("source_failure_sequence").notNull(), + actorId: varchar("actor_id", { length: 256 }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + }, + (table) => ({ + runOperationRequestKeyUnique: uniqueIndex( + "founder_weekly_review_operations_run_type_request_key_unique" + ).on(table.runId, table.operationType, table.requestKey), + companyRunCreatedIdx: index( + "founder_weekly_review_operations_company_run_created_at_idx" + ).on(table.companyId, table.runId, table.createdAt), + runTypeCreatedIdx: index("founder_weekly_review_operations_run_type_created_at_idx").on( + table.runId, + table.operationType, + table.createdAt + ), + }) +); + +export type FounderWeeklyReviewRunRow = InferSelectModel; +export type FounderWeeklyReviewOperationRow = InferSelectModel< + typeof founderWeeklyReviewOperations +>; diff --git a/packages/features/package.json b/packages/features/package.json index e59d538c6..5834233da 100644 --- a/packages/features/package.json +++ b/packages/features/package.json @@ -89,6 +89,10 @@ "types": "./src/client-prospector/db.ts", "default": "./src/client-prospector/db.ts" }, + "./founder-weekly-review": { + "types": "./src/founder-weekly-review/index.ts", + "default": "./src/founder-weekly-review/index.ts" + }, "./doc-ingestion": { "types": "./src/doc-ingestion/index.ts", "default": "./src/doc-ingestion/index.ts" diff --git a/packages/features/src/founder-weekly-review/README.md b/packages/features/src/founder-weekly-review/README.md new file mode 100644 index 000000000..50c3d7477 --- /dev/null +++ b/packages/features/src/founder-weekly-review/README.md @@ -0,0 +1,64 @@ +# Founder Weekly Review Lifecycle + +This module owns LAU-5 persistence and lifecycle foundations for company-scoped Founder Weekly Reviews. + +## Tables + +- `pdr_ai_v2_founder_weekly_review_runs`: one lifecycle row per review run. +- `pdr_ai_v2_founder_weekly_review_operations`: feature-scoped idempotency operations, currently `retry`. + +## Statuses + +- `queued` -> `generating` -> `draft` -> `published` +- `queued` -> `failed` +- `generating` -> `failed` +- `failed` -> `queued` via retry +- `draft` -> `draft` via manual edit +- `published` is terminal and immutable + +## Immutable fields + +- Always immutable after create: run ID, company ID, create request key, reporting period, evidence snapshot, evidence schema version, creator actor ID. +- Also immutable after publish: review payload, model metadata, lifecycle result fields. + +## Idempotency + +- Create idempotency: unique `(company_id, request_key)` on the run table. +- Retry idempotency: unique `(run_id, operation_type, request_key)` on the operations table, with the source failure sequence recorded to reject stale delayed retries from older failure cycles. + +## Service APIs + +- `FounderWeeklyReviewUserService` + - `createOrGetRun` + - `getRun` + - `listRuns` + - `retryFailedRun` + - `updateDraft` + - `publishDraft` +- `FounderWeeklyReviewWorkerService` + - `claimQueuedRun` + - `saveGeneratedDraft` + - `markGenerationFailed` + - `markQueuedRunFailed` + +## Company isolation + +Every repository method accepts `companyId` explicitly and includes it in SQL predicates. Wrong-company access resolves as not found rather than leaking existence. + +## Contract versions + +- Evidence snapshot: `founder-weekly-review-evidence/v1` +- Review payload: `founder-weekly-review/v1` + +## Current V1 product decision + +Multiple runs are allowed for the same company and reporting period. LAU-5 does not enforce period uniqueness or a single published review per period. + +## Deferred scope + +- LAU-6: evidence collection +- LAU-7: review generation payload population +- LAU-8: workflow orchestration +- LAU-9: HTTP APIs +- LAU-10: dashboard/UI +- LAU-11: readiness and source management UX diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts new file mode 100644 index 000000000..9ad0f1b59 --- /dev/null +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -0,0 +1,248 @@ +import { z } from "zod"; + +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION = + "founder-weekly-review-evidence/v1" as const; +export const FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION = + "founder-weekly-review/v1" as const; + +export const FounderWeeklyReviewStatusSchema = z.enum([ + "queued", + "generating", + "draft", + "published", + "failed", +]); +export type FounderWeeklyReviewStatus = z.infer; + +export const FounderWeeklyReviewOperationTypeSchema = z.enum(["retry"]); +export type FounderWeeklyReviewOperationType = z.infer< + typeof FounderWeeklyReviewOperationTypeSchema +>; + +export const ReportingPeriodSchema = z + .object({ + start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + end: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + }) + .refine((value) => value.start <= value.end, { + message: "Reporting period start must be on or before end", + path: ["end"], + }); +export type ReportingPeriod = z.infer; + +const SerializableMetadataPrimitiveSchema = z.union([ + z.string().max(512), + z.number().finite(), + z.boolean(), + z.null(), +]); + +const SerializableMetadataValueSchema: z.ZodType< + string | number | boolean | null | Array +> = z.union([ + SerializableMetadataPrimitiveSchema, + z.array(SerializableMetadataPrimitiveSchema).max(20), +]); + +export const FounderWeeklyReviewEvidenceItemSchema = z.object({ + sourceType: z.enum([ + "workspace_document", + "customer_feedback", + "github_activity", + "manual_note", + "other", + ]), + sourceId: z.string().min(1).max(256), + title: z.string().min(1).max(512), + sourceTimestamp: z.string().datetime({ offset: true }).optional(), + excerpt: z.string().min(1).max(4000), + canonicalUrl: z.string().url().max(2048).optional(), + workspaceDeepLink: z.string().max(2048).optional(), + metadata: z.record(z.string().max(64), SerializableMetadataValueSchema).default({}), +}); +export type FounderWeeklyReviewEvidenceItem = z.infer< + typeof FounderWeeklyReviewEvidenceItemSchema +>; + +export const FounderWeeklyReviewEvidenceWarningSchema = z.object({ + code: z.string().min(1).max(64), + message: z.string().min(1).max(512), + sourceType: FounderWeeklyReviewEvidenceItemSchema.shape.sourceType.optional(), +}); +export type FounderWeeklyReviewEvidenceWarning = z.infer< + typeof FounderWeeklyReviewEvidenceWarningSchema +>; + +export const FounderWeeklyReviewEvidenceSnapshotSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION), + capturedAt: z.string().datetime({ offset: true }), + reportingPeriod: ReportingPeriodSchema, + workspaceTimezone: z.string().min(1).max(128), + items: z.array(FounderWeeklyReviewEvidenceItemSchema).max(500), + sourceWarnings: z.array(FounderWeeklyReviewEvidenceWarningSchema).max(100).default([]), +}); +export type FounderWeeklyReviewEvidenceSnapshot = z.infer< + typeof FounderWeeklyReviewEvidenceSnapshotSchema +>; + +export const FounderWeeklyReviewConfidenceSchema = z.enum(["high", "medium", "low"]); +export type FounderWeeklyReviewConfidence = z.infer< + typeof FounderWeeklyReviewConfidenceSchema +>; + +export const FounderWeeklyReviewSectionItemSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("observed_fact"), + text: z.string().min(1).max(2000), + sourceIds: z.array(z.string().min(1).max(256)).min(1).max(20), + confidence: FounderWeeklyReviewConfidenceSchema, + }), + z.object({ + kind: z.literal("recommended_item"), + text: z.string().min(1).max(2000), + rationale: z.string().min(1).max(2000).optional(), + sourceIds: z.array(z.string().min(1).max(256)).max(20).default([]), + confidence: FounderWeeklyReviewConfidenceSchema.optional(), + }), + z.object({ + kind: z.literal("no_evidence"), + code: z.enum(["no_relevant_evidence", "source_unavailable", "not_assessed"]), + note: z.string().min(1).max(512).optional(), + }), + z.object({ + kind: z.literal("human_edit"), + markdown: z.string().min(1).max(12000), + }), +]); +export type FounderWeeklyReviewSectionItem = z.infer< + typeof FounderWeeklyReviewSectionItemSchema +>; + +const FounderWeeklyReviewSectionSchema = z.object({ + heading: z.string().min(1).max(128), + items: z.array(FounderWeeklyReviewSectionItemSchema).max(100), +}); + +export const FounderWeeklyReviewPayloadSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION), + sections: z.object({ + whatChanged: FounderWeeklyReviewSectionSchema, + whatShipped: FounderWeeklyReviewSectionSchema, + whatCustomersSaid: FounderWeeklyReviewSectionSchema, + currentBlockers: FounderWeeklyReviewSectionSchema, + nextPriorities: FounderWeeklyReviewSectionSchema, + }), +}); +export type FounderWeeklyReviewPayload = z.infer; + +export const FounderWeeklyReviewModelMetadataSchema = z.object({ + provider: z.string().min(1).max(128).optional(), + model: z.string().min(1).max(256).optional(), + promptVersion: z.string().min(1).max(128).optional(), + temperature: z.number().finite().optional(), + completionId: z.string().min(1).max(256).optional(), + notes: z.string().min(1).max(1024).optional(), + attributes: z.record(z.string().max(64), SerializableMetadataValueSchema).default({}), +}); +export type FounderWeeklyReviewModelMetadata = z.infer< + typeof FounderWeeklyReviewModelMetadataSchema +>; + +export interface FounderWeeklyReviewRunRecord { + id: string; + companyId: bigint; + requestKey: string; + reportingPeriod: ReportingPeriod; + status: FounderWeeklyReviewStatus; + reviewPayload: FounderWeeklyReviewPayload | null; + reviewSchemaVersion: typeof FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + evidenceSchemaVersion: typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION; + modelMetadata: FounderWeeklyReviewModelMetadata | null; + createdByActorId: string; + retryCount: number; + failureSequence: number; + generationAttempt: number; + generationClaimId: string | null; + generationJobId: string | null; + queuedAt: Date; + claimedAt: Date | null; + generationStartedAt: Date | null; + generatedAt: Date | null; + publishedAt: Date | null; + errorCode: string | null; + errorMessage: string | null; + createdAt: Date; + updatedAt: Date | null; +} + +export interface FounderWeeklyReviewOperationRecord { + id: string; + runId: string; + companyId: bigint; + operationType: FounderWeeklyReviewOperationType; + requestKey: string; + sourceFailureSequence: number; + actorId: string; + createdAt: Date; +} + +export interface CreateFounderWeeklyReviewRunInput { + id: string; + companyId: bigint; + requestKey: string; + reportingPeriod: ReportingPeriod; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + createdByActorId: string; +} + +export interface FounderWeeklyReviewRetryInput { + operationId: string; + companyId: bigint; + runId: string; + requestKey: string; + actorId: string; +} + +export interface FounderWeeklyReviewClaimInput { + companyId: bigint; + runId: string; + generationClaimId: string; + generationJobId?: string; +} + +export interface FounderWeeklyReviewGenerationFailure { + errorCode: string; + errorMessage?: string; +} + +export interface FounderWeeklyReviewUserActor { + externalUserId: string; + internalUserId?: bigint; + companyId: bigint; + role: string; +} + +export function buildFounderWeeklyReviewActorId( + actor: Pick +): string { + return `user:${actor.externalUserId}`; +} + +export function parseFounderWeeklyReviewPayload( + value: unknown +): FounderWeeklyReviewPayload { + return FounderWeeklyReviewPayloadSchema.parse(value); +} + +export function parseFounderWeeklyReviewEvidenceSnapshot( + value: unknown +): FounderWeeklyReviewEvidenceSnapshot { + return FounderWeeklyReviewEvidenceSnapshotSchema.parse(value); +} + +export function parseFounderWeeklyReviewModelMetadata( + value: unknown +): FounderWeeklyReviewModelMetadata { + return FounderWeeklyReviewModelMetadataSchema.parse(value); +} diff --git a/packages/features/src/founder-weekly-review/errors.ts b/packages/features/src/founder-weekly-review/errors.ts new file mode 100644 index 000000000..74658363e --- /dev/null +++ b/packages/features/src/founder-weekly-review/errors.ts @@ -0,0 +1,57 @@ +export class FounderWeeklyReviewError extends Error { + constructor( + message: string, + readonly code: + | "not_found" + | "forbidden" + | "invalid_transition" + | "conflict" + | "invalid_payload" + | "claim_ownership_mismatch" + ) { + super(message); + this.name = new.target.name; + } +} + +export class FounderWeeklyReviewNotFoundError extends FounderWeeklyReviewError { + constructor(runId: string) { + super(`Founder weekly review run "${runId}" was not found.`, "not_found"); + } +} + +export class FounderWeeklyReviewForbiddenError extends FounderWeeklyReviewError { + constructor(message = "The active workspace role cannot mutate founder weekly reviews.") { + super(message, "forbidden"); + } +} + +export class FounderWeeklyReviewInvalidTransitionError extends FounderWeeklyReviewError { + constructor(fromStatus: string, action: string) { + super( + `Cannot ${action} founder weekly review run from status "${fromStatus}".`, + "invalid_transition" + ); + } +} + +export class FounderWeeklyReviewConflictError extends FounderWeeklyReviewError { + constructor(message: string) { + super(message, "conflict"); + } +} + +export class FounderWeeklyReviewInvalidPayloadError extends FounderWeeklyReviewError { + constructor(message: string) { + super(message, "invalid_payload"); + } +} + +export class FounderWeeklyReviewClaimOwnershipMismatchError extends FounderWeeklyReviewError { + constructor(runId: string) { + super( + `Generation claim ownership mismatch for founder weekly review run "${runId}".`, + "claim_ownership_mismatch" + ); + } +} diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts new file mode 100644 index 000000000..b217ef302 --- /dev/null +++ b/packages/features/src/founder-weekly-review/index.ts @@ -0,0 +1,5 @@ +export * from "./contracts"; +export * from "./errors"; +export * from "./repository"; +export * from "./user-service"; +export * from "./worker-service"; diff --git a/packages/features/src/founder-weekly-review/repository.ts b/packages/features/src/founder-weekly-review/repository.ts new file mode 100644 index 000000000..8716fc553 --- /dev/null +++ b/packages/features/src/founder-weekly-review/repository.ts @@ -0,0 +1,564 @@ +import { and, desc, eq, sql } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { + founderWeeklyReviewOperations, + founderWeeklyReviewRuns, + type FounderWeeklyReviewOperationRow, + type FounderWeeklyReviewRunRow, +} from "@launchstack/core/db/schema"; + +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + type CreateFounderWeeklyReviewRunInput, + type FounderWeeklyReviewClaimInput, + type FounderWeeklyReviewGenerationFailure, + type FounderWeeklyReviewModelMetadata, + type FounderWeeklyReviewOperationRecord, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewRetryInput, + type FounderWeeklyReviewRunRecord, + parseFounderWeeklyReviewEvidenceSnapshot, + parseFounderWeeklyReviewModelMetadata, + parseFounderWeeklyReviewPayload, +} from "./contracts"; + +export interface ConditionalRunMutationResult { + updated: boolean; + run: FounderWeeklyReviewRunRecord | null; +} + +export interface RetryFounderWeeklyReviewResult { + outcome: "updated" | "idempotent" | "conflict" | "not_found"; + run: FounderWeeklyReviewRunRecord | null; + operation: FounderWeeklyReviewOperationRecord | null; +} + +function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord { + return { + id: row.id, + companyId: row.companyId, + requestKey: row.requestKey, + reportingPeriod: { + start: row.reportingPeriodStart, + end: row.reportingPeriodEnd, + }, + status: row.status, + reviewPayload: row.reviewPayload + ? parseFounderWeeklyReviewPayload(row.reviewPayload) + : null, + reviewSchemaVersion: row.reviewSchemaVersion as typeof FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + evidenceSnapshot: parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot), + evidenceSchemaVersion: + row.evidenceSchemaVersion as typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + modelMetadata: row.modelMetadata + ? parseFounderWeeklyReviewModelMetadata(row.modelMetadata) + : null, + createdByActorId: row.createdByActorId, + retryCount: row.retryCount, + failureSequence: row.failureSequence, + generationAttempt: row.generationAttempt, + generationClaimId: row.generationClaimId ?? null, + generationJobId: row.generationJobId ?? null, + queuedAt: row.queuedAt, + claimedAt: row.claimedAt ?? null, + generationStartedAt: row.generationStartedAt ?? null, + generatedAt: row.generatedAt ?? null, + publishedAt: row.publishedAt ?? null, + errorCode: row.errorCode ?? null, + errorMessage: row.errorMessage ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt ?? null, + }; +} + +function mapOperationRow(row: FounderWeeklyReviewOperationRow): FounderWeeklyReviewOperationRecord { + return { + id: row.id, + runId: row.runId, + companyId: row.companyId, + operationType: row.operationType, + requestKey: row.requestKey, + sourceFailureSequence: row.sourceFailureSequence, + actorId: row.actorId, + createdAt: row.createdAt, + }; +} + +function truncateErrorMessage(value?: string | null): string | null { + if (!value) { + return null; + } + return value.length <= 1024 ? value : value.slice(0, 1024); +} + +export class FounderWeeklyReviewRepository { + constructor(private readonly db: DbClient = getDb()) {} + + async createOrGetByRequestKey( + input: CreateFounderWeeklyReviewRunInput + ): Promise { + const [inserted] = await this.db + .insert(founderWeeklyReviewRuns) + .values({ + id: input.id, + companyId: input.companyId, + requestKey: input.requestKey, + reportingPeriodStart: input.reportingPeriod.start, + reportingPeriodEnd: input.reportingPeriod.end, + status: "queued", + reviewPayload: null, + reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + evidenceSnapshot: input.evidenceSnapshot, + evidenceSchemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + modelMetadata: null, + createdByActorId: input.createdByActorId, + queuedAt: new Date(), + updatedAt: new Date(), + }) + .onConflictDoNothing({ + target: [ + founderWeeklyReviewRuns.companyId, + founderWeeklyReviewRuns.requestKey, + ], + }) + .returning(); + + if (inserted) { + return mapRunRow(inserted); + } + + const existing = await this.getByCompanyAndRequestKey( + input.companyId, + input.requestKey + ); + if (!existing) { + throw new Error("Failed to create or retrieve founder weekly review run"); + } + return existing; + } + + async getByCompanyAndRunId( + companyId: bigint, + runId: string + ): Promise { + const [row] = await this.db + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId) + ) + ) + .limit(1); + + return row ? mapRunRow(row) : null; + } + + async getByCompanyAndRequestKey( + companyId: bigint, + requestKey: string + ): Promise { + const [row] = await this.db + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.requestKey, requestKey) + ) + ) + .limit(1); + + return row ? mapRunRow(row) : null; + } + + async listByCompany(companyId: bigint): Promise { + const rows = await this.db + .select() + .from(founderWeeklyReviewRuns) + .where(eq(founderWeeklyReviewRuns.companyId, companyId)) + .orderBy(desc(founderWeeklyReviewRuns.createdAt), desc(founderWeeklyReviewRuns.id)); + + return rows.map(mapRunRow); + } + + async updateDraftConditionally( + companyId: bigint, + runId: string, + reviewPayload: FounderWeeklyReviewPayload + ): Promise { + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + reviewPayload, + reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + updatedAt: new Date(), + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId), + eq(founderWeeklyReviewRuns.status, "draft") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(companyId, runId), + }; + } + + async publishConditionally( + companyId: bigint, + runId: string + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "published", + publishedAt: now, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId), + eq(founderWeeklyReviewRuns.status, "draft") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(companyId, runId), + }; + } + + async claimQueuedRun( + input: FounderWeeklyReviewClaimInput + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "generating", + generationAttempt: sql`${founderWeeklyReviewRuns.generationAttempt} + 1`, + generationClaimId: input.generationClaimId, + generationJobId: input.generationJobId ?? null, + claimedAt: now, + generationStartedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "queued") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async saveGeneratedDraftWithClaim( + input: FounderWeeklyReviewClaimInput, + reviewPayload: FounderWeeklyReviewPayload, + modelMetadata: FounderWeeklyReviewModelMetadata | null + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "draft", + reviewPayload, + reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + modelMetadata, + generatedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "generating"), + eq(founderWeeklyReviewRuns.generationClaimId, input.generationClaimId) + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async markGenerationFailedWithClaim( + input: FounderWeeklyReviewClaimInput, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "failed", + failureSequence: sql`${founderWeeklyReviewRuns.failureSequence} + 1`, + errorCode: failure.errorCode, + errorMessage: truncateErrorMessage(failure.errorMessage), + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "generating"), + eq(founderWeeklyReviewRuns.generationClaimId, input.generationClaimId) + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async markQueuedRunFailed( + companyId: bigint, + runId: string, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "failed", + failureSequence: sql`${founderWeeklyReviewRuns.failureSequence} + 1`, + errorCode: failure.errorCode, + errorMessage: truncateErrorMessage(failure.errorMessage), + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId), + eq(founderWeeklyReviewRuns.status, "queued") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(companyId, runId), + }; + } + + async retryFailedRun( + input: FounderWeeklyReviewRetryInput + ): Promise { + return this.db.transaction(async (tx) => { + const [currentRow] = await tx + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId) + ) + ) + .limit(1); + + if (!currentRow) { + return { + outcome: "not_found", + run: null, + operation: null, + }; + } + + const [existingOperationRow] = await tx + .select() + .from(founderWeeklyReviewOperations) + .where( + and( + eq(founderWeeklyReviewOperations.companyId, input.companyId), + eq(founderWeeklyReviewOperations.runId, input.runId), + eq(founderWeeklyReviewOperations.operationType, "retry"), + eq(founderWeeklyReviewOperations.requestKey, input.requestKey) + ) + ) + .limit(1); + + const currentRun = mapRunRow(currentRow); + + if (existingOperationRow) { + const operation = mapOperationRow(existingOperationRow); + if (operation.sourceFailureSequence !== currentRun.failureSequence) { + return { + outcome: "conflict", + run: currentRun, + operation, + }; + } + + return { + outcome: "idempotent", + run: currentRun, + operation, + }; + } + + if (currentRun.status !== "failed") { + return { + outcome: "conflict", + run: currentRun, + operation: null, + }; + } + + const [insertedOperationRow] = await tx + .insert(founderWeeklyReviewOperations) + .values({ + id: input.operationId, + runId: input.runId, + companyId: input.companyId, + operationType: "retry", + requestKey: input.requestKey, + sourceFailureSequence: currentRun.failureSequence, + actorId: input.actorId, + }) + .onConflictDoNothing({ + target: [ + founderWeeklyReviewOperations.runId, + founderWeeklyReviewOperations.operationType, + founderWeeklyReviewOperations.requestKey, + ], + }) + .returning(); + + if (!insertedOperationRow) { + const [conflictingOperationRow] = await tx + .select() + .from(founderWeeklyReviewOperations) + .where( + and( + eq(founderWeeklyReviewOperations.companyId, input.companyId), + eq(founderWeeklyReviewOperations.runId, input.runId), + eq(founderWeeklyReviewOperations.operationType, "retry"), + eq(founderWeeklyReviewOperations.requestKey, input.requestKey) + ) + ) + .limit(1); + + return { + outcome: + conflictingOperationRow && + conflictingOperationRow.sourceFailureSequence === + currentRun.failureSequence + ? "idempotent" + : "conflict", + run: await this.getRunInsideTransaction(tx, input.companyId, input.runId), + operation: conflictingOperationRow + ? mapOperationRow(conflictingOperationRow) + : null, + }; + } + + const now = new Date(); + const [updatedRow] = await tx + .update(founderWeeklyReviewRuns) + .set({ + status: "queued", + retryCount: sql`${founderWeeklyReviewRuns.retryCount} + 1`, + generationClaimId: null, + generationJobId: null, + claimedAt: null, + generationStartedAt: null, + generatedAt: null, + queuedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "failed"), + eq( + founderWeeklyReviewRuns.failureSequence, + currentRun.failureSequence + ) + ) + ) + .returning(); + + if (!updatedRow) { + return { + outcome: "conflict", + run: await this.getRunInsideTransaction(tx, input.companyId, input.runId), + operation: mapOperationRow(insertedOperationRow), + }; + } + + return { + outcome: "updated", + run: mapRunRow(updatedRow), + operation: mapOperationRow(insertedOperationRow), + }; + }); + } + + private async getRunInsideTransaction( + tx: Pick, + companyId: bigint, + runId: string + ): Promise { + const [row] = await tx + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId) + ) + ) + .limit(1); + return row ? mapRunRow(row) : null; + } +} diff --git a/packages/features/src/founder-weekly-review/user-service.ts b/packages/features/src/founder-weekly-review/user-service.ts new file mode 100644 index 000000000..2caeb899f --- /dev/null +++ b/packages/features/src/founder-weekly-review/user-service.ts @@ -0,0 +1,183 @@ +import { randomUUID } from "node:crypto"; +import { ZodError } from "zod"; + +import { + buildFounderWeeklyReviewActorId, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewRunRecord, + type FounderWeeklyReviewUserActor, + parseFounderWeeklyReviewEvidenceSnapshot, + parseFounderWeeklyReviewPayload, +} from "./contracts"; +import { + FounderWeeklyReviewConflictError, + FounderWeeklyReviewForbiddenError, + FounderWeeklyReviewInvalidPayloadError, + FounderWeeklyReviewInvalidTransitionError, + FounderWeeklyReviewNotFoundError, +} from "./errors"; +import { FounderWeeklyReviewRepository } from "./repository"; + +const ALLOWED_WORKSPACE_ROLES = new Set(["owner", "admin", "editor"]); + +export interface CreateFounderWeeklyReviewRunRequest { + requestKey: string; + reportingPeriod: { + start: string; + end: string; + }; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; +} + +function assertWorkspaceMutationRole(role: string): void { + if (!ALLOWED_WORKSPACE_ROLES.has(role)) { + throw new FounderWeeklyReviewForbiddenError(); + } +} + +function assertReportingPeriodMatchesSnapshot( + reportingPeriod: { start: string; end: string }, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): void { + if ( + reportingPeriod.start !== evidenceSnapshot.reportingPeriod.start || + reportingPeriod.end !== evidenceSnapshot.reportingPeriod.end + ) { + throw new FounderWeeklyReviewInvalidPayloadError( + "Evidence snapshot reporting period must match the run reporting period." + ); + } +} + +export class FounderWeeklyReviewUserService { + constructor( + private readonly repository = new FounderWeeklyReviewRepository() + ) {} + + async createOrGetRun( + actor: FounderWeeklyReviewUserActor, + input: CreateFounderWeeklyReviewRunRequest + ): Promise { + assertWorkspaceMutationRole(actor.role); + let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + try { + evidenceSnapshot = parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot); + } catch (error) { + if (error instanceof ZodError) { + throw new FounderWeeklyReviewInvalidPayloadError(error.message); + } + throw error; + } + assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); + + return this.repository.createOrGetByRequestKey({ + id: `fwr_${randomUUID()}`, + companyId: actor.companyId, + requestKey: input.requestKey, + reportingPeriod: input.reportingPeriod, + evidenceSnapshot, + createdByActorId: buildFounderWeeklyReviewActorId(actor), + }); + } + + async getRun( + actor: FounderWeeklyReviewUserActor, + runId: string + ): Promise { + const run = await this.repository.getByCompanyAndRunId(actor.companyId, runId); + if (!run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + return run; + } + + async listRuns(actor: FounderWeeklyReviewUserActor): Promise { + return this.repository.listByCompany(actor.companyId); + } + + async retryFailedRun( + actor: FounderWeeklyReviewUserActor, + runId: string, + requestKey: string + ): Promise { + assertWorkspaceMutationRole(actor.role); + const result = await this.repository.retryFailedRun({ + operationId: `fwrop_${randomUUID()}`, + companyId: actor.companyId, + runId, + requestKey, + actorId: buildFounderWeeklyReviewActorId(actor), + }); + + if (result.outcome === "not_found" || !result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + + if (result.outcome === "updated" || result.outcome === "idempotent") { + if (result.run.status === "failed" || result.run.status === "queued") { + return result.run; + } + } + + if (result.run.status !== "failed") { + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "retry"); + } + + throw new FounderWeeklyReviewConflictError( + `Retry request key "${requestKey}" belongs to a different failure cycle for run "${runId}".` + ); + } + + async updateDraft( + actor: FounderWeeklyReviewUserActor, + runId: string, + reviewPayload: FounderWeeklyReviewPayload + ): Promise { + assertWorkspaceMutationRole(actor.role); + let payload: FounderWeeklyReviewPayload; + try { + payload = parseFounderWeeklyReviewPayload(reviewPayload); + } catch (error) { + if (error instanceof ZodError) { + throw new FounderWeeklyReviewInvalidPayloadError(error.message); + } + throw error; + } + const result = await this.repository.updateDraftConditionally( + actor.companyId, + runId, + payload + ); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + if (result.updated) { + return result.run; + } + if (result.run.status === "published") { + throw new FounderWeeklyReviewInvalidTransitionError( + result.run.status, + "edit" + ); + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "edit"); + } + + async publishDraft( + actor: FounderWeeklyReviewUserActor, + runId: string + ): Promise { + assertWorkspaceMutationRole(actor.role); + const result = await this.repository.publishConditionally(actor.companyId, runId); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + if (result.updated || result.run.status === "published") { + return result.run; + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "publish"); + } +} diff --git a/packages/features/src/founder-weekly-review/worker-service.ts b/packages/features/src/founder-weekly-review/worker-service.ts new file mode 100644 index 000000000..f8f1c6fc7 --- /dev/null +++ b/packages/features/src/founder-weekly-review/worker-service.ts @@ -0,0 +1,128 @@ +import { ZodError } from "zod"; + +import { + type FounderWeeklyReviewClaimInput, + type FounderWeeklyReviewGenerationFailure, + type FounderWeeklyReviewModelMetadata, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewRunRecord, + parseFounderWeeklyReviewModelMetadata, + parseFounderWeeklyReviewPayload, +} from "./contracts"; +import { + FounderWeeklyReviewClaimOwnershipMismatchError, + FounderWeeklyReviewConflictError, + FounderWeeklyReviewInvalidPayloadError, + FounderWeeklyReviewInvalidTransitionError, + FounderWeeklyReviewNotFoundError, +} from "./errors"; +import { FounderWeeklyReviewRepository } from "./repository"; + +export interface FounderWeeklyReviewWorkerContext extends FounderWeeklyReviewClaimInput {} + +export class FounderWeeklyReviewWorkerService { + constructor( + private readonly repository = new FounderWeeklyReviewRepository() + ) {} + + async claimQueuedRun( + context: FounderWeeklyReviewWorkerContext + ): Promise { + const result = await this.repository.claimQueuedRun(context); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(context.runId); + } + if (result.updated) { + return result.run; + } + if ( + result.run.status === "generating" && + result.run.generationClaimId === context.generationClaimId + ) { + return result.run; + } + throw new FounderWeeklyReviewConflictError( + `Founder weekly review run "${context.runId}" is already owned by another generation claim or moved out of queue.` + ); + } + + async saveGeneratedDraft( + context: FounderWeeklyReviewWorkerContext, + reviewPayload: FounderWeeklyReviewPayload, + modelMetadata: FounderWeeklyReviewModelMetadata | null + ): Promise { + let payload: FounderWeeklyReviewPayload; + let metadata: FounderWeeklyReviewModelMetadata | null; + try { + payload = parseFounderWeeklyReviewPayload(reviewPayload); + metadata = modelMetadata + ? parseFounderWeeklyReviewModelMetadata(modelMetadata) + : null; + } catch (error) { + if (error instanceof ZodError) { + throw new FounderWeeklyReviewInvalidPayloadError(error.message); + } + throw error; + } + const result = await this.repository.saveGeneratedDraftWithClaim( + context, + payload, + metadata + ); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(context.runId); + } + if (result.updated) { + return result.run; + } + if (result.run.generationClaimId !== context.generationClaimId) { + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "save draft"); + } + + async markGenerationFailed( + context: FounderWeeklyReviewWorkerContext, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const result = await this.repository.markGenerationFailedWithClaim( + context, + failure + ); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(context.runId); + } + if (result.updated) { + return result.run; + } + if ( + result.run.status === "failed" && + result.run.generationClaimId === context.generationClaimId + ) { + return result.run; + } + if (result.run.generationClaimId !== context.generationClaimId) { + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "mark failed"); + } + + async markQueuedRunFailed( + companyId: bigint, + runId: string, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const result = await this.repository.markQueuedRunFailed(companyId, runId, failure); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + if (result.updated) { + return result.run; + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "mark failed"); + } +} diff --git a/packages/features/src/index.ts b/packages/features/src/index.ts index cb0ff5c3b..d9a0fe4d5 100644 --- a/packages/features/src/index.ts +++ b/packages/features/src/index.ts @@ -1 +1 @@ -export {}; +export * from "./founder-weekly-review"; From 587941e62a8add38311327b6dbd784c4840c5e03 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Wed, 22 Jul 2026 09:30:34 +0800 Subject: [PATCH 02/29] feat: extend founder review evidence source types --- .../lifecycle.integration.test.ts | 13 ++++++++++--- .../features/src/founder-weekly-review/contracts.ts | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts index e3410b511..5767f3b06 100644 --- a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts @@ -4,6 +4,7 @@ import { sql } from "drizzle-orm"; import { company } from "@launchstack/core/db/schema"; import { FounderWeeklyReviewConflictError, + FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewInvalidPayloadError, FounderWeeklyReviewInvalidTransitionError, FounderWeeklyReviewNotFoundError, @@ -22,7 +23,7 @@ const describeIfDatabase = : describe.skip; function createEvidenceSnapshot(): FounderWeeklyReviewEvidenceSnapshot { - return { + return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-18T10:00:00.000Z", reportingPeriod: { @@ -32,7 +33,7 @@ function createEvidenceSnapshot(): FounderWeeklyReviewEvidenceSnapshot { workspaceTimezone: "America/Los_Angeles", items: [ { - sourceType: "workspace_document", + sourceType: "document_change", sourceId: "doc-1", title: "Weekly Product Notes", sourceTimestamp: "2026-07-10T18:15:00.000Z", @@ -53,6 +54,12 @@ function createEvidenceSnapshot(): FounderWeeklyReviewEvidenceSnapshot { sentiment: "mixed", }, }, + { + sourceType: "founder_context", + sourceId: "founder-context-1", + title: "Founder weekly context", + excerpt: "Manual founder input: enterprise onboarding remains blocked on SSO setup.", + }, ], sourceWarnings: [ { @@ -61,7 +68,7 @@ function createEvidenceSnapshot(): FounderWeeklyReviewEvidenceSnapshot { sourceType: "github_activity", }, ], - }; + }); } function createPayload(seed: string): FounderWeeklyReviewPayload { diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts index 9ad0f1b59..2fddd9481 100644 --- a/packages/features/src/founder-weekly-review/contracts.ts +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -47,9 +47,11 @@ const SerializableMetadataValueSchema: z.ZodType< export const FounderWeeklyReviewEvidenceItemSchema = z.object({ sourceType: z.enum([ "workspace_document", + "document_change", "customer_feedback", "github_activity", "manual_note", + "founder_context", "other", ]), sourceId: z.string().min(1).max(256), From 12ac6dbe4dcc62ff51c86307f57d05dfb2d5638f Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Wed, 22 Jul 2026 15:44:40 +0800 Subject: [PATCH 03/29] feat: add structured founder weekly review generation --- .../founderWeeklyReview/generation.test.ts | 202 ++++++++++++++++++ .../lifecycle.integration.test.ts | 117 +++++++++- apps/web/config/llm-models.json | 18 ++ apps/web/src/lib/llm/config.ts | 6 + apps/web/src/lib/llm/generate.ts | 37 +++- apps/web/src/lib/llm/index.ts | 4 +- apps/web/src/lib/llm/types.ts | 18 +- .../generation-adapter.ts | 25 +++ .../src/founder-weekly-review/README.md | 14 +- .../src/founder-weekly-review/contracts.ts | 99 ++++++++- .../generation-validation.ts | 103 +++++++++ .../src/founder-weekly-review/generator.ts | 126 +++++++++++ .../src/founder-weekly-review/index.ts | 3 + .../src/founder-weekly-review/prompts.ts | 61 ++++++ .../src/founder-weekly-review/repository.ts | 20 +- 15 files changed, 839 insertions(+), 14 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/generation.test.ts create mode 100644 apps/web/src/server/founder-weekly-review/generation-adapter.ts create mode 100644 packages/features/src/founder-weekly-review/generation-validation.ts create mode 100644 packages/features/src/founder-weekly-review/generator.ts create mode 100644 packages/features/src/founder-weekly-review/prompts.ts diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts new file mode 100644 index 000000000..f6ec1985d --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -0,0 +1,202 @@ +jest.mock("~/lib/llm", () => ({ + generateStructuredWithMetadata: jest.fn(), +})); + +import { + FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + FounderWeeklyReviewV2PayloadSchema, + buildFounderWeeklyReviewPrompt, + generateFounderWeeklyReview, + parseFounderWeeklyReviewPayload, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewV2Payload, +} from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewGenerationValidationError } from "@launchstack/features/founder-weekly-review"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { generateStructuredWithMetadata } from "~/lib/llm"; + +const mockGenerateStructuredWithMetadata = generateStructuredWithMetadata as jest.Mock; + +function snapshot(items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderWeeklyReviewEvidenceSnapshot { + return { + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-18T10:00:00.000Z", + reportingPeriod: { start: "2026-07-07", end: "2026-07-13" }, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], + }; +} + +const source = (sourceId: string, sourceType: FounderWeeklyReviewEvidenceSnapshot["items"][number]["sourceType"], excerpt = "Evidence excerpt") => ({ + sourceId, + sourceType, + title: `${sourceType} title`, + excerpt, + metadata: {}, +}); + +function noEvidence(message = "No evidence", cta = "Add evidence") { + return { state: "no_evidence" as const, noEvidence: { code: "no_relevant_evidence", message, cta } }; +} + +function validPayload(): FounderWeeklyReviewV2Payload { + return { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "The plan changed.", sourceIds: ["doc-1"], confidence: 0 }] }, + whatShipped: { state: "evidence", items: [{ kind: "observed_fact", text: "A release shipped.", sourceIds: ["doc-1"], confidence: 1 }] }, + whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "A customer requested audit logs.", sourceIds: ["feedback-1"], confidence: 0.5 }] }, + currentBlockers: { state: "evidence", items: [{ kind: "observed_fact", text: "SSO remains blocked.", sourceIds: ["context-1"], confidence: 0.8 }] }, + nextPriorities: { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Prioritize SSO.", sourceIds: ["context-1"], confidence: 0.8 }] }, + }, + }; +} + +function fake(object: unknown, metadata = { provider: "openai", model: "test-model", capability: "founderWeeklyReview", temperature: 0 }) { + return jest.fn().mockResolvedValue({ object, metadata }); +} + +const completeSnapshot = () => snapshot([ + source("doc-1", "document_change", "A release shipped."), + source("feedback-1", "customer_feedback", "Please add audit logs."), + source("context-1", "founder_context", "SSO remains blocked."), +]); + +describe("Founder Weekly Review generation", () => { + it("generates complete evidence with numeric lower and upper confidence bounds", async () => { + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(validPayload()) }); + expect(result.reviewPayload).toEqual(validPayload()); + expect(result.modelMetadata).toMatchObject({ provider: "openai", temperature: 0, capability: "founderWeeklyReview" }); + expect(result.modelMetadata.promptHash).toMatch(/^[a-f0-9]{64}$/); + }); + + it("allows partial reviews with typed no-evidence sections", async () => { + const payload = validPayload(); + payload.sections.whatShipped = noEvidence(); + payload.sections.whatCustomersSaid = noEvidence(); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) }); + expect(result.reviewPayload.sections.whatShipped.state).toBe("no_evidence"); + }); + + it("preserves contradictory evidence rather than resolving it", async () => { + const payload = validPayload(); + payload.sections.currentBlockers = { state: "evidence", items: [{ kind: "contradictory_evidence", text: "The blocker status conflicts.", sourceIds: ["doc-1", "context-1"], confidence: 0.6 }] }; + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) }); + expect(result.reviewPayload.sections.currentBlockers).toMatchObject({ state: "evidence", items: [{ kind: "contradictory_evidence" }] }); + }); + + it("returns deterministic empty states and performs zero LLM calls for an empty snapshot", async () => { + const generate = fake(validPayload()); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: snapshot([]), generate }); + expect(generate).not.toHaveBeenCalled(); + expect(result.reviewPayload.sections.nextPriorities).toMatchObject({ state: "no_evidence" }); + expect(result.modelMetadata).toMatchObject({ provider: "skipped", model: "none", attributes: { generationSkipped: true } }); + for (const section of Object.values(result.reviewPayload.sections)) { + expect(section.state).toBe("no_evidence"); + if (section.state === "no_evidence") { + expect(section.noEvidence.cta.trim()).not.toBe(""); + } + } + }); + + it.each([ + ["unknown generated source ID", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["missing"], confidence: 0.5 }] }; }], + ["duplicate generated citations", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1", "doc-1"], confidence: 0.5 }] }; }], + ["customer section citing document_change", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }] }; }], + ["customer section citing founder_context", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["context-1"], confidence: 0.5 }] }; }], + ["contradiction with fewer than two citations", (p: FounderWeeklyReviewV2Payload) => { p.sections.currentBlockers = { state: "evidence", items: [{ kind: "contradictory_evidence", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }] }; }], + ["recommendation outside nextPriorities", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatChanged = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }] } as never; }], + ])("rejects %s", async (_name, mutate) => { + const payload = validPayload(); + mutate(payload); + await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) })).rejects.toBeInstanceOf(Error); + }); + + it("rejects duplicate input source IDs before calling the model", async () => { + const generate = fake(validPayload()); + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([source("same", "manual_note"), source("same", "document_change")]), generate })).rejects.toBeInstanceOf(FounderWeeklyReviewGenerationValidationError); + expect(generate).not.toHaveBeenCalled(); + }); + + it("enforces numeric confidence bounds", () => { + const payload = validPayload(); + payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: -0.01 }] }; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + (payload.sections.whatChanged as { state: "evidence"; items: Array<{ confidence: number }> }).items[0]!.confidence = 1.01; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + }); + + it("rejects mixed v2 section states instead of normalizing them", () => { + const payload = validPayload(); + payload.sections.whatChanged = { + state: "evidence", + items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }], + noEvidence: { code: "no_relevant_evidence", message: "x", cta: "x" }, + } as never; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + + payload.sections.whatChanged = { + state: "no_evidence", + noEvidence: { code: "no_relevant_evidence", message: "x", cta: "x" }, + items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }], + } as never; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + }); + + it("rejects observed facts in nextPriorities", () => { + const payload = validPayload(); + payload.sections.nextPriorities = { + state: "evidence", + items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }], + } as never; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + }); + + it("does not allow source warnings to be cited", async () => { + const evidenceSnapshot = completeSnapshot(); + evidenceSnapshot.sourceWarnings = [{ code: "warning-1", message: "Missing source", sourceType: "github_activity" }]; + const payload = validPayload(); + payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["warning-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot, generate: fake(payload) })).rejects.toBeInstanceOf(Error); + }); + + it("puts anti-invention requirements in the prompt and hashes a fixed fixture stably", async () => { + const generateA = fake(validPayload()); + const generateB = fake(validPayload()); + const first = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: generateA }); + const second = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: generateB }); + expect(generateA.mock.calls[0][0].system).toContain("Never invent"); + expect(generateA.mock.calls[0][0].system).toContain("whatCustomersSaid may cite only customer_feedback"); + for (const prohibitedFact of ["customers", "people", "dates", "metrics", "shipped work", "blockers", "outcomes", "source IDs"]) { + expect(generateA.mock.calls[0][0].system).toContain(prohibitedFact); + } + expect(first.modelMetadata.promptHash).toBe(second.modelMetadata.promptHash); + }); + + it("canonicalizes metadata key order before building the prompt and hash", async () => { + const firstSnapshot = completeSnapshot(); + firstSnapshot.items[0]!.metadata = { alpha: "a", beta: "b" }; + const secondSnapshot = completeSnapshot(); + secondSnapshot.items[0]!.metadata = { beta: "b", alpha: "a" }; + expect(buildFounderWeeklyReviewPrompt(firstSnapshot)).toBe(buildFounderWeeklyReviewPrompt(secondSnapshot)); + + const first = await generateFounderWeeklyReview({ evidenceSnapshot: firstSnapshot, generate: fake(validPayload()) }); + const second = await generateFounderWeeklyReview({ evidenceSnapshot: secondSnapshot, generate: fake(validPayload()) }); + expect(first.modelMetadata.promptHash).toBe(second.modelMetadata.promptHash); + }); + + it("adapts the configured web abstraction and returns its metadata", async () => { + mockGenerateStructuredWithMetadata.mockResolvedValue({ object: { ok: true }, metadata: { provider: "openai", model: "adapter-model", capability: "founderWeeklyReview", temperature: 0 } }); + const schema = require("zod").z.object({ ok: require("zod").z.boolean() }); + await expect(generateFounderWeeklyReviewStructured({ prompt: "p", schema })).resolves.toMatchObject({ metadata: { model: "adapter-model" } }); + expect(mockGenerateStructuredWithMetadata).toHaveBeenCalledWith(expect.objectContaining({ capability: "founderWeeklyReview" })); + }); + + it("continues to parse legacy v1 payloads", () => { + expect(parseFounderWeeklyReviewPayload({ + schemaVersion: "founder-weekly-review/v1", + sections: Object.fromEntries(["whatChanged", "whatShipped", "whatCustomersSaid", "currentBlockers", "nextPriorities"].map((key) => [key, { heading: key, items: [{ kind: "no_evidence", code: "not_assessed" }] }])), + }).schemaVersion).toBe("founder-weekly-review/v1"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts index 5767f3b06..43dcf3c03 100644 --- a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts @@ -13,6 +13,7 @@ import { FounderWeeklyReviewWorkerService, type FounderWeeklyReviewEvidenceSnapshot, type FounderWeeklyReviewPayload, + type FounderWeeklyReviewV2Payload, } from "@launchstack/features/founder-weekly-review"; import { createFounderWeeklyReviewTestDatabase } from "./testDb"; @@ -97,6 +98,60 @@ function createPayload(seed: string): FounderWeeklyReviewPayload { }; } +function createV2Payload(): FounderWeeklyReviewV2Payload { + return { + schemaVersion: "founder-weekly-review/v2", + sections: { + whatChanged: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Billing exports changed.", + sourceIds: ["doc-1"], + confidence: 0.9, + }], + }, + whatShipped: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Billing exports shipped.", + sourceIds: ["doc-1"], + confidence: 0.9, + }], + }, + whatCustomersSaid: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Prospects requested audit logging.", + sourceIds: ["feedback-1"], + confidence: 0.8, + }], + }, + currentBlockers: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "SSO setup remains blocked.", + sourceIds: ["founder-context-1"], + confidence: 0.8, + }], + }, + nextPriorities: { + state: "evidence", + items: [{ + kind: "recommendation", + label: "Recommendation", + text: "Prioritize SSO setup.", + sourceIds: ["founder-context-1"], + confidence: 0.8, + }], + }, + }, + }; +} + async function insertCompany( db: Awaited>["db"], name: string @@ -260,7 +315,16 @@ describeIfDatabase("Founder Weekly Review lifecycle integration", () => { createPayload("draft-2") ); expect(editedDraft.status).toBe("draft"); - expect(editedDraft.reviewPayload?.sections.whatChanged.items[0]).toMatchObject({ + expect(editedDraft.reviewPayload?.schemaVersion).toBe( + "founder-weekly-review/v1" + ); + if ( + !editedDraft.reviewPayload || + editedDraft.reviewPayload.schemaVersion !== "founder-weekly-review/v1" + ) { + throw new Error("Expected v1 draft payload"); + } + expect(editedDraft.reviewPayload.sections.whatChanged.items[0]).toMatchObject({ text: "Observed fact draft-2", }); @@ -449,4 +513,55 @@ describeIfDatabase("Founder Weekly Review lifecycle integration", () => { await testDb.close(); } }); + + it("round-trips v2 drafts and rejects mismatched persisted schema versions", async () => { + const testDb = await createFounderWeeklyReviewTestDatabase(); + try { + const companyId = await insertCompany(testDb.db, "V2 Review Co"); + const repository = new FounderWeeklyReviewRepository(testDb.db); + const userService = new FounderWeeklyReviewUserService(repository); + const worker = new FounderWeeklyReviewWorkerService(repository); + const actor = { + externalUserId: "clerk_v2", + companyId, + role: "owner", + }; + const evidenceSnapshot = createEvidenceSnapshot(); + const run = await userService.createOrGetRun(actor, { + requestKey: "v2-round-trip", + reportingPeriod: evidenceSnapshot.reportingPeriod, + evidenceSnapshot, + }); + const claim = { companyId, runId: run.id, generationClaimId: "v2-claim" }; + await worker.claimQueuedRun(claim); + + const payload = createV2Payload(); + const saved = await worker.saveGeneratedDraft(claim, payload, { + provider: "openai", + model: "gpt-4o-mini", + capability: "founderWeeklyReview", + temperature: 0, + attributes: {}, + }); + expect(saved.reviewPayload).toEqual(payload); + expect(saved.reviewPayload?.schemaVersion).toBe("founder-weekly-review/v2"); + expect(saved.reviewSchemaVersion).toBe("founder-weekly-review/v2"); + + const reread = await repository.getByCompanyAndRunId(companyId, run.id); + expect(reread?.reviewPayload).toEqual(payload); + expect(reread?.reviewPayload?.schemaVersion).toBe("founder-weekly-review/v2"); + expect(reread?.reviewSchemaVersion).toBe("founder-weekly-review/v2"); + + await testDb.db.execute(sql` + UPDATE "pdr_ai_v2_founder_weekly_review_runs" + SET "review_schema_version" = 'founder-weekly-review/v1' + WHERE "id" = ${run.id} + `); + await expect(repository.getByCompanyAndRunId(companyId, run.id)).rejects.toBeInstanceOf( + FounderWeeklyReviewInvalidPayloadError + ); + } finally { + await testDb.close(); + } + }); }); diff --git a/apps/web/config/llm-models.json b/apps/web/config/llm-models.json index 42b8f90f9..e1fd1b50e 100644 --- a/apps/web/config/llm-models.json +++ b/apps/web/config/llm-models.json @@ -19,6 +19,24 @@ "model": "llama3.2:3b", "temperature": 0 } + }, + "founderWeeklyReview": { + "openai": { + "model": "gpt-4o-mini", + "temperature": 0 + }, + "anthropic": { + "model": "claude-3-5-haiku-latest", + "temperature": 0 + }, + "google": { + "model": "gemini-2.0-flash", + "temperature": 0 + }, + "ollama": { + "model": "llama3.2:3b", + "temperature": 0 + } } } } diff --git a/apps/web/src/lib/llm/config.ts b/apps/web/src/lib/llm/config.ts index 2f17b84c8..1b8b714d3 100644 --- a/apps/web/src/lib/llm/config.ts +++ b/apps/web/src/lib/llm/config.ts @@ -75,6 +75,12 @@ const HARDCODED_DEFAULTS: LlmConfig = { google: { model: "gemini-2.0-flash", temperature: 0 }, ollama: { model: "llama3.2:3b", temperature: 0 }, }, + founderWeeklyReview: { + openai: { model: "gpt-4o-mini", temperature: 0 }, + anthropic: { model: "claude-3-5-haiku-latest", temperature: 0 }, + google: { model: "gemini-2.0-flash", temperature: 0 }, + ollama: { model: "llama3.2:3b", temperature: 0 }, + }, }, }; diff --git a/apps/web/src/lib/llm/generate.ts b/apps/web/src/lib/llm/generate.ts index be91ded83..09ce3f22b 100644 --- a/apps/web/src/lib/llm/generate.ts +++ b/apps/web/src/lib/llm/generate.ts @@ -15,7 +15,10 @@ import { generateObject } from "ai"; import type { ZodType } from "zod"; import { resolveModel } from "./providers"; -import type { GenerateStructuredInput } from "./types"; +import type { + GenerateStructuredInput, + StructuredGenerationResult, +} from "./types"; /** * Run a structured JSON-output LLM call against whichever provider is @@ -40,6 +43,18 @@ import type { GenerateStructuredInput } from "./types"; export async function generateStructured( input: GenerateStructuredInput, ): Promise> { + const result = await generateStructuredWithMetadata(input); + return result.object; +} + +/** + * Structured generation plus the single resolved model's reproducibility + * metadata. This is a companion API; generateStructured remains source and + * behavior compatible for existing callers. + */ +export async function generateStructuredWithMetadata( + input: GenerateStructuredInput, +): Promise>> { const resolved = resolveModel(input.capability, input.forceProvider); // Diagnostic logging: capture the chosen provider/model, prompt size, and @@ -76,7 +91,25 @@ export async function generateStructured( // when given a Zod schema. The `ZodType` generic constraint is a // pragmatic choice — it trades a small amount of type precision for // simpler call-site ergonomics. - return result.object as ReturnType; + const responseResult = result as unknown as { + finishReason?: string; + usage?: Record; + response?: { id?: string }; + }; + return { + object: result.object as ReturnType, + metadata: { + provider: resolved.provider, + model: resolved.modelId, + capability: input.capability, + temperature: resolved.temperature, + ...(responseResult.finishReason ? { finishReason: responseResult.finishReason } : {}), + ...(responseResult.usage ? { usage: responseResult.usage } : {}), + ...(responseResult.response?.id + ? { providerRequestId: responseResult.response.id } + : {}), + }, + }; } catch (err) { const elapsed = Date.now() - startedAt; console.error( diff --git a/apps/web/src/lib/llm/index.ts b/apps/web/src/lib/llm/index.ts index 1ecad9804..45a42d25a 100644 --- a/apps/web/src/lib/llm/index.ts +++ b/apps/web/src/lib/llm/index.ts @@ -11,7 +11,7 @@ * to this library or file an issue explaining why it can't be abstracted. */ -export { generateStructured } from "./generate"; +export { generateStructured, generateStructuredWithMetadata } from "./generate"; export { getAvailableProviders, resolveModel, @@ -29,4 +29,6 @@ export { type LlmConfig, type CapabilityModelConfig, type GenerateStructuredInput, + type StructuredGenerationMetadata, + type StructuredGenerationResult, } from "./types"; diff --git a/apps/web/src/lib/llm/types.ts b/apps/web/src/lib/llm/types.ts index 62f3a4085..15f7efae3 100644 --- a/apps/web/src/lib/llm/types.ts +++ b/apps/web/src/lib/llm/types.ts @@ -22,7 +22,7 @@ * Must support JSON schema output. Does NOT need vision, long context, * or high-quality free-form generation. */ -export const CAPABILITIES = ["smallExtraction"] as const; +export const CAPABILITIES = ["smallExtraction", "founderWeeklyReview"] as const; export type Capability = (typeof CAPABILITIES)[number]; @@ -105,3 +105,19 @@ export interface GenerateStructuredInput { */ schemaName?: string; } + +/** Resolved model and provider response details that are safe to persist for replay. */ +export interface StructuredGenerationMetadata { + provider: Provider; + model: string; + capability: Capability; + temperature: number; + finishReason?: string; + usage?: Record; + providerRequestId?: string; +} + +export interface StructuredGenerationResult { + object: T; + metadata: StructuredGenerationMetadata; +} diff --git a/apps/web/src/server/founder-weekly-review/generation-adapter.ts b/apps/web/src/server/founder-weekly-review/generation-adapter.ts new file mode 100644 index 000000000..bd032f5d4 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/generation-adapter.ts @@ -0,0 +1,25 @@ +import type { ZodType } from "zod"; + +import { + generateStructuredWithMetadata, + type StructuredGenerationMetadata, +} from "~/lib/llm"; + +/** + * Host adapter for LAU-7. The feature receives this function as an injected + * dependency and therefore remains independent of providers and web code. + */ +export function generateFounderWeeklyReviewStructured(input: { + system?: string; + prompt: string; + schema: TSchema; + schemaName?: string; +}): Promise<{ + object: ReturnType; + metadata: StructuredGenerationMetadata; +}> { + return generateStructuredWithMetadata({ + ...input, + capability: "founderWeeklyReview", + }); +} diff --git a/packages/features/src/founder-weekly-review/README.md b/packages/features/src/founder-weekly-review/README.md index 50c3d7477..bff08aa8c 100644 --- a/packages/features/src/founder-weekly-review/README.md +++ b/packages/features/src/founder-weekly-review/README.md @@ -41,6 +41,15 @@ This module owns LAU-5 persistence and lifecycle foundations for company-scoped - `markGenerationFailed` - `markQueuedRunFailed` +## LAU-7 confidence semantics + +V2 generated items use numeric `confidence` from 0 through 1. It measures how +strongly the generated review claim is supported by its cited supplied evidence. +It is not a confidence score for the truthfulness or reliability of an +underlying database row or source record. Unsupported claims must be omitted or +represented as `no_evidence`; they must not be emitted with an arbitrarily low +confidence value. + ## Company isolation Every repository method accepts `companyId` explicitly and includes it in SQL predicates. Wrong-company access resolves as not found rather than leaking existence. @@ -57,7 +66,10 @@ Multiple runs are allowed for the same company and reporting period. LAU-5 does ## Deferred scope - LAU-6: evidence collection -- LAU-7: review generation payload population +- LAU-7: structured review generation is provider-agnostic and consumes the + canonical evidence snapshot. It emits `founder-weekly-review/v2`; v1 payloads + remain readable for backwards compatibility. Citation validity is checked + against the supplied snapshot after structured-output parsing. - LAU-8: workflow orchestration - LAU-9: HTTP APIs - LAU-10: dashboard/UI diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts index 2fddd9481..2d9c47722 100644 --- a/packages/features/src/founder-weekly-review/contracts.ts +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -4,6 +4,8 @@ export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION = "founder-weekly-review-evidence/v1" as const; export const FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION = "founder-weekly-review/v1" as const; +export const FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION = + "founder-weekly-review/v2" as const; export const FounderWeeklyReviewStatusSchema = z.enum([ "queued", @@ -125,7 +127,11 @@ const FounderWeeklyReviewSectionSchema = z.object({ items: z.array(FounderWeeklyReviewSectionItemSchema).max(100), }); -export const FounderWeeklyReviewPayloadSchema = z.object({ +/** + * LAU-5 payload. Keep this schema byte-for-byte compatible with persisted v1 + * drafts; LAU-7 generation emits the separate v2 schema below. + */ +export const FounderWeeklyReviewV1PayloadSchema = z.object({ schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION), sections: z.object({ whatChanged: FounderWeeklyReviewSectionSchema, @@ -135,13 +141,102 @@ export const FounderWeeklyReviewPayloadSchema = z.object({ nextPriorities: FounderWeeklyReviewSectionSchema, }), }); +export type FounderWeeklyReviewV1Payload = z.infer; + +const V2ConfidenceSchema = z.number().min(0).max(1); +function v2SourceIdsSchema(minimum: number) { + return z.array(z.string().min(1).max(256)).min(minimum).max(20).superRefine( + (sourceIds, context) => { + if (new Set(sourceIds).size !== sourceIds.length) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "sourceIds must be unique" }); + } + } + ); +} + +export const FounderWeeklyReviewV2ObservedFactSchema = z.object({ + kind: z.literal("observed_fact"), + text: z.string().min(1).max(2000), + sourceIds: v2SourceIdsSchema(1), + confidence: V2ConfidenceSchema, +}).strict(); + +export const FounderWeeklyReviewV2ContradictoryEvidenceSchema = z.object({ + kind: z.literal("contradictory_evidence"), + text: z.string().min(1).max(2000), + sourceIds: v2SourceIdsSchema(2), + confidence: V2ConfidenceSchema, +}).strict(); + +export const FounderWeeklyReviewV2RecommendationSchema = z.object({ + kind: z.literal("recommendation"), + label: z.literal("Recommendation"), + text: z.string().min(1).max(2000), + rationale: z.string().min(1).max(2000).optional(), + sourceIds: v2SourceIdsSchema(1), + confidence: V2ConfidenceSchema, +}).strict(); + +export const FounderWeeklyReviewV2NoEvidenceSchema = z.object({ + code: z.string().min(1).max(64), + message: z.string().min(1).max(512), + cta: z.string().min(1).max(512), +}).strict(); + +const FounderWeeklyReviewV2NoEvidenceSectionSchema = z.object({ + state: z.literal("no_evidence"), + noEvidence: FounderWeeklyReviewV2NoEvidenceSchema, +}).strict(); + +const FounderWeeklyReviewV2FactualItemSchema = z.union([ + FounderWeeklyReviewV2ObservedFactSchema, + FounderWeeklyReviewV2ContradictoryEvidenceSchema, +]); +const FounderWeeklyReviewV2FactualSectionSchema = z.union([ + z.object({ + state: z.literal("evidence"), + items: z.array(FounderWeeklyReviewV2FactualItemSchema).min(1).max(100), + }).strict(), + FounderWeeklyReviewV2NoEvidenceSectionSchema, +]); +const FounderWeeklyReviewV2PrioritySectionSchema = z.union([ + z.object({ + state: z.literal("evidence"), + items: z.array(FounderWeeklyReviewV2RecommendationSchema).min(1).max(100), + }).strict(), + FounderWeeklyReviewV2NoEvidenceSectionSchema, +]); + +export const FounderWeeklyReviewV2PayloadSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION), + sections: z.object({ + whatChanged: FounderWeeklyReviewV2FactualSectionSchema, + whatShipped: FounderWeeklyReviewV2FactualSectionSchema, + whatCustomersSaid: FounderWeeklyReviewV2FactualSectionSchema, + currentBlockers: FounderWeeklyReviewV2FactualSectionSchema, + nextPriorities: FounderWeeklyReviewV2PrioritySectionSchema, + }).strict(), +}).strict(); +export type FounderWeeklyReviewV2Payload = z.infer; + +export const FounderWeeklyReviewPayloadSchema = z.union([ + FounderWeeklyReviewV1PayloadSchema, + FounderWeeklyReviewV2PayloadSchema, +]); export type FounderWeeklyReviewPayload = z.infer; +export type FounderWeeklyReviewPayloadSchemaVersion = + | typeof FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION + | typeof FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION; export const FounderWeeklyReviewModelMetadataSchema = z.object({ provider: z.string().min(1).max(128).optional(), model: z.string().min(1).max(256).optional(), promptVersion: z.string().min(1).max(128).optional(), temperature: z.number().finite().optional(), + capability: z.string().min(1).max(128).optional(), + promptHash: z.string().regex(/^[a-f0-9]{64}$/).optional(), + evidenceSchemaVersion: z.string().min(1).max(128).optional(), + reviewPayloadSchemaVersion: z.string().min(1).max(128).optional(), completionId: z.string().min(1).max(256).optional(), notes: z.string().min(1).max(1024).optional(), attributes: z.record(z.string().max(64), SerializableMetadataValueSchema).default({}), @@ -157,7 +252,7 @@ export interface FounderWeeklyReviewRunRecord { reportingPeriod: ReportingPeriod; status: FounderWeeklyReviewStatus; reviewPayload: FounderWeeklyReviewPayload | null; - reviewSchemaVersion: typeof FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION; + reviewSchemaVersion: FounderWeeklyReviewPayloadSchemaVersion; evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; evidenceSchemaVersion: typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION; modelMetadata: FounderWeeklyReviewModelMetadata | null; diff --git a/packages/features/src/founder-weekly-review/generation-validation.ts b/packages/features/src/founder-weekly-review/generation-validation.ts new file mode 100644 index 000000000..6e8d175d2 --- /dev/null +++ b/packages/features/src/founder-weekly-review/generation-validation.ts @@ -0,0 +1,103 @@ +import type { + FounderWeeklyReviewEvidenceSnapshot, + FounderWeeklyReviewV2Payload, +} from "./contracts"; + +export class FounderWeeklyReviewGenerationValidationError extends Error { + constructor(message: string) { + super(message); + this.name = "FounderWeeklyReviewGenerationValidationError"; + } +} + +export function assertUniqueSnapshotSourceIds( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): void { + const sourceIds = evidenceSnapshot.items.map((item) => item.sourceId); + if (new Set(sourceIds).size !== sourceIds.length) { + throw new FounderWeeklyReviewGenerationValidationError( + "Evidence snapshot contains duplicate source IDs." + ); + } +} + +export function validateFounderWeeklyReviewV2Citations( + payload: FounderWeeklyReviewV2Payload, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): FounderWeeklyReviewV2Payload { + const evidenceBySourceId = new Map( + evidenceSnapshot.items.map((item) => [item.sourceId, item]) + ); + const factualSections = [ + "whatChanged", + "whatShipped", + "whatCustomersSaid", + "currentBlockers", + ] as const; + + for (const sectionName of factualSections) { + const section = payload.sections[sectionName]; + if (section.state === "no_evidence") continue; + for (const item of section.items) { + assertCitations(item.sourceIds, evidenceBySourceId, item.kind); + if (item.kind === "contradictory_evidence" && item.sourceIds.length < 2) { + throw new FounderWeeklyReviewGenerationValidationError( + `${sectionName} contradictory_evidence must cite at least two sources.` + ); + } + if (sectionName === "whatCustomersSaid") { + for (const sourceId of item.sourceIds) { + const source = evidenceBySourceId.get(sourceId); + if (source?.sourceType === "founder_context") { + throw new FounderWeeklyReviewGenerationValidationError( + "founder_context must never be presented as customer feedback." + ); + } + if (source?.sourceType !== "customer_feedback") { + throw new FounderWeeklyReviewGenerationValidationError( + `whatCustomersSaid may cite only customer_feedback evidence; received "${sourceId}".` + ); + } + } + } + } + } + + const priorities = payload.sections.nextPriorities; + if (priorities.state === "evidence") { + for (const item of priorities.items) { + assertCitations(item.sourceIds, evidenceBySourceId, item.kind); + if (item.kind !== "recommendation") { + throw new FounderWeeklyReviewGenerationValidationError( + "nextPriorities may contain recommendations only." + ); + } + } + } + + return payload; +} + +function assertCitations( + sourceIds: readonly string[], + evidenceBySourceId: ReadonlyMap, + itemKind: string +): void { + if (sourceIds.length === 0) { + throw new FounderWeeklyReviewGenerationValidationError( + `${itemKind} must cite at least one evidence source.` + ); + } + if (new Set(sourceIds).size !== sourceIds.length) { + throw new FounderWeeklyReviewGenerationValidationError( + `${itemKind} contains duplicate source citations.` + ); + } + for (const sourceId of sourceIds) { + if (!evidenceBySourceId.has(sourceId)) { + throw new FounderWeeklyReviewGenerationValidationError( + `${itemKind} cites source ID "${sourceId}" that is absent from the evidence snapshot.` + ); + } + } +} diff --git a/packages/features/src/founder-weekly-review/generator.ts b/packages/features/src/founder-weekly-review/generator.ts new file mode 100644 index 000000000..c920aac79 --- /dev/null +++ b/packages/features/src/founder-weekly-review/generator.ts @@ -0,0 +1,126 @@ +import { createHash } from "node:crypto"; +import type { ZodType } from "zod"; + +import { + FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + FounderWeeklyReviewV2PayloadSchema, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewModelMetadata, + type FounderWeeklyReviewV2Payload, +} from "./contracts"; +import { + assertUniqueSnapshotSourceIds, + validateFounderWeeklyReviewV2Citations, +} from "./generation-validation"; +import { + buildFounderWeeklyReviewPrompt, + FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, +} from "./prompts"; + +export interface FounderWeeklyReviewResolvedGenerationMetadata { + provider: string; + model: string; + capability: string; + temperature: number; + finishReason?: string; + usage?: Record; + providerRequestId?: string; +} + +export type FounderWeeklyReviewStructuredGenerator = (input: { + system?: string; + prompt: string; + schema: TSchema; + schemaName?: string; +}) => Promise<{ + object: ReturnType; + metadata: FounderWeeklyReviewResolvedGenerationMetadata; +}>; + +export interface GenerateFounderWeeklyReviewInput { + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + generate: FounderWeeklyReviewStructuredGenerator; +} + +export interface GenerateFounderWeeklyReviewResult { + reviewPayload: FounderWeeklyReviewV2Payload; + modelMetadata: FounderWeeklyReviewModelMetadata; +} + +export async function generateFounderWeeklyReview( + input: GenerateFounderWeeklyReviewInput +): Promise { + const { evidenceSnapshot, generate } = input; + assertUniqueSnapshotSourceIds(evidenceSnapshot); + + const prompt = buildFounderWeeklyReviewPrompt(evidenceSnapshot); + const promptHash = createHash("sha256") + .update(FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT) + .update(prompt) + .digest("hex"); + + if (evidenceSnapshot.items.length === 0) { + return { + reviewPayload: buildEmptyReview(), + modelMetadata: buildMetadata( + { provider: "skipped", model: "none", capability: "founderWeeklyReview", temperature: 0 }, + promptHash, + true + ), + }; + } + + const result = await generate({ + system: FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, + prompt, + schema: FounderWeeklyReviewV2PayloadSchema, + schemaName: "founder_weekly_review_v2", + }); + const reviewPayload = validateFounderWeeklyReviewV2Citations( + FounderWeeklyReviewV2PayloadSchema.parse(result.object), + evidenceSnapshot + ); + + return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false) }; +} + +function buildMetadata( + metadata: FounderWeeklyReviewResolvedGenerationMetadata, + promptHash: string, + skipped: boolean +): FounderWeeklyReviewModelMetadata { + return { + provider: metadata.provider, + model: metadata.model, + capability: metadata.capability, + temperature: metadata.temperature, + promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + promptHash, + evidenceSchemaVersion: "founder-weekly-review-evidence/v1", + reviewPayloadSchemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + ...(metadata.providerRequestId ? { completionId: metadata.providerRequestId } : {}), + attributes: { + ...(skipped ? { generationSkipped: true } : {}), + ...(metadata.finishReason ? { finishReason: metadata.finishReason } : {}), + ...(metadata.usage ? { usage: JSON.stringify(metadata.usage) } : {}), + }, + }; +} + +function buildEmptyReview(): FounderWeeklyReviewV2Payload { + const noEvidence = (message: string, cta: string) => ({ + state: "no_evidence" as const, + noEvidence: { code: "no_relevant_evidence", message, cta }, + }); + return { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: noEvidence("No change evidence was supplied for this period.", "Add document changes or founder notes for this reporting period."), + whatShipped: noEvidence("No shipment evidence was supplied for this period.", "Add release notes, deployment records, or GitHub activity."), + whatCustomersSaid: noEvidence("No customer feedback was supplied for this period.", "Add customer calls, support feedback, or survey evidence."), + currentBlockers: noEvidence("No blocker evidence was supplied for this period.", "Add founder context, project notes, or issue evidence describing blockers."), + nextPriorities: noEvidence("No evidence is available to ground priorities.", "Add evidence for this period before requesting recommended priorities."), + }, + }; +} diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index b217ef302..b0c19516e 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -3,3 +3,6 @@ export * from "./errors"; export * from "./repository"; export * from "./user-service"; export * from "./worker-service"; +export * from "./generator"; +export * from "./generation-validation"; +export * from "./prompts"; diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts new file mode 100644 index 000000000..ee07f8a3c --- /dev/null +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -0,0 +1,61 @@ +import type { FounderWeeklyReviewEvidenceSnapshot } from "./contracts"; + +export const FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION = + "founder-weekly-review-generation/v1" as const; + +export const FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT = `You generate a structured Founder Weekly Review from supplied evidence only. + +Never invent, assume, infer, or embellish customers, dates, metrics, people, decisions, shipped work, blockers, outcomes, or source IDs. Every factual item must cite one or more supplied source IDs exactly as given. Do not create or modify source IDs. Confidence is how strongly the generated claim is supported by its cited supplied evidence; it is not a score for source reliability or truthfulness. Omit unsupported claims or use no_evidence rather than assigning them a low confidence. + +founder_context is internal manual input. It must never be represented as customer feedback. whatCustomersSaid may cite only customer_feedback evidence. + +When evidence conflicts, return contradictory_evidence with the conflicting source IDs. Do not choose a winner or reconcile it unless supplied evidence explicitly resolves the conflict. + +nextPriorities contains recommendations only. Every recommendation must have label "Recommendation" and be grounded in supplied evidence. If a section lacks relevant evidence, return its typed no_evidence state with a concrete CTA. sourceWarnings may inform the CTA but are not factual evidence and cannot be cited.`; + +/** Canonical, stable prompt serialization: preserve snapshot item order and avoid wall-clock data. */ +export function buildFounderWeeklyReviewPrompt( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): string { + return JSON.stringify(sortObjectKeysRecursively({ + promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + reportingPeriod: evidenceSnapshot.reportingPeriod, + workspaceTimezone: evidenceSnapshot.workspaceTimezone, + evidence: evidenceSnapshot.items.map((item) => ({ + sourceId: item.sourceId, + sourceType: item.sourceType, + title: item.title, + sourceTimestamp: item.sourceTimestamp ?? null, + excerpt: item.excerpt, + canonicalUrl: item.canonicalUrl ?? null, + workspaceDeepLink: item.workspaceDeepLink ?? null, + metadata: item.metadata, + })), + sourceWarnings: evidenceSnapshot.sourceWarnings, + requiredSections: [ + "whatChanged", + "whatShipped", + "whatCustomersSaid", + "currentBlockers", + "nextPriorities", + ], + })); +} + +/** Sort object keys recursively while retaining the exact supplied order of arrays. */ +function sortObjectKeysRecursively(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sortObjectKeysRecursively); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value as Record) + .sort() + .map((key) => [ + key, + sortObjectKeysRecursively((value as Record)[key]), + ]) + ); + } + return value; +} diff --git a/packages/features/src/founder-weekly-review/repository.ts b/packages/features/src/founder-weekly-review/repository.ts index 8716fc553..6390297d9 100644 --- a/packages/features/src/founder-weekly-review/repository.ts +++ b/packages/features/src/founder-weekly-review/repository.ts @@ -10,6 +10,7 @@ import { import { FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + type FounderWeeklyReviewPayloadSchemaVersion, type CreateFounderWeeklyReviewRunInput, type FounderWeeklyReviewClaimInput, type FounderWeeklyReviewGenerationFailure, @@ -22,6 +23,7 @@ import { parseFounderWeeklyReviewModelMetadata, parseFounderWeeklyReviewPayload, } from "./contracts"; +import { FounderWeeklyReviewInvalidPayloadError } from "./errors"; export interface ConditionalRunMutationResult { updated: boolean; @@ -35,6 +37,14 @@ export interface RetryFounderWeeklyReviewResult { } function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord { + const reviewPayload = row.reviewPayload + ? parseFounderWeeklyReviewPayload(row.reviewPayload) + : null; + if (reviewPayload && reviewPayload.schemaVersion !== row.reviewSchemaVersion) { + throw new FounderWeeklyReviewInvalidPayloadError( + `Founder weekly review run "${row.id}" has mismatched review payload and review schema versions.` + ); + } return { id: row.id, companyId: row.companyId, @@ -44,10 +54,8 @@ function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord end: row.reportingPeriodEnd, }, status: row.status, - reviewPayload: row.reviewPayload - ? parseFounderWeeklyReviewPayload(row.reviewPayload) - : null, - reviewSchemaVersion: row.reviewSchemaVersion as typeof FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + reviewPayload, + reviewSchemaVersion: row.reviewSchemaVersion as FounderWeeklyReviewPayloadSchemaVersion, evidenceSnapshot: parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot), evidenceSchemaVersion: row.evidenceSchemaVersion as typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, @@ -193,7 +201,7 @@ export class FounderWeeklyReviewRepository { .update(founderWeeklyReviewRuns) .set({ reviewPayload, - reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + reviewSchemaVersion: reviewPayload.schemaVersion, updatedAt: new Date(), }) .where( @@ -293,7 +301,7 @@ export class FounderWeeklyReviewRepository { .set({ status: "draft", reviewPayload, - reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + reviewSchemaVersion: reviewPayload.schemaVersion, modelMetadata, generatedAt: now, errorCode: null, From c29393eae953b866316ecf9f7b72408bf144912c Mon Sep 17 00:00:00 2001 From: Hank Li Date: Thu, 23 Jul 2026 18:46:00 -0500 Subject: [PATCH 04/29] add document change evidence collector and mapper for weekly review --- .../evidence-service.test.ts | 77 ++++++++++++++++++ .../founder-weekly-review/evidence-service.ts | 78 +++++++++++++++++++ .../src/founder-weekly-review/index.ts | 1 + 3 files changed, 156 insertions(+) create mode 100644 apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts create mode 100644 packages/features/src/founder-weekly-review/evidence-service.ts diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts new file mode 100644 index 000000000..a461c27f1 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts @@ -0,0 +1,77 @@ +import { + mapDocumentVersionToEvidenceItem, + FounderWeeklyReviewEvidenceItemSchema, + type DocumentVersionRow, +} from "@launchstack/features/founder-weekly-review"; + +function completeRow(): DocumentVersionRow { + return { + documentId: 42n, + documentTitle: "Weekly Product Notes", + documentCategory: "Product", + versionId: 100, + versionNumber: 3, + uploadedBy: "user_abc", + changelog: "Fixed onboarding activation delay.", + createdAt: new Date("2026-07-10T18:15:00.000Z"), + }; +} + +describe("mapDocumentVersionToEvidenceItem", () => { + it("maps a complete row to the exact expected evidence item", () => { + const item = mapDocumentVersionToEvidenceItem(completeRow()); + + expect(item).toEqual({ + sourceType: "document_change", + sourceId: "document-version:42:3", + title: "Weekly Product Notes", + sourceTimestamp: "2026-07-10T18:15:00.000Z", + excerpt: "Fixed onboarding activation delay.", + workspaceDeepLink: "/employer/documents/viewer?docId=42", + metadata: { + documentId: "42", + versionId: 100, + versionNumber: 3, + documentCategory: "Product", + uploadedBy: "user_abc", + hasChangelog: true, + }, + }); + }); + + it("produces an item that satisfies the evidence-item contract", () => { + const item = mapDocumentVersionToEvidenceItem(completeRow()); + expect(() => + FounderWeeklyReviewEvidenceItemSchema.parse(item) + ).not.toThrow(); + }); + + it("falls back to a factual excerpt when changelog is absent", () => { + const item = mapDocumentVersionToEvidenceItem({ + ...completeRow(), + changelog: null, + }); + + expect(item.excerpt).toBe("Version 3 uploaded"); + expect(item.metadata.hasChangelog).toBe(false); + }); + + it("treats a whitespace-only changelog as absent", () => { + const item = mapDocumentVersionToEvidenceItem({ + ...completeRow(), + changelog: " ", + }); + + expect(item.excerpt).toBe("Version 3 uploaded"); + expect(item.metadata.hasChangelog).toBe(false); + }); + + it("carries a null uploader through without inventing a value", () => { + const item = mapDocumentVersionToEvidenceItem({ + ...completeRow(), + uploadedBy: null, + }); + + expect(item.metadata.uploadedBy).toBeNull(); + }); +}); diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts new file mode 100644 index 000000000..60b185a74 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -0,0 +1,78 @@ +import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; +import { and, asc, eq, gte, lt } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { document, documentVersions } from "@launchstack/core/db/schema"; + +function normalizeText(value: string | null | undefined): string | null { + const trimmed = value?.trim(); + return trimmed ? trimmed : null; +} + +export interface DocumentVersionRow { + documentId: bigint; + documentTitle: string; + documentCategory: string | null; + versionId: number; + versionNumber: number; + uploadedBy: string | null; + changelog: string | null; + createdAt: Date; +} + +export function mapDocumentVersionToEvidenceItem( + row: DocumentVersionRow +): FounderWeeklyReviewEvidenceItem { + const changelog = normalizeText(row.changelog); + + return { + sourceType: "document_change", + sourceId: `document-version:${row.documentId}:${row.versionNumber}`, + title: row.documentTitle, + sourceTimestamp: row.createdAt.toISOString(), + excerpt: changelog ?? `Version ${row.versionNumber} uploaded`, + // canonicalUrl intentionally omitted: document.url is inconsistent across rows in document datatable + // relative paths (/api/files/115), storage urls, local dev urls + workspaceDeepLink: `/employer/documents/viewer?docId=${row.documentId}`, + metadata: { + documentId: row.documentId.toString(), + versionId: row.versionId, + versionNumber: row.versionNumber, + documentCategory: row.documentCategory, + uploadedBy: row.uploadedBy, + hasChangelog: changelog !== null, + }, + }; +} + +export class FounderWeeklyReviewEvidenceService { + constructor(private readonly db: DbClient = getDb()) {} + + async collectDocumentChangeEvidence( + companyId: bigint, + startInclusive: Date, + endExclusive: Date, + ): Promise { + const rows = await this.db + .select({ + documentId: documentVersions.documentId, + documentTitle: document.title, + documentCategory: document.category, + versionId: documentVersions.id, + versionNumber: documentVersions.versionNumber, + uploadedBy: documentVersions.uploadedBy, + changelog: documentVersions.changelog, + createdAt: documentVersions.createdAt, + }) + .from(documentVersions) + .innerJoin(document, eq(document.id, documentVersions.documentId)) + .where( + and( + gte(documentVersions.createdAt, startInclusive), + lt(documentVersions.createdAt, endExclusive), + eq(document.companyId, companyId) + ) + ) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id)); + return rows.map((row) => mapDocumentVersionToEvidenceItem(row)); + } +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index b217ef302..7fc76fd20 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -1,4 +1,5 @@ export * from "./contracts"; +export * from "./evidence-service"; export * from "./errors"; export * from "./repository"; export * from "./user-service"; From 38bc2a5bbfa052be398301aa4fdd5be4bacd6e8d Mon Sep 17 00:00:00 2001 From: Hank Li Date: Thu, 23 Jul 2026 20:36:53 -0500 Subject: [PATCH 05/29] implement helper to convert query date boundaries from workspace timezone to UTC --- .../reporting-period.test.ts | 45 +++++++++++++++++++ packages/features/package.json | 1 + .../src/founder-weekly-review/index.ts | 1 + .../founder-weekly-review/reporting-period.ts | 41 +++++++++++++++++ pnpm-lock.yaml | 10 ++++- 5 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts create mode 100644 packages/features/src/founder-weekly-review/reporting-period.ts diff --git a/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts b/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts new file mode 100644 index 000000000..0050fd95e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts @@ -0,0 +1,45 @@ +import { resolveReportingPeriodBounds } from "@launchstack/features/founder-weekly-review"; + +describe("resolveReportingPeriodBounds", () => { + it("resolves midnight-to-midnight bounds in the workspace timezone", () => { + // America/Los_Angeles in February is PST (UTC-8), so local midnight is 08:00 UTC. + const bounds = resolveReportingPeriodBounds( + { start: "2026-02-16", end: "2026-02-22" }, + "America/Los_Angeles" + ); + + expect(bounds.startInclusive.toISOString()).toBe("2026-02-16T08:00:00.000Z"); + // end is the last included day, so the exclusive bound is the next day's midnight. + expect(bounds.endExclusive.toISOString()).toBe("2026-02-23T08:00:00.000Z"); + }); + + it("accounts for daylight saving in the same timezone", () => { + // Same zone, but July is PDT (UTC-7), so local midnight is 07:00 UTC, not 08:00. + const bounds = resolveReportingPeriodBounds( + { start: "2026-07-06", end: "2026-07-12" }, + "America/Los_Angeles" + ); + + expect(bounds.startInclusive.toISOString()).toBe("2026-07-06T07:00:00.000Z"); + expect(bounds.endExclusive.toISOString()).toBe("2026-07-13T07:00:00.000Z"); + }); + + it("treats UTC boundaries as literal midnight", () => { + const bounds = resolveReportingPeriodBounds( + { start: "2026-02-16", end: "2026-02-22" }, + "UTC" + ); + + expect(bounds.startInclusive.toISOString()).toBe("2026-02-16T00:00:00.000Z"); + expect(bounds.endExclusive.toISOString()).toBe("2026-02-23T00:00:00.000Z"); + }); + + it("throws on an invalid timezone", () => { + expect(() => + resolveReportingPeriodBounds( + { start: "2026-02-16", end: "2026-02-22" }, + "Not/AZone" + ) + ).toThrow(/invalid workspace timezone/i); + }); +}); diff --git a/packages/features/package.json b/packages/features/package.json index 5834233da..bccc018c9 100644 --- a/packages/features/package.json +++ b/packages/features/package.json @@ -109,6 +109,7 @@ "@langchain/core": "^0.3.74", "@langchain/openai": "^0.6.11", "@launchstack/core": "workspace:*", + "dayjs": "^1.11.18", "docxtemplater": "^3.68.3", "drizzle-orm": "^0.45.1", "openai": "^4.62.1", diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 7fc76fd20..c49e51cb1 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -1,5 +1,6 @@ export * from "./contracts"; export * from "./evidence-service"; +export * from "./reporting-period"; export * from "./errors"; export * from "./repository"; export * from "./user-service"; diff --git a/packages/features/src/founder-weekly-review/reporting-period.ts b/packages/features/src/founder-weekly-review/reporting-period.ts new file mode 100644 index 000000000..61ae9ab11 --- /dev/null +++ b/packages/features/src/founder-weekly-review/reporting-period.ts @@ -0,0 +1,41 @@ +import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; +import timezone from "dayjs/plugin/timezone"; + +import type { ReportingPeriod } from "./contracts"; + +dayjs.extend(utc); +dayjs.extend(timezone); + +export interface ReportingPeriodBounds { + startInclusive: Date; + endExclusive: Date; +} + +// convert calendar dates in workspace timezone to UTC for querying +// end is the last included day +export function resolveReportingPeriodBounds( + period: ReportingPeriod, + workspaceTimezone: string +): ReportingPeriodBounds { + assertValidTimeZone(workspaceTimezone); + + return { + startInclusive: dayjs.tz(period.start, workspaceTimezone).startOf("day").toDate(), + endExclusive: dayjs + .tz(period.end, workspaceTimezone) + .add(1, "day") + .startOf("day") + .toDate(), + }; +} + +// dayjs.tz throws dayjs's own error for a bad zone; validate up front so callers +// get a clear, feature-specific message instead. +function assertValidTimeZone(timeZone: string): void { + try { + new Intl.DateTimeFormat("en-US", { timeZone }); + } catch { + throw new Error(`Invalid workspace timezone: ${timeZone}`); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6e3d46fc..647ab94ee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -608,6 +608,9 @@ importers: '@launchstack/core': specifier: workspace:* version: link:../core + dayjs: + specifier: ^1.11.18 + version: 1.11.19 docxtemplater: specifier: ^3.68.3 version: 3.68.3 @@ -1733,11 +1736,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.9': resolution: {integrity: sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==} @@ -2685,6 +2688,7 @@ packages: '@langchain/community@0.3.54': resolution: {integrity: sha512-p1lSlxu1ZBWBiW6PSQujutR03OfxI9R0s+CKPK2DRUjF8PFARe49OlPqHPlmGmkGnOlpZLF3gay5215p6ZkP8w==} engines: {node: '>=18'} + deprecated: This package has been deprecated. See https://github.com/langchain-ai/langchainjs-community/issues/61 for more info peerDependencies: '@arcjet/redact': ^v1.0.0-alpha.23 '@aws-crypto/sha256-js': ^5.0.0 @@ -5281,6 +5285,7 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@unrs/resolver-binding-android-arm-eabi@1.11.1': resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} @@ -9198,6 +9203,7 @@ packages: recharts@2.15.4: resolution: {integrity: sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==} engines: {node: '>=14'} + deprecated: 1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide peerDependencies: react: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 From 5754c62dfa7190a3b4929e413e0105d28309eddc Mon Sep 17 00:00:00 2001 From: Hank Li Date: Thu, 23 Jul 2026 21:22:49 -0500 Subject: [PATCH 06/29] implement class to assemble weekly review evidence snapshot with deduplication and ordering --- .../evidence-assembly.test.ts | 87 ++++++++++++++++++ .../founder-weekly-review/evidence-service.ts | 92 ++++++++++++++++++- 2 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts new file mode 100644 index 000000000..590813825 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts @@ -0,0 +1,87 @@ +import { + dedupeEvidenceItems, + orderEvidenceItems, + type FounderWeeklyReviewEvidenceItem, +} from "@launchstack/features/founder-weekly-review"; + +function makeItem( + sourceId: string, + sourceTimestamp?: string, + sourceType: FounderWeeklyReviewEvidenceItem["sourceType"] = "document_change" +): FounderWeeklyReviewEvidenceItem { + return { + sourceType, + sourceId, + title: `Title ${sourceId}`, + sourceTimestamp, + excerpt: `Excerpt ${sourceId}`, + metadata: {}, + }; +} + +describe("dedupeEvidenceItems", () => { + it("collapses items with the same sourceType+sourceId, keeping the first", () => { + const result = dedupeEvidenceItems([ + makeItem("a", "2026-01-01T00:00:00.000Z"), + makeItem("a", "2026-02-01T00:00:00.000Z"), // duplicate identity + makeItem("b"), + ]); + + expect(result.map((i) => i.sourceId)).toEqual(["a", "b"]); + // the FIRST "a" survives, so its timestamp is the January one + expect(result[0]?.sourceTimestamp).toBe("2026-01-01T00:00:00.000Z"); + }); + + it("keeps items that share a sourceId but differ in sourceType", () => { + const result = dedupeEvidenceItems([ + makeItem("x", undefined, "document_change"), + makeItem("x", undefined, "customer_feedback"), + ]); + + expect(result).toHaveLength(2); + }); + + it("returns an empty array unchanged", () => { + expect(dedupeEvidenceItems([])).toEqual([]); + }); +}); + +describe("orderEvidenceItems", () => { + it("sorts by sourceTimestamp ascending", () => { + const result = orderEvidenceItems([ + makeItem("b", "2026-03-01T00:00:00.000Z"), + makeItem("a", "2026-01-01T00:00:00.000Z"), + makeItem("c", "2026-02-01T00:00:00.000Z"), + ]); + + expect(result.map((i) => i.sourceId)).toEqual(["a", "c", "b"]); + }); + + it("breaks ties on identity when timestamps are equal", () => { + const ts = "2026-01-01T00:00:00.000Z"; + const result = orderEvidenceItems([makeItem("z", ts), makeItem("a", ts)]); + + expect(result.map((i) => i.sourceId)).toEqual(["a", "z"]); + }); + + it("sorts items without a timestamp first", () => { + const result = orderEvidenceItems([ + makeItem("withTs", "2026-01-01T00:00:00.000Z"), + makeItem("noTs"), + ]); + + expect(result.map((i) => i.sourceId)).toEqual(["noTs", "withTs"]); + }); + + it("does not mutate the input array", () => { + const input = [ + makeItem("b", "2026-02-01T00:00:00.000Z"), + makeItem("a", "2026-01-01T00:00:00.000Z"), + ]; + const originalOrder = input.map((i) => i.sourceId); + + orderEvidenceItems(input); + + expect(input.map((i) => i.sourceId)).toEqual(originalOrder); + }); +}); diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index 60b185a74..e21280d22 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -1,4 +1,11 @@ -import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + FounderWeeklyReviewEvidenceSnapshotSchema, + type FounderWeeklyReviewEvidenceItem, + type FounderWeeklyReviewEvidenceSnapshot, + type ReportingPeriod, +} from "./contracts"; +import { resolveReportingPeriodBounds } from "./reporting-period"; import { and, asc, eq, gte, lt } from "drizzle-orm"; import { getDb, type DbClient } from "@launchstack/core/db"; import { document, documentVersions } from "@launchstack/core/db/schema"; @@ -44,6 +51,14 @@ export function mapDocumentVersionToEvidenceItem( }; } +export interface BuildFounderWeeklyReviewEvidenceSnapshotInput { + companyId: bigint; + reportingPeriod: ReportingPeriod; + workspaceTimezone: string; + capturedAt?: Date; + maxItems?: number; +} + export class FounderWeeklyReviewEvidenceService { constructor(private readonly db: DbClient = getDb()) {} @@ -75,4 +90,79 @@ export class FounderWeeklyReviewEvidenceService { .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id)); return rows.map((row) => mapDocumentVersionToEvidenceItem(row)); } + + async buildEvidenceSnapshot( + input: BuildFounderWeeklyReviewEvidenceSnapshotInput + ): Promise { + const { startInclusive, endExclusive } = resolveReportingPeriodBounds( + input.reportingPeriod, + input.workspaceTimezone + ); + + // only document_change exists right now + const collected = await this.collectDocumentChangeEvidence( + input.companyId, + startInclusive, + endExclusive + ); + + // more evidence can be merged later + const merged = [...collected]; + + // 4. Drop exact duplicates, keep distinct citations. (AC #5) + const deduped = dedupeEvidenceItems(merged); + + // 5. Deterministic order, then cap at the schema's limit. + const maxItems = input.maxItems ?? 500; + const items = orderEvidenceItems(deduped).slice(0, maxItems); + + // 6. Wrap in the envelope. Empty `items` is valid — an empty/partial + // workspace returns a pack rather than throwing. (AC #6) + const snapshot = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + capturedAt: (input.capturedAt ?? new Date()).toISOString(), + reportingPeriod: input.reportingPeriod, + workspaceTimezone: input.workspaceTimezone, + items, + sourceWarnings: [], + }; + + // 7. Validate before returning — a malformed pack never leaves here. + return FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot); + } +} + +export function dedupeEvidenceItems( + items: FounderWeeklyReviewEvidenceItem[] +): FounderWeeklyReviewEvidenceItem[] { + const seenIdentities = new Set() + + return items.filter(item => { + const identity = `${item.sourceType}:${item.sourceId}` + + if (seenIdentities.has(identity)) { + return false + } + + seenIdentities.add(identity) + return true + }) +} + +// order primarily based on createdAt timestamp, sourceId as tie breaker +export function orderEvidenceItems( + items: FounderWeeklyReviewEvidenceItem[] +): FounderWeeklyReviewEvidenceItem[] { + return [...items].sort((a, b) => { + // if timestamp is missing use empty string + const aTime = a.sourceTimestamp ?? ""; + const bTime = b.sourceTimestamp ?? ""; + if (aTime !== bTime) { + return aTime.localeCompare(bTime); + } + + const identityA = `${a.sourceType}:${a.sourceId}`; + const identityB = `${b.sourceType}:${b.sourceId}`; + return identityA.localeCompare(identityB); + }); } \ No newline at end of file From 0bde15a3d990b128f93edc0c9b4ccd916c286775 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 25 Jul 2026 14:25:25 +0800 Subject: [PATCH 07/29] feat: add founder review async generation flow --- .../0017_founder_weekly_review_dispatches.sql | 26 ++++ .../[runId]/retry/route.ts | 17 +++ .../founder-weekly-reviews/[runId]/route.ts | 12 ++ .../app/api/founder-weekly-reviews/route.ts | 27 ++++ apps/web/src/app/api/inngest/route.ts | 4 + .../founder-weekly-review/dispatch-service.ts | 126 ++++++++++++++++++ .../evidence-collector.ts | 24 ++++ .../src/server/founder-weekly-review/http.ts | 22 +++ .../founder-weekly-review/observability.ts | 15 +++ apps/web/src/server/inngest/client.ts | 12 +- .../inngest/functions/founderWeeklyReview.ts | 68 ++++++++++ apps/web/src/server/metrics/registry.ts | 6 + .../runbooks/founder-weekly-review-staging.md | 29 ++++ .../src/db/schema/founder-weekly-review.ts | 44 ++++++ 14 files changed, 431 insertions(+), 1 deletion(-) create mode 100644 apps/web/drizzle/0017_founder_weekly_review_dispatches.sql create mode 100644 apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts create mode 100644 apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts create mode 100644 apps/web/src/app/api/founder-weekly-reviews/route.ts create mode 100644 apps/web/src/server/founder-weekly-review/dispatch-service.ts create mode 100644 apps/web/src/server/founder-weekly-review/evidence-collector.ts create mode 100644 apps/web/src/server/founder-weekly-review/http.ts create mode 100644 apps/web/src/server/founder-weekly-review/observability.ts create mode 100644 apps/web/src/server/inngest/functions/founderWeeklyReview.ts create mode 100644 docs/runbooks/founder-weekly-review-staging.md diff --git a/apps/web/drizzle/0017_founder_weekly_review_dispatches.sql b/apps/web/drizzle/0017_founder_weekly_review_dispatches.sql new file mode 100644 index 000000000..e23a31388 --- /dev/null +++ b/apps/web/drizzle/0017_founder_weekly_review_dispatches.sql @@ -0,0 +1,26 @@ +-- Durable outbox for Founder Weekly Review generation events (LAU-9). +CREATE TABLE IF NOT EXISTS "pdr_ai_v2_founder_weekly_review_dispatches" ( + "id" varchar(64) PRIMARY KEY, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "run_id" varchar(64) NOT NULL REFERENCES "pdr_ai_v2_founder_weekly_review_runs"("id") ON DELETE CASCADE, + "operation_type" varchar(16) NOT NULL, + "operation_key" varchar(128) NOT NULL, + "event_id" varchar(128) NOT NULL, + "generation_job_id" varchar(128) NOT NULL, + "generation_claim_id" varchar(128) NOT NULL, + "status" varchar(16) NOT NULL DEFAULT 'pending', + "attempt_count" integer NOT NULL DEFAULT 0, + "available_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "dispatched_at" timestamptz, + "last_error_code" varchar(128), + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_run_operation_key_unique" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("run_id", "operation_type", "operation_key"); +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_event_id_unique" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("event_id"); +CREATE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_pending_idx" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("status", "available_at", "created_at"); +CREATE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_company_run_idx" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("company_id", "run_id"); diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts new file mode 100644 index 000000000..76558bd68 --- /dev/null +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts @@ -0,0 +1,17 @@ +import { NextResponse } from "next/server"; +import { auth } from "@clerk/nextjs/server"; +import { z } from "zod"; +import { getActiveCompanyContext } from "~/lib/active-workspace"; +import { retryRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; +import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; +import { inngest } from "~/server/inngest/client"; +const RetrySchema = z.object({ requestKey: z.string().min(1).max(128) }); +export async function POST(request: Request, { params }: { params: Promise<{ runId: string }> }) { + const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + const parsed = RetrySchema.safeParse(await request.json().catch(() => null)); if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + try { const context = await getActiveCompanyContext(userId); const { runId } = await params; + const { run } = await retryRunWithDispatch({ actor: { externalUserId: userId, internalUserId: context.userId, companyId: context.companyId, role: context.role }, runId, requestKey: parsed.data.requestKey }); + await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); + return NextResponse.json({ run: safeRun(run) }, { status: 202 }); + } catch (error) { return safeFounderWeeklyReviewError(error); } +} diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts new file mode 100644 index 000000000..2a2dcca58 --- /dev/null +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts @@ -0,0 +1,12 @@ +import { NextResponse } from "next/server"; +import { auth } from "@clerk/nextjs/server"; +import { getActiveCompanyContext } from "~/lib/active-workspace"; +import { FounderWeeklyReviewUserService } from "@launchstack/features/founder-weekly-review"; +import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; +export async function GET(_request: Request, { params }: { params: Promise<{ runId: string }> }) { + const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + try { const context = await getActiveCompanyContext(userId); const { runId } = await params; + const run = await new FounderWeeklyReviewUserService().getRun({ externalUserId: userId, internalUserId: context.userId, companyId: context.companyId, role: context.role }, runId); + return NextResponse.json({ run: safeRun(run) }); + } catch (error) { return safeFounderWeeklyReviewError(error); } +} diff --git a/apps/web/src/app/api/founder-weekly-reviews/route.ts b/apps/web/src/app/api/founder-weekly-reviews/route.ts new file mode 100644 index 000000000..3cf5ac86f --- /dev/null +++ b/apps/web/src/app/api/founder-weekly-reviews/route.ts @@ -0,0 +1,27 @@ +import { NextResponse } from "next/server"; +import { auth } from "@clerk/nextjs/server"; +import { z } from "zod"; +import { getActiveCompanyContext } from "~/lib/active-workspace"; +import { unavailableFounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; +import { createRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; +import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; +import { inngest } from "~/server/inngest/client"; + +const CreateSchema = z.object({ requestKey: z.string().min(1).max(128), reportingPeriod: z.object({ start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), end: z.string().regex(/^\d{4}-\d{2}-\d{2}$/) }), workspaceTimezone: z.string().min(1).max(128), founderContext: z.string().max(4000).optional() }); + +export async function POST(request: Request) { + const { userId } = await auth(); + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + const parsed = CreateSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + try { + const context = await getActiveCompanyContext(userId); + const evidenceSnapshot = await unavailableFounderWeeklyReviewEvidenceCollector.collectFounderWeeklyReviewEvidence({ companyId: context.companyId, reportingPeriod: parsed.data.reportingPeriod, workspaceTimezone: parsed.data.workspaceTimezone, founderContext: parsed.data.founderContext }); + const { run } = await createRunWithDispatch({ actor: { externalUserId: userId, internalUserId: context.userId, companyId: context.companyId, role: context.role }, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, evidenceSnapshot }); + await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); + return NextResponse.json({ run: safeRun(run) }, { status: 202 }); + } catch (error) { + if (error && typeof error === "object" && "code" in error && (error as { code?: string }).code === "evidence_collector_unavailable") return NextResponse.json({ error: "Generation unavailable" }, { status: 503 }); + return safeFounderWeeklyReviewError(error); + } +} diff --git a/apps/web/src/app/api/inngest/route.ts b/apps/web/src/app/api/inngest/route.ts index d51fb96ed..2ba7707dd 100644 --- a/apps/web/src/app/api/inngest/route.ts +++ b/apps/web/src/app/api/inngest/route.ts @@ -18,6 +18,7 @@ import { reindexCompanyEmbeddingsJob } from "~/server/inngest/functions/reindexC import { modifyDocument } from "~/server/inngest/functions/modifyDocument"; import { crawlWebsite } from "~/server/inngest/functions/crawlWebsite"; import { rehydrateNoteAnchorsJob } from "~/server/inngest/functions/rehydrateNoteAnchors"; +import { founderWeeklyReviewDispatcher, founderWeeklyReviewDispatchReconciler, founderWeeklyReviewGenerationJob } from "~/server/inngest/functions/founderWeeklyReview"; // Register all Inngest functions const handler = serve({ @@ -32,6 +33,9 @@ const handler = serve({ modifyDocument, crawlWebsite, rehydrateNoteAnchorsJob, + founderWeeklyReviewDispatcher, + founderWeeklyReviewDispatchReconciler, + founderWeeklyReviewGenerationJob, ], }); diff --git a/apps/web/src/server/founder-weekly-review/dispatch-service.ts b/apps/web/src/server/founder-weekly-review/dispatch-service.ts new file mode 100644 index 000000000..6565e4a97 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/dispatch-service.ts @@ -0,0 +1,126 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, lte, or, sql } from "drizzle-orm"; +import { db } from "~/server/db"; +import type { DbClient } from "@launchstack/core/db"; +import { + founderWeeklyReviewDispatches, + type FounderWeeklyReviewDispatchRow, +} from "@launchstack/core/db/schema"; +import { + FounderWeeklyReviewRepository, + FounderWeeklyReviewUserService, + type FounderWeeklyReviewRunRecord, + type FounderWeeklyReviewUserActor, + type ReportingPeriod, + type FounderWeeklyReviewEvidenceSnapshot, +} from "@launchstack/features/founder-weekly-review"; + +export type FounderWeeklyReviewDispatch = Pick; + +function toDispatch(row: FounderWeeklyReviewDispatchRow): FounderWeeklyReviewDispatch { + return row; +} + +function identifiers(runId: string, operationType: "create" | "retry", operationKey: string) { + const suffix = `${operationType}:${operationKey}`; + return { + id: `fwrd_${randomUUID()}`, + eventId: `fwr-event:${runId}:${suffix}`.slice(0, 128), + generationJobId: `fwr-job:${runId}:${suffix}`.slice(0, 128), + generationClaimId: `fwr-claim:${runId}:${suffix}`.slice(0, 128), + }; +} + +async function createDispatch( + tx: Pick, + run: FounderWeeklyReviewRunRecord, + operationType: "create" | "retry", + operationKey: string, +): Promise { + const ids = identifiers(run.id, operationType, operationKey); + const [inserted] = await tx.insert(founderWeeklyReviewDispatches).values({ + id: ids.id, + companyId: run.companyId, + runId: run.id, + operationType, + operationKey, + eventId: ids.eventId, + generationJobId: ids.generationJobId, + generationClaimId: ids.generationClaimId, + status: "pending", + }).onConflictDoNothing().returning(); + if (inserted) return toDispatch(inserted); + const [existing] = await tx.select().from(founderWeeklyReviewDispatches).where(and( + eq(founderWeeklyReviewDispatches.runId, run.id), + eq(founderWeeklyReviewDispatches.operationType, operationType), + eq(founderWeeklyReviewDispatches.operationKey, operationKey), + )).limit(1); + if (!existing) throw new Error("Founder weekly review dispatch was not created"); + return toDispatch(existing); +} + +export async function createRunWithDispatch(input: { + actor: FounderWeeklyReviewUserActor; + requestKey: string; + reportingPeriod: ReportingPeriod; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; +}): Promise<{ run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch }> { + return db.transaction(async (tx) => { + const service = new FounderWeeklyReviewUserService(new FounderWeeklyReviewRepository(tx as unknown as DbClient)); + const run = await service.createOrGetRun(input.actor, input); + const dispatch = await createDispatch(tx, run, "create", input.requestKey); + return { run, dispatch }; + }); +} + +export async function retryRunWithDispatch(input: { + actor: FounderWeeklyReviewUserActor; + runId: string; + requestKey: string; +}): Promise<{ run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch }> { + return db.transaction(async (tx) => { + const service = new FounderWeeklyReviewUserService(new FounderWeeklyReviewRepository(tx as unknown as DbClient)); + const run = await service.retryFailedRun(input.actor, input.runId, input.requestKey); + const dispatch = await createDispatch(tx, run, "retry", input.requestKey); + return { run, dispatch }; + }); +} + +export async function claimPendingDispatches(limit = 20): Promise { + const now = new Date(); + const staleDispatchingBefore = new Date(now.getTime() - 5 * 60 * 1000); + const candidates = await db.select({ id: founderWeeklyReviewDispatches.id }) + .from(founderWeeklyReviewDispatches).where(and( + or( + and(inArray(founderWeeklyReviewDispatches.status, ["pending", "failed"]), lte(founderWeeklyReviewDispatches.availableAt, now)), + and(eq(founderWeeklyReviewDispatches.status, "dispatching"), lte(founderWeeklyReviewDispatches.updatedAt, staleDispatchingBefore)), + ), + )).limit(limit); + const claimed: FounderWeeklyReviewDispatch[] = []; + for (const candidate of candidates) { + const [row] = await db.update(founderWeeklyReviewDispatches).set({ + status: "dispatching", attemptCount: sql`${founderWeeklyReviewDispatches.attemptCount} + 1`, + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewDispatches.id, candidate.id), + or(inArray(founderWeeklyReviewDispatches.status, ["pending", "failed"]), and(eq(founderWeeklyReviewDispatches.status, "dispatching"), lte(founderWeeklyReviewDispatches.updatedAt, staleDispatchingBefore))), + )).returning(); + if (row) claimed.push(toDispatch(row)); + } + return claimed; +} + +export async function markDispatchDispatched(dispatchId: string): Promise { + await db.update(founderWeeklyReviewDispatches).set({ + status: "dispatched", dispatchedAt: new Date(), updatedAt: new Date(), + }).where(eq(founderWeeklyReviewDispatches.id, dispatchId)); +} + +export async function returnDispatchToPending(dispatchId: string, errorCode: string): Promise { + await db.update(founderWeeklyReviewDispatches).set({ + status: "pending", lastErrorCode: errorCode.slice(0, 128), availableAt: new Date(), + updatedAt: new Date(), + }).where(eq(founderWeeklyReviewDispatches.id, dispatchId)); +} diff --git a/apps/web/src/server/founder-weekly-review/evidence-collector.ts b/apps/web/src/server/founder-weekly-review/evidence-collector.ts new file mode 100644 index 000000000..58c7d4084 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/evidence-collector.ts @@ -0,0 +1,24 @@ +import type { FounderWeeklyReviewEvidenceSnapshot, ReportingPeriod } from "@launchstack/features/founder-weekly-review"; + +export interface FounderWeeklyReviewEvidenceCollector { + collectFounderWeeklyReviewEvidence(input: { + companyId: bigint; + reportingPeriod: ReportingPeriod; + workspaceTimezone: string; + founderContext?: string; + }): Promise; +} + +/** LAU-6 owns collection. This deliberately never creates an empty snapshot. */ +export class FounderWeeklyReviewEvidenceCollectorUnavailableError extends Error { + readonly code = "evidence_collector_unavailable"; + constructor() { + super("Founder weekly review evidence collection is not configured."); + } +} + +export const unavailableFounderWeeklyReviewEvidenceCollector: FounderWeeklyReviewEvidenceCollector = { + async collectFounderWeeklyReviewEvidence() { + throw new FounderWeeklyReviewEvidenceCollectorUnavailableError(); + }, +}; diff --git a/apps/web/src/server/founder-weekly-review/http.ts b/apps/web/src/server/founder-weekly-review/http.ts new file mode 100644 index 000000000..21324a08e --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/http.ts @@ -0,0 +1,22 @@ +import { NextResponse } from "next/server"; +import type { FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; + +export function safeRun(run: FounderWeeklyReviewRunRecord) { + return { + id: run.id, status: run.status, reportingPeriod: run.reportingPeriod, + generationAttempt: run.generationAttempt, retryCount: run.retryCount, + queuedAt: run.queuedAt.toISOString(), claimedAt: run.claimedAt?.toISOString() ?? null, + generatedAt: run.generatedAt?.toISOString() ?? null, publishedAt: run.publishedAt?.toISOString() ?? null, + errorCode: run.status === "failed" ? run.errorCode : null, + reviewPayload: run.reviewPayload, + }; +} + +export function safeFounderWeeklyReviewError(error: unknown) { + const code = error && typeof error === "object" && "code" in error ? (error as { code?: string }).code : undefined; + if (code === "forbidden") return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + if (code === "not_found") return NextResponse.json({ error: "Not found" }, { status: 404 }); + if (code === "invalid_transition" || code === "conflict") return NextResponse.json({ error: "Conflict" }, { status: 409 }); + if (code === "invalid_payload") return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + return NextResponse.json({ error: "Internal server error" }, { status: 500 }); +} diff --git a/apps/web/src/server/founder-weekly-review/observability.ts b/apps/web/src/server/founder-weekly-review/observability.ts new file mode 100644 index 000000000..58e836e35 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/observability.ts @@ -0,0 +1,15 @@ +import { createLogger } from "~/lib/logger"; +import { + founderWeeklyReviewGenerationTotal, + founderWeeklyReviewJobsEnqueued, + founderWeeklyReviewRetries, + founderWeeklyReviewStageDuration, + founderWeeklyReviewCitationFailures, +} from "~/server/metrics/registry"; + +const logger = createLogger("founder-weekly-review"); +export function logFounderWeeklyReview(fields: { + runId: string; companyId: string; stage: string; status: string; durationMs?: number; + generationAttempt?: number; retryCount?: number; errorClass?: string; provider?: string; model?: string; +}) { logger.info(fields, "founder weekly review stage"); } +export { founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRetries, founderWeeklyReviewStageDuration, founderWeeklyReviewCitationFailures }; diff --git a/apps/web/src/server/inngest/client.ts b/apps/web/src/server/inngest/client.ts index e7747a931..b299fa1f1 100644 --- a/apps/web/src/server/inngest/client.ts +++ b/apps/web/src/server/inngest/client.ts @@ -83,6 +83,14 @@ export type RehydrateNoteAnchorsEvent = { versionId: number; }; }; +export type FounderWeeklyReviewDispatchEvent = { + name: "founder-weekly-review/dispatch.requested"; + data: { dispatchId?: string }; +}; +export type FounderWeeklyReviewGenerationEvent = { + name: "founder-weekly-review/generation.requested"; + data: { runId: string; companyId: string; generationJobId: string; generationClaimId: string }; +}; export type Events = | ProcessDocumentEvent @@ -93,7 +101,9 @@ export type Events = | ReindexCompanyEmbeddingsEvent | DocumentModifyEvent | WebsiteCrawlEvent - | RehydrateNoteAnchorsEvent; + | RehydrateNoteAnchorsEvent + | FounderWeeklyReviewDispatchEvent + | FounderWeeklyReviewGenerationEvent; /** * Create the Inngest client. diff --git a/apps/web/src/server/inngest/functions/founderWeeklyReview.ts b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts new file mode 100644 index 000000000..baace4037 --- /dev/null +++ b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; +import { inngest } from "../client"; +import { FounderWeeklyReviewWorkerService, type FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; +import { generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewGenerationValidationError } from "@launchstack/features/founder-weekly-review"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { claimPendingDispatches, markDispatchDispatched, returnDispatchToPending } from "~/server/founder-weekly-review/dispatch-service"; +import { founderWeeklyReviewCitationFailures, founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; + +const GenerationEventSchema = z.object({ runId: z.string().min(1), companyId: z.string().min(1), generationJobId: z.string().min(1), generationClaimId: z.string().min(1) }); + +export const founderWeeklyReviewDispatcher = inngest.createFunction( + { id: "founder-weekly-review-dispatcher", retries: 3 }, + { event: "founder-weekly-review/dispatch.requested" }, + async ({ step }) => step.run("dispatch-pending", async () => { + const dispatches = await claimPendingDispatches(); + for (const dispatch of dispatches) { + try { + await inngest.send({ id: dispatch.eventId, name: "founder-weekly-review/generation.requested", data: { + runId: dispatch.runId, companyId: dispatch.companyId.toString(), generationJobId: dispatch.generationJobId, generationClaimId: dispatch.generationClaimId, + }}); + await markDispatchDispatched(dispatch.id); + founderWeeklyReviewJobsEnqueued.inc({ operation: dispatch.operationType }); + } catch { + await returnDispatchToPending(dispatch.id, "dispatch_failed"); + } + } + return { dispatched: dispatches.length }; + }) +); + +/** Periodic reconciliation covers a process failure after the DB outbox commit. */ +export const founderWeeklyReviewDispatchReconciler = inngest.createFunction( + { id: "founder-weekly-review-dispatch-reconciler", retries: 2 }, + { cron: "*/5 * * * *" }, + async () => { + await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); + return { requested: true }; + } +); + +export const founderWeeklyReviewGenerationJob = inngest.createFunction( + { id: "founder-weekly-review-generation", retries: 3, concurrency: { key: "event.data.runId", limit: 1 }, + onFailure: async ({ event }) => { + const parsed = GenerationEventSchema.safeParse(event.data); if (!parsed.success) return; + const worker = new FounderWeeklyReviewWorkerService(); + await worker.markGenerationFailed({ companyId: BigInt(parsed.data.companyId), runId: parsed.data.runId, generationJobId: parsed.data.generationJobId, generationClaimId: parsed.data.generationClaimId }, { errorCode: "generation_failed", errorMessage: "Generation failed after retries." }).catch(() => undefined); + } }, + { event: "founder-weekly-review/generation.requested" }, + async ({ event, step }) => { + const data = GenerationEventSchema.parse(event.data); + const context = { companyId: BigInt(data.companyId), runId: data.runId, generationJobId: data.generationJobId, generationClaimId: data.generationClaimId }; + const worker = new FounderWeeklyReviewWorkerService(); + const claimed = await step.run("claim", () => worker.claimQueuedRun(context)); + if (claimed.status !== "generating" || claimed.generationClaimId !== data.generationClaimId) return { skipped: true }; + try { + const generated = await step.run("generate", () => generateFounderWeeklyReview({ evidenceSnapshot: claimed.evidenceSnapshot, generate: generateFounderWeeklyReviewStructured })); + const saved = await step.run("persist", () => worker.saveGeneratedDraft(context, generated.reviewPayload, generated.modelMetadata)) as unknown as FounderWeeklyReviewRunRecord; + founderWeeklyReviewGenerationTotal.inc({ result: "success", error_class: "none" }); + logFounderWeeklyReview({ runId: saved.id, companyId: saved.companyId.toString(), stage: "persist", status: saved.status, generationAttempt: saved.generationAttempt, retryCount: saved.retryCount, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model }); + return { runId: saved.id, status: saved.status }; + } catch (error) { + if (error instanceof FounderWeeklyReviewGenerationValidationError) founderWeeklyReviewCitationFailures.inc(); + founderWeeklyReviewGenerationTotal.inc({ result: "failure", error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + throw error; + } + } +); diff --git a/apps/web/src/server/metrics/registry.ts b/apps/web/src/server/metrics/registry.ts index f788cf0f7..69cafa493 100644 --- a/apps/web/src/server/metrics/registry.ts +++ b/apps/web/src/server/metrics/registry.ts @@ -55,6 +55,12 @@ export const qaRequestCounter = new Counter({ registers: [metricsRegistry] }); +export const founderWeeklyReviewJobsEnqueued = new Counter({ name: "pdr_founder_weekly_review_jobs_enqueued_total", help: "Founder review jobs enqueued", labelNames: ["operation"], registers: [metricsRegistry] }); +export const founderWeeklyReviewGenerationTotal = new Counter({ name: "pdr_founder_weekly_review_generation_total", help: "Founder review generations by result", labelNames: ["result", "error_class"], registers: [metricsRegistry] }); +export const founderWeeklyReviewRetries = new Counter({ name: "pdr_founder_weekly_review_retries_total", help: "Founder review retries", registers: [metricsRegistry] }); +export const founderWeeklyReviewCitationFailures = new Counter({ name: "pdr_founder_weekly_review_citation_validation_failures_total", help: "Founder review citation validation failures", registers: [metricsRegistry] }); +export const founderWeeklyReviewStageDuration = new Histogram({ name: "pdr_founder_weekly_review_stage_duration_seconds", help: "Founder review stage duration", labelNames: ["stage", "result"], buckets: [0.1, 0.5, 1, 5, 15, 60, 300], registers: [metricsRegistry] }); + export async function getMetricsSnapshot(): Promise { return metricsRegistry.metrics(); } diff --git a/docs/runbooks/founder-weekly-review-staging.md b/docs/runbooks/founder-weekly-review-staging.md new file mode 100644 index 000000000..77abf7e92 --- /dev/null +++ b/docs/runbooks/founder-weekly-review-staging.md @@ -0,0 +1,29 @@ +# Founder Weekly Review staging smoke test + +Status: **pending execution**. This runbook requires an accessible staging +workspace, configured Inngest, and a configured generation provider. Never use +production evidence or commit generated review contents. + +1. Create an isolated staging workspace with synthetic complete, partial, and + empty evidence only. +2. Verify the Inngest endpoint and the configured `founderWeeklyReview` model. +3. Start a review with a fresh idempotency key and poll its run ID. +4. Verify `queued -> generating -> draft` and that every factual item cites a + supplied source ID; customer statements must cite customer feedback only. +5. Force a safe synthetic failure, retry with a new retry key, and verify the + same run ID/evidence snapshot is used. +6. Inspect only allowlisted logs and bounded metric labels. + +## Result record + +| Field | Value | +| --- | --- | +| Date / environment | | +| Synthetic workspace | | +| Run ID | | +| Status transitions | | +| Citation validation result | | +| Retry result | | +| Provider/model identifier | | +| Observer | | + diff --git a/packages/core/src/db/schema/founder-weekly-review.ts b/packages/core/src/db/schema/founder-weekly-review.ts index 21fed15ea..a99c368bb 100644 --- a/packages/core/src/db/schema/founder-weekly-review.ts +++ b/packages/core/src/db/schema/founder-weekly-review.ts @@ -23,6 +23,10 @@ export const founderWeeklyReviewRunStatusEnum = [ ] as const; export const founderWeeklyReviewOperationTypeEnum = ["retry"] as const; +export const founderWeeklyReviewDispatchOperationTypeEnum = ["create", "retry"] as const; +export const founderWeeklyReviewDispatchStatusEnum = [ + "pending", "dispatching", "dispatched", "failed", +] as const; export const founderWeeklyReviewRuns = pgTable( "founder_weekly_review_runs", @@ -130,7 +134,47 @@ export const founderWeeklyReviewOperations = pgTable( }) ); +/** Durable handoff from LAU-5 lifecycle writes to the Inngest event bus. */ +export const founderWeeklyReviewDispatches = pgTable( + "founder_weekly_review_dispatches", + { + id: varchar("id", { length: 64 }).primaryKey(), + companyId: bigint("company_id", { mode: "bigint" }).notNull() + .references(() => company.id, { onDelete: "cascade" }), + runId: varchar("run_id", { length: 64 }).notNull() + .references(() => founderWeeklyReviewRuns.id, { onDelete: "cascade" }), + operationType: varchar("operation_type", { + length: 16, + enum: founderWeeklyReviewDispatchOperationTypeEnum, + }).notNull(), + operationKey: varchar("operation_key", { length: 128 }).notNull(), + eventId: varchar("event_id", { length: 128 }).notNull(), + generationJobId: varchar("generation_job_id", { length: 128 }).notNull(), + generationClaimId: varchar("generation_claim_id", { length: 128 }).notNull(), + status: varchar("status", { length: 16, enum: founderWeeklyReviewDispatchStatusEnum }) + .notNull().default("pending"), + attemptCount: integer("attempt_count").notNull().default(0), + availableAt: timestamp("available_at", { withTimezone: true }).notNull() + .default(sql`CURRENT_TIMESTAMP`), + dispatchedAt: timestamp("dispatched_at", { withTimezone: true }), + lastErrorCode: varchar("last_error_code", { length: 128 }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: timestamp("updated_at", { withTimezone: true }).$onUpdate(() => new Date()), + }, + (table) => ({ + operationUnique: uniqueIndex("founder_weekly_review_dispatches_run_operation_key_unique") + .on(table.runId, table.operationType, table.operationKey), + eventUnique: uniqueIndex("founder_weekly_review_dispatches_event_id_unique").on(table.eventId), + pendingIdx: index("founder_weekly_review_dispatches_pending_idx") + .on(table.status, table.availableAt, table.createdAt), + companyRunIdx: index("founder_weekly_review_dispatches_company_run_idx") + .on(table.companyId, table.runId), + }) +); + export type FounderWeeklyReviewRunRow = InferSelectModel; export type FounderWeeklyReviewOperationRow = InferSelectModel< typeof founderWeeklyReviewOperations >; +export type FounderWeeklyReviewDispatchRow = InferSelectModel; From e414f0e506376afff5b41c44b80667fc9aa23945 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 25 Jul 2026 15:17:51 +0800 Subject: [PATCH 08/29] feat: complete founder review evidence collection --- .../customer-feedback.test.ts | 44 +++ .../evidence-assembly.test.ts | 25 +- .../evidence-service.test.ts | 5 +- .../founder-context.test.ts | 25 ++ .../reporting-period.test.ts | 9 + .../src/founder-weekly-review/README.md | 24 ++ .../founder-weekly-review/evidence-service.ts | 283 +++++++++++------- 7 files changed, 296 insertions(+), 119 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/founder-context.test.ts diff --git a/apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts b/apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts new file mode 100644 index 000000000..51f227ae9 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts @@ -0,0 +1,44 @@ +import { + mapCustomerFeedbackChunkToEvidenceItem, + type CustomerFeedbackChunkRow, +} from "@launchstack/features/founder-weekly-review"; + +function row(content: string | null): CustomerFeedbackChunkRow { + return { + documentId: 42n, + versionId: 100, + versionNumber: 3, + documentTitle: "Customer interview", + documentCategory: "Customer Feedback", + createdAt: new Date("2026-07-10T18:15:00.000Z"), + chunkId: 9, + chunkContent: content, + pageNumber: 4, + }; +} + +describe("Customer Feedback chunk mapping", () => { + it.each([null, "", " "])("does not create cited evidence for missing content", (content) => { + expect(mapCustomerFeedbackChunkToEvidenceItem(row(content))).toBeNull(); + }); + + it("maps valid content with only document-context-chunk provenance", () => { + expect(mapCustomerFeedbackChunkToEvidenceItem(row(" Export is too slow. "))).toEqual({ + sourceType: "customer_feedback", + sourceId: "customer_feedback:doc:42:version:100:section:9", + title: "Customer interview", + sourceTimestamp: "2026-07-10T18:15:00.000Z", + excerpt: "Export is too slow.", + workspaceDeepLink: "/employer/documents/viewer?docId=42", + metadata: { + documentId: "42", + documentVersionId: 100, + sectionId: 9, + versionNumber: 3, + documentCategory: "Customer Feedback", + pageNumber: 4, + excerptTruncated: false, + }, + }); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts index 590813825..916ddbfa0 100644 --- a/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts @@ -1,5 +1,6 @@ import { dedupeEvidenceItems, + FounderWeeklyReviewEvidenceConflictError, orderEvidenceItems, type FounderWeeklyReviewEvidenceItem, } from "@launchstack/features/founder-weekly-review"; @@ -20,25 +21,22 @@ function makeItem( } describe("dedupeEvidenceItems", () => { - it("collapses items with the same sourceType+sourceId, keeping the first", () => { + it("collapses canonically identical items with the same sourceId", () => { const result = dedupeEvidenceItems([ makeItem("a", "2026-01-01T00:00:00.000Z"), - makeItem("a", "2026-02-01T00:00:00.000Z"), // duplicate identity + makeItem("a", "2026-01-01T00:00:00.000Z"), makeItem("b"), ]); expect(result.map((i) => i.sourceId)).toEqual(["a", "b"]); - // the FIRST "a" survives, so its timestamp is the January one expect(result[0]?.sourceTimestamp).toBe("2026-01-01T00:00:00.000Z"); }); - it("keeps items that share a sourceId but differ in sourceType", () => { - const result = dedupeEvidenceItems([ + it("rejects sourceId conflicts even across source types", () => { + expect(() => dedupeEvidenceItems([ makeItem("x", undefined, "document_change"), makeItem("x", undefined, "customer_feedback"), - ]); - - expect(result).toHaveLength(2); + ])).toThrow(FounderWeeklyReviewEvidenceConflictError); }); it("returns an empty array unchanged", () => { @@ -64,6 +62,17 @@ describe("orderEvidenceItems", () => { expect(result.map((i) => i.sourceId)).toEqual(["a", "z"]); }); + it("orders source-type ties with ordinal comparison, not locale collation", () => { + const ts = "2026-01-01T00:00:00.000Z"; + const result = orderEvidenceItems([ + makeItem("same", ts, "github_activity"), + makeItem("same", ts, "customer_feedback"), + ]); + expect(result.map((item) => item.sourceType)).toEqual([ + "customer_feedback", "github_activity", + ]); + }); + it("sorts items without a timestamp first", () => { const result = orderEvidenceItems([ makeItem("withTs", "2026-01-01T00:00:00.000Z"), diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts index a461c27f1..328a90563 100644 --- a/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts @@ -23,18 +23,19 @@ describe("mapDocumentVersionToEvidenceItem", () => { expect(item).toEqual({ sourceType: "document_change", - sourceId: "document-version:42:3", + sourceId: "document_change:doc:42:version:100", title: "Weekly Product Notes", sourceTimestamp: "2026-07-10T18:15:00.000Z", excerpt: "Fixed onboarding activation delay.", workspaceDeepLink: "/employer/documents/viewer?docId=42", metadata: { documentId: "42", - versionId: 100, + documentVersionId: 100, versionNumber: 3, documentCategory: "Product", uploadedBy: "user_abc", hasChangelog: true, + changelogTruncated: false, }, }); }); diff --git a/apps/web/__tests__/founderWeeklyReview/founder-context.test.ts b/apps/web/__tests__/founderWeeklyReview/founder-context.test.ts new file mode 100644 index 000000000..87f933f11 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/founder-context.test.ts @@ -0,0 +1,25 @@ +import { + FounderWeeklyReviewEvidenceService, +} from "@launchstack/features/founder-weekly-review"; + +describe("founder context evidence", () => { + const service = new FounderWeeklyReviewEvidenceService({} as never); + + it("normalizes request-time founder context with a stable non-content id", () => { + const result = service.collectFounderContextEvidence({ + founderContext: " Customers need faster exports. ", + contextEntryId: "request-123", + actor: { externalUserId: "user_123" }, + }); + expect(result.items).toEqual([expect.objectContaining({ + sourceType: "founder_context", + sourceId: "founder_context:entry:request-123", + excerpt: "Customers need faster exports.", + metadata: expect.objectContaining({ enteredBy: "user_123", provenance: "request_time_founder_input" }), + })]); + }); + + it("does not create evidence for blank context", () => { + expect(service.collectFounderContextEvidence({ founderContext: " " }).items).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts b/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts index 0050fd95e..899eb9b6e 100644 --- a/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts @@ -24,6 +24,15 @@ describe("resolveReportingPeriodBounds", () => { expect(bounds.endExclusive.toISOString()).toBe("2026-07-13T07:00:00.000Z"); }); + it("uses the following local midnight across the DST spring transition", () => { + const bounds = resolveReportingPeriodBounds( + { start: "2026-03-08", end: "2026-03-08" }, + "America/Los_Angeles" + ); + expect(bounds.startInclusive.toISOString()).toBe("2026-03-08T08:00:00.000Z"); + expect(bounds.endExclusive.toISOString()).toBe("2026-03-09T07:00:00.000Z"); + }); + it("treats UTC boundaries as literal midnight", () => { const bounds = resolveReportingPeriodBounds( { start: "2026-02-16", end: "2026-02-22" }, diff --git a/packages/features/src/founder-weekly-review/README.md b/packages/features/src/founder-weekly-review/README.md index 50c3d7477..1720a5c46 100644 --- a/packages/features/src/founder-weekly-review/README.md +++ b/packages/features/src/founder-weekly-review/README.md @@ -50,6 +50,30 @@ Every repository method accepts `companyId` explicitly and includes it in SQL pr - Evidence snapshot: `founder-weekly-review-evidence/v1` - Review payload: `founder-weekly-review/v1` +## Evidence collection (LAU-6) + +Evidence `sourceType` is a provenance and citation-safety classification, not a +generic document label. The production collector currently emits: + +- `document_change` for every company-scoped document version created in the + reporting period; +- `customer_feedback` for cited, exact-version sections of documents whose + stored category is exactly `Customer Feedback`; and +- `founder_context` for non-empty request-time founder input, with the actor + and stable request/context entry identity retained as provenance. + +`manual_note` is intentionally not collected: notes are user-owned and the +current schema/API has no workspace-visible sharing policy. `github_activity` +is intentionally not collected: the repository has GitHub archive upload and +repo-explainer flows, but no company-scoped commit/PR event model. + +Every snapshot has globally unique `sourceId` values because citations refer to +bare source IDs. Exact duplicate items collapse; conflicting duplicates fail +collection. Optional-source absence and result truncation become bounded source +warnings, while successful evidence remains in the snapshot. The evidence +collector acquires and normalizes evidence only; it does not own review +lifecycle, dispatch, persistence, or generation. + ## Current V1 product decision Multiple runs are allowed for the same company and reporting period. LAU-5 does not enforce period uniqueness or a single published review per period. diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index e21280d22..5df6d5f6a 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -3,18 +3,47 @@ import { FounderWeeklyReviewEvidenceSnapshotSchema, type FounderWeeklyReviewEvidenceItem, type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewEvidenceWarning, type ReportingPeriod, } from "./contracts"; import { resolveReportingPeriodBounds } from "./reporting-period"; import { and, asc, eq, gte, lt } from "drizzle-orm"; import { getDb, type DbClient } from "@launchstack/core/db"; -import { document, documentVersions } from "@launchstack/core/db/schema"; +import { + document, + documentContextChunks, + documentVersions, +} from "@launchstack/core/db/schema"; + +/** Approved, stored category value. Do not fuzzy-match or seed this category. */ +export const CUSTOMER_FEEDBACK_CATEGORY = "Customer Feedback"; +const MAX_SNAPSHOT_ITEMS = 500; +const MAX_ITEMS_PER_SOURCE = 250; +const MAX_EXCERPT_LENGTH = 4000; +const MAX_WARNINGS = 100; function normalizeText(value: string | null | undefined): string | null { - const trimmed = value?.trim(); + const trimmed = value?.replace(/\s+/g, " ").trim(); return trimmed ? trimmed : null; } +function boundExcerpt(value: string): { excerpt: string; truncated: boolean } { + return value.length <= MAX_EXCERPT_LENGTH + ? { excerpt: value, truncated: false } + : { excerpt: value.slice(0, MAX_EXCERPT_LENGTH), truncated: true }; +} + +function warning(code: string, message: string, sourceType?: FounderWeeklyReviewEvidenceItem["sourceType"]): FounderWeeklyReviewEvidenceWarning { + return { code, message: message.slice(0, 512), ...(sourceType ? { sourceType } : {}) }; +} + +export class FounderWeeklyReviewEvidenceConflictError extends Error { + readonly code = "duplicate_evidence_source_id_conflict"; + constructor(readonly sourceId: string) { + super(`Conflicting evidence items share sourceId: ${sourceId}`); + } +} + export interface DocumentVersionRow { documentId: bigint; documentTitle: string; @@ -26,143 +55,179 @@ export interface DocumentVersionRow { createdAt: Date; } -export function mapDocumentVersionToEvidenceItem( - row: DocumentVersionRow -): FounderWeeklyReviewEvidenceItem { - const changelog = normalizeText(row.changelog); +export interface CustomerFeedbackChunkRow { + documentId: bigint; + versionId: number; + versionNumber: number; + documentTitle: string; + documentCategory: string; + createdAt: Date; + chunkId: number | null; + chunkContent: string | null; + pageNumber: number | null; +} +export function mapCustomerFeedbackChunkToEvidenceItem( + row: CustomerFeedbackChunkRow +): FounderWeeklyReviewEvidenceItem | null { + const content = row.chunkContent?.trim(); + if (!content || row.chunkId === null) return null; + const bounded = boundExcerpt(content); return { - sourceType: "document_change", - sourceId: `document-version:${row.documentId}:${row.versionNumber}`, + sourceType: "customer_feedback", + sourceId: `customer_feedback:doc:${row.documentId}:version:${row.versionId}:section:${row.chunkId}`, title: row.documentTitle, sourceTimestamp: row.createdAt.toISOString(), - excerpt: changelog ?? `Version ${row.versionNumber} uploaded`, - // canonicalUrl intentionally omitted: document.url is inconsistent across rows in document datatable - // relative paths (/api/files/115), storage urls, local dev urls + excerpt: bounded.excerpt, workspaceDeepLink: `/employer/documents/viewer?docId=${row.documentId}`, metadata: { documentId: row.documentId.toString(), - versionId: row.versionId, + documentVersionId: row.versionId, + sectionId: row.chunkId, versionNumber: row.versionNumber, documentCategory: row.documentCategory, - uploadedBy: row.uploadedBy, - hasChangelog: changelog !== null, + pageNumber: row.pageNumber, + excerptTruncated: bounded.truncated, + }, + }; +} + +export function mapDocumentVersionToEvidenceItem(row: DocumentVersionRow): FounderWeeklyReviewEvidenceItem { + const changelog = normalizeText(row.changelog); + const bounded = changelog ? boundExcerpt(changelog) : null; + return { + sourceType: "document_change", + sourceId: `document_change:doc:${row.documentId}:version:${row.versionId}`, + title: row.documentTitle, + sourceTimestamp: row.createdAt.toISOString(), + excerpt: bounded?.excerpt ?? `Version ${row.versionNumber} uploaded`, + workspaceDeepLink: `/employer/documents/viewer?docId=${row.documentId}`, + metadata: { + documentId: row.documentId.toString(), documentVersionId: row.versionId, + versionNumber: row.versionNumber, documentCategory: row.documentCategory, + uploadedBy: row.uploadedBy, hasChangelog: changelog !== null, + changelogTruncated: bounded?.truncated ?? false, }, }; } +export interface FounderWeeklyReviewEvidenceActor { externalUserId: string } export interface BuildFounderWeeklyReviewEvidenceSnapshotInput { companyId: bigint; reportingPeriod: ReportingPeriod; workspaceTimezone: string; + founderContext?: string; + actor?: FounderWeeklyReviewEvidenceActor; + /** Stable idempotency identity supplied by the request layer. */ + contextEntryId?: string; + requestKey?: string; capturedAt?: Date; maxItems?: number; } +export interface FounderWeeklyReviewEvidenceSourceResult { + items: FounderWeeklyReviewEvidenceItem[]; + warnings: FounderWeeklyReviewEvidenceWarning[]; +} + export class FounderWeeklyReviewEvidenceService { - constructor(private readonly db: DbClient = getDb()) {} - - async collectDocumentChangeEvidence( - companyId: bigint, - startInclusive: Date, - endExclusive: Date, - ): Promise { - const rows = await this.db - .select({ - documentId: documentVersions.documentId, - documentTitle: document.title, - documentCategory: document.category, - versionId: documentVersions.id, - versionNumber: documentVersions.versionNumber, - uploadedBy: documentVersions.uploadedBy, - changelog: documentVersions.changelog, - createdAt: documentVersions.createdAt, - }) - .from(documentVersions) - .innerJoin(document, eq(document.id, documentVersions.documentId)) - .where( - and( - gte(documentVersions.createdAt, startInclusive), - lt(documentVersions.createdAt, endExclusive), - eq(document.companyId, companyId) - ) - ) - .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id)); - return rows.map((row) => mapDocumentVersionToEvidenceItem(row)); + constructor(private readonly db: DbClient = getDb(), private readonly now: () => Date = () => new Date()) {} + + async collectDocumentChangeEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + return (await this.collectDocumentChangeEvidenceResult(companyId, startInclusive, endExclusive)).items; } - async buildEvidenceSnapshot( - input: BuildFounderWeeklyReviewEvidenceSnapshotInput - ): Promise { - const { startInclusive, endExclusive } = resolveReportingPeriodBounds( - input.reportingPeriod, - input.workspaceTimezone - ); - - // only document_change exists right now - const collected = await this.collectDocumentChangeEvidence( - input.companyId, - startInclusive, - endExclusive - ); - - // more evidence can be merged later - const merged = [...collected]; - - // 4. Drop exact duplicates, keep distinct citations. (AC #5) - const deduped = dedupeEvidenceItems(merged); - - // 5. Deterministic order, then cap at the schema's limit. - const maxItems = input.maxItems ?? 500; - const items = orderEvidenceItems(deduped).slice(0, maxItems); - - // 6. Wrap in the envelope. Empty `items` is valid — an empty/partial - // workspace returns a pack rather than throwing. (AC #6) - const snapshot = { - schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, - capturedAt: (input.capturedAt ?? new Date()).toISOString(), - reportingPeriod: input.reportingPeriod, - workspaceTimezone: input.workspaceTimezone, - items, - sourceWarnings: [], + private async collectDocumentChangeEvidenceResult(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + const rows = await this.db.select({ + documentId: documentVersions.documentId, documentTitle: document.title, + documentCategory: document.category, versionId: documentVersions.id, + versionNumber: documentVersions.versionNumber, uploadedBy: documentVersions.uploadedBy, + changelog: documentVersions.changelog, createdAt: documentVersions.createdAt, + }).from(documentVersions).innerJoin(document, eq(document.id, documentVersions.documentId)) + .where(and(gte(documentVersions.createdAt, startInclusive), lt(documentVersions.createdAt, endExclusive), eq(document.companyId, companyId))) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id)).limit(MAX_ITEMS_PER_SOURCE + 1); + return { + items: rows.slice(0, MAX_ITEMS_PER_SOURCE).map(mapDocumentVersionToEvidenceItem), + warnings: rows.length > MAX_ITEMS_PER_SOURCE + ? [warning("document_change_truncated", "Document change evidence was truncated to the per-source limit.", "document_change")] + : [], }; + } - // 7. Validate before returning — a malformed pack never leaves here. - return FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot); + async collectCustomerFeedbackEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + const rows = await this.db.select({ + documentId: documentVersions.documentId, versionId: documentVersions.id, + versionNumber: documentVersions.versionNumber, documentTitle: document.title, + documentCategory: document.category, createdAt: documentVersions.createdAt, + chunkId: documentContextChunks.id, + chunkContent: documentContextChunks.content, + pageNumber: documentContextChunks.pageNumber, + }).from(documentVersions).innerJoin(document, eq(document.id, documentVersions.documentId)) + .leftJoin(documentContextChunks, and(eq(documentContextChunks.documentId, documentVersions.documentId), eq(documentContextChunks.versionId, documentVersions.id))) + .where(and(gte(documentVersions.createdAt, startInclusive), lt(documentVersions.createdAt, endExclusive), eq(document.companyId, companyId), eq(document.category, CUSTOMER_FEEDBACK_CATEGORY))) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id), asc(documentContextChunks.id)).limit(MAX_ITEMS_PER_SOURCE + 1); + if (rows.length === 0) return { items: [], warnings: [warning("customer_feedback_unavailable", "No Customer Feedback documents were available for this reporting period.", "customer_feedback")] }; + const mapped = rows.map(mapCustomerFeedbackChunkToEvidenceItem); + const items = mapped.filter((item): item is FounderWeeklyReviewEvidenceItem => item !== null); + const warnings: FounderWeeklyReviewEvidenceWarning[] = []; + if (items.length === 0) warnings.push(warning("customer_feedback_missing_sections", "Customer Feedback document versions had no processed, citeable sections.", "customer_feedback")); + else if (items.length < rows.length) warnings.push(warning("customer_feedback_missing_sections", "Some Customer Feedback sections had no citeable content.", "customer_feedback")); + if (rows.length > MAX_ITEMS_PER_SOURCE) warnings.push(warning("customer_feedback_truncated", "Customer Feedback evidence was truncated to the per-source limit.", "customer_feedback")); + return { items: items.slice(0, MAX_ITEMS_PER_SOURCE), warnings }; } -} -export function dedupeEvidenceItems( - items: FounderWeeklyReviewEvidenceItem[] -): FounderWeeklyReviewEvidenceItem[] { - const seenIdentities = new Set() + collectFounderContextEvidence(input: Pick): FounderWeeklyReviewEvidenceSourceResult { + const context = normalizeText(input.founderContext); + if (!context) return { items: [], warnings: [] }; + const entryId = input.contextEntryId ?? input.requestKey; + if (!entryId) throw new Error("Founder context requires a stable contextEntryId or requestKey"); + if (entryId.length > 230) throw new Error("Founder context entry identity is too long"); + if (!input.actor) throw new Error("Founder context requires an authenticated actor"); + const bounded = boundExcerpt(context); + return { items: [{ sourceType: "founder_context", sourceId: `founder_context:entry:${entryId}`, + title: "Founder context", excerpt: bounded.excerpt, + metadata: { enteredBy: input.actor.externalUserId, provenance: "request_time_founder_input", excerptTruncated: bounded.truncated } }], warnings: [] }; + } + + async collectFounderWeeklyReviewEvidence(input: BuildFounderWeeklyReviewEvidenceSnapshotInput): Promise { + return this.buildEvidenceSnapshot(input); + } - return items.filter(item => { - const identity = `${item.sourceType}:${item.sourceId}` + async buildEvidenceSnapshot(input: BuildFounderWeeklyReviewEvidenceSnapshotInput): Promise { + const { startInclusive, endExclusive } = resolveReportingPeriodBounds(input.reportingPeriod, input.workspaceTimezone); + const documentChanges = await this.collectDocumentChangeEvidenceResult(input.companyId, startInclusive, endExclusive); + const feedback = await this.collectCustomerFeedbackEvidence(input.companyId, startInclusive, endExclusive); + const founderContext = this.collectFounderContextEvidence(input); + const sourceResults: FounderWeeklyReviewEvidenceSourceResult[] = [documentChanges, feedback, founderContext]; + const items = orderEvidenceItems(dedupeEvidenceItems(sourceResults.flatMap((result) => result.items))); + const requestedMax = input.maxItems ?? MAX_SNAPSHOT_ITEMS; + const maxItems = Math.max(0, Math.min(MAX_SNAPSHOT_ITEMS, requestedMax)); + const warnings = dedupeWarnings(sourceResults.flatMap((result) => result.warnings)); + if (items.length > maxItems) warnings.push(warning("evidence_snapshot_truncated", "Evidence snapshot was truncated to its configured maximum.")); + return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + capturedAt: (input.capturedAt ?? this.now()).toISOString(), reportingPeriod: input.reportingPeriod, + workspaceTimezone: input.workspaceTimezone, items: items.slice(0, maxItems), sourceWarnings: dedupeWarnings(warnings).slice(0, MAX_WARNINGS) }); + } +} - if (seenIdentities.has(identity)) { - return false - } +function canonicalItem(item: FounderWeeklyReviewEvidenceItem): string { return JSON.stringify(item); } +export function dedupeEvidenceItems(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceItem[] { + const bySourceId = new Map(); + for (const item of items) { + const existing = bySourceId.get(item.sourceId); + if (!existing) { bySourceId.set(item.sourceId, item); continue; } + if (canonicalItem(existing) !== canonicalItem(item)) throw new FounderWeeklyReviewEvidenceConflictError(item.sourceId); + } + return [...bySourceId.values()]; +} - seenIdentities.add(identity) - return true - }) +function compareOrdinal(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } +export function orderEvidenceItems(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceItem[] { + return [...items].sort((a, b) => compareOrdinal(a.sourceTimestamp ?? "", b.sourceTimestamp ?? "") || compareOrdinal(a.sourceType, b.sourceType) || compareOrdinal(a.sourceId, b.sourceId)); } -// order primarily based on createdAt timestamp, sourceId as tie breaker -export function orderEvidenceItems( - items: FounderWeeklyReviewEvidenceItem[] -): FounderWeeklyReviewEvidenceItem[] { - return [...items].sort((a, b) => { - // if timestamp is missing use empty string - const aTime = a.sourceTimestamp ?? ""; - const bTime = b.sourceTimestamp ?? ""; - if (aTime !== bTime) { - return aTime.localeCompare(bTime); - } - - const identityA = `${a.sourceType}:${a.sourceId}`; - const identityB = `${b.sourceType}:${b.sourceId}`; - return identityA.localeCompare(identityB); - }); -} \ No newline at end of file +export function dedupeWarnings(warnings: FounderWeeklyReviewEvidenceWarning[]): FounderWeeklyReviewEvidenceWarning[] { + const unique = new Map(); + for (const item of warnings) unique.set(`${item.code}\u0000${item.sourceType ?? ""}\u0000${item.message}`, item); + return [...unique.values()].sort((a, b) => compareOrdinal(a.code, b.code) || compareOrdinal(a.sourceType ?? "", b.sourceType ?? "") || compareOrdinal(a.message, b.message)); +} From 3ab8a761089993bad5a06fcafccef1dab99819c8 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 25 Jul 2026 16:32:16 +0800 Subject: [PATCH 09/29] feat: secure founder review async orchestration --- .../[runId]/retry/route.ts | 9 ++- .../founder-weekly-reviews/[runId]/route.ts | 6 +- .../app/api/founder-weekly-reviews/route.ts | 69 ++++++++++++++----- .../founder-weekly-review/actor-resolver.ts | 40 +++++++++++ .../evidence-collector.ts | 42 ++++++++++- .../src/server/founder-weekly-review/http.ts | 1 + .../founder-weekly-review/observability.ts | 6 +- .../inngest/functions/founderWeeklyReview.ts | 18 ++++- apps/web/src/server/metrics/registry.ts | 4 ++ .../runbooks/founder-weekly-review-staging.md | 32 ++++++--- 10 files changed, 189 insertions(+), 38 deletions(-) create mode 100644 apps/web/src/server/founder-weekly-review/actor-resolver.ts diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts index 76558bd68..115c0ca1d 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts @@ -1,16 +1,19 @@ import { NextResponse } from "next/server"; import { auth } from "@clerk/nextjs/server"; import { z } from "zod"; -import { getActiveCompanyContext } from "~/lib/active-workspace"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { retryRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; import { inngest } from "~/server/inngest/client"; +import { founderWeeklyReviewRetries, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; const RetrySchema = z.object({ requestKey: z.string().min(1).max(128) }); export async function POST(request: Request, { params }: { params: Promise<{ runId: string }> }) { const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); const parsed = RetrySchema.safeParse(await request.json().catch(() => null)); if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); - try { const context = await getActiveCompanyContext(userId); const { runId } = await params; - const { run } = await retryRunWithDispatch({ actor: { externalUserId: userId, internalUserId: context.userId, companyId: context.companyId, role: context.role }, runId, requestKey: parsed.data.requestKey }); + try { const actor = await productionFounderWeeklyReviewActorResolver.resolve(userId); const { runId } = await params; + logFounderWeeklyReview({ runId, companyId: actor.companyId.toString(), stage: "retry_requested", status: "failed" }); + const { run } = await retryRunWithDispatch({ actor, runId, requestKey: parsed.data.requestKey }); + if (run.status === "queued") { founderWeeklyReviewRetries.inc(); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "retry_queued", status: run.status, retryCount: run.retryCount }); } await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); return NextResponse.json({ run: safeRun(run) }, { status: 202 }); } catch (error) { return safeFounderWeeklyReviewError(error); } diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts index 2a2dcca58..3f4852f17 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts @@ -1,12 +1,12 @@ import { NextResponse } from "next/server"; import { auth } from "@clerk/nextjs/server"; -import { getActiveCompanyContext } from "~/lib/active-workspace"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { FounderWeeklyReviewUserService } from "@launchstack/features/founder-weekly-review"; import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; export async function GET(_request: Request, { params }: { params: Promise<{ runId: string }> }) { const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - try { const context = await getActiveCompanyContext(userId); const { runId } = await params; - const run = await new FounderWeeklyReviewUserService().getRun({ externalUserId: userId, internalUserId: context.userId, companyId: context.companyId, role: context.role }, runId); + try { const actor = await productionFounderWeeklyReviewActorResolver.resolve(userId); const { runId } = await params; + const run = await new FounderWeeklyReviewUserService().getRun(actor, runId); return NextResponse.json({ run: safeRun(run) }); } catch (error) { return safeFounderWeeklyReviewError(error); } } diff --git a/apps/web/src/app/api/founder-weekly-reviews/route.ts b/apps/web/src/app/api/founder-weekly-reviews/route.ts index 3cf5ac86f..ff086c53b 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/route.ts @@ -1,27 +1,62 @@ import { NextResponse } from "next/server"; import { auth } from "@clerk/nextjs/server"; import { z } from "zod"; -import { getActiveCompanyContext } from "~/lib/active-workspace"; -import { unavailableFounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; +import { FounderWeeklyReviewRepository } from "@launchstack/features/founder-weekly-review"; +import type { FounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; +import { canonicalFounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; +import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { createRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; import { inngest } from "~/server/inngest/client"; +import { founderWeeklyReviewRunsCreated, founderWeeklyReviewStageDuration, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; const CreateSchema = z.object({ requestKey: z.string().min(1).max(128), reportingPeriod: z.object({ start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), end: z.string().regex(/^\d{4}-\d{2}-\d{2}$/) }), workspaceTimezone: z.string().min(1).max(128), founderContext: z.string().max(4000).optional() }); -export async function POST(request: Request) { - const { userId } = await auth(); - if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - const parsed = CreateSchema.safeParse(await request.json().catch(() => null)); - if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); - try { - const context = await getActiveCompanyContext(userId); - const evidenceSnapshot = await unavailableFounderWeeklyReviewEvidenceCollector.collectFounderWeeklyReviewEvidence({ companyId: context.companyId, reportingPeriod: parsed.data.reportingPeriod, workspaceTimezone: parsed.data.workspaceTimezone, founderContext: parsed.data.founderContext }); - const { run } = await createRunWithDispatch({ actor: { externalUserId: userId, internalUserId: context.userId, companyId: context.companyId, role: context.role }, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, evidenceSnapshot }); - await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); - return NextResponse.json({ run: safeRun(run) }, { status: 202 }); - } catch (error) { - if (error && typeof error === "object" && "code" in error && (error as { code?: string }).code === "evidence_collector_unavailable") return NextResponse.json({ error: "Generation unavailable" }, { status: 503 }); - return safeFounderWeeklyReviewError(error); - } +export interface FounderWeeklyReviewRouteDependencies { + actorResolver: Pick; + evidenceCollector: FounderWeeklyReviewEvidenceCollector; + repository: Pick; + createRunWithDispatch: typeof createRunWithDispatch; + sendDispatchRequested: () => Promise; } + +export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRouteDependencies) { + return async function POST(request: Request) { + const { userId } = await auth(); + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + try { + // Authorization is deliberately before parsing/collection: invalid or + // unauthorized callers must never cause workspace evidence reads. + const actor = await deps.actorResolver.resolve(userId); + const parsed = CreateSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + const existing = await deps.repository.getByCompanyAndRequestKey(actor.companyId, parsed.data.requestKey); + if (existing) return NextResponse.json({ run: safeRun(existing) }, { status: 202 }); + const startedAt = performance.now(); + logFounderWeeklyReview({ runId: "pending", companyId: actor.companyId.toString(), stage: "evidence_collection_started", status: "pending" }); + const evidenceSnapshot = await deps.evidenceCollector.collectFounderWeeklyReviewEvidence({ companyId: actor.companyId, reportingPeriod: parsed.data.reportingPeriod, workspaceTimezone: parsed.data.workspaceTimezone, founderContext: parsed.data.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: parsed.data.requestKey }); + const durationMs = Math.round(performance.now() - startedAt); + founderWeeklyReviewStageDuration.observe({ stage: "evidence_collection", result: "success" }, durationMs / 1000); + logFounderWeeklyReview({ runId: "pending", companyId: actor.companyId.toString(), stage: "evidence_collection_completed", status: "pending", durationMs }); + const { run } = await deps.createRunWithDispatch({ actor, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, evidenceSnapshot }); + founderWeeklyReviewRunsCreated.inc(); + logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "run_created", status: run.status, retryCount: run.retryCount }); + logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "dispatch_created", status: run.status }); + await deps.sendDispatchRequested(); + return NextResponse.json({ run: safeRun(run) }, { status: 202 }); + } catch (error) { + founderWeeklyReviewStageDuration.observe({ stage: "evidence_collection", result: "failure" }, 0); + return safeFounderWeeklyReviewError(error); + } + }; +} + +const productionDependencies: FounderWeeklyReviewRouteDependencies = { + actorResolver: productionFounderWeeklyReviewActorResolver, + evidenceCollector: canonicalFounderWeeklyReviewEvidenceCollector, + repository: new FounderWeeklyReviewRepository(), + createRunWithDispatch, + sendDispatchRequested: () => inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }), +}; +export const POST = createFounderWeeklyReviewPostHandler(productionDependencies); diff --git a/apps/web/src/server/founder-weekly-review/actor-resolver.ts b/apps/web/src/server/founder-weekly-review/actor-resolver.ts new file mode 100644 index 000000000..e8a2ef9be --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/actor-resolver.ts @@ -0,0 +1,40 @@ +import { and, eq } from "drizzle-orm"; +import { users, userCompanyMemberships } from "@launchstack/core/db/schema"; +import { db } from "~/server/db"; +import { getActiveCompanyId } from "~/lib/active-workspace"; + +const GENERATION_ROLES = new Set(["owner", "admin", "editor"]); + +export class FounderWeeklyReviewAuthorizationError extends Error { + readonly code = "forbidden"; +} + +export interface FounderWeeklyReviewActor { + externalUserId: string; + internalUserId: bigint; + companyId: bigint; + role: "owner" | "admin" | "editor"; +} + +/** + * This is intentionally stricter than the legacy active-workspace context: + * a legacy users.role is never a substitute for an actual membership row. + */ +export class FounderWeeklyReviewActorResolver { + async resolve(externalUserId: string): Promise { + const companyId = await getActiveCompanyId(externalUserId); + const [user] = await db.select({ id: users.id }).from(users) + .where(eq(users.userId, externalUserId)).limit(1); + if (!user) throw new FounderWeeklyReviewAuthorizationError(); + const [membership] = await db.select({ role: userCompanyMemberships.role }) + .from(userCompanyMemberships) + .where(and(eq(userCompanyMemberships.userId, BigInt(user.id)), eq(userCompanyMemberships.companyId, companyId))) + .limit(1); + if (!membership || !GENERATION_ROLES.has(membership.role)) { + throw new FounderWeeklyReviewAuthorizationError(); + } + return { externalUserId, internalUserId: BigInt(user.id), companyId, role: membership.role as FounderWeeklyReviewActor["role"] }; + } +} + +export const productionFounderWeeklyReviewActorResolver = new FounderWeeklyReviewActorResolver(); diff --git a/apps/web/src/server/founder-weekly-review/evidence-collector.ts b/apps/web/src/server/founder-weekly-review/evidence-collector.ts index 58c7d4084..e699c410c 100644 --- a/apps/web/src/server/founder-weekly-review/evidence-collector.ts +++ b/apps/web/src/server/founder-weekly-review/evidence-collector.ts @@ -1,4 +1,9 @@ -import type { FounderWeeklyReviewEvidenceSnapshot, ReportingPeriod } from "@launchstack/features/founder-weekly-review"; +import { + FounderWeeklyReviewEvidenceService, + FounderWeeklyReviewEvidenceSnapshotSchema, + type FounderWeeklyReviewEvidenceSnapshot, + type ReportingPeriod, +} from "@launchstack/features/founder-weekly-review"; export interface FounderWeeklyReviewEvidenceCollector { collectFounderWeeklyReviewEvidence(input: { @@ -6,6 +11,10 @@ export interface FounderWeeklyReviewEvidenceCollector { reportingPeriod: ReportingPeriod; workspaceTimezone: string; founderContext?: string; + /** LAU-6 records request-time founder context against this actor. */ + actor: { externalUserId: string }; + /** Stable identity makes idempotent founder context source IDs stable. */ + requestKey: string; }): Promise; } @@ -22,3 +31,34 @@ export const unavailableFounderWeeklyReviewEvidenceCollector: FounderWeeklyRevie throw new FounderWeeklyReviewEvidenceCollectorUnavailableError(); }, }; + +/** + * App adapter over LAU-6's canonical workspace collector. This is kept as an + * interface boundary deliberately: LAU-8 can compose github_activity here + * without changing routes, persistence, or the generation worker. + */ +export class CanonicalFounderWeeklyReviewEvidenceCollector implements FounderWeeklyReviewEvidenceCollector { + constructor(private readonly service = new FounderWeeklyReviewEvidenceService()) {} + + async collectFounderWeeklyReviewEvidence(input: { + companyId: bigint; + reportingPeriod: ReportingPeriod; + workspaceTimezone: string; + founderContext?: string; + actor: { externalUserId: string }; + requestKey: string; + }): Promise { + const snapshot = await this.service.collectFounderWeeklyReviewEvidence({ + companyId: input.companyId, + reportingPeriod: input.reportingPeriod, + workspaceTimezone: input.workspaceTimezone, + founderContext: input.founderContext, + actor: input.actor, + requestKey: input.requestKey, + }); + return FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot); + } +} + +export const canonicalFounderWeeklyReviewEvidenceCollector = + new CanonicalFounderWeeklyReviewEvidenceCollector(); diff --git a/apps/web/src/server/founder-weekly-review/http.ts b/apps/web/src/server/founder-weekly-review/http.ts index 21324a08e..2d0eccf60 100644 --- a/apps/web/src/server/founder-weekly-review/http.ts +++ b/apps/web/src/server/founder-weekly-review/http.ts @@ -18,5 +18,6 @@ export function safeFounderWeeklyReviewError(error: unknown) { if (code === "not_found") return NextResponse.json({ error: "Not found" }, { status: 404 }); if (code === "invalid_transition" || code === "conflict") return NextResponse.json({ error: "Conflict" }, { status: 409 }); if (code === "invalid_payload") return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + if (code === "evidence_collector_unavailable" || code === "infrastructure_unavailable") return NextResponse.json({ error: "Generation unavailable" }, { status: 503 }); return NextResponse.json({ error: "Internal server error" }, { status: 500 }); } diff --git a/apps/web/src/server/founder-weekly-review/observability.ts b/apps/web/src/server/founder-weekly-review/observability.ts index 58e836e35..2a749819a 100644 --- a/apps/web/src/server/founder-weekly-review/observability.ts +++ b/apps/web/src/server/founder-weekly-review/observability.ts @@ -5,6 +5,10 @@ import { founderWeeklyReviewRetries, founderWeeklyReviewStageDuration, founderWeeklyReviewCitationFailures, + founderWeeklyReviewRunsCreated, + founderWeeklyReviewRunsCompleted, + founderWeeklyReviewRunsFailed, + founderWeeklyReviewDispatchFailures, } from "~/server/metrics/registry"; const logger = createLogger("founder-weekly-review"); @@ -12,4 +16,4 @@ export function logFounderWeeklyReview(fields: { runId: string; companyId: string; stage: string; status: string; durationMs?: number; generationAttempt?: number; retryCount?: number; errorClass?: string; provider?: string; model?: string; }) { logger.info(fields, "founder weekly review stage"); } -export { founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRetries, founderWeeklyReviewStageDuration, founderWeeklyReviewCitationFailures }; +export { founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRetries, founderWeeklyReviewStageDuration, founderWeeklyReviewCitationFailures, founderWeeklyReviewRunsCreated, founderWeeklyReviewRunsCompleted, founderWeeklyReviewRunsFailed, founderWeeklyReviewDispatchFailures }; diff --git a/apps/web/src/server/inngest/functions/founderWeeklyReview.ts b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts index baace4037..81300cfcb 100644 --- a/apps/web/src/server/inngest/functions/founderWeeklyReview.ts +++ b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts @@ -5,7 +5,7 @@ import { generateFounderWeeklyReview } from "@launchstack/features/founder-weekl import { FounderWeeklyReviewGenerationValidationError } from "@launchstack/features/founder-weekly-review"; import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; import { claimPendingDispatches, markDispatchDispatched, returnDispatchToPending } from "~/server/founder-weekly-review/dispatch-service"; -import { founderWeeklyReviewCitationFailures, founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; +import { founderWeeklyReviewCitationFailures, founderWeeklyReviewDispatchFailures, founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRunsCompleted, founderWeeklyReviewRunsFailed, founderWeeklyReviewStageDuration, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; const GenerationEventSchema = z.object({ runId: z.string().min(1), companyId: z.string().min(1), generationJobId: z.string().min(1), generationClaimId: z.string().min(1) }); @@ -21,8 +21,10 @@ export const founderWeeklyReviewDispatcher = inngest.createFunction( }}); await markDispatchDispatched(dispatch.id); founderWeeklyReviewJobsEnqueued.inc({ operation: dispatch.operationType }); + logFounderWeeklyReview({ runId: dispatch.runId, companyId: dispatch.companyId.toString(), stage: "dispatch_sent", status: "dispatched" }); } catch { await returnDispatchToPending(dispatch.id, "dispatch_failed"); + founderWeeklyReviewDispatchFailures.inc(); } } return { dispatched: dispatches.length }; @@ -51,17 +53,27 @@ export const founderWeeklyReviewGenerationJob = inngest.createFunction( const data = GenerationEventSchema.parse(event.data); const context = { companyId: BigInt(data.companyId), runId: data.runId, generationJobId: data.generationJobId, generationClaimId: data.generationClaimId }; const worker = new FounderWeeklyReviewWorkerService(); - const claimed = await step.run("claim", () => worker.claimQueuedRun(context)); + // Inngest's generic step inference intersects event return types; retain the + // concrete LAU-5 lifecycle record at this boundary. + const claimed = await step.run("claim", () => worker.claimQueuedRun(context)) as unknown as FounderWeeklyReviewRunRecord; if (claimed.status !== "generating" || claimed.generationClaimId !== data.generationClaimId) return { skipped: true }; + logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: "worker_claimed", status: claimed.status, generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount }); try { + const generationStartedAt = performance.now(); + logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: "generation_started", status: claimed.status, generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount }); const generated = await step.run("generate", () => generateFounderWeeklyReview({ evidenceSnapshot: claimed.evidenceSnapshot, generate: generateFounderWeeklyReviewStructured })); const saved = await step.run("persist", () => worker.saveGeneratedDraft(context, generated.reviewPayload, generated.modelMetadata)) as unknown as FounderWeeklyReviewRunRecord; founderWeeklyReviewGenerationTotal.inc({ result: "success", error_class: "none" }); - logFounderWeeklyReview({ runId: saved.id, companyId: saved.companyId.toString(), stage: "persist", status: saved.status, generationAttempt: saved.generationAttempt, retryCount: saved.retryCount, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model }); + founderWeeklyReviewRunsCompleted.inc(); + founderWeeklyReviewStageDuration.observe({ stage: "generation", result: "success" }, (performance.now() - generationStartedAt) / 1000); + founderWeeklyReviewStageDuration.observe({ stage: "end_to_end", result: "success" }, (Date.now() - saved.createdAt.getTime()) / 1000); + logFounderWeeklyReview({ runId: saved.id, companyId: saved.companyId.toString(), stage: "generation_completed", status: saved.status, durationMs: Math.round(performance.now() - generationStartedAt), generationAttempt: saved.generationAttempt, retryCount: saved.retryCount, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model }); return { runId: saved.id, status: saved.status }; } catch (error) { if (error instanceof FounderWeeklyReviewGenerationValidationError) founderWeeklyReviewCitationFailures.inc(); founderWeeklyReviewGenerationTotal.inc({ result: "failure", error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + founderWeeklyReviewRunsFailed.inc({ error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation_failed" : "generation_failed", status: "generating", generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount, errorClass: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); throw error; } } diff --git a/apps/web/src/server/metrics/registry.ts b/apps/web/src/server/metrics/registry.ts index 69cafa493..62b61a038 100644 --- a/apps/web/src/server/metrics/registry.ts +++ b/apps/web/src/server/metrics/registry.ts @@ -59,6 +59,10 @@ export const founderWeeklyReviewJobsEnqueued = new Counter({ name: "pdr_founder_ export const founderWeeklyReviewGenerationTotal = new Counter({ name: "pdr_founder_weekly_review_generation_total", help: "Founder review generations by result", labelNames: ["result", "error_class"], registers: [metricsRegistry] }); export const founderWeeklyReviewRetries = new Counter({ name: "pdr_founder_weekly_review_retries_total", help: "Founder review retries", registers: [metricsRegistry] }); export const founderWeeklyReviewCitationFailures = new Counter({ name: "pdr_founder_weekly_review_citation_validation_failures_total", help: "Founder review citation validation failures", registers: [metricsRegistry] }); +export const founderWeeklyReviewRunsCreated = new Counter({ name: "pdr_founder_weekly_review_runs_created_total", help: "Founder review runs created", registers: [metricsRegistry] }); +export const founderWeeklyReviewRunsCompleted = new Counter({ name: "pdr_founder_weekly_review_runs_completed_total", help: "Founder review runs completed", registers: [metricsRegistry] }); +export const founderWeeklyReviewRunsFailed = new Counter({ name: "pdr_founder_weekly_review_runs_failed_total", help: "Founder review runs failed", labelNames: ["error_class"], registers: [metricsRegistry] }); +export const founderWeeklyReviewDispatchFailures = new Counter({ name: "pdr_founder_weekly_review_dispatch_failures_total", help: "Founder review outbox dispatch failures", registers: [metricsRegistry] }); export const founderWeeklyReviewStageDuration = new Histogram({ name: "pdr_founder_weekly_review_stage_duration_seconds", help: "Founder review stage duration", labelNames: ["stage", "result"], buckets: [0.1, 0.5, 1, 5, 15, 60, 300], registers: [metricsRegistry] }); export async function getMetricsSnapshot(): Promise { diff --git a/docs/runbooks/founder-weekly-review-staging.md b/docs/runbooks/founder-weekly-review-staging.md index 77abf7e92..cac044326 100644 --- a/docs/runbooks/founder-weekly-review-staging.md +++ b/docs/runbooks/founder-weekly-review-staging.md @@ -4,15 +4,28 @@ Status: **pending execution**. This runbook requires an accessible staging workspace, configured Inngest, and a configured generation provider. Never use production evidence or commit generated review contents. -1. Create an isolated staging workspace with synthetic complete, partial, and - empty evidence only. -2. Verify the Inngest endpoint and the configured `founderWeeklyReview` model. -3. Start a review with a fresh idempotency key and poll its run ID. -4. Verify `queued -> generating -> draft` and that every factual item cites a - supplied source ID; customer statements must cite customer feedback only. -5. Force a safe synthetic failure, retry with a new retry key, and verify the - same run ID/evidence snapshot is used. -6. Inspect only allowlisted logs and bounded metric labels. +1. Create an isolated staging workspace and grant the tester an owner, admin, + or editor membership. Add a `Customer Feedback` category document with + processed sections; use synthetic data only. +2. Verify the Inngest endpoint and configured `founderWeeklyReview` provider. + Submit `POST /api/founder-weekly-reviews` with a fresh `requestKey`, local + reporting-period dates, IANA `workspaceTimezone`, and optional founder + context. Save the returned run ID. +3. Poll `GET /api/founder-weekly-reviews/{runId}` and verify + `queued -> generating -> draft`. Read the returned draft only after it is + `draft` and compare every cited ID with the persisted run EvidenceSnapshot; + customer statements may cite `customer_feedback` only. +4. Repeat with partial evidence (no Customer Feedback) and empty evidence. + Verify explicit no-evidence customer sections, no invented facts, and that + empty evidence performs no provider generation. +5. Force a safe provider failure in staging, then `POST + /api/founder-weekly-reviews/{runId}/retry` with a new retry request key. + Confirm the same run ID returns to queued, retry count increments once, and + the stored snapshot is byte-for-byte unchanged. +6. Inspect only allowlisted structured stage logs and bounded metrics + (`runs_created`, completed/failed, retries, citation failures, dispatch + failures, evidence/generation durations). Do not copy source contents from + logs or database tooling into the record. ## Result record @@ -26,4 +39,3 @@ production evidence or commit generated review contents. | Retry result | | | Provider/model identifier | | | Observer | | - From 227ed3ecf8f2d541c37e1f9dc872bd0ad62760f3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 29 Jul 2026 19:57:24 +0000 Subject: [PATCH 10/29] fix: align company Drizzle schema with migration 0011 Remove the legacy plaintext embedding credential columns from the company Drizzle schema and drop the runtime fallback reads that still queried them. Migration 0011 already drops those columns, so clean migration-replayed databases failed on full company selects/returning. Move the backfill script under apps/web and switch it to raw SQL so it can still migrate older db:push-shaped databases that retain the columns. Co-authored-by: Kien Le --- .../0010_company_embedding_credentials.sql | 6 +- ...1_drop_plaintext_embedding_credentials.sql | 6 +- apps/web/package.json | 1 + .../scripts/backfill-embedding-credentials.ts | 144 ++++++++++++++++++ package.json | 1 + packages/core/src/db/schema/base.ts | 11 +- .../core/src/db/schema/company-credentials.ts | 5 +- .../src/embeddings/company-credentials.ts | 85 +++-------- scripts/backfill-embedding-credentials.ts | 117 +------------- 9 files changed, 183 insertions(+), 193 deletions(-) create mode 100644 apps/web/scripts/backfill-embedding-credentials.ts diff --git a/apps/web/drizzle/0010_company_embedding_credentials.sql b/apps/web/drizzle/0010_company_embedding_credentials.sql index c77b6792d..17a9d6bac 100644 --- a/apps/web/drizzle/0010_company_embedding_credentials.sql +++ b/apps/web/drizzle/0010_company_embedding_credentials.sql @@ -10,9 +10,9 @@ -- Rollout plan: -- 1. Apply this migration (the new table is created empty). -- 2. Deploy application code that reads/writes via the new table. --- 3. Run `pnpm tsx scripts/backfill-embedding-credentials.ts` to encrypt --- existing plaintext values from `company` into this table and null --- out the old columns. +-- 3. Run `pnpm --filter @launchstack/web db:backfill:embedding-credentials` +-- to encrypt existing plaintext values from `company` into this table and +-- null out the old columns. -- 4. Once verified, apply 0011_drop_plaintext_embedding_credentials.sql -- to remove the legacy columns. diff --git a/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql b/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql index 9aa67f7bd..c37f08d0a 100644 --- a/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql +++ b/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql @@ -3,15 +3,15 @@ -- **Do not apply this migration until** the backfill script has run -- successfully against this environment: -- --- pnpm tsx scripts/backfill-embedding-credentials.ts +-- pnpm --filter @launchstack/web db:backfill:embedding-credentials -- -- The backfill copies every non-null plaintext key into -- `pdr_ai_v2_company_embedding_credentials` (encrypted) and then NULLs out -- the legacy columns. Applying this migration before the backfill will -- permanently destroy un-migrated company API keys. -- --- The application read path already falls back to the new credentials --- table, so once the backfill has run these columns are unused. +-- The application reads credentials only from the encrypted table; these +-- legacy columns must not remain in the Drizzle company schema. ALTER TABLE "pdr_ai_v2_company" DROP COLUMN IF EXISTS "embedding_openai_api_key", diff --git a/apps/web/package.json b/apps/web/package.json index 2e7761587..0faebc999 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,6 +12,7 @@ "db:migrate": "node ./scripts/ensure-pgvector.mjs && node ./scripts/migrate.mjs", "db:backfill:versions": "tsx ./scripts/backfill-document-versions.ts", "db:backfill:note-embeddings": "tsx ./scripts/backfill-note-embeddings.ts", + "db:backfill:embedding-credentials": "tsx ./scripts/backfill-embedding-credentials.ts", "db:studio": "drizzle-kit studio", "dev": "concurrently --names next,inngest --prefix-colors blue,magenta \"next dev --turbo\" \"pnpm dlx inngest-cli@latest dev -u http://localhost:3000/api/inngest\"", "dev:next": "next dev --turbo", diff --git a/apps/web/scripts/backfill-embedding-credentials.ts b/apps/web/scripts/backfill-embedding-credentials.ts new file mode 100644 index 000000000..69a71fbcf --- /dev/null +++ b/apps/web/scripts/backfill-embedding-credentials.ts @@ -0,0 +1,144 @@ +/** + * Backfill plaintext embedding credentials from the legacy columns on + * `pdr_ai_v2_company` into the encrypted `pdr_ai_v2_company_embedding_credentials` + * table, then NULL out the legacy columns. + * + * Migration 0011 drops those legacy columns. This script is only useful for + * databases that still have them (for example older `db:push`-shaped + * environments that never ran the SQL migration path). It uses raw SQL so it + * does not depend on the removed Drizzle schema fields. + * + * Safe to re-run: `upsertCompanyCredentials` upserts, and the legacy NULL + * step is idempotent. Requires `EMBEDDING_SECRETS_KEY` to be set. + * + * Run with: + * pnpm --filter @launchstack/web exec tsx ./scripts/backfill-embedding-credentials.ts + * + * Flags: + * --dry-run Log what would change without writing anything. + */ + +import "dotenv/config"; + +import { sql } from "drizzle-orm"; + +import { toRows } from "@launchstack/core/db"; +import { upsertCompanyCredentials } from "@launchstack/core/embeddings"; +import { db } from "../src/server/db"; + +const DRY_RUN = process.argv.includes("--dry-run"); + +type LegacyRow = { + id: number; + name: string; + openAIApiKey: string | null; + huggingFaceApiKey: string | null; + ollamaBaseUrl: string | null; + ollamaModel: string | null; +}; + +async function legacyColumnsExist(): Promise { + const rows = toRows<{ exists: number }>( + await db.execute(sql` + SELECT 1 AS exists + FROM information_schema.columns + WHERE table_name = 'pdr_ai_v2_company' + AND column_name = 'embedding_openai_api_key' + LIMIT 1 + `), + ); + return rows.length > 0; +} + +async function main() { + if (!process.env.EMBEDDING_SECRETS_KEY) { + console.error( + "Refusing to run: EMBEDDING_SECRETS_KEY is not set. Generate one with:\n" + + " node -e \"console.log(require('crypto').randomBytes(32).toString('base64'))\"", + ); + process.exit(1); + } + + if (!(await legacyColumnsExist())) { + console.log( + "Legacy plaintext embedding columns are already gone " + + "(migration 0011 applied). Nothing to backfill.", + ); + return; + } + + const rows = toRows( + await db.execute(sql` + SELECT + id, + name, + embedding_openai_api_key AS "openAIApiKey", + embedding_huggingface_api_key AS "huggingFaceApiKey", + embedding_ollama_base_url AS "ollamaBaseUrl", + embedding_ollama_model AS "ollamaModel" + FROM pdr_ai_v2_company + WHERE embedding_openai_api_key IS NOT NULL + OR embedding_huggingface_api_key IS NOT NULL + OR embedding_ollama_base_url IS NOT NULL + OR embedding_ollama_model IS NOT NULL + `), + ); + + console.log( + `Found ${rows.length} company row(s) with legacy embedding credentials${DRY_RUN ? " (dry-run)" : ""}.`, + ); + + let migrated = 0; + for (const row of rows) { + const input: { + openAIApiKey?: string | null; + huggingFaceApiKey?: string | null; + ollamaBaseUrl?: string | null; + ollamaModel?: string | null; + } = {}; + if (row.openAIApiKey) input.openAIApiKey = row.openAIApiKey; + if (row.huggingFaceApiKey) input.huggingFaceApiKey = row.huggingFaceApiKey; + if (row.ollamaBaseUrl) input.ollamaBaseUrl = row.ollamaBaseUrl; + if (row.ollamaModel) input.ollamaModel = row.ollamaModel; + + if (Object.keys(input).length === 0) continue; + + console.log( + ` company #${row.id} (${row.name}): fields = [${Object.keys(input).join(", ")}]`, + ); + + if (DRY_RUN) continue; + + await upsertCompanyCredentials(row.id, input); + + // Null out the legacy columns so this row won't be picked up on re-run. + await db.execute(sql` + UPDATE pdr_ai_v2_company + SET + embedding_openai_api_key = NULL, + embedding_huggingface_api_key = NULL, + embedding_ollama_base_url = NULL, + embedding_ollama_model = NULL + WHERE id = ${row.id} + `); + + migrated += 1; + } + + if (DRY_RUN) { + console.log( + `Dry run complete. ${rows.length} row(s) would be migrated. No changes written.`, + ); + } else { + console.log( + `Migrated ${migrated} row(s). Once verified, apply drizzle/0011_drop_plaintext_embedding_credentials.sql to remove the legacy columns.`, + ); + } +} + +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error("Backfill failed:", err); + process.exit(1); + }); diff --git a/package.json b/package.json index d46d769fe..b62f0ac96 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "db:migrate": "pnpm --filter @launchstack/web db:migrate", "db:studio": "pnpm --filter @launchstack/web db:studio", "db:backfill:versions": "pnpm --filter @launchstack/web db:backfill:versions", + "db:backfill:embedding-credentials": "pnpm --filter @launchstack/web db:backfill:embedding-credentials", "inngest:dev": "pnpm --filter @launchstack/web inngest:dev", "changeset": "changeset", "version": "changeset version", diff --git a/packages/core/src/db/schema/base.ts b/packages/core/src/db/schema/base.ts index f532e9b34..015b3676d 100644 --- a/packages/core/src/db/schema/base.ts +++ b/packages/core/src/db/schema/base.ts @@ -70,14 +70,9 @@ export const company = pgTable("company", { reindexStartedAt: timestamp("reindex_started_at", { withTimezone: true }), reindexCompletedAt: timestamp("reindex_completed_at", { withTimezone: true }), reindexError: text("reindex_error"), - // Legacy plaintext credential columns. Read-only during migration; - // writes go through src/lib/ai/company-credentials.ts into the - // encrypted `company_embedding_credentials` table. Dropped by - // drizzle/0011 once backfill has run. - embeddingOpenAIApiKey: text("embedding_openai_api_key"), - embeddingHuggingFaceApiKey: text("embedding_huggingface_api_key"), - embeddingOllamaBaseUrl: varchar("embedding_ollama_base_url", { length: 1024 }), - embeddingOllamaModel: varchar("embedding_ollama_model", { length: 256 }), + // Embedding provider credentials live in `company_embedding_credentials` + // (encrypted). The legacy plaintext columns on this table were dropped by + // drizzle/0011_drop_plaintext_embedding_credentials.sql. employerpasskey: varchar("employerPasskey", { length: 256 }).notNull().default(""), employeepasskey: varchar("employeePasskey", { length: 256 }).notNull().default(""), numberOfEmployees: varchar("numberOfEmployees", { length: 256 }).notNull(), diff --git a/packages/core/src/db/schema/company-credentials.ts b/packages/core/src/db/schema/company-credentials.ts index 39190809a..ad0262dc8 100644 --- a/packages/core/src/db/schema/company-credentials.ts +++ b/packages/core/src/db/schema/company-credentials.ts @@ -19,9 +19,8 @@ import { pgTable } from "./helpers"; * two real secrets (OpenAI, Hugging Face) are stored as base64 ciphertext * plus a 4-char suffix for UI feedback (`…ab12`). * - * Populated by `src/lib/ai/company-credentials.ts`. The legacy plaintext - * `company.embedding*` columns remain until backfill runs, then get dropped - * in a follow-up migration. + * Populated by `packages/core/src/embeddings/company-credentials.ts`. + * Legacy plaintext `company.embedding*` columns were dropped by migration 0011. */ export const companyEmbeddingCredentials = pgTable( "company_embedding_credentials", diff --git a/packages/core/src/embeddings/company-credentials.ts b/packages/core/src/embeddings/company-credentials.ts index 1dd35a159..0ccb85cb2 100644 --- a/packages/core/src/embeddings/company-credentials.ts +++ b/packages/core/src/embeddings/company-credentials.ts @@ -1,10 +1,7 @@ import { eq } from "drizzle-orm"; import { getDb } from "../db"; -import { - company, - companyEmbeddingCredentials, -} from "../db/schema"; +import { companyEmbeddingCredentials } from "../db/schema"; import { CiphertextDecodeError, decryptSecret, @@ -19,9 +16,8 @@ import { * API keys (OpenAI, Hugging Face) are AES-256-GCM ciphertext. Non-secret * config (Ollama base URL / model) is plaintext. * - * Read path falls back to the legacy plaintext columns on `company` so the - * application keeps working between the 0010 migration and the backfill. - * After backfill + 0011 those legacy columns will be gone. + * Credentials are read exclusively from `company_embedding_credentials`. + * Legacy plaintext columns on `company` were dropped by migration 0011. */ export interface CompanyCredentialsPlaintext { @@ -60,11 +56,9 @@ function last4(value: string | null | undefined): string | null { * Fetch the decrypted credentials for server-side use (embedding API calls). * Callers must never return these strings to the browser or log them. * - * Reads from the encrypted table first; falls back to legacy plaintext - * columns on `company` when the encrypted row is missing or has a null - * value for a given field. If the ciphertext is present but cannot be - * decrypted (bad key, corrupt data) we return null for that field and log — - * silently swallowing would mask rotation bugs. + * Reads from the encrypted credentials table only. If the ciphertext is + * present but cannot be decrypted (bad key, corrupt data) we return null for + * that field and log — silently swallowing would mask rotation bugs. */ export async function getCompanyCredentialsPlaintext( companyId: bigint | number | string, @@ -78,18 +72,7 @@ export async function getCompanyCredentialsPlaintext( .where(eq(companyEmbeddingCredentials.companyId, id)) .limit(1); - const [legacy] = await getDb() - .select({ - openAIApiKey: company.embeddingOpenAIApiKey, - huggingFaceApiKey: company.embeddingHuggingFaceApiKey, - ollamaBaseUrl: company.embeddingOllamaBaseUrl, - ollamaModel: company.embeddingOllamaModel, - }) - .from(company) - .where(eq(company.id, id)) - .limit(1); - - if (!encrypted && !legacy) return null; + if (!encrypted) return null; const decryptOrNull = (ciphertext: string | null | undefined): string | null => { if (!ciphertext) return null; @@ -107,17 +90,10 @@ export async function getCompanyCredentialsPlaintext( }; return { - openAIApiKey: - decryptOrNull(encrypted?.openAIApiKeyCiphertext) ?? - legacy?.openAIApiKey ?? - null, - huggingFaceApiKey: - decryptOrNull(encrypted?.huggingFaceApiKeyCiphertext) ?? - legacy?.huggingFaceApiKey ?? - null, - ollamaBaseUrl: - encrypted?.ollamaBaseUrl ?? legacy?.ollamaBaseUrl ?? null, - ollamaModel: encrypted?.ollamaModel ?? legacy?.ollamaModel ?? null, + openAIApiKey: decryptOrNull(encrypted.openAIApiKeyCiphertext), + huggingFaceApiKey: decryptOrNull(encrypted.huggingFaceApiKeyCiphertext), + ollamaBaseUrl: encrypted.ollamaBaseUrl ?? null, + ollamaModel: encrypted.ollamaModel ?? null, }; } @@ -152,39 +128,17 @@ export async function getRedactedCredentials( .where(eq(companyEmbeddingCredentials.companyId, id)) .limit(1); - const [legacy] = await getDb() - .select({ - openAIApiKey: company.embeddingOpenAIApiKey, - huggingFaceApiKey: company.embeddingHuggingFaceApiKey, - ollamaBaseUrl: company.embeddingOllamaBaseUrl, - ollamaModel: company.embeddingOllamaModel, - }) - .from(company) - .where(eq(company.id, id)) - .limit(1); - - const hasOpenAI = - Boolean(encrypted?.openAIApiKeyCiphertext) || - Boolean(legacy?.openAIApiKey); - const hasHF = - Boolean(encrypted?.huggingFaceApiKeyCiphertext) || - Boolean(legacy?.huggingFaceApiKey); - return { openAI: { - hasKey: hasOpenAI, - last4: - encrypted?.openAIApiKeyLast4 ?? - last4(legacy?.openAIApiKey ?? null), + hasKey: Boolean(encrypted?.openAIApiKeyCiphertext), + last4: encrypted?.openAIApiKeyLast4 ?? null, }, huggingFace: { - hasKey: hasHF, - last4: - encrypted?.huggingFaceApiKeyLast4 ?? - last4(legacy?.huggingFaceApiKey ?? null), + hasKey: Boolean(encrypted?.huggingFaceApiKeyCiphertext), + last4: encrypted?.huggingFaceApiKeyLast4 ?? null, }, - ollamaBaseUrl: encrypted?.ollamaBaseUrl ?? legacy?.ollamaBaseUrl ?? null, - ollamaModel: encrypted?.ollamaModel ?? legacy?.ollamaModel ?? null, + ollamaBaseUrl: encrypted?.ollamaBaseUrl ?? null, + ollamaModel: encrypted?.ollamaModel ?? null, }; } @@ -192,10 +146,7 @@ export async function getRedactedCredentials( * Upsert credentials for a company. Fields set to `undefined` are left * alone; fields set to `null` or empty string are cleared. * - * Writes to the encrypted table only; never to the legacy plaintext - * columns. If the caller updates a key that still exists in a legacy - * column for this company, that legacy value becomes stale — the backfill - * script is responsible for nulling it out as part of migration. + * Writes to the encrypted table only. */ export async function upsertCompanyCredentials( companyId: bigint | number | string, diff --git a/scripts/backfill-embedding-credentials.ts b/scripts/backfill-embedding-credentials.ts index f658cc456..bf8a3deda 100644 --- a/scripts/backfill-embedding-credentials.ts +++ b/scripts/backfill-embedding-credentials.ts @@ -1,113 +1,12 @@ -import "dotenv/config"; - -import { eq, isNotNull, or, sql } from "drizzle-orm"; - -import { db } from "../src/server/db"; -import { company } from "../src/server/db/schema"; -import { upsertCompanyCredentials } from "../src/lib/ai/company-credentials"; - /** - * Backfill plaintext embedding credentials from the legacy columns on - * `pdr_ai_v2_company` into the encrypted `pdr_ai_v2_company_embedding_credentials` - * table, then NULL out the legacy columns. - * - * Safe to re-run: `upsertCompanyCredentials` upserts, and the legacy NULL - * step is idempotent. Requires `EMBEDDING_SECRETS_KEY` to be set. - * - * Run with: pnpm tsx scripts/backfill-embedding-credentials.ts + * Moved to apps/web/scripts/backfill-embedding-credentials.ts. * - * Flags: - * --dry-run Log what would change without writing anything. + * Run with: + * pnpm db:backfill:embedding-credentials */ -const DRY_RUN = process.argv.includes("--dry-run"); - -async function main() { - if (!process.env.EMBEDDING_SECRETS_KEY) { - console.error( - "Refusing to run: EMBEDDING_SECRETS_KEY is not set. Generate one with:\n" + - " node -e \"console.log(require('crypto').randomBytes(32).toString('base64'))\"", - ); - process.exit(1); - } - - const rows = await db - .select({ - id: company.id, - name: company.name, - openAIApiKey: company.embeddingOpenAIApiKey, - huggingFaceApiKey: company.embeddingHuggingFaceApiKey, - ollamaBaseUrl: company.embeddingOllamaBaseUrl, - ollamaModel: company.embeddingOllamaModel, - }) - .from(company) - .where( - or( - isNotNull(company.embeddingOpenAIApiKey), - isNotNull(company.embeddingHuggingFaceApiKey), - isNotNull(company.embeddingOllamaBaseUrl), - isNotNull(company.embeddingOllamaModel), - ), - ); - - console.log( - `Found ${rows.length} company row(s) with legacy embedding credentials${DRY_RUN ? " (dry-run)" : ""}.`, - ); - - let migrated = 0; - for (const row of rows) { - const input: { - openAIApiKey?: string | null; - huggingFaceApiKey?: string | null; - ollamaBaseUrl?: string | null; - ollamaModel?: string | null; - } = {}; - if (row.openAIApiKey) input.openAIApiKey = row.openAIApiKey; - if (row.huggingFaceApiKey) input.huggingFaceApiKey = row.huggingFaceApiKey; - if (row.ollamaBaseUrl) input.ollamaBaseUrl = row.ollamaBaseUrl; - if (row.ollamaModel) input.ollamaModel = row.ollamaModel; - - if (Object.keys(input).length === 0) continue; - - console.log( - ` company #${row.id} (${row.name}): fields = [${Object.keys(input).join(", ")}]`, - ); - - if (DRY_RUN) continue; - - await upsertCompanyCredentials(row.id, input); - - // Null out the legacy columns so this row won't be picked up on re-run. - await db - .update(company) - .set({ - embeddingOpenAIApiKey: null, - embeddingHuggingFaceApiKey: null, - embeddingOllamaBaseUrl: null, - embeddingOllamaModel: null, - }) - .where(eq(company.id, row.id)); - - migrated += 1; - } - - if (DRY_RUN) { - console.log( - `Dry run complete. ${rows.length} row(s) would be migrated. No changes written.`, - ); - } else { - console.log( - `Migrated ${migrated} row(s). Once verified, apply drizzle/0011_drop_plaintext_embedding_credentials.sql to remove the legacy columns.`, - ); - } - - // Close the connection so the script exits cleanly. - await db.execute(sql`SELECT 1`); -} - -main() - .then(() => process.exit(0)) - .catch((err) => { - console.error("Backfill failed:", err); - process.exit(1); - }); +console.error( + "This script moved to apps/web/scripts/backfill-embedding-credentials.ts.\n" + + "Run: pnpm db:backfill:embedding-credentials", +); +process.exit(1); From 2415de15d703ee350984bbc269bac57ef73abe81 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 29 Jul 2026 19:57:49 +0000 Subject: [PATCH 11/29] chore: add changeset for company embedding schema fix Restore the missing Changesets config and record a patch release note for the @launchstack/core schema/runtime alignment with migration 0011. Co-authored-by: Kien Le --- .changeset/README.md | 8 ++++++++ .changeset/config.json | 11 +++++++++++ .changeset/fix-company-embedding-schema-drift.md | 5 +++++ 3 files changed, 24 insertions(+) create mode 100644 .changeset/README.md create mode 100644 .changeset/config.json create mode 100644 .changeset/fix-company-embedding-schema-drift.md diff --git a/.changeset/README.md b/.changeset/README.md new file mode 100644 index 000000000..654c6d475 --- /dev/null +++ b/.changeset/README.md @@ -0,0 +1,8 @@ +# Changesets + +Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works +with multi-package repos, or single-package repos to help you version and publish your code. You can +find the full documentation for it [in our repository](https://github.com/changesets/changesets). + +We have a quick list of common questions to get you started engaging with this project in +[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md). diff --git a/.changeset/config.json b/.changeset/config.json new file mode 100644 index 000000000..bd36e8be2 --- /dev/null +++ b/.changeset/config.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://unpkg.com/@changesets/config@3.1.4/schema.json", + "changelog": "@changesets/cli/changelog", + "commit": false, + "fixed": [], + "linked": [], + "access": "public", + "baseBranch": "main", + "updateInternalDependencies": "patch", + "ignore": ["@launchstack/features", "@launchstack/web"] +} diff --git a/.changeset/fix-company-embedding-schema-drift.md b/.changeset/fix-company-embedding-schema-drift.md new file mode 100644 index 000000000..1068ac71b --- /dev/null +++ b/.changeset/fix-company-embedding-schema-drift.md @@ -0,0 +1,5 @@ +--- +"@launchstack/core": patch +--- + +Align the company Drizzle schema with migration 0011 by removing dropped plaintext embedding credential columns and legacy credential read fallbacks. From 7b17019de6e4dc24b2bea9e1373971c15935a627 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 29 Jul 2026 20:01:57 +0000 Subject: [PATCH 12/29] test: add durable regression coverage for issue #297 Assert the company Drizzle schema and credential read path stay aligned with migration 0011, and add an optional live DB check that select/insert succeed after the plaintext embedding columns are dropped. Co-authored-by: Kien Le --- ...ny-embedding-schema-alignment.live.test.ts | 53 ++++++++++++ ...company-embedding-schema-alignment.test.ts | 85 +++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts create mode 100644 apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts diff --git a/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts new file mode 100644 index 000000000..043e12f1f --- /dev/null +++ b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts @@ -0,0 +1,53 @@ +/** + * Live DB verification for #297 against a migration-shaped company table. + * + * Requires DATABASE_URL pointing at a database where migration 0011 has + * already dropped the plaintext embedding columns. Skips otherwise so CI + * without Postgres still passes the unit suite. + */ + +import postgres from "postgres"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { company } from "@launchstack/core/db/schema"; + +const databaseUrl = process.env.DATABASE_URL; + +const describeIfDb = databaseUrl ? describe : describe.skip; + +describeIfDb("company select after migration 0011 (#297 live)", () => { + const client = postgres(databaseUrl!, { max: 1 }); + const db = drizzle(client); + + afterAll(async () => { + await client.end({ timeout: 5 }); + }); + + it("selects and inserts company rows without referencing dropped columns", async () => { + const missing = await client` + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'pdr_ai_v2_company' + AND column_name = ANY(${[ + "embedding_openai_api_key", + "embedding_huggingface_api_key", + "embedding_ollama_base_url", + "embedding_ollama_model", + ]}) + `; + expect(missing).toHaveLength(0); + + await expect(db.select().from(company).limit(1)).resolves.toBeDefined(); + + const inserted = await db + .insert(company) + .values({ + name: `issue-297-live-${Date.now()}`, + numberOfEmployees: "1", + }) + .returning(); + + expect(inserted).toHaveLength(1); + expect(inserted[0]).not.toHaveProperty("embeddingOpenAIApiKey"); + expect(inserted[0]?.id).toEqual(expect.any(Number)); + }); +}); diff --git a/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts new file mode 100644 index 000000000..e30ccafe8 --- /dev/null +++ b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts @@ -0,0 +1,85 @@ +/** + * Regression for https://github.com/Deodat-Lawson/LaunchStack/issues/297 + * + * Migration 0011 drops plaintext embedding credential columns from + * `pdr_ai_v2_company`. The Drizzle company schema and credential read path + * must stay aligned with that final shape — otherwise clean migration-replayed + * databases fail on `select()` / `.returning()` of the company row. + */ + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { getTableColumns } from "drizzle-orm"; +import { company } from "@launchstack/core/db/schema"; + +const DROPPED_EMBEDDING_COLUMNS = [ + "embedding_openai_api_key", + "embedding_huggingface_api_key", + "embedding_ollama_base_url", + "embedding_ollama_model", +] as const; + +/** Jest runs with cwd = apps/web; fall back to climbing from this file. */ +function resolveRepoRoot(): string { + const fromCwd = join(process.cwd(), "../.."); + if (existsSync(join(fromCwd, "packages/core/package.json"))) { + return fromCwd; + } + const fromFile = join(__dirname, "../../../../.."); + if (existsSync(join(fromFile, "packages/core/package.json"))) { + return fromFile; + } + throw new Error( + `Unable to resolve monorepo root from cwd=${process.cwd()} __dirname=${__dirname}`, + ); +} + +const repoRoot = resolveRepoRoot(); + +describe("company embedding credential schema alignment (#297)", () => { + it("does not declare columns dropped by migration 0011 on the company table", () => { + const columns = getTableColumns(company); + const sqlNames = Object.values(columns).map((column) => column.name); + + for (const dropped of DROPPED_EMBEDDING_COLUMNS) { + expect(sqlNames).not.toContain(dropped); + } + + expect(columns).not.toHaveProperty("embeddingOpenAIApiKey"); + expect(columns).not.toHaveProperty("embeddingHuggingFaceApiKey"); + expect(columns).not.toHaveProperty("embeddingOllamaBaseUrl"); + expect(columns).not.toHaveProperty("embeddingOllamaModel"); + }); + + it("keeps migration 0011 as the drop of those plaintext columns", () => { + const migrationPath = join( + repoRoot, + "apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql", + ); + const body = readFileSync(migrationPath, "utf8"); + + expect(body).toMatch(/ALTER TABLE\s+"pdr_ai_v2_company"/i); + for (const dropped of DROPPED_EMBEDDING_COLUMNS) { + expect(body).toContain(`DROP COLUMN IF EXISTS "${dropped}"`); + } + }); + + it("reads credentials only from the encrypted credentials table", () => { + const sourcePath = join( + repoRoot, + "packages/core/src/embeddings/company-credentials.ts", + ); + const source = readFileSync(sourcePath, "utf8"); + + expect(source).toContain("companyEmbeddingCredentials"); + expect(source).not.toMatch(/company\.embeddingOpenAIApiKey/); + expect(source).not.toMatch(/company\.embeddingHuggingFaceApiKey/); + expect(source).not.toMatch(/company\.embeddingOllamaBaseUrl/); + expect(source).not.toMatch(/company\.embeddingOllamaModel/); + expect(source).not.toMatch(/embedding_openai_api_key/); + expect(source).not.toMatch(/embedding_huggingface_api_key/); + expect(source).not.toMatch(/embedding_ollama_base_url/); + expect(source).not.toMatch(/embedding_ollama_model/); + }); +}); From ea2c436f9f16e232cb2536a4e40a15e2b9cf3d88 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Fri, 31 Jul 2026 22:26:34 +0800 Subject: [PATCH 13/29] feat: complete founder weekly review generation flow --- .gitignore | 1 + .../dispatch-atomicity.test.ts | 18 ++ .../evaluation-fixtures.test.ts | 65 ++++++ .../read-retry-route.test.ts | 20 ++ .../founderWeeklyReview/routes.test.ts | 52 +++++ ...ounder-weekly-review-synthetic-fixtures.ts | 24 +++ ...ounder-weekly-review-synthetic-baseline.ts | 192 ++++++++++++++++++ .../[runId]/retry/route.ts | 29 ++- .../founder-weekly-reviews/[runId]/route.ts | 30 ++- .../app/api/founder-weekly-reviews/route.ts | 8 +- .../founder-weekly-review/dispatch-service.ts | 48 +++-- .../evidence-collector.ts | 6 +- .../src/founder-weekly-review/repository.ts | 14 +- .../src/founder-weekly-review/user-service.ts | 31 ++- 14 files changed, 495 insertions(+), 43 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/routes.test.ts create mode 100644 apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts create mode 100644 apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts diff --git a/.gitignore b/.gitignore index d6a7f3a9a..e03ea0c04 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ packages/*/dist/ .kiro backups/ /models/ +/apps/web/.artifacts/ .claude diff --git a/apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts b/apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts new file mode 100644 index 000000000..18725f94e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts @@ -0,0 +1,18 @@ +jest.mock("~/server/db", () => ({ db: { transaction: jest.fn() } })); +import { eq } from "drizzle-orm"; +import { company, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; +const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }); +const actor = (companyId: bigint) => ({ externalUserId: "u", internalUserId: 1n, companyId, role: "owner" as const }); +async function companyId(db: Awaited>["db"]): Promise { const [row] = await db.insert(company).values({ name: "Atomic", numberOfEmployees: "1" }).returning(); return BigInt(row!.id); } + +describeDb("Founder Weekly Review dispatch atomicity", () => { + it("concurrent create persists one run and one dispatch", async () => { const test = await createFounderWeeklyReviewTestDatabase(); const a = await test.createSession(), b = await test.createSession(); try { const id = await companyId(test.db); const input = { actor: actor(id), requestKey: "create-race", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot }; const results = await Promise.all([createFounderWeeklyReviewDispatchService(a.db).createRunWithDispatch(input), createFounderWeeklyReviewDispatchService(b.db).createRunWithDispatch(input)]); const rows = await test.db.select().from(founderWeeklyReviewRuns); const dispatches = await test.db.select().from(founderWeeklyReviewDispatches); expect(rows).toHaveLength(1); expect(dispatches).toHaveLength(1); expect(results.map(x => x.created).sort()).toEqual([false, true]); expect(new Set(results.map(x => x.run.id)).size).toBe(1); expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(rows[0]!.evidenceSnapshot)).toEqual(snapshot); } finally { await a.close(); await b.close(); await test.close(); } }); + it("concurrent retry transitions once and preserves snapshot", async () => { const test = await createFounderWeeklyReviewTestDatabase(); const a = await test.createSession(), b = await test.createSession(); try { const id = await companyId(test.db); const repo = new FounderWeeklyReviewRepository(test.db); const created = await repo.createOrGetByRequestKey({ id: "fwr_failed", companyId: id, requestKey: "base", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot, createdByActorId: "user:u" }); await new FounderWeeklyReviewWorkerService(repo).markQueuedRunFailed(id, created.id, { errorCode: "x" }); const input = { actor: actor(id), runId: created.id, requestKey: "retry-race" }; const results = await Promise.all([createFounderWeeklyReviewDispatchService(a.db).retryRunWithDispatch(input), createFounderWeeklyReviewDispatchService(b.db).retryRunWithDispatch(input)]); const [row] = await test.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, created.id)); const dispatches = await test.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.operationType, "retry")); expect(results.map(x => x.transitionApplied).sort()).toEqual([false, true]); expect(row!.status).toBe("queued"); expect(row!.retryCount).toBe(1); expect(row!.evidenceSnapshot).toEqual(snapshot); expect(dispatches).toHaveLength(1); } finally { await a.close(); await b.close(); await test.close(); } }); + it("rolls back create when dispatch persistence fails", async () => { const test = await createFounderWeeklyReviewTestDatabase(); try { const id = await companyId(test.db); const service = createFounderWeeklyReviewDispatchService(test.db, { createDispatch: async () => { throw new Error("forced_dispatch_insert_failure"); } }); await expect(service.createRunWithDispatch({ actor: actor(id), requestKey: "rollback", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot })).rejects.toThrow("forced_dispatch_insert_failure"); expect(await test.db.select().from(founderWeeklyReviewRuns)).toHaveLength(0); expect(await test.db.select().from(founderWeeklyReviewDispatches)).toHaveLength(0); } finally { await test.close(); } }); + it("rolls back retry when dispatch persistence fails", async () => { const test = await createFounderWeeklyReviewTestDatabase(); try { const id = await companyId(test.db); const repo = new FounderWeeklyReviewRepository(test.db); const created = await repo.createOrGetByRequestKey({ id: "fwr_retry_rollback", companyId: id, requestKey: "base", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot, createdByActorId: "user:u" }); await new FounderWeeklyReviewWorkerService(repo).markQueuedRunFailed(id, created.id, { errorCode: "x" }); const service = createFounderWeeklyReviewDispatchService(test.db, { createDispatch: async () => { throw new Error("forced_dispatch_insert_failure"); } }); await expect(service.retryRunWithDispatch({ actor: actor(id), runId: created.id, requestKey: "retry" })).rejects.toThrow("forced_dispatch_insert_failure"); const [row] = await test.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, created.id)); expect(row!.status).toBe("failed"); expect(row!.retryCount).toBe(0); expect(row!.evidenceSnapshot).toEqual(snapshot); expect(await test.db.select().from(founderWeeklyReviewDispatches)).toHaveLength(0); } finally { await test.close(); } }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts b/apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts new file mode 100644 index 000000000..c6a7c2579 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts @@ -0,0 +1,65 @@ +import { + FounderWeeklyReviewEvidenceSnapshotSchema, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewV2Payload, + generateFounderWeeklyReview, +} from "@launchstack/features/founder-weekly-review"; + +const period = { start: "2026-07-06", end: "2026-07-12" }; +function fixture(items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderWeeklyReviewEvidenceSnapshot { + return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", + reportingPeriod: period, workspaceTimezone: "UTC", items, sourceWarnings: [], + }); +} +export const completeWorkspaceFixture = fixture([ + { sourceType: "document_change", sourceId: "doc:release", title: "Release notes", sourceTimestamp: "2026-07-10T00:00:00.000Z", excerpt: "Export shipped.", metadata: {} }, + { sourceType: "customer_feedback", sourceId: "feedback:1", title: "Customer call", excerpt: "Audit logs requested.", metadata: {} }, + { sourceType: "founder_context", sourceId: "context:1", title: "Founder context", excerpt: "SSO remains blocked.", metadata: {} }, +]); +export const partialWorkspaceFixture = fixture([ + { sourceType: "document_change", sourceId: "doc:release", title: "Release notes", sourceTimestamp: "2026-07-10T00:00:00.000Z", excerpt: "Export shipped.", metadata: {} }, + { sourceType: "founder_context", sourceId: "context:1", title: "Founder context", excerpt: "A founder heard customers ask about audit logs.", metadata: {} }, +]); +export const emptyWorkspaceFixture = fixture([]); + +function payload(customerEvidence: boolean): FounderWeeklyReviewV2Payload { + const noEvidence = { state: "no_evidence" as const, noEvidence: { code: "none", message: "No evidence", cta: "Add evidence" } }; + return { schemaVersion: "founder-weekly-review/v2", sections: { + whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "Exports shipped.", sourceIds: ["doc:release"], confidence: 0.9 }] }, + whatShipped: noEvidence, + whatCustomersSaid: customerEvidence ? { state: "evidence", items: [{ kind: "observed_fact", text: "Audit logs were requested.", sourceIds: ["feedback:1"], confidence: 0.8 }] } : noEvidence, + currentBlockers: noEvidence, + nextPriorities: noEvidence, + }}; +} +function assertGrounded(review: FounderWeeklyReviewV2Payload, snapshot: FounderWeeklyReviewEvidenceSnapshot) { + const ids = new Set(snapshot.items.map((item) => item.sourceId)); + for (const section of [review.sections.whatChanged, review.sections.whatShipped, review.sections.whatCustomersSaid, review.sections.currentBlockers]) { + if (section.state === "evidence") for (const item of section.items) { + expect(item.sourceIds.length).toBeGreaterThan(0); + for (const id of item.sourceIds) expect(ids.has(id)).toBe(true); + } + } +} + +describe("LAU-9 grounding evaluation fixtures", () => { + it("grounds the complete fixture, including customer attribution", async () => { + const generate = jest.fn().mockResolvedValue({ object: payload(true), metadata: { provider: "test", model: "fixed", capability: "founderWeeklyReview", temperature: 0 } }); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeWorkspaceFixture, generate }); + assertGrounded(result.reviewPayload, completeWorkspaceFixture); + expect(result.reviewPayload.sections.whatCustomersSaid).toMatchObject({ state: "evidence" }); + }); + it("keeps the partial fixture customer section at explicit no-evidence", async () => { + const generate = jest.fn().mockResolvedValue({ object: payload(false), metadata: { provider: "test", model: "fixed", capability: "founderWeeklyReview", temperature: 0 } }); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: partialWorkspaceFixture, generate }); + assertGrounded(result.reviewPayload, partialWorkspaceFixture); + expect(result.reviewPayload.sections.whatCustomersSaid.state).toBe("no_evidence"); + }); + it("completes the empty fixture without calling a provider or inventing facts", async () => { + const generate = jest.fn(); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: emptyWorkspaceFixture, generate }); + expect(generate).not.toHaveBeenCalled(); + for (const section of Object.values(result.reviewPayload.sections)) expect(section.state).toBe("no_evidence"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts b/apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts new file mode 100644 index 000000000..bd4cb976f --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts @@ -0,0 +1,20 @@ +jest.mock("@clerk/nextjs/server", () => ({ auth: jest.fn() })); +jest.mock("~/server/founder-weekly-review/actor-resolver", () => ({ productionFounderWeeklyReviewActorResolver: { resolve: jest.fn() } })); +jest.mock("~/server/founder-weekly-review/dispatch-service", () => ({ retryRunWithDispatch: jest.fn() })); +import { auth } from "@clerk/nextjs/server"; +import { createFounderWeeklyReviewGetHandler } from "~/app/api/founder-weekly-reviews/[runId]/route"; +import { createFounderWeeklyReviewRetryPostHandler } from "~/app/api/founder-weekly-reviews/[runId]/retry/route"; +import type { FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; +const mockAuth = auth as unknown as jest.Mock; +const actor = { externalUserId: "u", internalUserId: 1n, companyId: 1n, role: "owner" as const }; +function run(status: FounderWeeklyReviewRunRecord["status"] = "queued"): FounderWeeklyReviewRunRecord { return { id: "fwr_1", companyId: 1n, requestKey: "key", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, status, reviewPayload: status === "draft" ? { schemaVersion: "founder-weekly-review/v2", sections: {} } as never : null, reviewSchemaVersion: "founder-weekly-review/v1", evidenceSnapshot: { schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [{ sourceType: "founder_context", sourceId: "secret", title: "Founder context", excerpt: "DO NOT LEAK", metadata: {} }], sourceWarnings: [] }, evidenceSchemaVersion: "founder-weekly-review-evidence/v1", modelMetadata: null, createdByActorId: "user:u", retryCount: 0, failureSequence: 0, generationAttempt: 0, generationClaimId: null, generationJobId: null, queuedAt: new Date(), claimedAt: null, generationStartedAt: null, generatedAt: null, publishedAt: null, errorCode: "safe_code", errorMessage: "provider secret", createdAt: new Date(), updatedAt: null }; } +describe("Founder Weekly Review read and retry route handlers", () => { + beforeEach(() => mockAuth.mockResolvedValue({ userId: "u" })); + it("read authenticates before lookup and returns safe status payloads", async () => { const getRun = jest.fn().mockResolvedValue(run("failed")); const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, getRun }); const response = await handler(new Request("http://test"), { params: Promise.resolve({ runId: "fwr_1" }) }); const json = await response.json(); expect(response.status).toBe(200); expect(json.run.status).toBe("failed"); expect(JSON.stringify(json)).not.toContain("DO NOT LEAK"); expect(JSON.stringify(json)).not.toContain("provider secret"); }); + it("read returns 401 without lookup and maps company-scoped absence to 404", async () => { mockAuth.mockResolvedValue({ userId: null }); const getRun = jest.fn(); const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn() }, getRun }); expect((await handler(new Request("http://test"), { params: Promise.resolve({ runId: "x" }) })).status).toBe(401); expect(getRun).not.toHaveBeenCalled(); mockAuth.mockResolvedValue({ userId: "u" }); const missing = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, getRun: jest.fn().mockRejectedValue({ code: "not_found" }) }); expect((await missing(new Request("http://test"), { params: Promise.resolve({ runId: "x" }) })).status).toBe(404); }); + it.each(["queued", "generating", "draft", "failed", "published"] as const)("returns the %s status without snapshot internals", async (status) => { const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, getRun: jest.fn().mockResolvedValue(run(status)) }); const json = await (await handler(new Request("http://test"), { params: Promise.resolve({ runId: "fwr_1" }) })).json(); expect(json.run).toMatchObject({ id: "fwr_1", status }); expect(json.run).not.toHaveProperty("evidenceSnapshot"); expect(JSON.stringify(json)).not.toContain("DO NOT LEAK"); }); + it("does not look up a run when strict actor resolution fails", async () => { const getRun = jest.fn(); const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockRejectedValue({ code: "forbidden" }) }, getRun }); expect((await handler(new Request("http://test"), { params: Promise.resolve({ runId: "fwr_1" }) })).status).toBe(403); expect(getRun).not.toHaveBeenCalled(); }); + it("retry increments only an applied transition and does not notify rejected callers", async () => { const incrementRetry = jest.fn(), sendDispatchRequested = jest.fn().mockResolvedValue(undefined); const retryRunWithDispatch = jest.fn().mockResolvedValue({ run: run("queued"), dispatch: {}, transitionApplied: true }); const handler = createFounderWeeklyReviewRetryPostHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, retryRunWithDispatch, incrementRetry, sendDispatchRequested }); expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r1" }) }), { params: Promise.resolve({ runId: "fwr_1" }) })).status).toBe(202); expect(incrementRetry).toHaveBeenCalledTimes(1); expect(sendDispatchRequested).toHaveBeenCalledTimes(1); retryRunWithDispatch.mockResolvedValue({ run: run("queued"), dispatch: {}, transitionApplied: false }); await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r1" }) }), { params: Promise.resolve({ runId: "fwr_1" }) }); expect(incrementRetry).toHaveBeenCalledTimes(1); }); + it("retry rejects auth before mutation or notification", async () => { mockAuth.mockResolvedValue({ userId: null }); const retryRunWithDispatch = jest.fn(), sendDispatchRequested = jest.fn(), incrementRetry = jest.fn(); const handler = createFounderWeeklyReviewRetryPostHandler({ actorResolver: { resolve: jest.fn() }, retryRunWithDispatch, sendDispatchRequested, incrementRetry }); expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r" }) }), { params: Promise.resolve({ runId: "x" }) })).status).toBe(401); expect(retryRunWithDispatch).not.toHaveBeenCalled(); expect(sendDispatchRequested).not.toHaveBeenCalled(); expect(incrementRetry).not.toHaveBeenCalled(); }); + it.each(["queued", "generating", "draft", "published"] as const)("maps %s retry conflict safely", async (status) => { const retryRunWithDispatch = jest.fn().mockRejectedValue({ code: "invalid_transition", status }); const sendDispatchRequested = jest.fn(), incrementRetry = jest.fn(); const handler = createFounderWeeklyReviewRetryPostHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, retryRunWithDispatch, sendDispatchRequested, incrementRetry }); const response = await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r" }) }), { params: Promise.resolve({ runId: "fwr_1" }) }); expect(response.status).toBe(409); expect(incrementRetry).not.toHaveBeenCalled(); expect(sendDispatchRequested).not.toHaveBeenCalled(); }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/routes.test.ts b/apps/web/__tests__/founderWeeklyReview/routes.test.ts new file mode 100644 index 000000000..0dba8c899 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/routes.test.ts @@ -0,0 +1,52 @@ +jest.mock("@clerk/nextjs/server", () => ({ auth: jest.fn() })); +jest.mock("~/server/founder-weekly-review/actor-resolver", () => ({ + productionFounderWeeklyReviewActorResolver: { resolve: jest.fn() }, +})); +jest.mock("~/server/founder-weekly-review/dispatch-service", () => ({ + createRunWithDispatch: jest.fn(), +})); + +import { createFounderWeeklyReviewPostHandler } from "~/app/api/founder-weekly-reviews/route"; +import type { FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; +import { auth } from "@clerk/nextjs/server"; +const mockAuth = auth as unknown as jest.Mock; + +const run = (id = "fwr_1"): FounderWeeklyReviewRunRecord => ({ + id, companyId: 1n, requestKey: "key", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, status: "queued", reviewPayload: null, + reviewSchemaVersion: "founder-weekly-review/v1", evidenceSnapshot: { schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }, evidenceSchemaVersion: "founder-weekly-review-evidence/v1", modelMetadata: null, createdByActorId: "user:u", retryCount: 0, failureSequence: 0, generationAttempt: 0, generationClaimId: null, generationJobId: null, queuedAt: new Date(), claimedAt: null, generationStartedAt: null, generatedAt: null, publishedAt: null, errorCode: null, errorMessage: null, createdAt: new Date(), updatedAt: null, +}); +const body = { requestKey: "key", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", founderContext: "Context" }; +function setup(overrides: Partial[0]> = {}) { + const collector = { collectFounderWeeklyReviewEvidence: jest.fn().mockResolvedValue(run().evidenceSnapshot) }; + const createRunWithDispatch = jest.fn().mockResolvedValue({ run: run(), dispatch: {}, created: true }); + const deps = { actorResolver: { resolve: jest.fn().mockResolvedValue({ externalUserId: "u", internalUserId: 1n, companyId: 1n, role: "owner" }) }, evidenceCollector: collector, repository: { getByCompanyAndRequestKey: jest.fn().mockResolvedValue(null) }, createRunWithDispatch, sendDispatchRequested: jest.fn().mockResolvedValue(undefined), recordRunCreated: jest.fn(), ...overrides } as Parameters[0]; + return { deps, collector, createRunWithDispatch, handler: createFounderWeeklyReviewPostHandler(deps) }; +} +describe("Founder Weekly Review create route", () => { + beforeEach(() => mockAuth.mockResolvedValue({ userId: "u" })); + it("returns 401 without auth and never resolves or collects", async () => { + mockAuth.mockResolvedValue({ userId: null }); const { handler, collector, deps } = setup(); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(401); + expect(deps.actorResolver.resolve).not.toHaveBeenCalled(); expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); + }); + it("authorizes before validation and never collects rejected requests", async () => { + const { handler, collector, createRunWithDispatch } = setup({ actorResolver: { resolve: jest.fn().mockRejectedValue({ code: "forbidden" }) } }); + expect((await handler(new Request("http://test", { method: "POST", body: "{" }))).status).toBe(403); + expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); expect(createRunWithDispatch).not.toHaveBeenCalled(); + }); + it("returns existing company-scoped run before collection and does not record creation", async () => { + const existing = run("existing"); const { handler, collector, createRunWithDispatch, deps } = setup({ repository: { getByCompanyAndRequestKey: jest.fn().mockResolvedValue(existing) } }); + const response = await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) })); + expect(response.status).toBe(202); expect((await response.json()).run.id).toBe("existing"); expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); expect(createRunWithDispatch).not.toHaveBeenCalled(); expect(deps.recordRunCreated).not.toHaveBeenCalled(); + }); + it("forwards canonical collector inputs, creates a queued run, and records exactly one creation", async () => { + const { handler, collector, createRunWithDispatch, deps } = setup(); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(202); + expect(collector.collectFounderWeeklyReviewEvidence).toHaveBeenCalledWith(expect.objectContaining({ companyId: 1n, reportingPeriod: body.reportingPeriod, workspaceTimezone: "UTC", founderContext: "Context", actor: { externalUserId: "u" }, requestKey: "key" })); + expect(createRunWithDispatch).toHaveBeenCalledTimes(1); expect(deps.recordRunCreated).toHaveBeenCalledTimes(1); + }); + it("does not create or record when collection fails", async () => { + const { handler, createRunWithDispatch, deps } = setup({ evidenceCollector: { collectFounderWeeklyReviewEvidence: jest.fn().mockRejectedValue(new Error("nope")) } }); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(500); expect(createRunWithDispatch).not.toHaveBeenCalled(); expect(deps.recordRunCreated).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts b/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts new file mode 100644 index 000000000..ce458688f --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts @@ -0,0 +1,24 @@ +import { FounderWeeklyReviewEvidenceSnapshotSchema } from "@launchstack/features/founder-weekly-review"; + +const period = { start: "2026-02-16", end: "2026-02-28" }; +const make = (items: unknown[]) => FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-03-01T00:00:00.000Z", + reportingPeriod: period, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], +}); + +export const syntheticFounderWeeklyReviewFixtures = { + partial: make([ + { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1" } }, + { sourceType: "founder_context", sourceId: "synthetic:context:priority", title: "Founder context", excerpt: "Prioritize onboarding reliability.", metadata: { fixture: "synthetic-v1" } }, + ]), + full: make([ + { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1" } }, + { sourceType: "customer_feedback", sourceId: "synthetic:feedback:export", title: "Customer feedback", sourceTimestamp: "2026-02-21T12:00:00.000Z", excerpt: "A customer requested saved export filters.", metadata: { fixture: "synthetic-v1" } }, + { sourceType: "founder_context", sourceId: "synthetic:context:priority", title: "Founder context", excerpt: "Prioritize onboarding reliability.", metadata: { fixture: "synthetic-v1" } }, + ]), + empty: make([]), +} as const; diff --git a/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts b/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts new file mode 100644 index 000000000..79e668cb3 --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts @@ -0,0 +1,192 @@ +import "dotenv/config"; +import { createHash, randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import { mkdir, rename, writeFile, access } from "node:fs/promises"; +import { resolve } from "node:path"; +import { ZodError } from "zod"; +import { zodSchema } from "ai"; +import { eq } from "drizzle-orm"; +import { company, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, FounderWeeklyReviewV2PayloadSchema, generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { syntheticFounderWeeklyReviewFixtures } from "./founder-weekly-review-synthetic-fixtures"; + +const require = createRequire(import.meta.url); +const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); + +function canonicalizeJson(value: unknown): unknown { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Cannot canonicalize non-finite number."); + return value; + } + if (Array.isArray(value)) return value.map(canonicalizeJson); + if (typeof value === "object") { + return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalizeJson((value as Record)[key])])); + } + throw new Error("Cannot canonicalize unsupported JSON value."); +} + +function digestJson(value: unknown): string { + return createHash("sha256").update(JSON.stringify(canonicalizeJson(value)), "utf8").digest("hex"); +} + +async function writeAtomically(path: string, content: string) { + try { await access(path); throw new Error("Refusing to overwrite an existing export."); } catch (error) { if (!(error instanceof Error) || !error.message.includes("ENOENT")) { if (error instanceof Error && error.message.includes("overwrite")) throw error; } } + const temporary = `${path}.${randomUUID()}.tmp`; + await writeFile(temporary, content, "utf8"); + await rename(temporary, path); +} + +function markdownFor(run: { reportingPeriod: { start: string; end: string }; modelMetadata: { provider?: string; model?: string } | null; reviewPayload: any; evidenceSnapshot: { items: Array<{ sourceId: string; sourceType: string; title: string; sourceTimestamp?: string }> } }) { + const labels: Record = { whatChanged: "Other Meaningful Changes", whatShipped: "Shipped This Period", whatCustomersSaid: "Customer Signals", currentBlockers: "Risks & Blockers", nextPriorities: "Priorities for the Next Period" }; + const none: Record = { whatChanged: "No other evidence-backed changes were identified during this reporting period.", whatShipped: "No completed product releases were identified during this reporting period.", whatCustomersSaid: "No customer feedback was available for this reporting period.", currentBlockers: "No evidence-backed blockers were identified during this reporting period.", nextPriorities: "No evidence-backed priorities were identified for the next reporting period." }; + const reference = new Map(); const source = new Map(run.evidenceSnapshot.items.map((item) => [item.sourceId, item])); const references: string[] = []; + const cite = (ids: string[] = []) => ids.map((id) => { let n = reference.get(id); if (!n) { n = reference.size + 1; reference.set(id, n); const item = source.get(id); references.push(`[${n}] ${item?.title ?? item?.sourceType ?? "Evidence"}`); } return `[${n}]`; }).join(""); + const period = `${new Date(`${run.reportingPeriod.start}T00:00:00Z`).toLocaleDateString("en-US", { month: "long", day: "numeric" })}–${new Date(`${run.reportingPeriod.end}T00:00:00Z`).toLocaleDateString("en-US", { day: "numeric", year: "numeric" })}`; + const shippedTexts = new Set((run.reviewPayload.sections.whatShipped?.items ?? []).map((item: any) => String(item.text ?? "").trim().replace(/\s+/g, " "))); + const lines = ["# Founder Weekly Review", "", `**Reporting period:** ${period}`, "", "## Key Outcomes", ""]; + for (const key of ["whatShipped", "whatChanged", "whatCustomersSaid", "currentBlockers", "nextPriorities"]) { const section = run.reviewPayload.sections[key]; lines.push(`## ${labels[key]!}`); if (section?.state === "no_evidence") lines.push(none[key]!); else { const items = (section?.items ?? []).filter((item: any) => key !== "whatChanged" || !shippedTexts.has(String(item.text ?? "").trim().replace(/\s+/g, " "))); if (!items.length) lines.push(none[key]!); else items.forEach((item: any, index: number) => lines.push(key === "nextPriorities" ? `${index + 1}. ${item.text}${cite(item.sourceIds)}` : `- ${item.text}${cite(item.sourceIds)}`)); } lines.push(""); } + if (references.length) lines.push("## Evidence References", "", ...references, ""); + lines.push("---", "", `*Generated with ${run.modelMetadata?.model ?? "the configured model"}*`); return lines.join("\n"); +} + +if (process.env.SYNTHETIC_FWR_LOCAL !== "1") throw new Error("Refusing synthetic baseline: set SYNTHETIC_FWR_LOCAL=1."); +const syntheticProvider = process.env.SYNTHETIC_FWR_PROVIDER; +const deterministicMode = (process.argv[2]?.startsWith("negative-") ?? false) || process.argv[2] === "retry-snapshot-immutability"; +if (!deterministicMode && syntheticProvider !== "ollama" && syntheticProvider !== "kimi") throw new Error("Refusing synthetic baseline: set SYNTHETIC_FWR_PROVIDER=ollama or kimi."); +if (process.env.NODE_ENV === "production") throw new Error("Refusing synthetic baseline in production."); +const url = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? ""; +if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(url)) throw new Error("Refusing non-local database."); +if (syntheticProvider === "ollama") { + process.env.OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434/v1"; + process.env.LLM_PROVIDER_PRIORITY = "ollama"; + process.env.LLM_MODEL_FOUNDERWEEKLYREVIEW_OLLAMA = process.env.SYNTHETIC_FWR_MODEL ?? "llama3.1:8b"; +} +if (syntheticProvider === "kimi" && !process.env.MOONSHOT_API_KEY?.trim()) throw new Error("Refusing Kimi synthetic baseline: MOONSHOT_API_KEY is required."); + +async function generateWithLocalOllama(input: { system?: string; prompt: string; schema: TSchema; schemaName?: string }) { + console.log(JSON.stringify({ stage: "ollama_request_constructed", result: "pass" })); + const response = await fetch(`${process.env.OLLAMA_BASE_URL!.replace(/\/$/, "")}/chat/completions`, { + method: "POST", headers: { "content-type": "application/json", authorization: "Bearer ollama" }, + body: JSON.stringify({ model: process.env.SYNTHETIC_FWR_MODEL ?? "llama3.1:8b", temperature: 0, stream: false, messages: [{ role: "system", content: input.system ?? "" }, { role: "user", content: input.prompt }], response_format: { type: "json_schema", json_schema: { name: input.schemaName ?? "founder_weekly_review", strict: true, schema: zodSchema(input.schema).jsonSchema } } }), + }); + console.log(JSON.stringify({ stage: "ollama_http_response", result: response.ok ? "pass" : "fail", httpStatus: response.status })); + if (!response.ok) throw new Error(`ollama_http_error:${response.status}`); + const body = await response.json() as { choices?: Array<{ message?: { content?: string } }> }; + const content = body.choices?.[0]?.message?.content; + if (!content?.trim()) throw new Error("ollama_missing_content"); + console.log(JSON.stringify({ stage: "ollama_content_extracted", result: "pass" })); + let parsed: unknown; + try { parsed = JSON.parse(content); console.log(JSON.stringify({ stage: "ollama_json_parsed", result: "pass" })); } + catch { throw new Error("ollama_json_parse_failed"); } + try { const object = input.schema.parse(parsed); console.log(JSON.stringify({ stage: "review_schema_valid", result: "pass" })); return { object, metadata: { provider: "ollama", model: process.env.SYNTHETIC_FWR_MODEL ?? "llama3.1:8b", capability: "founderWeeklyReview", temperature: 0 } }; } + catch (error) { const issues = error instanceof ZodError ? error.issues.map((issue) => ({ path: issue.path, code: issue.code, ...("expected" in issue ? { expected: issue.expected } : {}), ...("received" in issue ? { received: issue.received } : {}) })) : []; console.log(JSON.stringify({ stage: "review_schema_valid", result: "fail", topLevelKeys: parsed && typeof parsed === "object" && !Array.isArray(parsed) ? Object.keys(parsed as object) : [], issues })); throw new Error("review_schema_invalid"); } +} + +async function generateWithKimi(input: { system?: string; prompt: string; schema: TSchema; schemaName?: string }) { + const base = (process.env.MOONSHOT_BASE_URL ?? "https://api.moonshot.ai/v1").replace(/\/$/, ""); + const schemaGuide = JSON.stringify(zodSchema(input.schema).jsonSchema); + console.log(JSON.stringify({ stage: "kimi_request_constructed", result: "pass" })); + const response = await fetch(`${base}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${process.env.MOONSHOT_API_KEY}` }, body: JSON.stringify({ model: process.env.SYNTHETIC_FWR_MODEL ?? "kimi-k2.6", messages: [{ role: "system", content: `${input.system ?? ""}\nReturn one JSON object only. Its complete required structural schema is: ${schemaGuide}` }, { role: "user", content: input.prompt }], stream: false, thinking: { type: "disabled" }, response_format: { type: "json_object" } }), signal: AbortSignal.timeout(45_000) }); + console.log(JSON.stringify({ stage: "kimi_http_response", result: response.ok ? "pass" : "fail", httpStatus: response.status })); + if (!response.ok) throw new Error(`kimi_http_error:${response.status}`); + const body = await response.json() as { model?: string; usage?: Record; choices?: Array<{ message?: { content?: string }; finish_reason?: string }> }; + const content = body.choices?.[0]?.message?.content; + if (!content?.trim()) throw new Error("kimi_missing_content"); + console.log(JSON.stringify({ stage: "kimi_content_extracted", result: "pass" })); + let parsed: unknown; try { parsed = JSON.parse(content); console.log(JSON.stringify({ stage: "kimi_json_parsed", result: "pass" })); } catch { throw new Error("kimi_json_parse_failed"); } + return { object: parsed as ReturnType, metadata: { provider: "kimi", model: body.model ?? (process.env.SYNTHETIC_FWR_MODEL ?? "kimi-k2.6"), capability: "founderWeeklyReview", temperature: undefined as unknown as number, ...(body.choices?.[0]?.finish_reason ? { finishReason: body.choices[0].finish_reason } : {}), ...(body.usage ? { usage: body.usage } : {}) } }; +} + +const fixtureName = process.argv[2] ?? "partial"; +const isNegativeUnknownCitation = fixtureName === "negative-unknown-citation"; +const isNegativeFounderContextCustomer = fixtureName === "negative-founder-context-customer"; +const isDeterministicNegative = isNegativeUnknownCitation || isNegativeFounderContextCustomer; +const isRetrySnapshotImmutability = fixtureName === "retry-snapshot-immutability"; +const snapshot = fixtureName === "negative-founder-context-customer" ? syntheticFounderWeeklyReviewFixtures.partial : (fixtureName.startsWith("negative-") || isRetrySnapshotImmutability) ? syntheticFounderWeeklyReviewFixtures.full : syntheticFounderWeeklyReviewFixtures[fixtureName as keyof typeof syntheticFounderWeeklyReviewFixtures]; +if (!snapshot) throw new Error("Fixture must be partial, full, empty, negative-unknown-citation, negative-founder-context-customer, or retry-snapshot-immutability."); +const testDb = await createFounderWeeklyReviewTestDatabase(); +try { + await testDb.db.insert(company).values({ id: 3, name: "Synthetic FWR", numberOfEmployees: "1" }); + const actor = { companyId: 3n, userId: 1n, externalUserId: "synthetic-owner", role: "owner" as const, workspaceTimezone: "UTC" }; + const requestKey = `synthetic-${fixtureName}-${randomUUID()}`; + const dispatchService = createFounderWeeklyReviewDispatchService(testDb.db); + const { run, dispatch } = await dispatchService.createRunWithDispatch({ actor, requestKey, reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot }); + const event = { runId: dispatch.runId, companyId: dispatch.companyId.toString(), generationJobId: dispatch.generationJobId, generationClaimId: dispatch.generationClaimId }; + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); + const claimed = await worker.claimQueuedRun({ companyId: BigInt(event.companyId), runId: event.runId, generationJobId: event.generationJobId, generationClaimId: event.generationClaimId }); + let generated: Awaited> | null = null; + try { + const none = { state: "no_evidence" as const, noEvidence: { code: "none", message: "No evidence", cta: "Add evidence" } }; + const validPayload = FounderWeeklyReviewV2PayloadSchema.parse({ schemaVersion: "founder-weekly-review/v2", sections: { whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "Release evidence.", sourceIds: ["synthetic:doc:release"], confidence: 0.8 }] }, whatShipped: none, whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "Customer signal.", sourceIds: ["synthetic:feedback:export"], confidence: 0.8 }] }, currentBlockers: none, nextPriorities: none } }); + const payload = structuredClone(validPayload); + if (isNegativeUnknownCitation || isRetrySnapshotImmutability) payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "Release evidence.", sourceIds: ["synthetic:missing:unknown-citation"], confidence: 0.8 }] }; + if (isNegativeFounderContextCustomer) payload.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "Customer signal.", sourceIds: ["synthetic:context:priority"], confidence: 0.8 }] }; + generated = await generateFounderWeeklyReview({ evidenceSnapshot: claimed.evidenceSnapshot, generate: (isDeterministicNegative || isRetrySnapshotImmutability) ? async () => ({ object: payload, metadata: { provider: "synthetic", model: "deterministic", capability: "founderWeeklyReview", temperature: 0 } }) : syntheticProvider === "kimi" ? generateWithKimi : generateWithLocalOllama }); + } catch (error) { + const message = error instanceof Error ? error.message : "unknown_generation_failure"; + const failureKind = message.includes("absent from the evidence snapshot") ? "citation_validation_failed" : message.includes("must never be presented as customer feedback") ? "source_semantic_validation_failed" : ["ollama_http_error", "ollama_missing_content", "ollama_json_parse_failed", "review_schema_invalid"].find((kind) => message.startsWith(kind)) ?? "unknown_generation_failure"; + const failed = await worker.markGenerationFailed({ companyId: BigInt(event.companyId), runId: event.runId, generationJobId: event.generationJobId, generationClaimId: event.generationClaimId }, { errorCode: "generation_failed", errorMessage: "Synthetic baseline provider generation failed." }); + const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(3n, failed.id); + const unchanged = readBack ? digestJson(readBack.evidenceSnapshot) === digestJson(snapshot) : false; + const citation = failureKind === "citation_validation_failed" ? "failed" : failureKind === "source_semantic_validation_failed" ? "passed" : "not_reached"; + const semantic = failureKind === "source_semantic_validation_failed" ? "failed" : "not_reached"; + if (isRetrySnapshotImmutability) { + if (!readBack || failureKind !== "citation_validation_failed" || readBack.status !== "failed" || readBack.retryCount !== 0 || readBack.errorCode !== "generation_failed" || readBack.reviewPayload || !unchanged) throw new Error("Retry baseline precondition failed."); + const before = readBack; + const retryRequestKey = `synthetic-retry-${randomUUID()}`; + const firstRetry = await dispatchService.retryRunWithDispatch({ actor, runId: before.id, requestKey: retryRequestKey }); + const afterFirstRetry = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, before.id); + let secondRetryBehavior: "idempotent_queued_return" | "lifecycle_conflict"; + let secondRetry: Awaited> | null = null; + try { + secondRetry = await dispatchService.retryRunWithDispatch({ actor, runId: before.id, requestKey: retryRequestKey }); + secondRetryBehavior = "idempotent_queued_return"; + } catch { + secondRetryBehavior = "lifecycle_conflict"; + } + const afterSecondRetry = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, before.id); + const allRuns = await new FounderWeeklyReviewRepository(testDb.db).listByCompany(actor.companyId); + const matchingRuns = allRuns.filter((candidate) => candidate.requestKey === before.requestKey); + const retryDispatches = (await testDb.db.select().from(founderWeeklyReviewDispatches)).filter((candidate) => candidate.runId === before.id && candidate.operationType === "retry"); + const retryClaim = await worker.claimQueuedRun({ companyId: actor.companyId, runId: firstRetry.run.id, generationJobId: firstRetry.dispatch.generationJobId, generationClaimId: firstRetry.dispatch.generationClaimId }); + const afterClaim = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, before.id); + const sameIdentity = Boolean(afterFirstRetry && afterSecondRetry && afterClaim) && before.id === afterFirstRetry!.id && before.id === afterSecondRetry!.id && before.id === afterClaim!.id; + const snapshotUnchanged = Boolean(afterFirstRetry && afterSecondRetry && afterClaim) && digestJson(before.evidenceSnapshot) === digestJson(afterFirstRetry!.evidenceSnapshot) && digestJson(before.evidenceSnapshot) === digestJson(afterSecondRetry!.evidenceSnapshot) && digestJson(before.evidenceSnapshot) === digestJson(afterClaim!.evidenceSnapshot); + const requestKeyUnchanged = Boolean(afterClaim) && before.requestKey === afterClaim!.requestKey; + const companyUnchanged = Boolean(afterClaim) && before.companyId === afterClaim!.companyId; + const periodUnchanged = Boolean(afterClaim) && before.reportingPeriod.start === afterClaim!.reportingPeriod.start && before.reportingPeriod.end === afterClaim!.reportingPeriod.end; + const schemaVersionsUnchanged = Boolean(afterClaim) && before.evidenceSchemaVersion === afterClaim!.evidenceSchemaVersion && before.reviewSchemaVersion === afterClaim!.reviewSchemaVersion; + const reviewPayloadAbsent = !afterClaim?.reviewPayload; + const retryCounts = [before.retryCount, afterFirstRetry?.retryCount, afterSecondRetry?.retryCount]; + const retryDispatchCreated = firstRetry.dispatch.operationType === "retry" && retryDispatches.length === 1 && (!secondRetry || secondRetry.dispatch.id === firstRetry.dispatch.id); + console.log(JSON.stringify({ fixture: fixtureName, runId: before.id, lifecycle: [run.status, claimed.status, failed.status, firstRetry.run.status, retryClaim.status], retryCounts, sameRow: sameIdentity, matchingRunRowCount: matchingRuns.length, snapshotUnchanged, requestKeyUnchanged, companyUnchanged, periodUnchanged, schemaVersionsUnchanged, reviewPayloadAbsent, duplicateRetryBehavior: secondRetryBehavior, outboxBehavior: retryDispatchCreated ? "creates_one_retry_dispatch_and_reuses_it_for_duplicate_request_key" : "unexpected_retry_dispatch_state", postRetryClaimSucceeded: retryClaim.status === "generating", externalProviderCalled: false })); + if (!sameIdentity || matchingRuns.length !== 1 || afterFirstRetry?.retryCount !== 1 || afterSecondRetry?.retryCount !== 1 || afterClaim?.retryCount !== 1 || !snapshotUnchanged || !requestKeyUnchanged || !companyUnchanged || !periodUnchanged || !schemaVersionsUnchanged || !reviewPayloadAbsent || !retryDispatchCreated || retryClaim.status !== "generating") throw new Error("Retry snapshot immutability invariant failed."); + } else { + console.log(JSON.stringify({ fixture: fixtureName, runId: failed.id, lifecycle: [run.status, claimed.status, failed.status], canonicalSchemaValidation: "passed", citationValidation: citation, sourceSemanticValidation: semantic, firstFailingBoundary: failureKind, finalStatus: readBack?.status, errorCode: readBack?.errorCode, draftPersisted: Boolean(readBack?.reviewPayload), draftRetrievable: Boolean(readBack?.reviewPayload), snapshotUnchanged: unchanged })); + if (!isDeterministicNegative || !readBack || failureKind !== (isNegativeUnknownCitation ? "citation_validation_failed" : "source_semantic_validation_failed") || readBack.status !== "failed" || readBack.errorCode !== "generation_failed" || readBack.reviewPayload || !unchanged) throw new Error("Negative safety invariant failed."); + } + } + if (!isDeterministicNegative && !isRetrySnapshotImmutability) { + const saved = await worker.saveGeneratedDraft({ companyId: BigInt(event.companyId), runId: event.runId, generationJobId: event.generationJobId, generationClaimId: event.generationClaimId }, generated!.reviewPayload, generated!.modelMetadata); + const [persisted] = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); + const [persistedDispatch] = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, dispatch.id)); + if (!persisted || !persistedDispatch) throw new Error("Synthetic baseline persistence verification failed."); + const ids = new Set(snapshot.items.map((item) => item.sourceId)); + for (const section of Object.values(saved.reviewPayload!.sections)) if (section.state === "evidence") for (const item of section.items) for (const id of item.sourceIds) if (!ids.has(id)) throw new Error("Persisted citation is not in snapshot."); + const customerSection = saved.reviewPayload!.sections.whatCustomersSaid; + const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(3n, saved.id); + if (!readBack?.reviewPayload || readBack.status !== "draft") throw new Error("Validated draft read-back failed."); + if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { + const directory = resolve(process.cwd(), process.env.SYNTHETIC_FWR_EXPORT_DIR ?? ".artifacts/founder-weekly-review"); + await mkdir(directory, { recursive: true }); + const fileId = saved.id.replace(/[^A-Za-z0-9_-]/g, "_"); + const envelope = { runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, periodStart: readBack.reportingPeriod.start, periodEnd: readBack.reportingPeriod.end, generatedAt: readBack.generatedAt?.toISOString() ?? null, review: readBack.reviewPayload }; + const markdownPath = resolve(directory, `${fileId}.md`); const jsonPath = resolve(directory, `${fileId}.json`); + await writeAtomically(markdownPath, markdownFor(readBack)); await writeAtomically(jsonPath, JSON.stringify(envelope, null, 2)); + console.log(JSON.stringify({ runId: readBack.id, status: readBack.status, markdownPath, jsonPath, filesWritten: true })); + } + console.log(JSON.stringify({ label: "Synthetic-evidence integration baseline", fixture: fixtureName, runId: saved.id, lifecycle: [run.status, claimed.status, saved.status], event, provider: generated!.modelMetadata.provider, model: generated!.modelMetadata.model, snapshotUnchanged: digestJson(persisted.evidenceSnapshot) === digestJson(snapshot), outbox: { status: persistedDispatch.status, hasEvidence: false }, customerSection: "state" in customerSection ? customerSection.state : "legacy", draftReturnedByRepository: saved.status === "draft" })); + } +} finally { await testDb.close(); } diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts index 115c0ca1d..3b4ba8dba 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts @@ -1,20 +1,29 @@ import { NextResponse } from "next/server"; import { auth } from "@clerk/nextjs/server"; import { z } from "zod"; +import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { retryRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; import { inngest } from "~/server/inngest/client"; import { founderWeeklyReviewRetries, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; const RetrySchema = z.object({ requestKey: z.string().min(1).max(128) }); -export async function POST(request: Request, { params }: { params: Promise<{ runId: string }> }) { - const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - const parsed = RetrySchema.safeParse(await request.json().catch(() => null)); if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); - try { const actor = await productionFounderWeeklyReviewActorResolver.resolve(userId); const { runId } = await params; - logFounderWeeklyReview({ runId, companyId: actor.companyId.toString(), stage: "retry_requested", status: "failed" }); - const { run } = await retryRunWithDispatch({ actor, runId, requestKey: parsed.data.requestKey }); - if (run.status === "queued") { founderWeeklyReviewRetries.inc(); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "retry_queued", status: run.status, retryCount: run.retryCount }); } - await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); - return NextResponse.json({ run: safeRun(run) }, { status: 202 }); - } catch (error) { return safeFounderWeeklyReviewError(error); } +export interface FounderWeeklyReviewRetryRouteDependencies { + actorResolver: Pick; + retryRunWithDispatch: typeof retryRunWithDispatch; + sendDispatchRequested: () => Promise; + incrementRetry: () => void; } +export function createFounderWeeklyReviewRetryPostHandler(deps: FounderWeeklyReviewRetryRouteDependencies) { + return async function POST(request: Request, { params }: { params: Promise<{ runId: string }> }) { + const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + const parsed = RetrySchema.safeParse(await request.json().catch(() => null)); if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + try { const actor = await deps.actorResolver.resolve(userId); const { runId } = await params; + logFounderWeeklyReview({ runId, companyId: actor.companyId.toString(), stage: "retry_requested", status: "failed" }); + const { run, transitionApplied } = await deps.retryRunWithDispatch({ actor, runId, requestKey: parsed.data.requestKey }); + if (transitionApplied) { deps.incrementRetry(); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "retry_queued", status: run.status, retryCount: run.retryCount }); } + await deps.sendDispatchRequested(); return NextResponse.json({ run: safeRun(run) }, { status: 202 }); + } catch (error) { return safeFounderWeeklyReviewError(error); } + }; +} +export const POST = createFounderWeeklyReviewRetryPostHandler({ actorResolver: productionFounderWeeklyReviewActorResolver, retryRunWithDispatch, sendDispatchRequested: () => inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }), incrementRetry: () => founderWeeklyReviewRetries.inc() }); diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts index 3f4852f17..6d970ef71 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts @@ -1,12 +1,28 @@ import { NextResponse } from "next/server"; import { auth } from "@clerk/nextjs/server"; -import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { FounderWeeklyReviewUserService } from "@launchstack/features/founder-weekly-review"; +import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; -export async function GET(_request: Request, { params }: { params: Promise<{ runId: string }> }) { - const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - try { const actor = await productionFounderWeeklyReviewActorResolver.resolve(userId); const { runId } = await params; - const run = await new FounderWeeklyReviewUserService().getRun(actor, runId); - return NextResponse.json({ run: safeRun(run) }); - } catch (error) { return safeFounderWeeklyReviewError(error); } + +export interface FounderWeeklyReviewGetRouteDependencies { + actorResolver: Pick; + getRun: (actor: Parameters[0], runId: string) => ReturnType; +} + +export function createFounderWeeklyReviewGetHandler(deps: FounderWeeklyReviewGetRouteDependencies) { + return async function GET(_request: Request, { params }: { params: Promise<{ runId: string }> }) { + const { userId } = await auth(); + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + try { + const actor = await deps.actorResolver.resolve(userId); + const { runId } = await params; + const run = await deps.getRun(actor, runId); + return NextResponse.json({ run: safeRun(run) }); + } catch (error) { return safeFounderWeeklyReviewError(error); } + }; } +export const GET = createFounderWeeklyReviewGetHandler({ + actorResolver: productionFounderWeeklyReviewActorResolver, + getRun: (actor, runId) => new FounderWeeklyReviewUserService().getRun(actor, runId), +}); diff --git a/apps/web/src/app/api/founder-weekly-reviews/route.ts b/apps/web/src/app/api/founder-weekly-reviews/route.ts index ff086c53b..e9c16c9dd 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/route.ts @@ -19,6 +19,7 @@ export interface FounderWeeklyReviewRouteDependencies { repository: Pick; createRunWithDispatch: typeof createRunWithDispatch; sendDispatchRequested: () => Promise; + recordRunCreated: () => void; } export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRouteDependencies) { @@ -39,8 +40,8 @@ export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRo const durationMs = Math.round(performance.now() - startedAt); founderWeeklyReviewStageDuration.observe({ stage: "evidence_collection", result: "success" }, durationMs / 1000); logFounderWeeklyReview({ runId: "pending", companyId: actor.companyId.toString(), stage: "evidence_collection_completed", status: "pending", durationMs }); - const { run } = await deps.createRunWithDispatch({ actor, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, evidenceSnapshot }); - founderWeeklyReviewRunsCreated.inc(); + const { run, created } = await deps.createRunWithDispatch({ actor, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, evidenceSnapshot }); + if (created) deps.recordRunCreated(); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "run_created", status: run.status, retryCount: run.retryCount }); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "dispatch_created", status: run.status }); await deps.sendDispatchRequested(); @@ -55,8 +56,9 @@ export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRo const productionDependencies: FounderWeeklyReviewRouteDependencies = { actorResolver: productionFounderWeeklyReviewActorResolver, evidenceCollector: canonicalFounderWeeklyReviewEvidenceCollector, - repository: new FounderWeeklyReviewRepository(), + repository: { getByCompanyAndRequestKey: (companyId, requestKey) => new FounderWeeklyReviewRepository().getByCompanyAndRequestKey(companyId, requestKey) }, createRunWithDispatch, sendDispatchRequested: () => inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }), + recordRunCreated: () => founderWeeklyReviewRunsCreated.inc(), }; export const POST = createFounderWeeklyReviewPostHandler(productionDependencies); diff --git a/apps/web/src/server/founder-weekly-review/dispatch-service.ts b/apps/web/src/server/founder-weekly-review/dispatch-service.ts index 6565e4a97..8e9645940 100644 --- a/apps/web/src/server/founder-weekly-review/dispatch-service.ts +++ b/apps/web/src/server/founder-weekly-review/dispatch-service.ts @@ -33,6 +33,16 @@ function identifiers(runId: string, operationType: "create" | "retry", operation }; } +export type FounderWeeklyReviewTransactionClient = Pick; +export interface CreateFounderWeeklyReviewDispatchInput { + run: FounderWeeklyReviewRunRecord; + operationType: "create" | "retry"; + operationKey: string; +} +export interface FounderWeeklyReviewDispatchServiceDependencies { + createDispatch?: (transaction: FounderWeeklyReviewTransactionClient, input: CreateFounderWeeklyReviewDispatchInput) => Promise; +} + async function createDispatch( tx: Pick, run: FounderWeeklyReviewRunRecord, @@ -61,33 +71,37 @@ async function createDispatch( return toDispatch(existing); } -export async function createRunWithDispatch(input: { +export type CreateRunWithDispatchInput = { actor: FounderWeeklyReviewUserActor; requestKey: string; reportingPeriod: ReportingPeriod; evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; -}): Promise<{ run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch }> { - return db.transaction(async (tx) => { +}; +export type RetryRunWithDispatchInput = { actor: FounderWeeklyReviewUserActor; runId: string; requestKey: string }; +export type CreateRunWithDispatchResult = { run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch; created: boolean }; +export type RetryRunWithDispatchResult = { run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch; transitionApplied: boolean }; + +export function createFounderWeeklyReviewDispatchService(database: Pick, dependencies: FounderWeeklyReviewDispatchServiceDependencies = {}) { + const writeDispatch = dependencies.createDispatch ?? ((transaction, input) => createDispatch(transaction, input.run, input.operationType, input.operationKey)); + const createRunWithDispatch = async (input: CreateRunWithDispatchInput): Promise => database.transaction(async (tx) => { const service = new FounderWeeklyReviewUserService(new FounderWeeklyReviewRepository(tx as unknown as DbClient)); - const run = await service.createOrGetRun(input.actor, input); - const dispatch = await createDispatch(tx, run, "create", input.requestKey); - return { run, dispatch }; + const { run, created } = await service.createOrGetRunWithMetadata(input.actor, input); + const dispatch = await writeDispatch(tx, { run, operationType: "create", operationKey: input.requestKey }); + return { run, dispatch, created }; }); -} - -export async function retryRunWithDispatch(input: { - actor: FounderWeeklyReviewUserActor; - runId: string; - requestKey: string; -}): Promise<{ run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch }> { - return db.transaction(async (tx) => { + const retryRunWithDispatch = async (input: RetryRunWithDispatchInput): Promise => database.transaction(async (tx) => { const service = new FounderWeeklyReviewUserService(new FounderWeeklyReviewRepository(tx as unknown as DbClient)); - const run = await service.retryFailedRun(input.actor, input.runId, input.requestKey); - const dispatch = await createDispatch(tx, run, "retry", input.requestKey); - return { run, dispatch }; + const { run, transitionApplied } = await service.retryFailedRunWithMetadata(input.actor, input.runId, input.requestKey); + const dispatch = await writeDispatch(tx, { run, operationType: "retry", operationKey: input.requestKey }); + return { run, dispatch, transitionApplied }; }); + return { createRunWithDispatch, retryRunWithDispatch }; } +const productionDispatchService = createFounderWeeklyReviewDispatchService(db); +export const createRunWithDispatch = productionDispatchService.createRunWithDispatch; +export const retryRunWithDispatch = productionDispatchService.retryRunWithDispatch; + export async function claimPendingDispatches(limit = 20): Promise { const now = new Date(); const staleDispatchingBefore = new Date(now.getTime() - 5 * 60 * 1000); diff --git a/apps/web/src/server/founder-weekly-review/evidence-collector.ts b/apps/web/src/server/founder-weekly-review/evidence-collector.ts index e699c410c..474d8db0e 100644 --- a/apps/web/src/server/founder-weekly-review/evidence-collector.ts +++ b/apps/web/src/server/founder-weekly-review/evidence-collector.ts @@ -38,7 +38,9 @@ export const unavailableFounderWeeklyReviewEvidenceCollector: FounderWeeklyRevie * without changing routes, persistence, or the generation worker. */ export class CanonicalFounderWeeklyReviewEvidenceCollector implements FounderWeeklyReviewEvidenceCollector { - constructor(private readonly service = new FounderWeeklyReviewEvidenceService()) {} + // Lazy construction keeps route composition importable in tests and does + // not open a DB dependency until production collection is actually used. + constructor(private service?: FounderWeeklyReviewEvidenceService) {} async collectFounderWeeklyReviewEvidence(input: { companyId: bigint; @@ -48,7 +50,7 @@ export class CanonicalFounderWeeklyReviewEvidenceCollector implements FounderWee actor: { externalUserId: string }; requestKey: string; }): Promise { - const snapshot = await this.service.collectFounderWeeklyReviewEvidence({ + const snapshot = await (this.service ??= new FounderWeeklyReviewEvidenceService()).collectFounderWeeklyReviewEvidence({ companyId: input.companyId, reportingPeriod: input.reportingPeriod, workspaceTimezone: input.workspaceTimezone, diff --git a/packages/features/src/founder-weekly-review/repository.ts b/packages/features/src/founder-weekly-review/repository.ts index 6390297d9..5c3c78154 100644 --- a/packages/features/src/founder-weekly-review/repository.ts +++ b/packages/features/src/founder-weekly-review/repository.ts @@ -29,6 +29,10 @@ export interface ConditionalRunMutationResult { updated: boolean; run: FounderWeeklyReviewRunRecord | null; } +export interface CreateFounderWeeklyReviewResult { + run: FounderWeeklyReviewRunRecord; + created: boolean; +} export interface RetryFounderWeeklyReviewResult { outcome: "updated" | "idempotent" | "conflict" | "not_found"; @@ -106,6 +110,12 @@ export class FounderWeeklyReviewRepository { async createOrGetByRequestKey( input: CreateFounderWeeklyReviewRunInput ): Promise { + return (await this.createOrGetByRequestKeyWithResult(input)).run; + } + + async createOrGetByRequestKeyWithResult( + input: CreateFounderWeeklyReviewRunInput + ): Promise { const [inserted] = await this.db .insert(founderWeeklyReviewRuns) .values({ @@ -133,7 +143,7 @@ export class FounderWeeklyReviewRepository { .returning(); if (inserted) { - return mapRunRow(inserted); + return { run: mapRunRow(inserted), created: true }; } const existing = await this.getByCompanyAndRequestKey( @@ -143,7 +153,7 @@ export class FounderWeeklyReviewRepository { if (!existing) { throw new Error("Failed to create or retrieve founder weekly review run"); } - return existing; + return { run: existing, created: false }; } async getByCompanyAndRunId( diff --git a/packages/features/src/founder-weekly-review/user-service.ts b/packages/features/src/founder-weekly-review/user-service.ts index 2caeb899f..e6ae6e675 100644 --- a/packages/features/src/founder-weekly-review/user-service.ts +++ b/packages/features/src/founder-weekly-review/user-service.ts @@ -29,6 +29,14 @@ export interface CreateFounderWeeklyReviewRunRequest { }; evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; } +export interface CreateFounderWeeklyReviewRunResult { + run: FounderWeeklyReviewRunRecord; + created: boolean; +} +export interface RetryFounderWeeklyReviewRunResult { + run: FounderWeeklyReviewRunRecord; + transitionApplied: boolean; +} function assertWorkspaceMutationRole(role: string): void { if (!ALLOWED_WORKSPACE_ROLES.has(role)) { @@ -71,14 +79,23 @@ export class FounderWeeklyReviewUserService { } assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); - return this.repository.createOrGetByRequestKey({ + return (await this.repository.createOrGetByRequestKeyWithResult({ id: `fwr_${randomUUID()}`, companyId: actor.companyId, requestKey: input.requestKey, reportingPeriod: input.reportingPeriod, evidenceSnapshot, createdByActorId: buildFounderWeeklyReviewActorId(actor), - }); + })).run; + } + + async createOrGetRunWithMetadata(actor: FounderWeeklyReviewUserActor, input: CreateFounderWeeklyReviewRunRequest): Promise { + assertWorkspaceMutationRole(actor.role); + let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + try { evidenceSnapshot = parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot); } + catch (error) { if (error instanceof ZodError) throw new FounderWeeklyReviewInvalidPayloadError(error.message); throw error; } + assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); + return this.repository.createOrGetByRequestKeyWithResult({ id: `fwr_${randomUUID()}`, companyId: actor.companyId, requestKey: input.requestKey, reportingPeriod: input.reportingPeriod, evidenceSnapshot, createdByActorId: buildFounderWeeklyReviewActorId(actor) }); } async getRun( @@ -129,6 +146,16 @@ export class FounderWeeklyReviewUserService { ); } + async retryFailedRunWithMetadata(actor: FounderWeeklyReviewUserActor, runId: string, requestKey: string): Promise { + assertWorkspaceMutationRole(actor.role); + const result = await this.repository.retryFailedRun({ operationId: `fwrop_${randomUUID()}`, companyId: actor.companyId, runId, requestKey, actorId: buildFounderWeeklyReviewActorId(actor) }); + if (result.outcome === "not_found" || !result.run) throw new FounderWeeklyReviewNotFoundError(runId); + if (result.outcome === "updated") return { run: result.run, transitionApplied: true }; + if (result.outcome === "idempotent" && (result.run.status === "failed" || result.run.status === "queued")) return { run: result.run, transitionApplied: false }; + if (result.run.status !== "failed") throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "retry"); + throw new FounderWeeklyReviewConflictError(`Retry request key "${requestKey}" belongs to a different failure cycle for run "${runId}".`); + } + async updateDraft( actor: FounderWeeklyReviewUserActor, runId: string, From bfade6ca102f0504b005824f45e09e98dae17342 Mon Sep 17 00:00:00 2001 From: Peace Odetola Date: Sat, 1 Aug 2026 01:28:37 -0500 Subject: [PATCH 14/29] Add founder weekly review evaluator benchmark --- package.json | 4 +- .../benchmarks/baseline-output.json | 447 +++++++++ .../benchmarks/baseline-report.md | 90 ++ .../founder-weekly-review/benchmarks/cases.ts | 864 ++++++++++++++++++ .../benchmarks/runner.ts | 232 +++++ .../src/founder-weekly-review/evaluation.ts | 521 +++++++++++ 6 files changed, 2157 insertions(+), 1 deletion(-) create mode 100644 packages/features/src/founder-weekly-review/benchmarks/baseline-output.json create mode 100644 packages/features/src/founder-weekly-review/benchmarks/baseline-report.md create mode 100644 packages/features/src/founder-weekly-review/benchmarks/cases.ts create mode 100644 packages/features/src/founder-weekly-review/benchmarks/runner.ts create mode 100644 packages/features/src/founder-weekly-review/evaluation.ts diff --git a/package.json b/package.json index d46d769fe..59baaaca9 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "inngest:dev": "pnpm --filter @launchstack/web inngest:dev", "changeset": "changeset", "version": "changeset version", - "release": "pnpm --filter @launchstack/core build && changeset publish" + "release": "pnpm --filter @launchstack/core build && changeset publish", + "eval:founder-weekly-review": "tsx packages/features/src/founder-weekly-review/benchmarks/runner.ts" }, "devDependencies": { "@changesets/cli": "^2.31.0", @@ -38,6 +39,7 @@ "@typescript-eslint/parser": "^8.42.0", "eslint": "^9.34.0", "prettier": "^3.6.2", + "tsx": "^4.21.0", "typescript": "^5.9.2" }, "packageManager": "pnpm@10.15.1", diff --git a/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json b/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json new file mode 100644 index 000000000..08dfcab9b --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json @@ -0,0 +1,447 @@ +{ + "summary": { + "totalCases": 19, + "passed": 19, + "failed": 0, + "hardFailures": 9, + "passRate": 1, + "overallScore": 0.48157894736842105 + }, + "metrics": { + "citationValidity": 0.8823529411764706, + "citationCoverage": 1, + "unsupportedClaimRate": 0.058823529411764705, + "unsupportedShippedClaimRate": 0.11764705882352941, + "sourceTypeViolationRate": 0.11764705882352941, + "evidenceCoverage": 0.7156862745098039, + "emptySectionCorrectness": 1, + "duplicateClaimRate": 0.058823529411764705 + }, + "weakestCases": [ + { + "case": "invalid_citation_report", + "score": 0, + "failures": [ + "invalid_citation" + ] + }, + { + "case": "invalid_missing_source_citation", + "score": 0, + "failures": [ + "invalid_citation" + ] + }, + { + "case": "founder_context_as_customer_feedback", + "score": 0, + "failures": [ + "invalid_source_type" + ] + } + ], + "commonFailures": { + "invalid_citation": 2, + "invalid_source_type": 3, + "duplicate_claim": 1, + "malformed_payload": 2, + "unsupported_shipped_claim": 2, + "unsupported_claim": 1, + "conflicting_evidence": 1 + }, + "cases": [ + { + "case": "valid_customer_feedback_report", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "invalid_citation_report", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 0, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_citation", + "section": "whatCustomersSaid", + "explanation": "Unknown sourceId: fake_feedback_999" + } + ] + }, + { + "case": "empty_workspace", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "complete_workspace", + "passed": true, + "score": 0.8999999999999999, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0.6666666666666666, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "partial_workspace", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "invalid_missing_source_citation", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 0, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_citation", + "section": "whatCustomersSaid", + "explanation": "Unknown sourceId: missing_source" + } + ] + }, + { + "case": "founder_context_as_customer_feedback", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 1, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatCustomersSaid", + "explanation": "Source type \"founder_context\" is not valid for section \"whatCustomersSaid\"" + } + ] + }, + { + "case": "founder_context_only_valid", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "multi_source_customer_feedback", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "duplicate_customer_claims", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0.5, + "evidenceCoverage": 1, + "duplicateClaimRate": 1, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "duplicate_claim", + "section": "whatChanged", + "claim": "customer requested csv export", + "explanation": "Same factual claim appears multiple times." + }, + { + "category": "invalid_source_type", + "section": "whatChanged", + "explanation": "Source type \"customer_feedback\" is not valid for section \"whatChanged\"" + } + ] + }, + { + "case": "duplicate_source_ids_schema_validation", + "passed": true, + "score": 0, + "hasHardFailure": true, + "failures": [ + { + "category": "malformed_payload", + "explanation": "Report failed schema validation." + } + ] + }, + { + "case": "unsupported_shipped_claim", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 1, + "sourceTypeViolationRate": 0.5, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatShipped", + "explanation": "Source type \"customer_feedback\" is not valid for section \"whatShipped\"" + }, + { + "category": "unsupported_shipped_claim", + "section": "whatShipped", + "claim": "CSV export was shipped.", + "explanation": "Evidence does not contain a shipping signal." + } + ] + }, + { + "case": "malformed_payload", + "passed": true, + "score": 0, + "hasHardFailure": true, + "failures": [ + { + "category": "malformed_payload", + "explanation": "Report failed schema validation." + } + ] + }, + { + "case": "evidence_omitted", + "passed": true, + "score": 0.8500000000000001, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0.5, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "exaggerated_claim", + "passed": true, + "score": 0.7, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 1, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "unsupported_claim", + "section": "whatCustomersSaid", + "claim": "Customers cannot use the product without CSV export.", + "explanation": "Claim is not directly supported by cited evidence." + } + ] + }, + { + "case": "conflicting_evidence", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "conflicting_evidence", + "section": "whatCustomersSaid", + "claim": "All customers want CSV export.", + "explanation": "Claim ignores conflicting cited evidence." + } + ] + }, + { + "case": "document_change_as_shipped", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 1, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "unsupported_shipped_claim", + "section": "whatShipped", + "claim": "CSV export was shipped.", + "explanation": "Evidence does not contain a shipping signal." + } + ] + }, + { + "case": "unavailable_source_warning", + "passed": true, + "score": 0.7, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + }, + { + "case": "valid_contradictory_evidence", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [] + } + ] +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/baseline-report.md b/packages/features/src/founder-weekly-review/benchmarks/baseline-report.md new file mode 100644 index 000000000..1de3daf0e --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/baseline-report.md @@ -0,0 +1,90 @@ +# Founder Weekly Review Evaluator Baseline + +## Purpose + +This benchmark evaluates the Founder Weekly Review generation pipeline against predefined evidence scenarios. + +It measures: +- citation correctness +- evidence coverage +- unsupported claims +- source type correctness +- handling of empty or conflicting evidence + +## Running the Benchmark + +```bash +pnpm eval:founder-weekly-review +``` + +This regenerates the benchmark output at: + +packages/features/src/founder-weekly-review/benchmarks/baseline-output.json + +## Coverage + +The benchmark currently includes: +- valid evidence-backed reports +- missing/invalid citations +- unsupported shipped claims +- source type misuse +- conflicting evidence +- empty evidence states +- malformed payload handling + +## Summary + +- Benchmark cases: 19 +- Passing cases: 19 +- Failing cases: 0 +- Overall score: 0.482 +- Hard failures: 9 + +## Metrics + +| Metric | Score | +|---|---:| +| Citation validity | 0.88 | +| Citation coverage | 1.0 | +| Unsupported claim rate | 0.059 | +| Source type violation rate | 0.118 | +| Evidence coverage | 0.716 | +| Empty section correctness | 1.0 | + +## Weakest Cases + +1. invalid_citation_report + - Failure: invalid_citation + +2. invalid_missing_source_citation + - Failure: invalid_citation + +3. founder_context_as_customer_feedback + - Failure: invalid_source_type + +## Common Failure Categories + +- invalid_source_type: 3 +- invalid_citation: 2 +- malformed_payload: 2 +- unsupported_shipped_claim: 2 + +## Recommended Improvements + +### Prompt improvements +- Encourage explicit evidence references for factual claims. +- Avoid converting founder context into customer feedback. + +### Evidence improvements +- Include stronger shipped signals from GitHub/document sources. +- Preserve source type metadata. + +### Validator improvements +- Improve semantic matching for evidence coverage. +- Add stricter detection of unsupported generalizations. + +## Notes + +Passing cases indicate that expected evaluator behavior matched the benchmark expectations. + +Hard failures represent intentionally invalid scenarios (for example malformed payloads or invalid citations) that the evaluator correctly detected. \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/cases.ts b/packages/features/src/founder-weekly-review/benchmarks/cases.ts new file mode 100644 index 000000000..1acfec6f9 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/cases.ts @@ -0,0 +1,864 @@ +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + FounderWeeklyReviewV2Payload, + type FounderWeeklyReviewEvidenceSnapshot, +} from "../contracts"; + +export type BenchmarkCase = { + id: string; + description: string; + runThroughGeneration: boolean; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + generatedReport?: FounderWeeklyReviewV2Payload; + + expectations: { + shouldPass: boolean; + requiredClaims?: string[]; + forbiddenClaims?: string[]; + expectedEmptySections?: string[]; + expectedFailureCategories?: string[]; + }; +}; + +const noEvidenceSection = { + state: "no_evidence" as const, + noEvidence: { + code: "none", + message: "No evidence available.", + cta: "Add evidence.", + }, +}; + +const validEvidence = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + capturedAt: "2026-07-01T00:00:00.000Z", + reportingPeriod: { + start: "2026-07-01", + end: "2026-07-07", + }, + workspaceTimezone: "America/New_York", + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Customer requested export feature", + excerpt: "Customer asked for CSV export.", + metadata: {}, + }, + ], + sourceWarnings: [], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const emptyEvidence = { + ...validEvidence, + items: [], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const founderContextEvidence = { + ...validEvidence, + items: [ + { + sourceType: "founder_context", + sourceId: "context_1", + title: "Founder notes", + excerpt: "Founder heard customers asking for export.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const founderContextOnlyEvidence = { + ...validEvidence, + items: [ + { + sourceType: "founder_context", + sourceId: "context_1", + title: "Founder priorities", + excerpt: "Founder wants to improve onboarding.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const multipleCustomerEvidence = { + ...validEvidence, + items: [ + ...validEvidence.items, + { + sourceType: "customer_feedback", + sourceId: "feedback_2", + title: "Second customer", + excerpt: "Another customer requested CSV export.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const completeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Customer request", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + { + sourceType: "github_activity", + sourceId: "github_1", + title: "Released export feature", + excerpt: "CSV export was shipped.", + metadata: {}, + }, + { + sourceType: "founder_context", + sourceId: "context_1", + title: "Founder priorities", + excerpt: "Focus on improving onboarding.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const multipleThemeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Export request", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback_2", + title: "Onboarding complaint", + excerpt: "Customer reported onboarding was confusing.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const weakEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Feature request", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const conflictingEvidence = { + ...validEvidence, + items:[ + { + sourceType:"customer_feedback", + sourceId:"feedback_1", + title:"Customer wants export", + excerpt:"Customer requested CSV export.", + metadata:{}, + }, + { + sourceType:"customer_feedback", + sourceId:"feedback_2", + title:"Customer does not need export", + excerpt:"Customer said CSV export is unnecessary.", + metadata:{}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const validContradictoryEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Customer wants export", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback_2", + title: "Customer does not need export", + excerpt: "Customer said CSV export is unnecessary.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const documentChangeEvidence = { + ...validEvidence, + items:[ + { + sourceType:"document_change", + sourceId:"docs_1", + title:"Updated README", + excerpt:"README updated with export instructions.", + metadata:{}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const unavailableSourceEvidence = { + ...validEvidence, + items:[ + { + sourceType:"customer_feedback", + sourceId:"feedback_missing", + title:"Unavailable feedback", + excerpt:"Source unavailable.", + metadata:{}, + }, + ], + sourceWarnings:[ + { + code:"source_unavailable", + message:"Source unavailable", + sourceType:"customer_feedback", + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const validReport = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "A customer requested CSV export.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + + whatChanged: { + state: "no_evidence", + noEvidence: { + code: "no_change_reported", + message: "No changes were reported for this period.", + cta: "Check with the team if there were any internal changes.", + }, + }, + + whatShipped: { + state: "no_evidence", + noEvidence: { + code: "nothing_shipped", + message: "No shipped work was reported this period.", + cta: "Confirm release notes or deployment logs.", + }, + }, + + currentBlockers: { + state: "no_evidence", + noEvidence: { + code: "no_blockers", + message: "No current blockers were reported.", + cta: "Reach out if any impediments arise.", + }, + }, + + nextPriorities: { + state: "no_evidence", + noEvidence: { + code: "no_priorities", + message: "No next priorities were specified.", + cta: "Discuss upcoming focuses with the team.", + }, + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const invalidCitationReport = ({ + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "A customer requested CSV export.", + sourceIds: ["fake_feedback_999"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload); + +const emptyReport = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatCustomersSaid: noEvidenceSection, + whatChanged: noEvidenceSection, + whatShipped: noEvidenceSection, + currentBlockers: noEvidenceSection, + nextPriorities: noEvidenceSection, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const missingCitationReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "A customer requested CSV export.", + sourceIds: ["missing_source"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const founderContextCustomerReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customers requested export.", + sourceIds: ["context_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const founderContextOnlyReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + + nextPriorities: { + state: "evidence", + items: [ + { + kind: "recommendation", + text: "Improve onboarding.", + sourceIds: ["context_1"], + confidence: 0.9, + label: "Recommendation", + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const multiSourceReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Multiple customers requested CSV export.", + sourceIds: [ + "feedback_1", + "feedback_2", + ], + confidence: 0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const completeReport = { + ...validReport, + sections: { + ...validReport.sections, + + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customer requested CSV export.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export was shipped.", + sourceIds: ["github_1"], + confidence: 0.9, + }, + ], + }, + + nextPriorities: { + state: "evidence", + items: [ + { + kind: "recommendation", + text: "Improve onboarding.", + sourceIds: ["context_1"], + confidence: 0.9, + label: "Recommendation", + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + + +const duplicateClaimsReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customer requested CSV export.", + sourceIds:["feedback_1"], + confidence:0.8, + } + ] + }, + + whatChanged:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customer requested CSV export.", + sourceIds:["feedback_1"], + confidence:0.8, + } + ] + } + } +} satisfies FounderWeeklyReviewV2Payload; + +const duplicateSourcesReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Multiple customers requested CSV export.", + sourceIds: [ + "feedback_1", + "feedback_1", + ], + confidence: 0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const unsupportedShippedReport = { + ...validReport, + sections: { + ...validReport.sections, + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export was shipped.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const malformedReport = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "invalid_kind" as any, + text: "Invalid item kind.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + whatChanged: noEvidenceSection, + whatShipped: noEvidenceSection, + currentBlockers: noEvidenceSection, + nextPriorities: noEvidenceSection, + }, +} as unknown as FounderWeeklyReviewV2Payload; + +const omittedEvidenceReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customer requested CSV export.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const exaggeratedClaimReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customers cannot use the product without CSV export.", + sourceIds:["feedback_1"], + confidence:0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const conflictingEvidenceReport = { + ...validReport, + sections:{ + ...validReport.sections, + whatCustomersSaid:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"All customers want CSV export.", + sourceIds:[ + "feedback_1", + "feedback_2", + ], + confidence:0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const validContradictoryEvidenceReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "contradictory_evidence", + text: "Customers have mixed opinions about CSV export. One customer requested it while another said it was unnecessary.", + sourceIds: [ + "feedback_1", + "feedback_2", + ], + confidence: 0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const documentChangeShippedReport = { + ...emptyReport, + sections:{ + ...emptyReport.sections, + whatShipped:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"CSV export was shipped.", + sourceIds:["docs_1"], + confidence:0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const unavailableSourceSafeReport = { + ...emptyReport, +} satisfies FounderWeeklyReviewV2Payload; + +export const benchmarkCases: BenchmarkCase[] = [ + { + id: "valid_customer_feedback_report", + runThroughGeneration: true, + description: "A report with valid customer feedback citation", + evidenceSnapshot: validEvidence, + generatedReport: validReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "invalid_citation_report", + runThroughGeneration: false, + description: "A report citing evidence that does not exist", + evidenceSnapshot: validEvidence, + generatedReport: invalidCitationReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_citation", + ], + }, + }, + + { + id: "empty_workspace", + runThroughGeneration: true, + description: "No evidence produces an all no_evidence report.", + evidenceSnapshot: emptyEvidence, + generatedReport: emptyReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "complete_workspace", + runThroughGeneration: true, + description: "Workspace has complete evidence coverage across sections.", + evidenceSnapshot: completeEvidence, + generatedReport: completeReport, + expectations: { + shouldPass: true, + } + }, + + { + id: "partial_workspace", + runThroughGeneration: true, + description: "Workspace has some evidence but not every section.", + evidenceSnapshot: validEvidence, + generatedReport: validReport, + expectations:{ + shouldPass:true, + }, + }, + + { + id: "invalid_missing_source_citation", + runThroughGeneration: false, + description: "Observed fact references a missing evidence source.", + evidenceSnapshot: validEvidence, + generatedReport: missingCitationReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_citation", + ], + }, + }, + + { + id: "founder_context_as_customer_feedback", + runThroughGeneration: false, + description: "Founder context is cited as customer feedback.", + evidenceSnapshot: founderContextEvidence, + generatedReport: founderContextCustomerReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_source_type", + ], + }, + }, + + { + id: "founder_context_only_valid", + runThroughGeneration: true, + description: "Founder context can support recommendations when used in the correct section.", + evidenceSnapshot: founderContextOnlyEvidence, + generatedReport: founderContextOnlyReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "multi_source_customer_feedback", + runThroughGeneration: true, + description: "Customer feedback cites multiple sources.", + evidenceSnapshot: multipleCustomerEvidence, + generatedReport: multiSourceReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "duplicate_customer_claims", + runThroughGeneration: false, + description: "Report contains duplicate claims across sections.", + evidenceSnapshot: validEvidence, + generatedReport: duplicateClaimsReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "duplicate_claim" + ] + } + }, + + { + id: "duplicate_source_ids_schema_validation", + runThroughGeneration: false, + description: "Schema rejects reports with duplicate source IDs.", + evidenceSnapshot: validEvidence, + generatedReport: duplicateSourcesReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "malformed_payload" + ] + } + }, + + { + id: "unsupported_shipped_claim", + runThroughGeneration: false, + description: "Customer feedback cannot prove shipped work.", + evidenceSnapshot: validEvidence, + generatedReport: unsupportedShippedReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "unsupported_shipped_claim", + ], + }, + }, + + { + id: "malformed_payload", + runThroughGeneration: false, + description: "Report does not satisfy schema.", + evidenceSnapshot: validEvidence, + generatedReport: malformedReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "malformed_payload", + ], + }, + }, + + { + id: "evidence_omitted", + runThroughGeneration: false, + description: "Report ignores a major evidence theme.", + evidenceSnapshot: multipleThemeEvidence, + generatedReport: omittedEvidenceReport, + expectations: { + shouldPass: true, + }, + }, + + { + id:"exaggerated_claim", + runThroughGeneration: false, + description:"Report makes a stronger claim than evidence supports.", + evidenceSnapshot:weakEvidence, + generatedReport:exaggeratedClaimReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "unsupported_claim", + ], + }, + }, + + { + id:"conflicting_evidence", + runThroughGeneration: false, + description:"Report ignores conflicting evidence.", + evidenceSnapshot:conflictingEvidence, + generatedReport:conflictingEvidenceReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "conflicting_evidence", + ], + }, + }, + + { + id:"document_change_as_shipped", + runThroughGeneration: false, + description:"Documentation updates cannot prove shipped features.", + evidenceSnapshot:documentChangeEvidence, + generatedReport:documentChangeShippedReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "unsupported_shipped_claim", + ], + }, + }, + + { + id:"unavailable_source_warning", + runThroughGeneration: true, + description:"Unavailable evidence should not force hallucination.", + evidenceSnapshot:unavailableSourceEvidence, + generatedReport:unavailableSourceSafeReport, + expectations:{ + shouldPass:true, + }, + }, + + { + id: "valid_contradictory_evidence", + runThroughGeneration: true, + description: "Report correctly represents conflicting evidence.", + evidenceSnapshot: validContradictoryEvidence, + generatedReport: validContradictoryEvidenceReport, + expectations: { + shouldPass: true, + }, + }, +]; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/runner.ts b/packages/features/src/founder-weekly-review/benchmarks/runner.ts new file mode 100644 index 000000000..d3a7ae2a4 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/runner.ts @@ -0,0 +1,232 @@ +import { + evaluateFounderWeeklyReview, + type EvaluationFailure, +} from "../evaluation"; +import { benchmarkCases } from "./cases"; +import { + FounderWeeklyReviewV2PayloadSchema, + type FounderWeeklyReviewV2Payload, +} from "../contracts"; +import { generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { writeFileSync } from "fs"; + +let passedCount = 0; +let failedCount = 0; + +let hardFailureCount = 0; + +type BenchmarkMetrics = { + citationValidity: number; + citationCoverage: number; + unsupportedClaimRate: number; + unsupportedShippedClaimRate: number; + sourceTypeViolationRate: number; + evidenceCoverage: number; + duplicateClaimRate: number; + emptySectionCorrectness: number; +}; + +type BenchmarkResult = { + case: string; + passed: boolean; + score: number; + hasHardFailure: boolean; + metrics?: BenchmarkMetrics; + failures: EvaluationFailure[]; +}; + +const results: BenchmarkResult[] = []; + +for (const testCase of benchmarkCases) { + + if (!testCase.generatedReport) { + continue; + } + + let generatedReport: FounderWeeklyReviewV2Payload; + + if(testCase.runThroughGeneration) { + const generated = await generateFounderWeeklyReview({ + evidenceSnapshot: testCase.evidenceSnapshot, + generate: async ({schema}) => ({ + object: schema.parse(testCase.generatedReport), + metadata: { + provider: "benchmark", + model: "fixture", + capability: "founderWeeklyReview", + temperature: 0, + }, + }), + }); + + generatedReport = generated.reviewPayload; + + } else { + generatedReport = testCase.generatedReport; + } + + const schemaResult = FounderWeeklyReviewV2PayloadSchema.safeParse( + generatedReport + ); + + if (!schemaResult.success) { + + const passed = + testCase.expectations.expectedFailureCategories?.includes( + "malformed_payload" + ) ?? false; + + console.log( + `${passed ? "✅" : "❌"} ${testCase.id} (schema validation failure)` + ); + + if (!passed) { + console.log(JSON.stringify({ + case: testCase.id, + expected: testCase.expectations.expectedFailureCategories, + actual: "malformed_payload", + zodErrors: schemaResult.error.issues, + }, null, 2)); + } + + results.push({ + case: testCase.id, + passed, + score: 0, + hasHardFailure: true, + failures: [ + { + category: "malformed_payload", + explanation: "Report failed schema validation.", + }, + ], + }); + + // Malformed payload cases are expected failures, but still count as hard failures. + hardFailureCount++; + + if (passed) { + passedCount++; + } else { + failedCount++; + } + + continue; + } + + const result = evaluateFounderWeeklyReview( + testCase.evidenceSnapshot, + generatedReport + ); + + const actualFailures = result.failures.map(f => f.category); + + const passed = + testCase.expectations.shouldPass + ? actualFailures.length === 0 + : testCase.expectations.expectedFailureCategories?.every( + category => actualFailures.includes(category) + ) ?? false; + + if (result.hasHardFailure) { + hardFailureCount++; + } + + if (passed) { + passedCount++; + } else { + failedCount++; + } + + results.push({ + case: testCase.id, + passed, + score: result.overallScore, + hasHardFailure: result.hasHardFailure, + metrics: result.deterministic, + failures: result.failures, + }); + + console.log( + `${passed ? "✅" : "❌"} ${testCase.id}` + ); + + if (!passed) { + console.log(JSON.stringify({ + case:testCase.id, + passed: passed, + score:result.overallScore, + failures:result.failures + },null,2)); + } +} + +const averageScore = + results.length === 0 + ? 0 + : results.reduce( + (sum, result) => sum + result.score, + 0 + ) / results.length; + +const averageMetric = (key: keyof BenchmarkMetrics) => { + const values = results + .map(result => result.metrics?.[key]) + .filter((value): value is number => value !== undefined); + + return values.length === 0 + ? 0 + : values.reduce((sum, value) => sum + value, 0) / values.length; +}; + +const weakestCases = [...results] + .sort((a,b) => a.score - b.score) + .slice(0,3) + .map(result => ({ + case: result.case, + score: result.score, + failures: result.failures.map(f => f.category), + })); + +const failureCounts = results + .flatMap(result => result.failures) + .reduce>((acc, failure) => { + acc[failure.category] = + (acc[failure.category] ?? 0) + 1; + + return acc; + }, {}); + +const benchmarkOutput = { + summary: { + totalCases: benchmarkCases.length, + passed: passedCount, + failed: failedCount, + hardFailures: hardFailureCount, + passRate: benchmarkCases.length === 0 + ? 0 + : passedCount / benchmarkCases.length, + overallScore: averageScore, + }, + metrics: { + citationValidity: averageMetric("citationValidity"), + citationCoverage: averageMetric("citationCoverage"), + unsupportedClaimRate: averageMetric("unsupportedClaimRate"), + unsupportedShippedClaimRate: averageMetric("unsupportedShippedClaimRate"), + sourceTypeViolationRate: averageMetric("sourceTypeViolationRate"), + evidenceCoverage: averageMetric("evidenceCoverage"), + emptySectionCorrectness: averageMetric("emptySectionCorrectness"), + duplicateClaimRate: averageMetric("duplicateClaimRate"), + }, + weakestCases, + commonFailures: failureCounts, + cases: results, +}; + +console.log("\n===== Benchmark Summary ====="); + +console.log(JSON.stringify(benchmarkOutput, null, 2)); + +writeFileSync("packages/features/src/founder-weekly-review/benchmarks/baseline-output.json", JSON.stringify(benchmarkOutput, null, 2)); + +process.exitCode = failedCount > 0 ? 1 : 0; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/evaluation.ts b/packages/features/src/founder-weekly-review/evaluation.ts new file mode 100644 index 000000000..bfbabfc37 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evaluation.ts @@ -0,0 +1,521 @@ +import{ + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewPayload, + FounderWeeklyReviewV2PayloadSchema, +} from "./contracts"; + +export interface EvaluationResult { + passed: boolean; + + hasHardFailure: boolean; + + deterministic: { + canonicalSchemaValid: boolean; + citationValidity: number; + citationCoverage: number; + unsupportedClaimRate: number; + unsupportedShippedClaimRate: number; + sourceTypeViolationRate: number; + evidenceCoverage: number; + duplicateClaimRate: number; + emptySectionCorrectness: number; + }; + + overallScore: number; + + failures: EvaluationFailure[]; +} + +export type EvaluationFailure = { + category: string; + section?: string; + claim?: string; + explanation: string; +}; + +const SECTION_SOURCE_RULES: Record = { + whatCustomersSaid: ["customer_feedback"], + whatChanged: [ + "document_change", + "workspace_document", + "github_activity", + ], + whatShipped: [ + "github_activity", + "document_change", + ], + currentBlockers: [ + "founder_context", + "manual_note", + ], + nextPriorities: [ + "founder_context", + "manual_note", + ], +}; + +type EvidenceSourceType = + FounderWeeklyReviewEvidenceSnapshot["items"][number]["sourceType"]; + +function isValidSourceForSection( + sectionName: string, + sourceType: EvidenceSourceType +) { + const allowed = SECTION_SOURCE_RULES[sectionName]; + + if (!allowed) return true; + + return allowed.includes(sourceType); +} + +function getSectionItems(section: unknown): unknown[] { + if (typeof section !== "object" || section === null) { + return []; + } + + if ( + "items" in section && + Array.isArray(section.items) + ) { + return section.items; + } + + return []; +} + +function getItemKind(item: unknown): string | undefined { + if ( + typeof item === "object" && + item !== null && + "kind" in item && + typeof item.kind === "string" + ) { + return item.kind; + } + + return undefined; +} + +function normalizeClaim(text:string) { + return text + .toLowerCase() + .replace(/[.,!?]/g, "") + .replace(/\s+/g," ") + .trim(); +} + +function claimSupportedByEvidence( + claim: string, + evidenceText: string +) { + const claimWords = new Set( + normalizeClaim(claim) + .split(" ") + .filter(word => word.length > 3) + ); + + const evidenceWords = new Set( + normalizeClaim(evidenceText) + .split(" ") + .filter(word => word.length > 3) + ); + + const overlap = [...claimWords].filter(word => + evidenceWords.has(word) + ); + + return overlap.length >= 2; +} + +function evaluateEmptySections( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + report: FounderWeeklyReviewPayload, + failures: EvaluationFailure[] +): number { + if (evidenceSnapshot.items.length !== 0) return 1; + + let incorrect = 0; + + for (const [sectionName, section] of Object.entries(report.sections)) { + if (section.state !== "no_evidence") { + incorrect++; + + failures.push({ + category: "invalid_empty_section", + section: sectionName, + explanation: + "Empty evidence snapshot must produce no_evidence sections.", + }); + } + } + + return incorrect === 0 ? 1 : 0; +} + +function evidenceIndicatesShipped( + evidence: FounderWeeklyReviewEvidenceSnapshot["items"][number] +) { + const text = normalizeClaim( + `${evidence.title} ${evidence.excerpt}` + ); + + return [ + "released", + "deployed", + "launched", + "shipped", + "merged", + "implemented", + "completed" + ].some(keyword => text.includes(keyword)); +} + +function evidenceConflicts( + evidenceItems: FounderWeeklyReviewEvidenceSnapshot["items"][number][] +) { + const texts = evidenceItems.map(e => + normalizeClaim(`${e.title} ${e.excerpt}`) + ); + + const supporting = texts.some(text => + [ + "want", + "requested", + "needs", + "need", + ].some(keyword => text.includes(keyword)) + ); + + const opposing = texts.some(text => + [ + "unnecessary", + "does not need", + "doesn't need", + "don't need", + "not needed", + ].some(keyword => text.includes(keyword)) + ); + + return supporting && opposing; +} + +export function evaluateFounderWeeklyReview( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + report: FounderWeeklyReviewPayload, +): EvaluationResult { + const failures: EvaluationFailure[] = []; + + const evidenceById = new Map( + evidenceSnapshot.items.map((item) => [ + item.sourceId, + item, + ]) + ); + + const claims = new Set(); + + let totalCitations = 0; + let validCitations = 0; + + let totalClaimsWithCitationIds = 0; + let totalClaimsRequiringCitation = 0; + + let sourceTypeViolations = 0; + let totalSourceTypeChecks = 0; + + let emptySectionCorrectness = 1; + + let unsupportedClaims = 0; + let unsupportedClaimChecks = 0; + + let unsupportedShippedClaims = 0; + let shippedClaimChecks = 0; + + let canonicalSchemaValid = true; + + try { + FounderWeeklyReviewV2PayloadSchema.parse(report); + } catch { + canonicalSchemaValid = false; + + failures.push({ + category:"malformed_payload", + explanation: + "Report does not satisfy Founder Weekly Review V2 schema.", + }); + } + + if (!canonicalSchemaValid) { + return { + passed: false, + hasHardFailure: true, + deterministic: { + canonicalSchemaValid, + citationValidity: 0, + citationCoverage: 0, + unsupportedShippedClaimRate: 0, + unsupportedClaimRate: 0, + sourceTypeViolationRate: 1, + evidenceCoverage: 0, + duplicateClaimRate: 0, + emptySectionCorrectness, + }, + overallScore: 0, + failures, + }; + } + + emptySectionCorrectness = evaluateEmptySections( + evidenceSnapshot, + report, + failures + ); + + const reportClaims = new Set(); + + for (const [sectionName, section] of Object.entries(report.sections)) { + for (const item of getSectionItems(section)) { + const text = typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" + ? normalizeClaim(item.text) + : null; + + if (text) { + reportClaims.add(text); + if (claims.has(text)) { + failures.push({ + category: "duplicate_claim", + section: sectionName, + claim: text, + explanation: "Same factual claim appears multiple times.", + }); + } + claims.add(text); + } + + if ( + typeof item === "object" && + item !== null && + "kind" in item + ) { + const kind = getItemKind(item); + if ( + kind === "observed_fact" || + kind === "contradictory_evidence" || + kind === "recommendation" + ) { + totalClaimsRequiringCitation++; + if ( + "sourceIds" in item && + Array.isArray(item.sourceIds) && + item.sourceIds.length > 0 + ) { + totalClaimsWithCitationIds++; + } + } + } + + if ( + typeof item === "object" && + item !== null && + "sourceIds" in item && + Array.isArray(item.sourceIds) + ) { + + for (const sourceId of item.sourceIds) { + + const evidence = evidenceById.get(sourceId); + + if (evidence) { + totalSourceTypeChecks++; + + if(!isValidSourceForSection(sectionName, evidence.sourceType)) { + sourceTypeViolations++; + + failures.push({ + category: "invalid_source_type", + section: sectionName, + explanation: `Source type "${evidence.sourceType}" is not valid for section "${sectionName}"`, + }); + } + + if ( + sectionName === "whatShipped" && + typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" + ) { + shippedClaimChecks++; + + if (!evidenceIndicatesShipped(evidence)) { + unsupportedShippedClaims++; + + failures.push({ + category: "unsupported_shipped_claim", + section: sectionName, + claim: item.text, + explanation: + "Evidence does not contain a shipping signal.", + }); + } + } + } + + totalCitations++; + if (evidenceById.has(sourceId)) { + validCitations++; + } else { + failures.push({ + category: "invalid_citation", + section: sectionName, + explanation: `Unknown sourceId: ${sourceId}`, + }); + } + } + if ( + typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" + ) { + + const kind = getItemKind(item); + + if (kind !== "recommendation" && + kind !== "contradictory_evidence" && + sectionName !== "whatShipped") { + unsupportedClaimChecks++; + + const citedEvidence = item.sourceIds + .map((id:string) => evidenceById.get(id)) + .filter( + (e): e is FounderWeeklyReviewEvidenceSnapshot["items"][number] => + Boolean(e) + ); + + const combinedEvidenceText = citedEvidence + .map(e => `${e.title} ${e.excerpt}`) + .join(" "); + + if ( + citedEvidence.length > 1 && + evidenceConflicts(citedEvidence) + ) { + failures.push({ + category: "conflicting_evidence", + section: sectionName, + claim: item.text, + explanation: + "Claim ignores conflicting cited evidence.", + }); + } else if ( + citedEvidence.length > 0 && + !claimSupportedByEvidence( + item.text, + combinedEvidenceText + ) + ) { + unsupportedClaims++; + + failures.push({ + category: "unsupported_claim", + section: sectionName, + claim: item.text, + explanation: + "Claim is not directly supported by cited evidence.", + }); + } + } + } + } + } + } + + const citationValidity = + totalCitations === 0 + ? 1 + : validCitations / totalCitations; + + const citationCoverage = + totalClaimsRequiringCitation === 0 + ? 1 + : totalClaimsWithCitationIds / totalClaimsRequiringCitation; + + const sourceTypeViolationRate = + totalSourceTypeChecks === 0 + ? 0 + : sourceTypeViolations / totalSourceTypeChecks; + + const hasHardFailure = failures.some((f) => + [ // duplicate_claim not included: more of a quality issue + "malformed_payload", + "invalid_citation", + "invalid_source_type", + "unsupported_shipped_claim", + "invalid_empty_section", + "conflicting_evidence" + ].includes(f.category) + ); + + const coveredEvidence = + evidenceSnapshot.items.filter((evidence) => { + const evidenceText = normalizeClaim( + `${evidence.title} ${evidence.excerpt}`); + + return [...reportClaims].some((claim) => + claimSupportedByEvidence(claim, evidenceText) + ); + }).length; + + const evidenceCoverage = + evidenceSnapshot.items.length === 0 + ? 1 + : coveredEvidence / evidenceSnapshot.items.length; + + const overallScore = hasHardFailure + ? 0 + : citationValidity * 0.30 + + citationCoverage * 0.20 + + evidenceCoverage * 0.30 + + (1 - sourceTypeViolationRate) * 0.20; + + return { + passed: failures.length === 0, + + hasHardFailure, + + deterministic: { + canonicalSchemaValid, + citationValidity, + citationCoverage, + unsupportedClaimRate: + unsupportedClaimChecks === 0 + ? 0 + : unsupportedClaims / unsupportedClaimChecks, + + unsupportedShippedClaimRate: + shippedClaimChecks === 0 + ? 0 + : unsupportedShippedClaims / shippedClaimChecks, + sourceTypeViolationRate, + evidenceCoverage, + duplicateClaimRate: + claims.size === 0 + ? 0 + : Math.min( + 1, + failures.filter( + (f) => f.category === "duplicate_claim" + ).length / claims.size + ), + emptySectionCorrectness, + }, + + overallScore, + + failures, + }; +} \ No newline at end of file From 714e20a3c7df68bb6027512ccaebdadfd9614a9f Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 1 Aug 2026 19:13:57 +0800 Subject: [PATCH 15/29] Complete founder weekly review generation flow --- HANDOFF.md | 260 ++++++++ .../async-collection.test.ts | 49 ++ .../founderWeeklyReview/generation.test.ts | 49 +- .../kimi-chat-completions.test.ts | 123 ++++ .../founderWeeklyReview/markdown.test.ts | 55 ++ .../founderWeeklyReview/migration.test.ts | 2 + .../founderWeeklyReview/routes.test.ts | 9 +- .../__tests__/founderWeeklyReview/testDb.ts | 46 ++ ...founder_weekly_review_async_collection.sql | 22 + ...founder-weekly-review-inngest-transport.ts | 588 ++++++++++++++++++ ...run-founder-weekly-review-realistic-e2e.ts | 49 ++ ...ounder-weekly-review-synthetic-baseline.ts | 21 +- .../app/api/founder-weekly-reviews/route.ts | 20 +- apps/web/src/lib/llm/generate.ts | 45 +- apps/web/src/lib/llm/providers.ts | 71 ++- apps/web/src/lib/llm/types.ts | 6 +- .../founder-weekly-review/dispatch-service.ts | 3 +- .../generation-adapter.ts | 1 + .../server/founder-weekly-review/markdown.ts | 85 +++ .../inngest/functions/founderWeeklyReview.ts | 53 +- .../realistic-company/seed.json | 10 + .../src/db/schema/founder-weekly-review.ts | 11 +- .../src/founder-weekly-review/contracts.ts | 29 +- .../generation-validation.ts | 20 +- .../src/founder-weekly-review/generator.ts | 73 ++- .../src/founder-weekly-review/prompts.ts | 10 +- .../src/founder-weekly-review/repository.ts | 83 ++- .../src/founder-weekly-review/user-service.ts | 27 +- .../founder-weekly-review/worker-service.ts | 32 + 29 files changed, 1773 insertions(+), 79 deletions(-) create mode 100644 HANDOFF.md create mode 100644 apps/web/__tests__/founderWeeklyReview/async-collection.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/markdown.test.ts create mode 100644 apps/web/drizzle/0018_founder_weekly_review_async_collection.sql create mode 100644 apps/web/scripts/run-founder-weekly-review-inngest-transport.ts create mode 100644 apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts create mode 100644 apps/web/src/server/founder-weekly-review/markdown.ts create mode 100644 apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 000000000..5e1066bea --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,260 @@ +# Founder Weekly Review (LAU-9) Handoff + +## Current state + +- Branch: `lau-9-founder-review-generation-flow` +- Do not commit or push without explicit approval. +- Preserve all existing uncommitted work and the four untracked local Ollama probe files. +- Do not use shared Neon for this work. Use the local PostgreSQL URL only through the isolated-schema helper. +- Do not change embeddings, retrieval, frontend/LAU-10, PDF ingestion, Peace's evaluator, or provider defaults unless a later task explicitly authorizes it. + +## What has been completed locally + +### Async Inngest-owned workflow + +The prior synchronous POST collection flow has been refactored locally toward: + +```text +POST -> queued run without snapshot + transactional outbox +Inngest workflow -> collecting -> attach immutable snapshot -> queued + -> generating -> draft | failed +``` + +Key implementation files: + +- `apps/web/src/app/api/founder-weekly-reviews/route.ts` + - POST authorizes and validates, then persists durable collection input instead of collecting evidence synchronously. +- `packages/features/src/founder-weekly-review/contracts.ts` + - Adds `collecting` status and bounded `FounderWeeklyReviewCollectionInputSchema`: + - `workspaceTimezone` + - optional bounded `founderContext` + - `actorExternalUserId` +- `packages/core/src/db/schema/founder-weekly-review.ts` + - Allows a null `evidenceSnapshot` before collection; adds collection claim/timestamps/input fields. +- `apps/web/drizzle/0018_founder_weekly_review_async_collection.sql` + - Forward migration for nullable snapshot and durable collection fields. + - Backfills existing rows with bounded collection input. It must not be applied to shared Neon without separate approval. +- `packages/features/src/founder-weekly-review/repository.ts` + - Adds conditional collection claim, insert-once snapshot attachment, and collection failure methods. +- `packages/features/src/founder-weekly-review/worker-service.ts` + - Adds collection claim/attach/failure service methods. +- `apps/web/src/server/inngest/functions/founderWeeklyReview.ts` + - Uses the canonical evidence collector when no snapshot exists, then runs existing generation/validation. +- `apps/web/src/server/founder-weekly-review/dispatch-service.ts` + - Initial and retry outbox contracts accept optional snapshot plus collection input; event remains identifier-only. + +Important invariants implemented: + +- A generation claim requires an existing immutable snapshot. +- A collection claim only succeeds for `queued` runs whose snapshot is absent. +- Snapshot attachment requires the current collection claim and only writes when the snapshot remains absent. +- Failed retry remains the same run row and can recollect only if it still has no snapshot; otherwise it reuses the snapshot. +- Events do not contain evidence, founder context, prompts, credentials, or report text. + +### Local synthetic safety baselines + +Existing runner: + +- `apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts` + +It supports local-only synthetic fixtures, deterministic negative citation/source-semantic cases, retry snapshot immutability checks, bounded Kimi/Ollama diagnostics, optional export, and `FWR_PRINT_REPORT=1` terminal printing of a persisted/re-read report. + +Known validated local outcomes from earlier work: + +- Kimi partial and full synthetic cases reached persisted `draft`. +- Unknown citation IDs become sanitized `generation_failed` with no draft. +- Founder context cannot support customer testimony. +- Retry reused one row; retry count behaved `0 -> 1 -> 1`; snapshot digest was unchanged. + +### Realistic isolated-database E2E baseline + +New local-only assets: + +- `apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json` +- `apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts` +- `apps/web/__tests__/founderWeeklyReview/async-collection.test.ts` + +The fixture is fictional and seeds real collector source tables: + +- product release document/version; +- internal planning document/version; +- exact-category `Customer Feedback` document/version with processed context chunks; +- request-time founder context through durable collection input; +- an out-of-period control and another-company control. + +The driver uses the isolated schema helper, creates a queued run via the dispatch service, claims collection, invokes `FounderWeeklyReviewEvidenceService`, attaches the actual collector output, claims generation, invokes Kimi through the local runner adapter, validates via `generateFounderWeeklyReview`, persists/re-reads the draft, optionally prints and exports it, and then drops the isolated schema. + +Last successful realistic run: + +- Run ID: `fwr_0029d66b-99cd-4ebf-9f7a-3da904381be6` +- Lifecycle: `queued -> collecting -> queued -> generating -> draft` +- Provider/model: `kimi` / `kimi-k2.6` +- Evidence counts: `document_change=3`, `customer_feedback=3`, `founder_context=1` +- Warning codes: none +- Snapshot digest unchanged after attachment and repository read-back. +- Canonical schema, citation validation, and source-semantic validation passed. +- One run row and one initial outbox row were present. + +Export artifacts from that run are local and Git-ignored: + +- `apps/web/.artifacts/founder-weekly-review/fwr_0029d66b-99cd-4ebf-9f7a-3da904381be6.md` +- `apps/web/.artifacts/founder-weekly-review/fwr_0029d66b-99cd-4ebf-9f7a-3da904381be6.json` + +The Markdown did not contain the run ID or `evidenceSnapshot`. The JSON envelope did not contain the snapshot, prompts, credentials, raw provider output, or database URL. + +## Current quality-improvement task (interrupted before edits) + +The active next task is **Phase 1 report-quality improvement** within the existing Founder Weekly Review v2 payload schema. Do not add a V3 schema or change persisted payload shape. + +Requested improvements: + +1. Improve the generation prompt in `packages/features/src/founder-weekly-review/prompts.ts` so output synthesizes related evidence, explains why it matters, states limitations/evidence gaps, separates shipped work from preparation, treats customer feedback as customer-only evidence, and creates distinct grounded priorities. +2. Inspect and increase the founder-review output token budget only if an actual budget exists at the provider boundary. At the last inspection, `apps/web/src/lib/llm/generate.ts` delegates to AI SDK `generateObject` without an explicit founder-review max-output setting; the Kimi local adapters also do not currently send a max-token parameter. Do not invent a production-wide provider refactor. +3. Improve Markdown rendering in both runners (or extract a safe shared renderer) so it: + - has no empty `Key Outcomes` heading; + - omits or uses useful text for no-evidence sections; + - keeps citations next to claims; + - uses distinct evidence-reference labels for chunks from the same feedback document; + - uses title plus bounded existing metadata (page/section) rather than category-only labels; + - clearly labels founder context as founder-provided; + - does not show internal IDs, snapshots, prompts, provider responses, credentials, or database URLs. +4. Update only the fictional feedback **title** in the realistic fixture to something descriptive such as `Customer Interviews - February 2026`; retain its exact category `Customer Feedback` and retain the actual fixture facts. +5. Add deterministic renderer tests; do not call a provider in those tests. +6. Rerun the local realistic Kimi E2E with `FWR_PRINT_REPORT=1` and export enabled. + +Current prompt location and behavior: + +- `packages/features/src/founder-weekly-review/prompts.ts` + - system prompt currently emphasizes non-invention, citations, customer-source semantics, and no-evidence behavior but does not sufficiently ask for multi-evidence synthesis or natural founder-facing prose. +- `packages/features/src/founder-weekly-review/generator.ts` + - parses the current V2 schema, validates citations and semantics, and builds metadata. Do not weaken this path. +- `packages/features/src/founder-weekly-review/generation-validation.ts` + - contains the citation and source-semantic enforcement; preserve it unchanged. + +Current Markdown renderers are duplicated: + +- `markdownFor()` in `apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts` +- `markdown()` in `apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts` + +Both currently use simple title-only references and render a `## Key Outcomes` container even when only child headings carry content. A shared local renderer helper is preferable if extraction stays narrow and preserves the persisted-draft-only boundary. + +## Local database and test setup + +Use only: + +```powershell +$env:DATABASE_URL='postgresql://postgres:password@127.0.0.1:5433/pdr_ai_v2' +$env:LAUNCHSTACK_TEST_DATABASE_URL=$env:DATABASE_URL +``` + +The isolated helper is: + +- `apps/web/__tests__/founderWeeklyReview/testDb.ts` + +It creates a temporary schema, applies migration files, sets a schema-first search path, and drops the schema during cleanup. It was expanded to include local `document_versions` and `document_context_chunks` columns required by current Drizzle inserts; this prevents accidental fallback reads/writes to `public` during realistic seeding. + +Focused verification most recently passed: + +```powershell +pnpm.cmd --filter @launchstack/core typecheck +pnpm.cmd --filter @launchstack/features typecheck +pnpm.cmd --filter @launchstack/web typecheck + +Set-Location apps/web +pnpm.cmd exec jest --runInBand --detectOpenHandles founderWeeklyReview +``` + +Result: 13 suites passed, 74 tests passed, 0 skipped, no open-handle warning. + +`git diff --check` also passed. + +## Run commands + +### Deterministic async collection test + +```powershell +Set-Location apps/web +pnpm.cmd exec jest --runInBand --detectOpenHandles founderWeeklyReview/async-collection.test.ts +``` + +### Realistic isolated Kimi E2E + +`MOONSHOT_API_KEY` is loaded by the project `.env` mechanism. Never print it or `.env` content. + +```powershell +Set-Location apps/web +$env:SYNTHETIC_FWR_LOCAL='1' +$env:FWR_GENERATION_PROVIDER='kimi' +$env:KIMI_MODEL_ID='kimi-k2.6' +$env:MOONSHOT_BASE_URL='https://api.moonshot.ai/v1' +$env:FWR_PRINT_REPORT='1' +$env:SYNTHETIC_FWR_EXPORT_REPORT='1' +$env:SYNTHETIC_FWR_EXPORT_DIR='.artifacts/founder-weekly-review' +$env:DATABASE_URL='postgresql://postgres:password@127.0.0.1:5433/pdr_ai_v2' +$env:LAUNCHSTACK_TEST_DATABASE_URL=$env:DATABASE_URL +pnpm.cmd exec tsx ./scripts/run-founder-weekly-review-realistic-e2e.ts +``` + +The command performs one external Kimi generation request, but all database writes are inside a temporary local schema that the driver removes. + +### Founder Weekly Review generation provider + +Founder Weekly Review now selects its generation provider explicitly: + +```powershell +# Normal/default production behavior (also used when unset) +$env:FWR_GENERATION_PROVIDER='openai' +# Requires OPENAI_API_KEY; OPENAI_MODEL_ID is optional. + +# Local Moonshot/Kimi alternative +$env:FWR_GENERATION_PROVIDER='kimi' +$env:KIMI_MODEL_ID='kimi-k2.6' # optional; this is the default +# Requires MOONSHOT_API_KEY; MOONSHOT_BASE_URL is optional. +``` + +`FWR_GENERATION_PROVIDER` accepts only `openai` or `kimi`; an unset value is +`openai`. The Kimi path uses Moonshot Chat Completions with JSON-object output +and local canonical validation. It does not change the normal OpenAI protocol. +Kimi thinking is explicitly disabled for this path. Keep its current 1,800 +output-token budget for the next baseline; revisit approximately 2,500–3,000 +only if non-thinking real reports demonstrably approach the current limit. + +## Working-tree precautions + +Expected tracked modifications are the async workflow and test/E2E work listed by `git status -sb`. Expected untracked files include: + +- `apps/web/__tests__/founderWeeklyReview/async-collection.test.ts` +- `apps/web/drizzle/0018_founder_weekly_review_async_collection.sql` +- `apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts` +- `apps/web/test-fixtures/` +- four `ollama-*.json` local probe artifacts at repository root. + +The Ollama probe artifacts and `.artifacts/` exports must not be committed. No commit or push has been made for the current work. + +## Known limitations / next work + +### Latest local transport baseline + +The real isolated local transport reached a persisted draft for +`fwr_79958e72-d05c-4bc7-ad21-f6e76d490e2e` using `kimi` / `kimi-k2.6`: + +```text +queued without snapshot -> collecting -> queued with immutable snapshot +-> generating -> draft +``` + +Canonical schema, citation, and source-semantic validation passed, with +`retryCount=0`. Founder-context-as-customer-feedback remains forbidden. A +repairable post-generation validation failure now receives exactly one explicit +same-snapshot semantic-repair call; a second failure is non-retriable for +Inngest and fails normally. The local harness now has a bounded terminal +callback drain before scoped process-tree cleanup; rerun the local-only smoke +or transport baseline to confirm it with an explicitly local database URL. + +Next work is workflow comprehension and Peace evaluator integration. A broader +provider registry for Llama, Hugging Face, and Ollama remains a post-PR follow-up. + +- The realistic driver invokes production-equivalent collection/worker service boundaries directly, not a live Inngest dev-server callback. +- The flat V2 schema has only sections, items, text, source IDs, confidence, and typed no-evidence states. It cannot add an executive summary, overall status, owners, metrics, or decision records without a future schema version. +- The existing collector emits the feedback document version as `document_change` in addition to its processed feedback chunks. Customer-facing claims are still constrained to `customer_feedback` evidence. +- Peace evaluator integration, real PDF ingestion, shared-dev deployment, and frontend work remain out of scope. diff --git a/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts b/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts new file mode 100644 index 000000000..455c4c8ce --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts @@ -0,0 +1,49 @@ +jest.mock("~/server/db", () => ({ db: { transaction: jest.fn() } })); +import { company } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; +const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }); + +describeDb("Founder Weekly Review async evidence workflow", () => { + it("creates without a snapshot, attaches it once, and then permits generation claim", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [companyRow] = await test.db.insert(company).values({ name: "Workflow", numberOfEmployees: "1" }).returning(); + const actor = { externalUserId: "u", internalUserId: 1n, companyId: BigInt(companyRow!.id), role: "owner" }; + const created = await createFounderWeeklyReviewDispatchService(test.db).createRunWithDispatch({ actor, requestKey: "workflow", reportingPeriod: snapshot.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", founderContext: "bounded", actorExternalUserId: "u" } }); + expect(created.run.status).toBe("queued"); + expect(created.run.evidenceSnapshot).toBeNull(); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(test.db)); + const context = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: "collection-1" }; + const collecting = await worker.claimEvidenceCollection(context); + expect(collecting.status).toBe("collecting"); + const attached = await worker.attachEvidenceSnapshotIfAbsent(context, snapshot); + expect(attached.status).toBe("queued"); + expect(attached.evidenceSnapshot).toEqual(snapshot); + await expect(worker.attachEvidenceSnapshotIfAbsent(context, snapshot)).resolves.toMatchObject({ id: created.run.id }); + const generating = await worker.claimQueuedRun({ companyId: actor.companyId, runId: created.run.id, generationClaimId: "generation-1", generationJobId: "job-1" }); + expect(generating.status).toBe("generating"); + } finally { await test.close(); } + }); + + it("retries collection failures without creating a snapshot or another run", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [companyRow] = await test.db.insert(company).values({ name: "Recovery", numberOfEmployees: "1" }).returning(); + const actor = { externalUserId: "u", internalUserId: 1n, companyId: BigInt(companyRow!.id), role: "owner" }; + const service = createFounderWeeklyReviewDispatchService(test.db); + const created = await service.createRunWithDispatch({ actor, requestKey: "collection-failure", reportingPeriod: snapshot.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", actorExternalUserId: "u" } }); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(test.db)); + const collecting = await worker.claimEvidenceCollection({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: "collection-failure-claim" }); + expect(collecting.status).toBe("collecting"); + const failed = await worker.markCollectionFailed({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: "collection-failure-claim" }, { errorCode: "evidence_collection_failed" }); + expect(failed.status).toBe("failed"); expect(failed.evidenceSnapshot).toBeNull(); + const retried = await service.retryRunWithDispatch({ actor, runId: created.run.id, requestKey: "collection-retry" }); + expect(retried.run.id).toBe(created.run.id); expect(retried.run.retryCount).toBe(1); expect(retried.run.evidenceSnapshot).toBeNull(); + await expect(worker.claimEvidenceCollection({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: retried.dispatch.generationClaimId })).resolves.toMatchObject({ status: "collecting" }); + } finally { await test.close(); } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts index f6ec1985d..550d8c78c 100644 --- a/apps/web/__tests__/founderWeeklyReview/generation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -65,10 +65,57 @@ const completeSnapshot = () => snapshot([ describe("Founder Weekly Review generation", () => { it("generates complete evidence with numeric lower and upper confidence bounds", async () => { - const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(validPayload()) }); + const generate = fake(validPayload()); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate }); expect(result.reviewPayload).toEqual(validPayload()); expect(result.modelMetadata).toMatchObject({ provider: "openai", temperature: 0, capability: "founderWeeklyReview" }); expect(result.modelMetadata.promptHash).toMatch(/^[a-f0-9]{64}$/); + expect(generate).toHaveBeenCalledTimes(1); + expect(generate.mock.calls[0][0]).toMatchObject({ generationPhase: "initial" }); + }); + + it("performs exactly one semantic repair against the same immutable snapshot", async () => { + const invalid = validPayload(); + invalid.sections.whatCustomersSaid = { + state: "evidence", + items: [{ kind: "observed_fact", text: "Founder direction was presented as customer feedback.", sourceIds: ["context-1"], confidence: 0.5 }], + }; + const repaired = validPayload(); + const generate = jest.fn() + .mockResolvedValueOnce({ object: invalid, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }) + .mockResolvedValueOnce({ object: repaired, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }); + const evidenceSnapshot = completeSnapshot(); + + await expect(generateFounderWeeklyReview({ evidenceSnapshot, generate })).resolves.toMatchObject({ reviewPayload: repaired }); + + expect(generate).toHaveBeenCalledTimes(2); + expect(generate.mock.calls[0][0]).toMatchObject({ generationPhase: "initial" }); + expect(generate.mock.calls[1][0]).toMatchObject({ generationPhase: "semantic-repair" }); + expect(generate.mock.calls[1][0].prompt).toContain("founder_context is founder-provided direction, not customer testimony."); + expect(generate.mock.calls[1][0].prompt).toContain("context-1"); + expect(evidenceSnapshot).toEqual(completeSnapshot()); + }); + + it("fails normally after one invalid semantic repair and never makes a third call", async () => { + const invalid = validPayload(); + invalid.sections.whatCustomersSaid = { + state: "evidence", + items: [{ kind: "observed_fact", text: "Invalid customer claim.", sourceIds: ["context-1"], confidence: 0.5 }], + }; + const generate = jest.fn() + .mockResolvedValueOnce({ object: invalid, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }) + .mockResolvedValueOnce({ object: invalid, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }); + + await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate })).rejects.toMatchObject({ + name: "FounderWeeklyReviewGenerationValidationError", + }); + expect(generate).toHaveBeenCalledTimes(2); + }); + + it("does not semantic-repair provider failures", async () => { + const generate = jest.fn().mockRejectedValue(new Error("provider unavailable")); + await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate })).rejects.toThrow("provider unavailable"); + expect(generate).toHaveBeenCalledTimes(1); }); it("allows partial reviews with typed no-evidence sections", async () => { diff --git a/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts new file mode 100644 index 000000000..dd0828aae --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts @@ -0,0 +1,123 @@ +import { z } from "zod"; + +import { __resetLlmConfigForTests } from "~/lib/llm/config"; +import { generateStructuredWithMetadata } from "~/lib/llm/generate"; +import { __resetProviderCacheForTests } from "~/lib/llm/providers"; +import { resolveModel } from "~/lib/llm/providers"; + +describe("Founder Weekly Review Kimi transport", () => { + const originalEnv = { ...process.env }; + const originalFetch = global.fetch; + + beforeEach(() => { + process.env = { + ...originalEnv, + FWR_GENERATION_PROVIDER: "kimi", + MOONSHOT_API_KEY: "test-key-not-logged", + MOONSHOT_BASE_URL: "https://api.moonshot.ai/v1", + KIMI_MODEL_ID: "kimi-k2.6", + }; + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_MODEL_ID; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + }); + + afterEach(() => { + process.env = originalEnv; + global.fetch = originalFetch; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + }); + + it("uses Moonshot Chat Completions JSON mode and locally validates the result", async () => { + const logSpy = jest.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = jest.spyOn(console, "error").mockImplementation(() => undefined); + const fetchMock = jest.fn(async (url: string | URL, init?: RequestInit) => { + expect(String(url)).toMatch(/\/chat\/completions$/); + expect(String(url)).not.toContain("/responses"); + const body = JSON.parse(String(init?.body)) as Record; + expect(body.model).toBe("kimi-k2.6"); + expect(body.max_tokens).toBe(1800); + expect(body.messages).toEqual(expect.any(Array)); + expect(body).not.toHaveProperty("temperature"); + expect(body.response_format).toEqual({ type: "json_object" }); + expect(body.thinking).toEqual({ type: "disabled" }); + return new Response(JSON.stringify({ + id: "chatcmpl-test", + object: "chat.completion", + created: 0, + model: "kimi-k2.6", + choices: [{ index: 0, finish_reason: "stop", logprobs: null, message: { role: "assistant", content: "{\"ok\":true}" } }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + global.fetch = fetchMock as typeof fetch; + + await expect(generateStructuredWithMetadata({ + capability: "founderWeeklyReview", + system: "Return valid JSON.", + prompt: "Generate the object.", + schema: z.object({ ok: z.boolean() }), + schemaName: "founder_weekly_review", + })).resolves.toMatchObject({ object: { ok: true }, metadata: { provider: "kimi", model: "kimi-k2.6" } }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(`${logSpy.mock.calls.flat().join(" ")} ${errorSpy.mock.calls.flat().join(" ")}`).not.toContain("test-key-not-logged"); + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it("defaults to OpenAI and does not infer Kimi from Moonshot credentials", () => { + delete process.env.FWR_GENERATION_PROVIDER; + process.env.OPENAI_API_KEY = "openai-test-key"; + process.env.OPENAI_MODEL_ID = "gpt-test"; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + const resolved = resolveModel("founderWeeklyReview"); + expect(resolved).toMatchObject({ provider: "openai", modelId: "gpt-test" }); + expect(resolved.temperature).toBe(0); + expect(resolved.structuredOutputMode).toBeUndefined(); + expect(resolved).not.toHaveProperty("thinking"); + }); + + it("uses explicit OpenAI without requiring Moonshot configuration", () => { + process.env.FWR_GENERATION_PROVIDER = "openai"; + process.env.OPENAI_API_KEY = "openai-test-key"; + process.env.OPENAI_MODEL_ID = "gpt-openai"; + delete process.env.MOONSHOT_API_KEY; + delete process.env.MOONSHOT_BASE_URL; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + + const resolved = resolveModel("founderWeeklyReview"); + expect(resolved).toMatchObject({ provider: "openai", modelId: "gpt-openai" }); + expect(resolved.structuredOutputMode).toBeUndefined(); + }); + + it("fails before any request for invalid or missing selected-provider configuration", () => { + process.env.FWR_GENERATION_PROVIDER = "other"; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + expect(() => resolveModel("founderWeeklyReview")).toThrow('FWR_GENERATION_PROVIDER must be "openai" or "kimi"'); + process.env.FWR_GENERATION_PROVIDER = "kimi"; + delete process.env.MOONSHOT_API_KEY; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + expect(() => resolveModel("founderWeeklyReview")).toThrow("FWR_GENERATION_PROVIDER=kimi requires MOONSHOT_API_KEY"); + + process.env.FWR_GENERATION_PROVIDER = "openai"; + process.env.MOONSHOT_API_KEY = "moonshot-secret-that-must-not-appear"; + delete process.env.OPENAI_API_KEY; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + let thrown: Error | null = null; + try { + resolveModel("founderWeeklyReview"); + } catch (error) { + thrown = error as Error; + } + expect(thrown?.message).toContain("FWR_GENERATION_PROVIDER=openai requires OPENAI_API_KEY"); + expect(thrown?.message).not.toContain("moonshot-secret-that-must-not-appear"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/markdown.test.ts b/apps/web/__tests__/founderWeeklyReview/markdown.test.ts new file mode 100644 index 000000000..ffbd29b76 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/markdown.test.ts @@ -0,0 +1,55 @@ +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; + +function reviewRun() { + return { + reportingPeriod: { start: "2026-02-16", end: "2026-02-28" }, + modelMetadata: { model: "kimi-k2.6" }, + evidenceSnapshot: { + items: [ + { sourceId: "customer_feedback:doc:7:version:3:section:15", sourceType: "customer_feedback", title: "Customer Interviews — February 2026", excerpt: "first raw excerpt", metadata: { pageNumber: 2, sectionId: 15, documentCategory: "Customer Feedback" } }, + { sourceId: "customer_feedback:doc:7:version:3:section:16", sourceType: "customer_feedback", title: "Customer Interviews — February 2026", excerpt: "second raw excerpt", metadata: { pageNumber: 3, sectionId: 16, documentCategory: "Customer Feedback" } }, + { sourceId: "document_change:doc:4:version:2", sourceType: "document_change", title: "Onboarding reliability plan", excerpt: "document changelog", metadata: { versionNumber: 2 } }, + { sourceId: "founder_context:request:secret", sourceType: "founder_context", title: "Founder context", excerpt: "private context", metadata: {} }, + ], + }, + reviewPayload: { + schemaVersion: "founder-weekly-review/v2", + sections: { + whatShipped: { state: "no_evidence", noEvidence: { code: "none", message: "No completed release evidence.", cta: "Add release evidence." } }, + whatChanged: { state: "no_evidence", noEvidence: { code: "none", message: "No other change evidence.", cta: "Add change evidence." } }, + whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "Two distinct interview excerpts identify reporting setup and recovery concerns.", sourceIds: ["customer_feedback:doc:7:version:3:section:15", "customer_feedback:doc:7:version:3:section:16"], confidence: 0.8 }] }, + currentBlockers: { state: "evidence", items: [{ kind: "observed_fact", text: "The documented plan is operational preparation, not proof that retries are fixed.", sourceIds: ["document_change:doc:4:version:2"], confidence: 0.8 }] }, + nextPriorities: { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Improve recovery feedback.", rationale: "Founder-provided context identifies it as a priority.", sourceIds: ["founder_context:request:secret"], confidence: 0.7 }] }, + }, + }, + } as any; +} + +describe("Founder Weekly Review Markdown", () => { + it("renders useful no-evidence sections without an empty Key Outcomes container", () => { + const rendered = renderFounderWeeklyReviewMarkdown(reviewRun()); + + expect(rendered).not.toContain("## Key Outcomes"); + expect(rendered).toContain("## Shipped This Period\n\nNo completed release evidence.\n\nNext: Add release evidence."); + }); + + it("uses deterministic, distinct human-readable evidence labels with metadata", () => { + const rendered = renderFounderWeeklyReviewMarkdown(reviewRun()); + + expect(rendered).toContain("[1] Customer Interviews — February 2026 — page 2 — section 15"); + expect(rendered).toContain("[2] Customer Interviews — February 2026 — page 3 — section 16"); + expect(rendered).toContain("[3] Document change — Onboarding reliability plan"); + expect(rendered).toContain("[4] Founder-provided context"); + expect(rendered).not.toContain("[1] Customer Feedback"); + }); + + it("uses the exact rendered Markdown for terminal and export paths without operational internals", () => { + const terminalMarkdown = renderFounderWeeklyReviewMarkdown(reviewRun()); + const exportedMarkdown = renderFounderWeeklyReviewMarkdown(reviewRun()); + + expect(exportedMarkdown).toBe(terminalMarkdown); + for (const forbidden of ["customer_feedback:doc:7", "founder_context:request:secret", "first raw excerpt", "evidenceSnapshot", "MOONSHOT_API_KEY", "DATABASE_URL", "provider raw response", "internal error"]) { + expect(terminalMarkdown).not.toContain(forbidden); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/migration.test.ts b/apps/web/__tests__/founderWeeklyReview/migration.test.ts index e7f1f8c7f..fd11b5acf 100644 --- a/apps/web/__tests__/founderWeeklyReview/migration.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/migration.test.ts @@ -55,6 +55,7 @@ describeIfDatabase("Founder Weekly Review migrations", () => { "review_schema_version", "evidence_snapshot", "evidence_schema_version", + "collection_input", "created_by_actor_id" ) VALUES @@ -68,6 +69,7 @@ describeIfDatabase("Founder Weekly Review migrations", () => { 'founder-weekly-review/v1', '{"schemaVersion":"founder-weekly-review-evidence/v1","capturedAt":"2026-07-18T00:00:00.000Z","reportingPeriod":{"start":"2026-07-07","end":"2026-07-13"},"workspaceTimezone":"UTC","items":[],"sourceWarnings":[]}'::jsonb, 'founder-weekly-review-evidence/v1', + '{"workspaceTimezone":"UTC","actorExternalUserId":"test"}'::jsonb, 'user:test' ) `); diff --git a/apps/web/__tests__/founderWeeklyReview/routes.test.ts b/apps/web/__tests__/founderWeeklyReview/routes.test.ts index 0dba8c899..2c57d8c41 100644 --- a/apps/web/__tests__/founderWeeklyReview/routes.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/routes.test.ts @@ -39,14 +39,15 @@ describe("Founder Weekly Review create route", () => { const response = await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) })); expect(response.status).toBe(202); expect((await response.json()).run.id).toBe("existing"); expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); expect(createRunWithDispatch).not.toHaveBeenCalled(); expect(deps.recordRunCreated).not.toHaveBeenCalled(); }); - it("forwards canonical collector inputs, creates a queued run, and records exactly one creation", async () => { + it("persists durable collection inputs, creates a queued run, and records exactly one creation", async () => { const { handler, collector, createRunWithDispatch, deps } = setup(); expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(202); - expect(collector.collectFounderWeeklyReviewEvidence).toHaveBeenCalledWith(expect.objectContaining({ companyId: 1n, reportingPeriod: body.reportingPeriod, workspaceTimezone: "UTC", founderContext: "Context", actor: { externalUserId: "u" }, requestKey: "key" })); + expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); + expect(createRunWithDispatch).toHaveBeenCalledWith(expect.objectContaining({ requestKey: "key", reportingPeriod: body.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", founderContext: "Context", actorExternalUserId: "u" } })); expect(createRunWithDispatch).toHaveBeenCalledTimes(1); expect(deps.recordRunCreated).toHaveBeenCalledTimes(1); }); - it("does not create or record when collection fails", async () => { + it("does not synchronously collect evidence", async () => { const { handler, createRunWithDispatch, deps } = setup({ evidenceCollector: { collectFounderWeeklyReviewEvidence: jest.fn().mockRejectedValue(new Error("nope")) } }); - expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(500); expect(createRunWithDispatch).not.toHaveBeenCalled(); expect(deps.recordRunCreated).not.toHaveBeenCalled(); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(202); expect(createRunWithDispatch).toHaveBeenCalledTimes(1); expect(deps.recordRunCreated).toHaveBeenCalledTimes(1); }); }); diff --git a/apps/web/__tests__/founderWeeklyReview/testDb.ts b/apps/web/__tests__/founderWeeklyReview/testDb.ts index b8ad11c5d..831b9a3ef 100644 --- a/apps/web/__tests__/founderWeeklyReview/testDb.ts +++ b/apps/web/__tests__/founderWeeklyReview/testDb.ts @@ -74,6 +74,52 @@ async function bootstrapIsolatedSchema( "url" varchar(256), "category" varchar(256), "title" varchar(256), + "ocr_enabled" boolean DEFAULT false, + "ocr_processed" boolean DEFAULT false, + "ocr_metadata" jsonb, + "ocr_job_id" varchar(256), + "ocr_provider" varchar(50), + "ocr_confidence_score" integer, + "ocr_cost_cents" integer, + "mime_type" varchar(128), + "source_archive_name" varchar(256), + "file_type" varchar(128), + "current_version_id" bigint, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_versions" ( + "id" serial PRIMARY KEY, + "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, + "version_number" integer NOT NULL, + "url" varchar(512) NOT NULL, + "mime_type" varchar(128) NOT NULL, + "file_size" bigint, + "uploaded_by" varchar(256), + "changelog" text, + "ocr_job_id" varchar(256), + "ocr_provider" varchar(50), + "ocr_processed" boolean DEFAULT false, + "ocr_metadata" jsonb, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE ("document_id", "version_number") + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_context_chunks" ( + "id" serial PRIMARY KEY, + "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, + "version_id" bigint REFERENCES "pdr_ai_v2_document_versions"("id") ON DELETE CASCADE, + "structure_id" bigint, + "content" text NOT NULL, + "token_count" integer NOT NULL DEFAULT 0, + "char_count" integer NOT NULL DEFAULT 0, + "embedding" vector(1536), + "content_hash" varchar(64), + "semantic_type" varchar(50), + "page_number" integer, + "line_start" integer, + "line_end" integer, "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, "updated_at" timestamptz ); diff --git a/apps/web/drizzle/0018_founder_weekly_review_async_collection.sql b/apps/web/drizzle/0018_founder_weekly_review_async_collection.sql new file mode 100644 index 000000000..1b738b23e --- /dev/null +++ b/apps/web/drizzle/0018_founder_weekly_review_async_collection.sql @@ -0,0 +1,22 @@ +-- Move Founder Weekly Review evidence collection into the durable workflow. +ALTER TABLE "pdr_ai_v2_founder_weekly_review_runs" + ALTER COLUMN "evidence_snapshot" DROP NOT NULL; +ALTER TABLE "pdr_ai_v2_founder_weekly_review_runs" + ADD COLUMN "collection_input" jsonb, + ADD COLUMN "collection_claim_id" varchar(128), + ADD COLUMN "collection_started_at" timestamptz, + ADD COLUMN "evidence_collected_at" timestamptz; + +-- Existing pre-workflow rows already have a persisted snapshot. Their actor is +-- retained only as a bounded durable collection input; it is never exposed. +UPDATE "pdr_ai_v2_founder_weekly_review_runs" +SET "collection_input" = jsonb_build_object( + 'workspaceTimezone', COALESCE("evidence_snapshot"->>'workspaceTimezone', 'UTC'), + 'actorExternalUserId', regexp_replace("created_by_actor_id", '^user:', '') +) +WHERE "collection_input" IS NULL; + +ALTER TABLE "pdr_ai_v2_founder_weekly_review_runs" + ALTER COLUMN "collection_input" SET NOT NULL; +CREATE INDEX "founder_weekly_review_runs_collection_claim_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "id", "status", "collection_claim_id"); diff --git a/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts b/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts new file mode 100644 index 000000000..390802f95 --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts @@ -0,0 +1,588 @@ +import "dotenv/config"; +import { createHash, randomUUID } from "node:crypto"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { createWriteStream, type WriteStream } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import net from "node:net"; +import { relative, resolve } from "node:path"; + +import { eq } from "drizzle-orm"; +import { + company, + document, + documentContextChunks, + documentVersions, + founderWeeklyReviewDispatches, +} from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewRepository } from "@launchstack/features/founder-weekly-review"; + +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; + +const require = createRequire(import.meta.url); +const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); +const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/realistic-company/seed.json"); +const STARTUP_TIMEOUT_MS = 45_000; +const SOCKET_RELEASE_WAIT_MS = 1_500; +const STARTUP_SMOKE_ONLY = process.env.FWR_TRANSPORT_STARTUP_SMOKE_ONLY === "1"; +const VERBOSE_CHILD_LOGS = process.env.FWR_TRANSPORT_VERBOSE_LOGS === "1"; +const RECENT_DIAGNOSTIC_LIMIT = 160; +const SHUTDOWN_DRAIN_CAP_MS = 5_000; +const CALLBACK_QUIET_MS = 1_000; + +type Fixture = { + reportingPeriod: { start: string; end: string }; + workspaceTimezone: string; + founderContext: string; + documents: Array<{ + title: string; + category: string; + changelog: string; + timestamp: string; + chunks?: string[]; + }>; +}; + +interface ManagedChild { + name: "next" | "inngest" | string; + process: ChildProcess; +} + +interface ShutdownDrainState { + terminalStatus: "draft" | "failed" | null; + finalDiagnosticsCaptured: boolean; + cleanupStarted: boolean; + callbackDrained: boolean; + lastCallbackActivityAt: number; + functionFinishedAfterTerminal: boolean; + knownMissingBodyDuringShutdown: boolean; +} + +function digest(value: unknown) { + return createHash("sha256").update(JSON.stringify(value), "utf8").digest("hex"); +} + +function sleep(ms: number) { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + +function lifecycleTimeoutMs(): number { + const raw = process.env.FWR_TRANSPORT_TIMEOUT_MS; + if (!raw) return 720_000; + if (!/^\d+$/.test(raw)) { + throw new Error("FWR_TRANSPORT_TIMEOUT_MS must be a positive safe integer."); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error("FWR_TRANSPORT_TIMEOUT_MS must be a positive safe integer."); + } + return value; +} + +async function allocateLoopbackPort(): Promise { + const server = net.createServer(); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen({ host: "127.0.0.1", port: 0, exclusive: true }, () => { + server.off("error", rejectListen); + resolveListen(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("Unable to allocate a loopback TCP port."); + } + await new Promise((resolveClose, rejectClose) => server.close((error) => error ? rejectClose(error) : resolveClose())); + return address.port; +} + +async function canConnect(port: number, host: string): Promise { + return new Promise((resolveConnect) => { + const socket = net.createConnection({ host, port }); + const settle = (listening: boolean) => { + socket.removeAllListeners(); + socket.destroy(); + resolveConnect(listening); + }; + socket.once("connect", () => settle(true)); + socket.once("error", () => settle(false)); + socket.setTimeout(500, () => settle(false)); + }); +} + +async function isPortListening(port: number): Promise { + // `next dev -H localhost` may choose IPv6 loopback on Windows. Check both + // loopback families so stale listeners are never mistaken for a free port. + return (await canConnect(port, "127.0.0.1")) || await canConnect(port, "::1"); +} + +async function assertPortClosed(port: number, label: string): Promise { + if (await isPortListening(port)) { + throw new Error(`${label} port ${port} is already listening; refusing to attach to an existing process.`); + } +} + +function childIsAlive(child: ManagedChild | undefined): boolean { + return Boolean(child && child.process.exitCode === null && child.process.signalCode === null); +} + +function boundedLogs(logs: string[]): string { + return logs.slice(-100).join("").slice(-8_000); +} + +function appendRecentDiagnostic(logs: string[], line: string): void { + logs.push(line); + if (logs.length > RECENT_DIAGNOSTIC_LIMIT) logs.splice(0, logs.length - RECENT_DIAGNOSTIC_LIMIT); +} + +function isInngestDevUiNoise(line: string): boolean { + try { + const parsed = JSON.parse(line) as { level?: unknown; event?: unknown; event_name?: unknown }; + return parsed.level === "INFO" && (parsed.event === "cli/dev_ui.loaded" || parsed.event_name === "cli/dev_ui.loaded"); + } catch { + return false; + } +} + +function isUsefulDiagnostic(name: string, line: string): boolean { + return /warn|error|fail|exception|stack|retry|founder-weekly-review|claim-evidence|collect-evidence|persist-evidence|\bclaim\b|\bgenerate\b|\bpersist\b|database|relation|column|non-2\d\d|PUT \/api\/inngest 200/i.test(line); +} + +function isTerminalNoise(name: string, line: string): boolean { + if (name === "inngest") return isInngestDevUiNoise(line); + // Keep registration failures and non-2xx callbacks visible, but omit the + // steady-state successful callback poll noise from the terminal. + return /^(GET|PUT) \/api\/inngest.*\s2\d\d\s/i.test(line) + || /^responseBody:/i.test(line); +} + +function isKnownShutdownMissingBody( + name: "next" | "inngest", + line: string, + state: ShutdownDrainState +): boolean { + return name === "next" + && line.includes("[Inngest] error - Missing body when executing, possibly due to missing request body middleware") + && state.terminalStatus !== null + && state.finalDiagnosticsCaptured + && state.cleanupStarted + && state.callbackDrained; +} + +function printRecentDiagnostics(name: "next" | "inngest", logs: string[]): void { + console.log(`===== RECENT ${name.toUpperCase()} DIAGNOSTICS =====`); + for (const line of logs) console.log(line); + console.log(`===== END ${name.toUpperCase()} DIAGNOSTICS =====`); +} + +async function closeLogStream(stream: WriteStream): Promise { + await new Promise((resolveClose) => stream.end(resolveClose)); +} + +function startManagedChild( + managedChildren: ManagedChild[], + command: string, + args: string[], + cwd: string, + env: NodeJS.ProcessEnv, + logs: string[], + rawLog: WriteStream, + name: "next" | "inngest", + shutdownDrain: ShutdownDrainState +): ManagedChild { + // pnpm.cmd requires cmd.exe on Windows. taskkill /T in cleanup targets this shell + // PID and its pnpm/Node descendants as one scoped process tree. + const child = spawn(command, args, { + cwd, + env, + shell: process.platform === "win32", + windowsHide: true, + }); + const managed: ManagedChild = { name, process: child }; + managedChildren.push(managed); + const emit = (stream: "stdout" | "stderr", data: Buffer) => { + const output = data.toString(); + rawLog.write(output); + for (const line of output.split(/\r?\n/)) { + if (!line.trim()) continue; + if (name === "next" && /\/api\/inngest/i.test(line)) shutdownDrain.lastCallbackActivityAt = Date.now(); + if (name === "inngest" && shutdownDrain.terminalStatus && line.includes("inngest/function.finished")) shutdownDrain.functionFinishedAfterTerminal = true; + const formatted = `[${name}:${stream}] ${line}`; + const expectedShutdownNoise = isKnownShutdownMissingBody(name, line, shutdownDrain); + const suppressedNoise = isTerminalNoise(name, line) || expectedShutdownNoise; + if (isUsefulDiagnostic(name, line) && !suppressedNoise) appendRecentDiagnostic(logs, formatted); + if (expectedShutdownNoise) { + shutdownDrain.knownMissingBodyDuringShutdown = true; + console.log("[next] known shutdown-only Inngest callback race observed; retained in raw log"); + } + if (VERBOSE_CHILD_LOGS || !suppressedNoise) console.log(formatted); + } + }; + child.stdout?.on("data", (data: Buffer) => emit("stdout", data)); + child.stderr?.on("data", (data: Buffer) => emit("stderr", data)); + child.once("exit", (code, signal) => { + const line = `[${name}] exited code=${code ?? "null"} signal=${signal ?? "none"}`; + appendRecentDiagnostic(logs, line); + console.log(line); + }); + return managed; +} + +async function drainTerminalCallbackActivity(state: ShutdownDrainState): Promise { + const deadline = Date.now() + SHUTDOWN_DRAIN_CAP_MS; + console.log(`[cleanup] draining terminal callback activity capMs=${SHUTDOWN_DRAIN_CAP_MS}`); + while (Date.now() < deadline) { + const callbackQuiet = Date.now() - state.lastCallbackActivityAt >= CALLBACK_QUIET_MS; + if (state.functionFinishedAfterTerminal && callbackQuiet) { + state.callbackDrained = true; + console.log("[cleanup] terminal callback drain settled"); + return; + } + await sleep(100); + } + console.warn(`[cleanup] terminal callback drain cap reached functionFinished=${state.functionFinishedAfterTerminal}`); +} + +function terminateProcessTree(child: ManagedChild): void { + const pid = child.process.pid; + if (!pid || !childIsAlive(child)) return; + if (process.platform === "win32") { + const result = spawnSync("taskkill", ["/PID", String(pid), "/T", "/F"], { + encoding: "utf8", + windowsHide: true, + }); + if (result.error || (result.status !== 0 && childIsAlive(child))) { + console.warn(`[cleanup] ${child.name} pid=${pid} taskkill did not confirm termination (already exited is non-fatal)`); + } + return; + } + child.process.kill("SIGTERM"); +} + +async function waitForChildExit(child: ManagedChild): Promise { + if (!childIsAlive(child)) return; + await Promise.race([ + new Promise((resolveExit) => child.process.once("exit", () => resolveExit())), + sleep(5_000), + ]); +} + +function owningPid(port: number): string | undefined { + if (process.platform !== "win32") return undefined; + const result = spawnSync("netstat", ["-ano", "-p", "tcp"], { encoding: "utf8", windowsHide: true }); + const match = result.stdout?.split(/\r?\n/).find((line) => new RegExp(`127\\.0\\.0\\.1:${port}\\s+.*LISTENING`, "i").test(line)); + return match?.trim().split(/\s+/).at(-1); +} + +async function waitForCallback( + callbackUrl: string, + managedChildren: ManagedChild[], + logs: string[], + shouldAbort: () => boolean +): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (shouldAbort()) throw new Error("Startup interrupted before callback readiness."); + const next = managedChildren.find((child) => child.name === "next"); + if (!childIsAlive(next)) throw new Error(`Next exited before callback readiness: ${boundedLogs(logs)}`); + try { + const response = await fetch(callbackUrl); + const registration = await response.text(); + const functionsInGetResponse = registration.includes("founder-weekly-review-dispatcher") && registration.includes("founder-weekly-review-generation"); + // The dev CLI performs the authoritative registration with PUT. A + // bare GET is still required to respond successfully, but its body + // is protocol-dependent and does not always echo function IDs. + const functionsRegisteredByCli = logs.some((line) => line.includes("PUT /api/inngest 200")); + if (response.ok && (functionsInGetResponse || functionsRegisteredByCli)) { + console.log(`[harness] callback ready url=${callbackUrl}; Founder Weekly Review functions registered=${functionsInGetResponse ? "GET" : "CLI PUT"}`); + return; + } + } catch { + // Next is still booting; child liveness is checked every iteration. + } + await sleep(500); + } + throw new Error(`Timed out waiting for registered callback ${callbackUrl}: ${boundedLogs(logs)}`); +} + +async function waitForHttp(url: string, child: ManagedChild, logs: string[], shouldAbort: () => boolean): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (shouldAbort()) throw new Error("Startup interrupted before Inngest readiness."); + if (!childIsAlive(child)) throw new Error(`Inngest exited before readiness: ${boundedLogs(logs)}`); + try { + if ((await fetch(url)).ok) return; + } catch { + // Inngest is still booting. + } + await sleep(500); + } + throw new Error(`Timed out waiting for ${url}: ${boundedLogs(logs)}`); +} + +async function waitForPort(port: number, child: ManagedChild, logs: string[], shouldAbort: () => boolean): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (shouldAbort()) throw new Error("Startup interrupted before Next was listening."); + if (!childIsAlive(child)) throw new Error(`Next exited before listening: ${boundedLogs(logs)}`); + if (await isPortListening(port)) return; + await sleep(250); + } + throw new Error(`Timed out waiting for Next port ${port}: ${boundedLogs(logs)}`); +} + +function createCleanupController( + managedChildren: ManagedChild[], + nextPort: number, + inngestPort: number, + shutdownDrain: ShutdownDrainState +): () => Promise { + let cleanupPromise: Promise | undefined; + return () => cleanupPromise ??= (async () => { + shutdownDrain.cleanupStarted = true; + console.log("[cleanup] stopping managed child process trees"); + for (const child of [...managedChildren].reverse()) { + console.log(`[cleanup] stopping ${child.name} pid=${child.process.pid ?? "unknown"}`); + terminateProcessTree(child); + await waitForChildExit(child); + } + await sleep(SOCKET_RELEASE_WAIT_MS); + let portsClosed = true; + for (const [name, port] of [["next", nextPort], ["inngest", inngestPort]] as const) { + if (await isPortListening(port)) { + portsClosed = false; + const child = managedChildren.find((candidate) => candidate.name === name); + console.warn(`[cleanup] ${name}Port=${port} still listening pid=${owningPid(port) ?? "unknown"} childAlive=${childIsAlive(child)}`); + } else { + console.log(`[cleanup] ${name}Port=closed`); + } + } + if (shutdownDrain.knownMissingBodyDuringShutdown && portsClosed) { + console.log("[cleanup] known Missing body message classified as shutdown-only after terminal callback drain and port closure"); + } + })(); +} + +async function main(): Promise { + if (process.env.SYNTHETIC_FWR_LOCAL !== "1" || process.env.NODE_ENV === "production") { + throw new Error("Refusing transport E2E outside explicit local mode."); + } + const localUrl = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? ""; + if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(localUrl)) { + throw new Error("Refusing non-local database."); + } + if (!STARTUP_SMOKE_ONLY && !process.env.MOONSHOT_API_KEY?.trim()) { + throw new Error("MOONSHOT_API_KEY is required for the real callback generation."); + } + + const testDb = await createFounderWeeklyReviewTestDatabase(); + const managedChildren: ManagedChild[] = []; + const nextLogs: string[] = []; + const inngestLogs: string[] = []; + const artifactDirectory = resolve(process.cwd(), ".artifacts/founder-weekly-review"); + await mkdir(artifactDirectory, { recursive: true }); + const nextLogPath = resolve(artifactDirectory, "transport-next.log"); + const inngestLogPath = resolve(artifactDirectory, "transport-inngest.log"); + const nextRawLog = createWriteStream(nextLogPath, { flags: "w" }); + const inngestRawLog = createWriteStream(inngestLogPath, { flags: "w" }); + const nextPort = await allocateLoopbackPort(); + let inngestPort = await allocateLoopbackPort(); + while (inngestPort === nextPort) inngestPort = await allocateLoopbackPort(); + const shutdownDrain: ShutdownDrainState = { + terminalStatus: null, + finalDiagnosticsCaptured: false, + cleanupStarted: false, + callbackDrained: false, + lastCallbackActivityAt: Date.now(), + functionFinishedAfterTerminal: false, + knownMissingBodyDuringShutdown: false, + }; + const cleanup = createCleanupController(managedChildren, nextPort, inngestPort, shutdownDrain); + let shutdownRequested = false; + const requestShutdown = (reason: string) => { + shutdownRequested = true; + console.warn(`[cleanup] ${reason} received`); + void cleanup().catch((error: unknown) => console.warn(`[cleanup] ${String(error)}`)); + }; + const onSigint = () => requestShutdown("SIGINT"); + const onSigterm = () => requestShutdown("SIGTERM"); + const onUncaughtException = (error: Error) => { + console.error(`[cleanup] uncaughtException: ${error.message}`); + process.exitCode = 1; + requestShutdown("uncaughtException"); + }; + const onUnhandledRejection = (reason: unknown) => { + console.error(`[cleanup] unhandledRejection: ${String(reason)}`); + process.exitCode = 1; + requestShutdown("unhandledRejection"); + }; + process.once("SIGINT", onSigint); + process.once("SIGTERM", onSigterm); + process.once("uncaughtException", onUncaughtException); + process.once("unhandledRejection", onUnhandledRejection); + + try { + const schemaUrl = new URL(localUrl); + schemaUrl.searchParams.set("options", `-c search_path=${testDb.schemaName},public`); + // The current Next/Inngest adapter normalizes its callback host to + // localhost. Explicitly bind there (loopback-only) so the adapter and + // CLI address the same listener; allocation remains 127.0.0.1-only. + const callbackUrl = `http://localhost:${nextPort}/api/inngest`; + const inngestUrl = `http://127.0.0.1:${inngestPort}`; + const runtimeEnv: NodeJS.ProcessEnv = { + ...process.env, + DATABASE_URL: schemaUrl.toString(), + LAUNCHSTACK_TEST_DATABASE_URL: schemaUrl.toString(), + // The current Inngest SDK treats a URL-valued INNGEST_DEV as the + // explicit local dev-server target. INNGEST_BASE_URL configures the + // API/event base and is not a replacement for this transport URL. + INNGEST_DEV: inngestUrl, + INNGEST_EVENT_KEY: "local-transport", + }; + console.log(`[harness] database host=${schemaUrl.hostname} port=${schemaUrl.port || "5432"} schema=${testDb.schemaName}`); + console.log(`[harness] selected nextPort=${nextPort} inngestPort=${inngestPort}`); + console.log(`[harness] verbose child logs: ${VERBOSE_CHILD_LOGS ? "enabled" : "disabled"}`); + console.log(`[harness] Next log: ${relative(process.cwd(), nextLogPath)}`); + console.log(`[harness] Inngest log: ${relative(process.cwd(), inngestLogPath)}`); + await assertPortClosed(nextPort, "Next"); + await assertPortClosed(inngestPort, "Inngest"); + const next = startManagedChild(managedChildren, "pnpm.cmd", ["exec", "next", "dev", "--turbo", "-H", "localhost", "-p", String(nextPort)], process.cwd(), runtimeEnv, nextLogs, nextRawLog, "next", shutdownDrain); + console.log(`[next] expected schema=${testDb.schemaName}`); + // Inngest's serve handler proxies its registration response through the + // dev server. Before that server exists, a GET can reset even though the + // current child owns and is listening on the allocated callback port. + await waitForPort(nextPort, next, nextLogs, () => shutdownRequested); + await assertPortClosed(inngestPort, "Inngest"); + const inngest = startManagedChild(managedChildren, "pnpm.cmd", ["dlx", "inngest-cli@latest", "dev", "--no-discovery", "-u", callbackUrl, "--port", String(inngestPort)], process.cwd(), runtimeEnv, inngestLogs, inngestRawLog, "inngest", shutdownDrain); + await waitForHttp(inngestUrl, inngest, inngestLogs, () => shutdownRequested); + await waitForCallback(callbackUrl, managedChildren, nextLogs, () => shutdownRequested); + console.log(`[harness] Inngest ready url=${inngestUrl} callback=${callbackUrl}`); + if (STARTUP_SMOKE_ONLY) { + console.log("[harness] startup/cleanup smoke completed; no data seeded, event dispatched, or provider called"); + return; + } + + const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as Fixture; + const [target] = await testDb.db.insert(company).values({ name: "Northstar Analytics", numberOfEmployees: "24" }).returning(); + const [other] = await testDb.db.insert(company).values({ name: "Other Company", numberOfEmployees: "4" }).returning(); + for (const entry of fixture.documents) { + const [doc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: `local://${entry.title}`, category: entry.category, title: entry.title }).returning(); + const [version] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${entry.title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: entry.changelog, createdAt: new Date(entry.timestamp) }).returning(); + for (const [index, content] of (entry.chunks ?? []).entries()) { + await testDb.db.insert(documentContextChunks).values({ documentId: BigInt(doc!.id), versionId: BigInt(version!.id), content, tokenCount: content.split(/\s+/).length, charCount: content.length, pageNumber: index + 1 }); + } + } + for (const [companyId, title, timestamp] of [[BigInt(target!.id), "Outside period", "2026-03-01T00:00:00.000Z"], [BigInt(other!.id), "Other company control", "2026-02-21T00:00:00.000Z"]] as const) { + const [doc] = await testDb.db.insert(document).values({ companyId, url: `local://${title}`, category: "Product", title }).returning(); + await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: title, createdAt: new Date(timestamp) }); + } + const [seededDocuments, seededVersions, seededChunks] = await Promise.all([ + testDb.db.select().from(document), + testDb.db.select().from(documentVersions), + testDb.db.select().from(documentContextChunks), + ]); + console.log(`[harness] fixture counts companies=2 documents=${seededDocuments.length} versions=${seededVersions.length} contextChunks=${seededChunks.length}`); + const actor = { externalUserId: "realistic-owner", internalUserId: 1n, companyId: BigInt(target!.id), role: "owner" as const }; + const created = await createFounderWeeklyReviewDispatchService(testDb.db).createRunWithDispatch({ actor, requestKey: `transport-${randomUUID()}`, reportingPeriod: fixture.reportingPeriod, collectionInput: { workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actorExternalUserId: actor.externalUserId } }); + if (created.run.status !== "queued" || created.run.evidenceSnapshot) throw new Error("Initial queued-without-snapshot invariant failed."); + const initialDispatches = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, created.run.id)); + if (initialDispatches.length !== 1) throw new Error("Expected exactly one initial outbox dispatch."); + process.env.INNGEST_DEV = inngestUrl; + process.env.INNGEST_EVENT_KEY = "local-transport"; + const { inngest: client } = await import("~/server/inngest/client"); + const requested = await client.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); + console.log(`[harness] dispatch event sent at=${new Date().toISOString()}`); + const repository = new FounderWeeklyReviewRepository(testDb.db); + const deadline = Date.now() + lifecycleTimeoutMs(); + const startedAt = Date.now(); + let heartbeatAt = 0; + const seen: string[] = ["queued without snapshot"]; + let previousState = seen[0]!; + let final: Awaited> | null = null; + while (Date.now() < deadline) { + if (shutdownRequested) throw new Error("Transport run interrupted during lifecycle polling."); + if (!childIsAlive(next) || !childIsAlive(inngest)) { + throw new Error("A managed child exited before the Founder Weekly Review reached a terminal state."); + } + const current = await repository.getByCompanyAndRunId(actor.companyId, created.run.id); + const currentDispatch = (await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, created.dispatch.id)))[0]; + if (current) { + const state = `${current.status}${current.evidenceSnapshot ? " with snapshot" : " without snapshot"}`; + if (state !== previousState) { + previousState = state; + seen.push(state); + console.log(`[harness] lifecycle ${state} at=${new Date().toISOString()}`); + } + if (Date.now() - heartbeatAt >= 10_000) { + heartbeatAt = Date.now(); + console.log(`[harness +${Math.floor((Date.now() - startedAt) / 1000)}s] nextAlive=${childIsAlive(next)} inngestAlive=${childIsAlive(inngest)} callbackReady=true runStatus=${current.status} snapshot=${current.evidenceSnapshot ? "present" : "absent"} dispatchStatus=${currentDispatch?.status ?? "unknown"} collectionClaim=${current.collectionClaimId ? "present" : "absent"} generationClaim=${current.generationClaimId ? "present" : "absent"}`); + } + if (current.status === "draft") { + final = current; + shutdownDrain.terminalStatus = "draft"; + console.log("[harness] terminal status=draft observed; stopping lifecycle polling and starting cleanup"); + break; + } + if (current.status === "failed") { + console.error(JSON.stringify({ + runId: current.id, + status: current.status, + failureCode: current.errorCode, + failureMessage: current.errorMessage?.slice(0, 1_024) ?? null, + snapshot: current.evidenceSnapshot ? "present" : "absent", + collectionClaim: current.collectionClaimId ? "present" : "absent", + collectionStartedAt: current.collectionStartedAt?.toISOString() ?? null, + evidenceCollectedAt: current.evidenceCollectedAt?.toISOString() ?? null, + generationClaim: current.generationClaimId ? "present" : "absent", + retryCount: current.retryCount, + dispatchStatus: currentDispatch?.status ?? null, + dispatchAttempts: currentDispatch?.attemptCount ?? null, + lastLifecycleTransition: current.updatedAt?.toISOString() ?? current.createdAt.toISOString(), + })); + shutdownDrain.terminalStatus = "failed"; + shutdownDrain.finalDiagnosticsCaptured = true; + console.log("[harness] terminal status=failed observed; stopping lifecycle polling and starting cleanup"); + throw new Error(`Founder Weekly Review run ${current.id} failed before draft persistence.`); + } + } + await sleep(500); + } + if (!final?.reviewPayload || !final.evidenceSnapshot) throw new Error("Timed out before persisted draft read-back."); + const dispatch = (await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, created.dispatch.id)))[0]; + const rendered = renderFounderWeeklyReviewMarkdown(final); + const directory = resolve(process.cwd(), ".artifacts/founder-weekly-review"); + await mkdir(directory, { recursive: true }); + const markdownPath = resolve(directory, `${final.id}-transport.md`); + await writeFile(markdownPath, rendered, "utf8"); + if (process.env.FWR_PRINT_REPORT === "1") { + console.log("===== FOUNDER WEEKLY REVIEW ====="); + console.log(rendered); + console.log("===== END FOUNDER WEEKLY REVIEW ====="); + } + const generationEventKeys = ["runId", "companyId", "generationJobId", "generationClaimId"]; + const forbiddenEventKeys = ["evidenceSnapshot", "founderContext", "documentContent", "customerFeedback", "prompt", "reviewPayload", "providerResponse", "databaseUrl", "credentials", "token"]; + console.log(JSON.stringify({ runId: final.id, lifecycle: seen, dispatch: { initialStatus: initialDispatches[0]!.status, finalStatus: dispatch?.status, attempts: dispatch?.attemptCount }, ingressEventIds: requested.ids, callbackUrl, functionId: "founder-weekly-review-generation", transportEvent: { name: "founder-weekly-review/generation.requested", keys: generationEventKeys, companyIdSerializedAsString: true, forbiddenKeysAbsent: forbiddenEventKeys.every((key) => !generationEventKeys.includes(key)) }, steps: ["claim-evidence", "collect-evidence", "persist-evidence", "claim", "generate", "persist"], evidenceCounts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, final!.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])), snapshotDigest: digest(final.evidenceSnapshot), retryCount: final.retryCount, generationAttempt: final.generationAttempt, provider: final.modelMetadata?.provider, model: final.modelMetadata?.model, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, markdownPath, terminalEqualsExport: (await readFile(markdownPath, "utf8")) === rendered, devLogsMentionFunction: inngestLogs.join("").includes("founder-weekly-review-generation") })); + shutdownDrain.finalDiagnosticsCaptured = true; + } catch (error) { + printRecentDiagnostics("next", nextLogs); + printRecentDiagnostics("inngest", inngestLogs); + throw error; + } finally { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + process.off("uncaughtException", onUncaughtException); + process.off("unhandledRejection", onUnhandledRejection); + if (shutdownDrain.terminalStatus && shutdownDrain.finalDiagnosticsCaptured) { + await drainTerminalCallbackActivity(shutdownDrain); + } + await cleanup(); + await closeLogStream(nextRawLog); + await closeLogStream(inngestRawLog); + // testDb.close closes client connections and drops the schema only after callback processes and ports are handled. + await testDb.close(); + } +} + +await main(); diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts new file mode 100644 index 000000000..80b733641 --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -0,0 +1,49 @@ +import "dotenv/config"; +import { createHash, randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import { readFile, mkdir, rename, writeFile, access } from "node:fs/promises"; +import { resolve } from "node:path"; +import { eq } from "drizzle-orm"; +import { company, document, documentContextChunks, documentVersions, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; + +const require = createRequire(import.meta.url); +const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); +const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/realistic-company/seed.json"); + +type Fixture = { reportingPeriod: { start: string; end: string }; workspaceTimezone: string; founderContext: string; documents: Array<{ title: string; category: string; changelog: string; timestamp: string; chunks?: string[] }> }; +function canonicalize(value: unknown): unknown { if (value === null || ["string", "boolean"].includes(typeof value)) return value; if (typeof value === "number" && Number.isFinite(value)) return value; if (Array.isArray(value)) return value.map(canonicalize); if (typeof value === "object") return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalize((value as Record)[key])])); throw new Error("Cannot canonicalize snapshot."); } +function digest(value: unknown) { return createHash("sha256").update(JSON.stringify(canonicalize(value)), "utf8").digest("hex"); } +async function writeAtomic(path: string, body: string) { try { await access(path); throw new Error("Refusing to overwrite export."); } catch (error) { if (error instanceof Error && error.message.includes("overwrite")) throw error; } const temp = `${path}.${randomUUID()}.tmp`; await writeFile(temp, body, "utf8"); await rename(temp, path); } +if (process.env.SYNTHETIC_FWR_LOCAL !== "1" || process.env.NODE_ENV === "production") throw new Error("Refusing realistic E2E outside explicit local mode."); +const localUrl = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? ""; +if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(localUrl)) throw new Error("Refusing non-local database."); + +const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as Fixture; +const testDb = await createFounderWeeklyReviewTestDatabase(); +try { + const [target] = await testDb.db.insert(company).values({ name: "Northstar Analytics", numberOfEmployees: "24" }).returning(); + const [other] = await testDb.db.insert(company).values({ name: "Other Company", numberOfEmployees: "4" }).returning(); + for (const entry of fixture.documents) { const [doc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: `local://${entry.title}`, category: entry.category, title: entry.title }).returning(); const [version] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${entry.title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: entry.changelog, createdAt: new Date(entry.timestamp) }).returning(); for (const [index, content] of (entry.chunks ?? []).entries()) await testDb.db.insert(documentContextChunks).values({ documentId: BigInt(doc!.id), versionId: BigInt(version!.id), content, tokenCount: content.split(/\s+/).length, charCount: content.length, pageNumber: index + 1 }); } + const [outsideDoc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: "local://outside", category: "Product", title: "Outside period" }).returning(); await testDb.db.insert(documentVersions).values({ documentId: BigInt(outsideDoc!.id), versionNumber: 1, url: "local://outside/v1", mimeType: "text/plain", uploadedBy: "seed", changelog: "Outside-period control.", createdAt: new Date("2026-03-01T00:00:00.000Z") }); + const [otherDoc] = await testDb.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://other", category: "Product", title: "Other company control" }).returning(); await testDb.db.insert(documentVersions).values({ documentId: BigInt(otherDoc!.id), versionNumber: 1, url: "local://other/v1", mimeType: "text/plain", uploadedBy: "seed", changelog: "Cross-company control.", createdAt: new Date("2026-02-21T00:00:00.000Z") }); + const actor = { externalUserId: "realistic-owner", internalUserId: 1n, companyId: BigInt(target!.id), role: "owner" }; + const dispatchService = createFounderWeeklyReviewDispatchService(testDb.db); + const created = await dispatchService.createRunWithDispatch({ actor, requestKey: `realistic-${randomUUID()}`, reportingPeriod: fixture.reportingPeriod, collectionInput: { workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actorExternalUserId: actor.externalUserId } }); + if (created.run.evidenceSnapshot) throw new Error("Workflow run unexpectedly has an initial snapshot."); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); const collectionContext = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: created.dispatch.generationClaimId }; + const collecting = await worker.claimEvidenceCollection(collectionContext); + const collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-01T00:00:00.000Z")); + const snapshot = await collector.collectFounderWeeklyReviewEvidence({ companyId: actor.companyId, reportingPeriod: fixture.reportingPeriod, workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: created.run.requestKey }); + const beforeDigest = digest(snapshot); const attached = await worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot); const afterDigest = digest(attached.evidenceSnapshot); + const counts = Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, attached.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])); + if (attached.status !== "queued" || !attached.evidenceSnapshot || beforeDigest !== afterDigest || !counts.document_change || !counts.customer_feedback || counts.founder_context !== 1 || attached.evidenceSnapshot.items.some((item) => item.title === "Outside period" || item.title === "Other company control") || new Set(attached.evidenceSnapshot.items.map((item) => item.sourceId)).size !== attached.evidenceSnapshot.items.length) throw new Error("Realistic collector assertions failed."); + const generationContext = { companyId: actor.companyId, runId: attached.id, generationJobId: created.dispatch.generationJobId, generationClaimId: created.dispatch.generationClaimId }; const generating = await worker.claimQueuedRun(generationContext); if (!generating.evidenceSnapshot) throw new Error("Generation began without snapshot."); + const generated = await generateFounderWeeklyReview({ evidenceSnapshot: generating.evidenceSnapshot, generate: generateFounderWeeklyReviewStructured }); const saved = await worker.saveGeneratedDraft(generationContext, generated.reviewPayload, generated.modelMetadata); const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft" || !readBack.evidenceSnapshot) throw new Error("Validated draft read-back failed."); + const rendered = renderFounderWeeklyReviewMarkdown(readBack); let markdownPath: string | null = null; let jsonPath: string | null = null; if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const directory = resolve(process.cwd(), process.env.SYNTHETIC_FWR_EXPORT_DIR ?? ".artifacts/founder-weekly-review"); await mkdir(directory, { recursive: true }); markdownPath = resolve(directory, `${saved.id}.md`); jsonPath = resolve(directory, `${saved.id}.json`); await writeAtomic(markdownPath, rendered); await writeAtomic(jsonPath, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, periodStart: readBack.reportingPeriod.start, periodEnd: readBack.reportingPeriod.end, review: readBack.reviewPayload }, null, 2)); } + if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); console.log("===== END FOUNDER WEEKLY REVIEW ====="); } + const dispatchRows = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, saved.id)); const runRows = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); console.log(JSON.stringify({ runId: saved.id, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: counts, warningCodes: attached.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, dispatchCount: dispatchRows.length, runRowCount: runRows.length, markdownPath, jsonPath })); +} finally { await testDb.close(); } diff --git a/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts b/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts index 79e668cb3..5f182d466 100644 --- a/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts +++ b/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts @@ -9,6 +9,7 @@ import { eq } from "drizzle-orm"; import { company, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; import { FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, FounderWeeklyReviewV2PayloadSchema, generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; import { syntheticFounderWeeklyReviewFixtures } from "./founder-weekly-review-synthetic-fixtures"; const require = createRequire(import.meta.url); @@ -88,7 +89,7 @@ async function generateWithKimi(input: { const base = (process.env.MOONSHOT_BASE_URL ?? "https://api.moonshot.ai/v1").replace(/\/$/, ""); const schemaGuide = JSON.stringify(zodSchema(input.schema).jsonSchema); console.log(JSON.stringify({ stage: "kimi_request_constructed", result: "pass" })); - const response = await fetch(`${base}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${process.env.MOONSHOT_API_KEY}` }, body: JSON.stringify({ model: process.env.SYNTHETIC_FWR_MODEL ?? "kimi-k2.6", messages: [{ role: "system", content: `${input.system ?? ""}\nReturn one JSON object only. Its complete required structural schema is: ${schemaGuide}` }, { role: "user", content: input.prompt }], stream: false, thinking: { type: "disabled" }, response_format: { type: "json_object" } }), signal: AbortSignal.timeout(45_000) }); + const response = await fetch(`${base}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${process.env.MOONSHOT_API_KEY}` }, body: JSON.stringify({ model: process.env.SYNTHETIC_FWR_MODEL ?? "kimi-k2.6", max_tokens: 1800, messages: [{ role: "system", content: `${input.system ?? ""}\nReturn one JSON object only. Its complete required structural schema is: ${schemaGuide}` }, { role: "user", content: input.prompt }], stream: false, thinking: { type: "disabled" }, response_format: { type: "json_object" } }), signal: AbortSignal.timeout(90_000) }); console.log(JSON.stringify({ stage: "kimi_http_response", result: response.ok ? "pass" : "fail", httpStatus: response.status })); if (!response.ok) throw new Error(`kimi_http_error:${response.status}`); const body = await response.json() as { model?: string; usage?: Record; choices?: Array<{ message?: { content?: string }; finish_reason?: string }> }; @@ -123,6 +124,7 @@ try { const payload = structuredClone(validPayload); if (isNegativeUnknownCitation || isRetrySnapshotImmutability) payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "Release evidence.", sourceIds: ["synthetic:missing:unknown-citation"], confidence: 0.8 }] }; if (isNegativeFounderContextCustomer) payload.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "Customer signal.", sourceIds: ["synthetic:context:priority"], confidence: 0.8 }] }; + if (!claimed.evidenceSnapshot) throw new Error("Synthetic baseline requires a persisted evidence snapshot."); generated = await generateFounderWeeklyReview({ evidenceSnapshot: claimed.evidenceSnapshot, generate: (isDeterministicNegative || isRetrySnapshotImmutability) ? async () => ({ object: payload, metadata: { provider: "synthetic", model: "deterministic", capability: "founderWeeklyReview", temperature: 0 } }) : syntheticProvider === "kimi" ? generateWithKimi : generateWithLocalOllama }); } catch (error) { const message = error instanceof Error ? error.message : "unknown_generation_failure"; @@ -178,14 +180,25 @@ try { const customerSection = saved.reviewPayload!.sections.whatCustomersSaid; const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(3n, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft") throw new Error("Validated draft read-back failed."); - if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { + const shouldExport = process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1"; + const shouldPrint = process.env.FWR_PRINT_REPORT === "1"; + if (shouldExport || shouldPrint) { + if (!readBack.evidenceSnapshot) throw new Error("Validated draft has no evidence snapshot."); + const renderedMarkdown = renderFounderWeeklyReviewMarkdown(readBack as typeof readBack & { evidenceSnapshot: NonNullable }); + if (shouldPrint) { + console.log("===== FOUNDER WEEKLY REVIEW ====="); + console.log(renderedMarkdown); + console.log("===== END FOUNDER WEEKLY REVIEW ====="); + } + if (shouldExport) { const directory = resolve(process.cwd(), process.env.SYNTHETIC_FWR_EXPORT_DIR ?? ".artifacts/founder-weekly-review"); await mkdir(directory, { recursive: true }); const fileId = saved.id.replace(/[^A-Za-z0-9_-]/g, "_"); - const envelope = { runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, periodStart: readBack.reportingPeriod.start, periodEnd: readBack.reportingPeriod.end, generatedAt: readBack.generatedAt?.toISOString() ?? null, review: readBack.reviewPayload }; + const envelope = { runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, periodStart: readBack.reportingPeriod.start, periodEnd: readBack.reportingPeriod.end, review: readBack.reviewPayload }; const markdownPath = resolve(directory, `${fileId}.md`); const jsonPath = resolve(directory, `${fileId}.json`); - await writeAtomically(markdownPath, markdownFor(readBack)); await writeAtomically(jsonPath, JSON.stringify(envelope, null, 2)); + await writeAtomically(markdownPath, renderedMarkdown); await writeAtomically(jsonPath, JSON.stringify(envelope, null, 2)); console.log(JSON.stringify({ runId: readBack.id, status: readBack.status, markdownPath, jsonPath, filesWritten: true })); + } } console.log(JSON.stringify({ label: "Synthetic-evidence integration baseline", fixture: fixtureName, runId: saved.id, lifecycle: [run.status, claimed.status, saved.status], event, provider: generated!.modelMetadata.provider, model: generated!.modelMetadata.model, snapshotUnchanged: digestJson(persisted.evidenceSnapshot) === digestJson(snapshot), outbox: { status: persistedDispatch.status, hasEvidence: false }, customerSection: "state" in customerSection ? customerSection.state : "legacy", draftReturnedByRepository: saved.status === "draft" })); } diff --git a/apps/web/src/app/api/founder-weekly-reviews/route.ts b/apps/web/src/app/api/founder-weekly-reviews/route.ts index e9c16c9dd..580929f5e 100644 --- a/apps/web/src/app/api/founder-weekly-reviews/route.ts +++ b/apps/web/src/app/api/founder-weekly-reviews/route.ts @@ -3,19 +3,19 @@ import { auth } from "@clerk/nextjs/server"; import { z } from "zod"; import { FounderWeeklyReviewRepository } from "@launchstack/features/founder-weekly-review"; import type { FounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; -import { canonicalFounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; import { createRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; import { inngest } from "~/server/inngest/client"; -import { founderWeeklyReviewRunsCreated, founderWeeklyReviewStageDuration, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; +import { founderWeeklyReviewRunsCreated, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; const CreateSchema = z.object({ requestKey: z.string().min(1).max(128), reportingPeriod: z.object({ start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), end: z.string().regex(/^\d{4}-\d{2}-\d{2}$/) }), workspaceTimezone: z.string().min(1).max(128), founderContext: z.string().max(4000).optional() }); export interface FounderWeeklyReviewRouteDependencies { actorResolver: Pick; - evidenceCollector: FounderWeeklyReviewEvidenceCollector; + /** Retained only for test construction compatibility; collection is worker-owned. */ + evidenceCollector?: FounderWeeklyReviewEvidenceCollector; repository: Pick; createRunWithDispatch: typeof createRunWithDispatch; sendDispatchRequested: () => Promise; @@ -27,27 +27,20 @@ export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRo const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); try { - // Authorization is deliberately before parsing/collection: invalid or - // unauthorized callers must never cause workspace evidence reads. + // Authorization precedes all workflow persistence: invalid callers must + // never create a company-scoped collection job. const actor = await deps.actorResolver.resolve(userId); const parsed = CreateSchema.safeParse(await request.json().catch(() => null)); if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); const existing = await deps.repository.getByCompanyAndRequestKey(actor.companyId, parsed.data.requestKey); if (existing) return NextResponse.json({ run: safeRun(existing) }, { status: 202 }); - const startedAt = performance.now(); - logFounderWeeklyReview({ runId: "pending", companyId: actor.companyId.toString(), stage: "evidence_collection_started", status: "pending" }); - const evidenceSnapshot = await deps.evidenceCollector.collectFounderWeeklyReviewEvidence({ companyId: actor.companyId, reportingPeriod: parsed.data.reportingPeriod, workspaceTimezone: parsed.data.workspaceTimezone, founderContext: parsed.data.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: parsed.data.requestKey }); - const durationMs = Math.round(performance.now() - startedAt); - founderWeeklyReviewStageDuration.observe({ stage: "evidence_collection", result: "success" }, durationMs / 1000); - logFounderWeeklyReview({ runId: "pending", companyId: actor.companyId.toString(), stage: "evidence_collection_completed", status: "pending", durationMs }); - const { run, created } = await deps.createRunWithDispatch({ actor, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, evidenceSnapshot }); + const { run, created } = await deps.createRunWithDispatch({ actor, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, collectionInput: { workspaceTimezone: parsed.data.workspaceTimezone, founderContext: parsed.data.founderContext, actorExternalUserId: actor.externalUserId } }); if (created) deps.recordRunCreated(); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "run_created", status: run.status, retryCount: run.retryCount }); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "dispatch_created", status: run.status }); await deps.sendDispatchRequested(); return NextResponse.json({ run: safeRun(run) }, { status: 202 }); } catch (error) { - founderWeeklyReviewStageDuration.observe({ stage: "evidence_collection", result: "failure" }, 0); return safeFounderWeeklyReviewError(error); } }; @@ -55,7 +48,6 @@ export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRo const productionDependencies: FounderWeeklyReviewRouteDependencies = { actorResolver: productionFounderWeeklyReviewActorResolver, - evidenceCollector: canonicalFounderWeeklyReviewEvidenceCollector, repository: { getByCompanyAndRequestKey: (companyId, requestKey) => new FounderWeeklyReviewRepository().getByCompanyAndRequestKey(companyId, requestKey) }, createRunWithDispatch, sendDispatchRequested: () => inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }), diff --git a/apps/web/src/lib/llm/generate.ts b/apps/web/src/lib/llm/generate.ts index 09ce3f22b..ea30da15f 100644 --- a/apps/web/src/lib/llm/generate.ts +++ b/apps/web/src/lib/llm/generate.ts @@ -11,7 +11,7 @@ * as call sites need them. */ -import { generateObject } from "ai"; +import { generateObject, zodSchema } from "ai"; import type { ZodType } from "zod"; import { resolveModel } from "./providers"; @@ -69,22 +69,39 @@ export async function generateStructuredWithMetadata( `provider=${resolved.provider} model=${resolved.modelId} ` + `prompt=${promptChars} chars`, ); + if (input.capability === "founderWeeklyReview") { + console.log(`[llm] FWR generation phase=${input.generationPhase ?? "initial"} provider=${resolved.provider} model=${resolved.modelId} protocol=${resolved.structuredOutputMode === "json_object" ? "chat-completions" : "responses"}`); + } try { - const result = await generateObject({ + const common = { model: resolved.model, - temperature: resolved.temperature, - schema: input.schema, - schemaName: input.schemaName, - system: input.system, + ...(resolved.temperature === undefined ? {} : { temperature: resolved.temperature }), + ...(input.capability === "founderWeeklyReview" ? { maxOutputTokens: 1800 } : {}), + ...(resolved.structuredOutputMode === "json_object" ? { abortSignal: AbortSignal.timeout(90_000) } : {}), prompt: input.prompt, - }); + }; + // Moonshot/Kimi supports Chat Completions JSON-object mode, not the + // Responses API JSON-schema protocol. Keep schema validation local and + // deterministic after parsing the provider's JSON object. + const result = resolved.structuredOutputMode === "json_object" + ? await generateObject({ + ...common, + output: "no-schema", + system: `${input.system ?? ""}\nReturn one JSON object only. Required structural schema: ${JSON.stringify(zodSchema(input.schema).jsonSchema)}`, + }) + : await generateObject({ + ...common, + schema: input.schema, + schemaName: input.schemaName, + system: input.system, + }); const elapsed = Date.now() - startedAt; console.log( `[llm] generateStructured ok capability=${input.capability} ` + - `provider=${resolved.provider} model=${resolved.modelId} ` + - `${elapsed}ms`, + `provider=${resolved.provider} model=${resolved.modelId} ` + + `phase=${input.generationPhase ?? "initial"} ${elapsed}ms`, ); // Cast is safe: `generateObject` returns `{ object: z.infer }` @@ -97,12 +114,14 @@ export async function generateStructuredWithMetadata( response?: { id?: string }; }; return { - object: result.object as ReturnType, + object: resolved.structuredOutputMode === "json_object" + ? input.schema.parse(result.object) as ReturnType + : result.object as ReturnType, metadata: { provider: resolved.provider, model: resolved.modelId, capability: input.capability, - temperature: resolved.temperature, + ...(resolved.temperature === undefined ? {} : { temperature: resolved.temperature }), ...(responseResult.finishReason ? { finishReason: responseResult.finishReason } : {}), ...(responseResult.usage ? { usage: responseResult.usage } : {}), ...(responseResult.response?.id @@ -114,8 +133,8 @@ export async function generateStructuredWithMetadata( const elapsed = Date.now() - startedAt; console.error( `[llm] generateStructured FAIL capability=${input.capability} ` + - `provider=${resolved.provider} model=${resolved.modelId} ` + - `${elapsed}ms err=${err instanceof Error ? err.message : String(err)}`, + `provider=${resolved.provider} model=${resolved.modelId} ` + + `phase=${input.generationPhase ?? "initial"} ${elapsed}ms err=${err instanceof Error ? err.message : String(err)}`, ); throw err; } diff --git a/apps/web/src/lib/llm/providers.ts b/apps/web/src/lib/llm/providers.ts index b17c870d6..09a589830 100644 --- a/apps/web/src/lib/llm/providers.ts +++ b/apps/web/src/lib/llm/providers.ts @@ -105,6 +105,11 @@ export function getAvailableProviders(): ProviderAvailability[] { /** Reset cached availability. Test-only. */ export function __resetProviderCacheForTests(): void { cachedAvailability = null; + openaiInstance = null; + anthropicInstance = null; + googleInstance = null; + ollamaInstance = null; + kimiInstance = null; } /** @@ -123,6 +128,10 @@ function checkCredentials( return process.env.OPENAI_API_KEY ? { ok: true } : { ok: false, reason: "OPENAI_API_KEY not set" }; + case "kimi": + return process.env.MOONSHOT_API_KEY + ? { ok: true } + : { ok: false, reason: "MOONSHOT_API_KEY not set" }; case "anthropic": return process.env.ANTHROPIC_API_KEY ? { ok: true } @@ -149,6 +158,7 @@ let openaiInstance: OpenAIProvider | null = null; let anthropicInstance: AnthropicProvider | null = null; let googleInstance: GoogleGenerativeAIProvider | null = null; let ollamaInstance: OpenAIProvider | null = null; +let kimiInstance: OpenAIProvider | null = null; function getOpenAIProvider(): OpenAIProvider { openaiInstance ??= createOpenAI({ @@ -157,6 +167,36 @@ function getOpenAIProvider(): OpenAIProvider { return openaiInstance; } +/** + * Moonshot's OpenAI-compatible API accepts a top-level `thinking` field that + * the OpenAI adapter's typed provider options intentionally do not expose. + * Keep this narrow to Kimi Chat Completions; OpenAI requests never use it. + */ +async function fetchKimiChatCompletions( + input: Parameters[0], + init?: Parameters[1], +): Promise { + const url = input instanceof Request ? input.url : String(input); + if (!url.endsWith("/chat/completions") || typeof init?.body !== "string") { + return fetch(input, init); + } + + const body = JSON.parse(init.body) as Record; + return fetch(input, { + ...init, + body: JSON.stringify({ ...body, thinking: { type: "disabled" } }), + }); +} + +function getKimiProvider(): OpenAIProvider { + kimiInstance ??= createOpenAI({ + apiKey: process.env.MOONSHOT_API_KEY, + baseURL: (process.env.MOONSHOT_BASE_URL ?? "https://api.moonshot.ai/v1").replace(/\/+$/, ""), + fetch: fetchKimiChatCompletions, + }); + return kimiInstance; +} + function getAnthropicProvider(): AnthropicProvider { anthropicInstance ??= createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY, @@ -238,8 +278,32 @@ export interface ResolvedModel { provider: Provider; modelId: string; model: LanguageModel; - /** Carries through the temperature from config so `generate.ts` can pass it. */ - temperature: number; + /** + * Provider-supported sampling option. Undefined means the request must omit + * temperature rather than applying a cross-provider default. + * TODO: add provider-specific request capabilities as more providers need them. + */ + temperature?: number; + structuredOutputMode?: "json_object"; +} + +export type FounderWeeklyReviewGenerationProvider = "openai" | "kimi"; + +function resolveFounderWeeklyReviewModel(): ResolvedModel { + const selected = process.env.FWR_GENERATION_PROVIDER ?? "openai"; + if (selected !== "openai" && selected !== "kimi") { + throw new Error(`FWR_GENERATION_PROVIDER must be "openai" or "kimi"; received "${selected}".`); + } + const config = getLlmConfig(); + if (selected === "openai") { + if (!process.env.OPENAI_API_KEY) throw new Error("FWR_GENERATION_PROVIDER=openai requires OPENAI_API_KEY."); + const modelId = process.env.OPENAI_MODEL_ID ?? config.capabilities.founderWeeklyReview.openai?.model; + if (!modelId) throw new Error("FWR_GENERATION_PROVIDER=openai requires OPENAI_MODEL_ID or a configured founderWeeklyReview OpenAI model."); + return { provider: "openai", modelId, model: getOpenAIProvider()(modelId), temperature: config.capabilities.founderWeeklyReview.openai?.temperature ?? 0 }; + } + if (!process.env.MOONSHOT_API_KEY) throw new Error("FWR_GENERATION_PROVIDER=kimi requires MOONSHOT_API_KEY."); + const modelId = process.env.KIMI_MODEL_ID ?? "kimi-k2.6"; + return { provider: "kimi", modelId, model: getKimiProvider().chat(modelId), structuredOutputMode: "json_object" }; } /** @@ -258,6 +322,7 @@ export function resolveModel( capability: Capability, forceProvider?: Provider, ): ResolvedModel { + if (capability === "founderWeeklyReview" && !forceProvider) return resolveFounderWeeklyReviewModel(); const config = getLlmConfig(); const availability = getAvailableProviders(); const availabilityByProvider = new Map( @@ -324,6 +389,8 @@ function instantiate( model: getOpenAIProvider()(modelConfig.model), temperature, }; + case "kimi": + return { provider, modelId: modelConfig.model, model: getKimiProvider().chat(modelConfig.model), structuredOutputMode: "json_object" }; case "anthropic": return { provider, diff --git a/apps/web/src/lib/llm/types.ts b/apps/web/src/lib/llm/types.ts index 15f7efae3..c8e7fadb0 100644 --- a/apps/web/src/lib/llm/types.ts +++ b/apps/web/src/lib/llm/types.ts @@ -35,7 +35,7 @@ export type Capability = (typeof CAPABILITIES)[number]; * - `ollama`: local inference via Ollama's OpenAI-compatible endpoint. * Requires `OLLAMA_BASE_URL` to be set. Free but quality varies by model. */ -export const PROVIDERS = ["openai", "anthropic", "google", "ollama"] as const; +export const PROVIDERS = ["openai", "kimi", "anthropic", "google", "ollama"] as const; export type Provider = (typeof PROVIDERS)[number]; @@ -104,6 +104,8 @@ export interface GenerateStructuredInput { * / tool name. Purely cosmetic but helps with provider-side logging. */ schemaName?: string; + /** Bounded FWR-only attempt label for safe operational logging. */ + generationPhase?: "initial" | "semantic-repair"; } /** Resolved model and provider response details that are safe to persist for replay. */ @@ -111,7 +113,7 @@ export interface StructuredGenerationMetadata { provider: Provider; model: string; capability: Capability; - temperature: number; + temperature?: number; finishReason?: string; usage?: Record; providerRequestId?: string; diff --git a/apps/web/src/server/founder-weekly-review/dispatch-service.ts b/apps/web/src/server/founder-weekly-review/dispatch-service.ts index 8e9645940..0fa5edcb2 100644 --- a/apps/web/src/server/founder-weekly-review/dispatch-service.ts +++ b/apps/web/src/server/founder-weekly-review/dispatch-service.ts @@ -75,7 +75,8 @@ export type CreateRunWithDispatchInput = { actor: FounderWeeklyReviewUserActor; requestKey: string; reportingPeriod: ReportingPeriod; - evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + evidenceSnapshot?: FounderWeeklyReviewEvidenceSnapshot; + collectionInput?: import("@launchstack/features/founder-weekly-review").FounderWeeklyReviewCollectionInput; }; export type RetryRunWithDispatchInput = { actor: FounderWeeklyReviewUserActor; runId: string; requestKey: string }; export type CreateRunWithDispatchResult = { run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch; created: boolean }; diff --git a/apps/web/src/server/founder-weekly-review/generation-adapter.ts b/apps/web/src/server/founder-weekly-review/generation-adapter.ts index bd032f5d4..7cea36e43 100644 --- a/apps/web/src/server/founder-weekly-review/generation-adapter.ts +++ b/apps/web/src/server/founder-weekly-review/generation-adapter.ts @@ -14,6 +14,7 @@ export function generateFounderWeeklyReviewStructured(i prompt: string; schema: TSchema; schemaName?: string; + generationPhase?: "initial" | "semantic-repair"; }): Promise<{ object: ReturnType; metadata: StructuredGenerationMetadata; diff --git a/apps/web/src/server/founder-weekly-review/markdown.ts b/apps/web/src/server/founder-weekly-review/markdown.ts new file mode 100644 index 000000000..8666adab9 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/markdown.ts @@ -0,0 +1,85 @@ +import type { + FounderWeeklyReviewEvidenceItem, +} from "@launchstack/features/founder-weekly-review"; + +type RenderableRun = { + reportingPeriod: { start: string; end: string }; + modelMetadata: { model?: string } | null; + reviewPayload: any; + evidenceSnapshot: { items: FounderWeeklyReviewEvidenceItem[] } | null; +}; + +const SECTION_ORDER = ["whatShipped", "whatChanged", "whatCustomersSaid", "currentBlockers", "nextPriorities"] as const; +const SECTION_LABELS: Record<(typeof SECTION_ORDER)[number], string> = { + whatShipped: "Shipped This Period", + whatChanged: "Other Meaningful Changes", + whatCustomersSaid: "Customer Signals", + currentBlockers: "Risks & Blockers", + nextPriorities: "Priorities for the Next Period", +}; +const REFERENCE_SEPARATOR = String.fromCharCode(0x2014); + +function metadataNumber(item: FounderWeeklyReviewEvidenceItem, key: string): string | null { + const value = item.metadata[key]; + return typeof value === "number" || typeof value === "string" ? String(value) : null; +} + +export function formatFounderWeeklyReviewEvidenceReference(item: FounderWeeklyReviewEvidenceItem): string { + if (item.sourceType === "founder_context") return "Founder-provided context"; + const details: string[] = []; + const page = metadataNumber(item, "pageNumber"); + const section = metadataNumber(item, "sectionId"); + if (page) details.push(`page ${page}`); + if (section) details.push(`section ${section}`); + const title = item.title.trim() || "Untitled evidence"; + const prefix = item.sourceType === "document_change" ? `Document change ${REFERENCE_SEPARATOR} ` : ""; + return [prefix + title, ...details].join(` ${REFERENCE_SEPARATOR} `); +} + +/** Renders only a persisted, validated draft and intentionally excludes operational internals. */ +export function renderFounderWeeklyReviewMarkdown(run: RenderableRun): string { + if (!run.reviewPayload || !run.evidenceSnapshot) { + throw new Error("A persisted review payload and evidence snapshot are required for Markdown rendering."); + } + const sources = new Map(run.evidenceSnapshot.items.map((item) => [item.sourceId, item])); + const referenceNumbers = new Map(); + const references: string[] = []; + const cite = (sourceIds: readonly string[]) => sourceIds.map((sourceId) => { + let number = referenceNumbers.get(sourceId); + if (!number) { + number = referenceNumbers.size + 1; + referenceNumbers.set(sourceId, number); + const source = sources.get(sourceId); + if (source) references.push(`[${number}] ${formatFounderWeeklyReviewEvidenceReference(source)}`); + } + return `[${number}]`; + }).join(""); + const shipped = new Set(run.reviewPayload.sections.whatShipped.state === "evidence" + ? run.reviewPayload.sections.whatShipped.items.map((item: any) => item.text.trim().replace(/\s+/g, " ")) + : []); + const lines = ["# Founder Weekly Review", "", `**Reporting period:** ${run.reportingPeriod.start} to ${run.reportingPeriod.end}`, ""]; + for (const key of SECTION_ORDER) { + const section = run.reviewPayload.sections[key]; + const items = section.state === "evidence" + ? section.items.filter((item: any) => key !== "whatChanged" || !shipped.has(item.text.trim().replace(/\s+/g, " "))) + : []; + const noEvidence = section.state === "no_evidence" ? section.noEvidence : null; + if (!items.length && !noEvidence) continue; + lines.push(`## ${SECTION_LABELS[key]}`, ""); + if (noEvidence) { + lines.push(noEvidence.message, "", `Next: ${noEvidence.cta}`, ""); + continue; + } + items.forEach((item: any, index: number) => { + const marker = key === "nextPriorities" ? `${index + 1}.` : "-"; + lines.push(`${marker} ${item.text}${cite(item.sourceIds)}`); + if (item.kind === "recommendation" && item.rationale) { + lines.push(` - Why now: ${item.rationale}${cite(item.sourceIds)}`); + } + }); + lines.push(""); + } + if (references.length) lines.push("## Evidence References", "", ...references, ""); + lines.push("---", "", `*Generated with ${run.modelMetadata?.model ?? "the configured model"}*`); + return lines.join("\n"); +} diff --git a/apps/web/src/server/inngest/functions/founderWeeklyReview.ts b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts index 81300cfcb..ebe49ebd8 100644 --- a/apps/web/src/server/inngest/functions/founderWeeklyReview.ts +++ b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts @@ -1,9 +1,11 @@ import { z } from "zod"; +import { NonRetriableError } from "inngest"; import { inngest } from "../client"; import { FounderWeeklyReviewWorkerService, type FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; import { generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; import { FounderWeeklyReviewGenerationValidationError } from "@launchstack/features/founder-weekly-review"; import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { canonicalFounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; import { claimPendingDispatches, markDispatchDispatched, returnDispatchToPending } from "~/server/founder-weekly-review/dispatch-service"; import { founderWeeklyReviewCitationFailures, founderWeeklyReviewDispatchFailures, founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRunsCompleted, founderWeeklyReviewRunsFailed, founderWeeklyReviewStageDuration, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; @@ -46,23 +48,63 @@ export const founderWeeklyReviewGenerationJob = inngest.createFunction( onFailure: async ({ event }) => { const parsed = GenerationEventSchema.safeParse(event.data); if (!parsed.success) return; const worker = new FounderWeeklyReviewWorkerService(); - await worker.markGenerationFailed({ companyId: BigInt(parsed.data.companyId), runId: parsed.data.runId, generationJobId: parsed.data.generationJobId, generationClaimId: parsed.data.generationClaimId }, { errorCode: "generation_failed", errorMessage: "Generation failed after retries." }).catch(() => undefined); + const companyId = BigInt(parsed.data.companyId); + const current = await worker.getRun(companyId, parsed.data.runId).catch(() => null); + if (current?.status === "collecting" && !current.evidenceSnapshot) { + await worker.markCollectionFailed({ companyId, runId: parsed.data.runId, collectionClaimId: parsed.data.generationClaimId }, { errorCode: "evidence_collection_failed", errorMessage: "Evidence collection failed after retries." }).catch(() => undefined); + } else { + await worker.markGenerationFailed({ companyId, runId: parsed.data.runId, generationJobId: parsed.data.generationJobId, generationClaimId: parsed.data.generationClaimId }, { errorCode: "generation_failed", errorMessage: "Generation failed after retries." }).catch(() => undefined); + } } }, { event: "founder-weekly-review/generation.requested" }, async ({ event, step }) => { const data = GenerationEventSchema.parse(event.data); const context = { companyId: BigInt(data.companyId), runId: data.runId, generationJobId: data.generationJobId, generationClaimId: data.generationClaimId }; const worker = new FounderWeeklyReviewWorkerService(); + let workflowRun = await worker.getRun(context.companyId, context.runId); + if (workflowRun.status === "draft" || workflowRun.status === "published") return { skipped: true }; + if (!workflowRun.evidenceSnapshot) { + const collectionContext = { companyId: context.companyId, runId: context.runId, collectionClaimId: data.generationClaimId }; + try { + // Step results are JSON transport values; a run record contains bigint + // IDs, so read the canonical record back rather than consuming a + // lossy serialized step result in the callback. + await step.run("claim-evidence", () => worker.claimEvidenceCollection(collectionContext)); + const collecting = await worker.getRun(context.companyId, context.runId); + if (!collecting.evidenceSnapshot && collecting.status === "collecting" && collecting.collectionClaimId === collectionContext.collectionClaimId) { + logFounderWeeklyReview({ runId: collecting.id, companyId: collecting.companyId.toString(), stage: "evidence_collection_started", status: collecting.status, retryCount: collecting.retryCount }); + const snapshot = await step.run("collect-evidence", () => canonicalFounderWeeklyReviewEvidenceCollector.collectFounderWeeklyReviewEvidence({ + companyId: collecting.companyId, + reportingPeriod: collecting.reportingPeriod, + workspaceTimezone: collecting.collectionInput?.workspaceTimezone ?? "UTC", + founderContext: collecting.collectionInput?.founderContext, + actor: { externalUserId: collecting.collectionInput?.actorExternalUserId ?? "unknown" }, + requestKey: collecting.requestKey, + })); + await step.run("persist-evidence", () => worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot)); + workflowRun = await worker.getRun(context.companyId, context.runId); + logFounderWeeklyReview({ runId: workflowRun.id, companyId: workflowRun.companyId.toString(), stage: "evidence_collection_completed", status: workflowRun.status, retryCount: workflowRun.retryCount }); + } else workflowRun = collecting; + } catch (error) { + await worker.markCollectionFailed(collectionContext, { errorCode: "evidence_collection_failed", errorMessage: "Evidence collection failed after retries." }).catch(() => undefined); + throw error; + } + } + if (!workflowRun.evidenceSnapshot) return { skipped: true }; // Inngest's generic step inference intersects event return types; retain the // concrete LAU-5 lifecycle record at this boundary. - const claimed = await step.run("claim", () => worker.claimQueuedRun(context)) as unknown as FounderWeeklyReviewRunRecord; + await step.run("claim", () => worker.claimQueuedRun(context)); + const claimed = await worker.getRun(context.companyId, context.runId); if (claimed.status !== "generating" || claimed.generationClaimId !== data.generationClaimId) return { skipped: true }; logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: "worker_claimed", status: claimed.status, generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount }); try { const generationStartedAt = performance.now(); logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: "generation_started", status: claimed.status, generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount }); - const generated = await step.run("generate", () => generateFounderWeeklyReview({ evidenceSnapshot: claimed.evidenceSnapshot, generate: generateFounderWeeklyReviewStructured })); - const saved = await step.run("persist", () => worker.saveGeneratedDraft(context, generated.reviewPayload, generated.modelMetadata)) as unknown as FounderWeeklyReviewRunRecord; + const evidenceSnapshot = claimed.evidenceSnapshot; + if (!evidenceSnapshot) throw new Error("Evidence snapshot is required before generation."); + const generated = await step.run("generate", () => generateFounderWeeklyReview({ evidenceSnapshot, generate: generateFounderWeeklyReviewStructured })); + await step.run("persist", () => worker.saveGeneratedDraft(context, generated.reviewPayload, generated.modelMetadata)); + const saved = await worker.getRun(context.companyId, context.runId); founderWeeklyReviewGenerationTotal.inc({ result: "success", error_class: "none" }); founderWeeklyReviewRunsCompleted.inc(); founderWeeklyReviewStageDuration.observe({ stage: "generation", result: "success" }, (performance.now() - generationStartedAt) / 1000); @@ -74,6 +116,9 @@ export const founderWeeklyReviewGenerationJob = inngest.createFunction( founderWeeklyReviewGenerationTotal.inc({ result: "failure", error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); founderWeeklyReviewRunsFailed.inc({ error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation_failed" : "generation_failed", status: "generating", generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount, errorClass: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + if (error instanceof FounderWeeklyReviewGenerationValidationError) { + throw new NonRetriableError("Founder Weekly Review validation failed after bounded semantic repair."); + } throw error; } } diff --git a/apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json b/apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json new file mode 100644 index 000000000..824ef0ff1 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json @@ -0,0 +1,10 @@ +{ + "reportingPeriod": { "start": "2026-02-16", "end": "2026-02-28" }, + "workspaceTimezone": "UTC", + "founderContext": "Prioritize onboarding reliability and follow up on saved filter adoption.", + "documents": [ + { "title": "Saved export filters release", "category": "Product", "changelog": "Released saved export filters so teams can reuse recurring reporting views.", "timestamp": "2026-02-20T10:00:00.000Z" }, + { "title": "Onboarding reliability plan", "category": "Planning", "changelog": "Documented retry telemetry and ownership for onboarding reliability work.", "timestamp": "2026-02-24T11:00:00.000Z" }, + { "title": "Customer Interviews — February 2026", "category": "Customer Feedback", "changelog": "Processed customer feedback for the reporting period.", "timestamp": "2026-02-22T12:00:00.000Z", "chunks": ["Saved export filters would remove repetitive setup from our weekly reporting.", "Onboarding occasionally retries before completing and needs clearer recovery feedback.", "The recently released approval workflow reduced our manual follow-up work."] } + ] +} diff --git a/packages/core/src/db/schema/founder-weekly-review.ts b/packages/core/src/db/schema/founder-weekly-review.ts index a99c368bb..3fd90e187 100644 --- a/packages/core/src/db/schema/founder-weekly-review.ts +++ b/packages/core/src/db/schema/founder-weekly-review.ts @@ -16,6 +16,7 @@ import { pgTable } from "./helpers"; export const founderWeeklyReviewRunStatusEnum = [ "queued", + "collecting", "generating", "draft", "published", @@ -47,9 +48,12 @@ export const founderWeeklyReviewRuns = pgTable( reviewPayload: jsonb("review_payload").$type | null>(), reviewSchemaVersion: varchar("review_schema_version", { length: 64 }).notNull(), evidenceSnapshot: jsonb("evidence_snapshot") - .$type>() - .notNull(), + .$type | null>(), evidenceSchemaVersion: varchar("evidence_schema_version", { length: 64 }).notNull(), + collectionInput: jsonb("collection_input").$type>().notNull(), + collectionClaimId: varchar("collection_claim_id", { length: 128 }), + collectionStartedAt: timestamp("collection_started_at", { withTimezone: true }), + evidenceCollectedAt: timestamp("evidence_collected_at", { withTimezone: true }), modelMetadata: jsonb("model_metadata").$type | null>(), createdByActorId: varchar("created_by_actor_id", { length: 256 }).notNull(), retryCount: integer("retry_count").notNull().default(0), @@ -95,6 +99,9 @@ export const founderWeeklyReviewRuns = pgTable( table.status, table.generationClaimId ), + collectionClaimIdx: index("founder_weekly_review_runs_collection_claim_idx").on( + table.companyId, table.id, table.status, table.collectionClaimId + ), }) ); diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts index 2d9c47722..d32626b7b 100644 --- a/packages/features/src/founder-weekly-review/contracts.ts +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -9,6 +9,7 @@ export const FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION = export const FounderWeeklyReviewStatusSchema = z.enum([ "queued", + "collecting", "generating", "draft", "published", @@ -16,6 +17,14 @@ export const FounderWeeklyReviewStatusSchema = z.enum([ ]); export type FounderWeeklyReviewStatus = z.infer; +/** Durable, request-derived inputs needed to collect evidence after the HTTP response. */ +export const FounderWeeklyReviewCollectionInputSchema = z.object({ + workspaceTimezone: z.string().min(1).max(128), + founderContext: z.string().min(1).max(4000).optional(), + actorExternalUserId: z.string().min(1).max(256), +}).strict(); +export type FounderWeeklyReviewCollectionInput = z.infer; + export const FounderWeeklyReviewOperationTypeSchema = z.enum(["retry"]); export type FounderWeeklyReviewOperationType = z.infer< typeof FounderWeeklyReviewOperationTypeSchema @@ -253,8 +262,12 @@ export interface FounderWeeklyReviewRunRecord { status: FounderWeeklyReviewStatus; reviewPayload: FounderWeeklyReviewPayload | null; reviewSchemaVersion: FounderWeeklyReviewPayloadSchemaVersion; - evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | null; evidenceSchemaVersion: typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION; + collectionInput?: FounderWeeklyReviewCollectionInput; + collectionClaimId?: string | null; + collectionStartedAt?: Date | null; + evidenceCollectedAt?: Date | null; modelMetadata: FounderWeeklyReviewModelMetadata | null; createdByActorId: string; retryCount: number; @@ -289,7 +302,9 @@ export interface CreateFounderWeeklyReviewRunInput { companyId: bigint; requestKey: string; reportingPeriod: ReportingPeriod; - evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + /** Existing callers may provide a snapshot; workflow callers intentionally do not. */ + evidenceSnapshot?: FounderWeeklyReviewEvidenceSnapshot; + collectionInput?: FounderWeeklyReviewCollectionInput; createdByActorId: string; } @@ -308,6 +323,12 @@ export interface FounderWeeklyReviewClaimInput { generationJobId?: string; } +export interface FounderWeeklyReviewCollectionClaimInput { + companyId: bigint; + runId: string; + collectionClaimId: string; +} + export interface FounderWeeklyReviewGenerationFailure { errorCode: string; errorMessage?: string; @@ -338,6 +359,10 @@ export function parseFounderWeeklyReviewEvidenceSnapshot( return FounderWeeklyReviewEvidenceSnapshotSchema.parse(value); } +export function parseFounderWeeklyReviewCollectionInput(value: unknown): FounderWeeklyReviewCollectionInput { + return FounderWeeklyReviewCollectionInputSchema.parse(value); +} + export function parseFounderWeeklyReviewModelMetadata( value: unknown ): FounderWeeklyReviewModelMetadata { diff --git a/packages/features/src/founder-weekly-review/generation-validation.ts b/packages/features/src/founder-weekly-review/generation-validation.ts index 6e8d175d2..b3b0f936d 100644 --- a/packages/features/src/founder-weekly-review/generation-validation.ts +++ b/packages/features/src/founder-weekly-review/generation-validation.ts @@ -4,12 +4,22 @@ import type { } from "./contracts"; export class FounderWeeklyReviewGenerationValidationError extends Error { - constructor(message: string) { + constructor( + message: string, + public readonly details: ReadonlyArray = [] + ) { super(message); this.name = "FounderWeeklyReviewGenerationValidationError"; } } +export interface FounderWeeklyReviewValidationDetail { + code: string; + section?: string; + itemIndex?: number; + sourceId?: string; +} + export function assertUniqueSnapshotSourceIds( evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot ): void { @@ -38,7 +48,7 @@ export function validateFounderWeeklyReviewV2Citations( for (const sectionName of factualSections) { const section = payload.sections[sectionName]; if (section.state === "no_evidence") continue; - for (const item of section.items) { + for (const [itemIndex, item] of section.items.entries()) { assertCitations(item.sourceIds, evidenceBySourceId, item.kind); if (item.kind === "contradictory_evidence" && item.sourceIds.length < 2) { throw new FounderWeeklyReviewGenerationValidationError( @@ -50,12 +60,14 @@ export function validateFounderWeeklyReviewV2Citations( const source = evidenceBySourceId.get(sourceId); if (source?.sourceType === "founder_context") { throw new FounderWeeklyReviewGenerationValidationError( - "founder_context must never be presented as customer feedback." + "founder_context must never be presented as customer feedback.", + [{ code: "founder_context_used_as_customer_feedback", section: sectionName, itemIndex, sourceId }] ); } if (source?.sourceType !== "customer_feedback") { throw new FounderWeeklyReviewGenerationValidationError( - `whatCustomersSaid may cite only customer_feedback evidence; received "${sourceId}".` + `whatCustomersSaid may cite only customer_feedback evidence; received "${sourceId}".`, + [{ code: "customer_signals_requires_customer_feedback", section: sectionName, itemIndex, sourceId }] ); } } diff --git a/packages/features/src/founder-weekly-review/generator.ts b/packages/features/src/founder-weekly-review/generator.ts index c920aac79..9a963ae91 100644 --- a/packages/features/src/founder-weekly-review/generator.ts +++ b/packages/features/src/founder-weekly-review/generator.ts @@ -10,6 +10,7 @@ import { } from "./contracts"; import { assertUniqueSnapshotSourceIds, + FounderWeeklyReviewGenerationValidationError, validateFounderWeeklyReviewV2Citations, } from "./generation-validation"; import { @@ -22,7 +23,7 @@ export interface FounderWeeklyReviewResolvedGenerationMetadata { provider: string; model: string; capability: string; - temperature: number; + temperature?: number; finishReason?: string; usage?: Record; providerRequestId?: string; @@ -33,6 +34,7 @@ export type FounderWeeklyReviewStructuredGenerator = (i prompt: string; schema: TSchema; schemaName?: string; + generationPhase?: "initial" | "semantic-repair"; }) => Promise<{ object: ReturnType; metadata: FounderWeeklyReviewResolvedGenerationMetadata; @@ -71,20 +73,77 @@ export async function generateFounderWeeklyReview( }; } - const result = await generate({ + const initial = await generate({ system: FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, prompt, schema: FounderWeeklyReviewV2PayloadSchema, schemaName: "founder_weekly_review_v2", + generationPhase: "initial", }); - const reviewPayload = validateFounderWeeklyReviewV2Citations( - FounderWeeklyReviewV2PayloadSchema.parse(result.object), - evidenceSnapshot - ); + let result = initial; + let reviewPayload: FounderWeeklyReviewV2Payload; + try { + reviewPayload = validateFounderWeeklyReviewV2Citations( + FounderWeeklyReviewV2PayloadSchema.parse(initial.object), + evidenceSnapshot + ); + logGenerationValidation("initial", initial.metadata, "passed"); + } catch (error) { + if (!(error instanceof FounderWeeklyReviewGenerationValidationError)) throw error; + logGenerationValidation("initial", initial.metadata, "failed"); + const repaired = await generate({ + system: FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, + prompt: buildSemanticRepairPrompt(initial.object, evidenceSnapshot, error), + schema: FounderWeeklyReviewV2PayloadSchema, + schemaName: "founder_weekly_review_v2", + generationPhase: "semantic-repair", + }); + result = repaired; + try { + reviewPayload = validateFounderWeeklyReviewV2Citations( + FounderWeeklyReviewV2PayloadSchema.parse(repaired.object), + evidenceSnapshot + ); + logGenerationValidation("semantic-repair", repaired.metadata, "passed"); + } catch (repairError) { + logGenerationValidation("semantic-repair", repaired.metadata, "failed"); + throw repairError; + } + } return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false) }; } +function buildSemanticRepairPrompt( + candidate: FounderWeeklyReviewV2Payload, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + error: FounderWeeklyReviewGenerationValidationError +): string { + const errors = error.details.length > 0 + ? error.details + : [{ code: "report_validation_failed" }]; + const sources = evidenceSnapshot.items.map(({ sourceId, sourceType }) => ({ sourceId, sourceType })); + return [ + "Correct the complete canonical Founder Weekly Review JSON candidate below.", + "Customer Signals may cite only customer_feedback sources.", + "founder_context is founder-provided direction, not customer testimony.", + "Remove a customer claim if it lacks customer_feedback support.", + "Do not invent or substitute a source ID.", + "Return the complete corrected canonical JSON object only.", + `Validation errors: ${JSON.stringify(errors)}`, + `Source IDs and source types: ${JSON.stringify(sources)}`, + `Previous canonical candidate: ${JSON.stringify(candidate)}`, + ].join("\n"); +} + +function logGenerationValidation( + phase: "initial" | "semantic-repair", + metadata: FounderWeeklyReviewResolvedGenerationMetadata, + result: "passed" | "failed" +): void { + console.info(`[fwr] generation phase=${phase} provider=${metadata.provider} model=${metadata.model} validation=${result}`); +} + function buildMetadata( metadata: FounderWeeklyReviewResolvedGenerationMetadata, promptHash: string, @@ -94,7 +153,7 @@ function buildMetadata( provider: metadata.provider, model: metadata.model, capability: metadata.capability, - temperature: metadata.temperature, + ...(metadata.temperature === undefined ? {} : { temperature: metadata.temperature }), promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, promptHash, evidenceSchemaVersion: "founder-weekly-review-evidence/v1", diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts index ee07f8a3c..01aca49a7 100644 --- a/packages/features/src/founder-weekly-review/prompts.ts +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -7,7 +7,15 @@ export const FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT = `You generate a structured Fo Never invent, assume, infer, or embellish customers, dates, metrics, people, decisions, shipped work, blockers, outcomes, or source IDs. Every factual item must cite one or more supplied source IDs exactly as given. Do not create or modify source IDs. Confidence is how strongly the generated claim is supported by its cited supplied evidence; it is not a score for source reliability or truthfulness. Omit unsupported claims or use no_evidence rather than assigning them a low confidence. -founder_context is internal manual input. It must never be represented as customer feedback. whatCustomersSaid may cite only customer_feedback evidence. +Write a concise, natural, professional review for a founder. Prefer a few substantive items over many one-sentence paraphrases. When related evidence supports it, synthesize the relationship into one focused item: describe the development or signal, why it matters, what the evidence does and does not establish, and an evidence-backed next action where the section permits it. Aim for roughly 2–4 sentences per substantive item when the supplied evidence supports that depth, and for approximately 600–1,000 words overall when the evidence supports it. Do not add filler to reach a length target. + +Keep the distinctions below explicit. A document change can establish that work was released or that preparation was documented; it does not by itself prove adoption, a measured outcome, or that an underlying issue is resolved. Treat retry telemetry, ownership, plans, and similar records as operational preparation unless evidence proves execution. Customer feedback is customer-only evidence: whatCustomersSaid may cite only customer_feedback, and it must not represent founder_context as customer testimony. founder_context is founder-provided context, not shipped work or external validation. Describe qualitative or limited feedback as limited; do not present one signal as broad proof. + +Use whatShipped only for work that the evidence establishes as released during this reporting period. Use whatChanged only for separate, non-shipped developments such as operational preparation; do not restate a shipped item there. Do not turn the administrative processing of a feedback document into a meaningful change when the customer signals themselves are already covered in whatCustomersSaid. + +Use currentBlockers for evidence-backed execution blockers and for material product, customer, or operational risks. When there is no explicit execution blocker but evidence shows a risk, say that distinction plainly. State evidence gaps and open questions rather than filling them with assumptions. Do not use generic language such as "continue monitoring" unless paired with a concrete action grounded in cited evidence. + +For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must be evidence-backed and explain its rationale in the optional rationale field when useful. Avoid repetitive wording across sections. When evidence conflicts, return contradictory_evidence with the conflicting source IDs. Do not choose a winner or reconcile it unless supplied evidence explicitly resolves the conflict. diff --git a/packages/features/src/founder-weekly-review/repository.ts b/packages/features/src/founder-weekly-review/repository.ts index 5c3c78154..c873b2cd8 100644 --- a/packages/features/src/founder-weekly-review/repository.ts +++ b/packages/features/src/founder-weekly-review/repository.ts @@ -13,6 +13,7 @@ import { type FounderWeeklyReviewPayloadSchemaVersion, type CreateFounderWeeklyReviewRunInput, type FounderWeeklyReviewClaimInput, + type FounderWeeklyReviewCollectionClaimInput, type FounderWeeklyReviewGenerationFailure, type FounderWeeklyReviewModelMetadata, type FounderWeeklyReviewOperationRecord, @@ -20,6 +21,7 @@ import { type FounderWeeklyReviewRetryInput, type FounderWeeklyReviewRunRecord, parseFounderWeeklyReviewEvidenceSnapshot, + parseFounderWeeklyReviewCollectionInput, parseFounderWeeklyReviewModelMetadata, parseFounderWeeklyReviewPayload, } from "./contracts"; @@ -60,9 +62,15 @@ function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord status: row.status, reviewPayload, reviewSchemaVersion: row.reviewSchemaVersion as FounderWeeklyReviewPayloadSchemaVersion, - evidenceSnapshot: parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot), + evidenceSnapshot: row.evidenceSnapshot + ? parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot) + : null, evidenceSchemaVersion: row.evidenceSchemaVersion as typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + collectionInput: parseFounderWeeklyReviewCollectionInput(row.collectionInput), + collectionClaimId: row.collectionClaimId ?? null, + collectionStartedAt: row.collectionStartedAt ?? null, + evidenceCollectedAt: row.evidenceCollectedAt ?? null, modelMetadata: row.modelMetadata ? parseFounderWeeklyReviewModelMetadata(row.modelMetadata) : null, @@ -127,8 +135,12 @@ export class FounderWeeklyReviewRepository { status: "queued", reviewPayload: null, reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, - evidenceSnapshot: input.evidenceSnapshot, + evidenceSnapshot: input.evidenceSnapshot ?? null, evidenceSchemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + collectionInput: input.collectionInput ?? { + workspaceTimezone: input.evidenceSnapshot?.workspaceTimezone ?? "UTC", + actorExternalUserId: input.createdByActorId.replace(/^user:/, ""), + }, modelMetadata: null, createdByActorId: input.createdByActorId, queuedAt: new Date(), @@ -285,7 +297,8 @@ export class FounderWeeklyReviewRepository { and( eq(founderWeeklyReviewRuns.companyId, input.companyId), eq(founderWeeklyReviewRuns.id, input.runId), - eq(founderWeeklyReviewRuns.status, "queued") + eq(founderWeeklyReviewRuns.status, "queued"), + sql`${founderWeeklyReviewRuns.evidenceSnapshot} IS NOT NULL` ) ) .returning(); @@ -300,6 +313,70 @@ export class FounderWeeklyReviewRepository { }; } + async claimEvidenceCollection(input: FounderWeeklyReviewCollectionClaimInput): Promise { + const now = new Date(); + const [row] = await this.db.update(founderWeeklyReviewRuns).set({ + status: "collecting", + collectionClaimId: input.collectionClaimId, + collectionStartedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "queued"), + sql`${founderWeeklyReviewRuns.evidenceSnapshot} IS NULL`, + )).returning(); + return row ? { updated: true, run: mapRunRow(row) } : { + updated: false, run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async attachEvidenceSnapshotIfAbsent( + input: FounderWeeklyReviewCollectionClaimInput, + evidenceSnapshot: import("./contracts").FounderWeeklyReviewEvidenceSnapshot, + ): Promise { + const now = new Date(); + const [row] = await this.db.update(founderWeeklyReviewRuns).set({ + status: "queued", + evidenceSnapshot, + evidenceSchemaVersion: evidenceSnapshot.schemaVersion, + evidenceCollectedAt: now, + collectionClaimId: null, + queuedAt: now, + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "collecting"), + eq(founderWeeklyReviewRuns.collectionClaimId, input.collectionClaimId), + sql`${founderWeeklyReviewRuns.evidenceSnapshot} IS NULL`, + )).returning(); + return row ? { updated: true, run: mapRunRow(row) } : { + updated: false, run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async markCollectionFailed(input: FounderWeeklyReviewCollectionClaimInput, failure: FounderWeeklyReviewGenerationFailure): Promise { + const now = new Date(); + const [row] = await this.db.update(founderWeeklyReviewRuns).set({ + status: "failed", + failureSequence: sql`${founderWeeklyReviewRuns.failureSequence} + 1`, + errorCode: failure.errorCode, + errorMessage: truncateErrorMessage(failure.errorMessage), + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "collecting"), + eq(founderWeeklyReviewRuns.collectionClaimId, input.collectionClaimId), + )).returning(); + return row ? { updated: true, run: mapRunRow(row) } : { + updated: false, run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + async saveGeneratedDraftWithClaim( input: FounderWeeklyReviewClaimInput, reviewPayload: FounderWeeklyReviewPayload, diff --git a/packages/features/src/founder-weekly-review/user-service.ts b/packages/features/src/founder-weekly-review/user-service.ts index e6ae6e675..0ebc5d7cc 100644 --- a/packages/features/src/founder-weekly-review/user-service.ts +++ b/packages/features/src/founder-weekly-review/user-service.ts @@ -4,6 +4,7 @@ import { ZodError } from "zod"; import { buildFounderWeeklyReviewActorId, type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewCollectionInput, type FounderWeeklyReviewPayload, type FounderWeeklyReviewRunRecord, type FounderWeeklyReviewUserActor, @@ -27,7 +28,8 @@ export interface CreateFounderWeeklyReviewRunRequest { start: string; end: string; }; - evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + evidenceSnapshot?: FounderWeeklyReviewEvidenceSnapshot; + collectionInput?: FounderWeeklyReviewCollectionInput; } export interface CreateFounderWeeklyReviewRunResult { run: FounderWeeklyReviewRunRecord; @@ -68,16 +70,10 @@ export class FounderWeeklyReviewUserService { input: CreateFounderWeeklyReviewRunRequest ): Promise { assertWorkspaceMutationRole(actor.role); - let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; - try { - evidenceSnapshot = parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot); - } catch (error) { - if (error instanceof ZodError) { - throw new FounderWeeklyReviewInvalidPayloadError(error.message); - } - throw error; - } - assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); + let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | undefined; + try { evidenceSnapshot = input.evidenceSnapshot ? parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot) : undefined; } + catch (error) { if (error instanceof ZodError) throw new FounderWeeklyReviewInvalidPayloadError(error.message); throw error; } + if (evidenceSnapshot) assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); return (await this.repository.createOrGetByRequestKeyWithResult({ id: `fwr_${randomUUID()}`, @@ -85,17 +81,18 @@ export class FounderWeeklyReviewUserService { requestKey: input.requestKey, reportingPeriod: input.reportingPeriod, evidenceSnapshot, + collectionInput: input.collectionInput ?? { workspaceTimezone: evidenceSnapshot?.workspaceTimezone ?? "UTC", actorExternalUserId: actor.externalUserId }, createdByActorId: buildFounderWeeklyReviewActorId(actor), })).run; } async createOrGetRunWithMetadata(actor: FounderWeeklyReviewUserActor, input: CreateFounderWeeklyReviewRunRequest): Promise { assertWorkspaceMutationRole(actor.role); - let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; - try { evidenceSnapshot = parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot); } + let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | undefined; + try { evidenceSnapshot = input.evidenceSnapshot ? parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot) : undefined; } catch (error) { if (error instanceof ZodError) throw new FounderWeeklyReviewInvalidPayloadError(error.message); throw error; } - assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); - return this.repository.createOrGetByRequestKeyWithResult({ id: `fwr_${randomUUID()}`, companyId: actor.companyId, requestKey: input.requestKey, reportingPeriod: input.reportingPeriod, evidenceSnapshot, createdByActorId: buildFounderWeeklyReviewActorId(actor) }); + if (evidenceSnapshot) assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); + return this.repository.createOrGetByRequestKeyWithResult({ id: `fwr_${randomUUID()}`, companyId: actor.companyId, requestKey: input.requestKey, reportingPeriod: input.reportingPeriod, evidenceSnapshot, collectionInput: input.collectionInput ?? { workspaceTimezone: evidenceSnapshot?.workspaceTimezone ?? "UTC", actorExternalUserId: actor.externalUserId }, createdByActorId: buildFounderWeeklyReviewActorId(actor) }); } async getRun( diff --git a/packages/features/src/founder-weekly-review/worker-service.ts b/packages/features/src/founder-weekly-review/worker-service.ts index f8f1c6fc7..a3d5544dc 100644 --- a/packages/features/src/founder-weekly-review/worker-service.ts +++ b/packages/features/src/founder-weekly-review/worker-service.ts @@ -2,6 +2,8 @@ import { ZodError } from "zod"; import { type FounderWeeklyReviewClaimInput, + type FounderWeeklyReviewCollectionClaimInput, + type FounderWeeklyReviewEvidenceSnapshot, type FounderWeeklyReviewGenerationFailure, type FounderWeeklyReviewModelMetadata, type FounderWeeklyReviewPayload, @@ -19,12 +21,19 @@ import { import { FounderWeeklyReviewRepository } from "./repository"; export interface FounderWeeklyReviewWorkerContext extends FounderWeeklyReviewClaimInput {} +export interface FounderWeeklyReviewCollectionContext extends FounderWeeklyReviewCollectionClaimInput {} export class FounderWeeklyReviewWorkerService { constructor( private readonly repository = new FounderWeeklyReviewRepository() ) {} + async getRun(companyId: bigint, runId: string): Promise { + const run = await this.repository.getByCompanyAndRunId(companyId, runId); + if (!run) throw new FounderWeeklyReviewNotFoundError(runId); + return run; + } + async claimQueuedRun( context: FounderWeeklyReviewWorkerContext ): Promise { @@ -47,6 +56,29 @@ export class FounderWeeklyReviewWorkerService { ); } + async claimEvidenceCollection(context: FounderWeeklyReviewCollectionContext): Promise { + const result = await this.repository.claimEvidenceCollection(context); + if (!result.run) throw new FounderWeeklyReviewNotFoundError(context.runId); + if (result.updated) return result.run; + if (result.run.status === "collecting" && result.run.collectionClaimId === context.collectionClaimId) return result.run; + if (result.run.evidenceSnapshot) return result.run; + throw new FounderWeeklyReviewConflictError(`Founder weekly review run "${context.runId}" is already owned by another collection claim.`); + } + + async attachEvidenceSnapshotIfAbsent(context: FounderWeeklyReviewCollectionContext, snapshot: FounderWeeklyReviewEvidenceSnapshot): Promise { + const result = await this.repository.attachEvidenceSnapshotIfAbsent(context, snapshot); + if (!result.run) throw new FounderWeeklyReviewNotFoundError(context.runId); + if (result.updated || result.run.evidenceSnapshot) return result.run; + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + + async markCollectionFailed(context: FounderWeeklyReviewCollectionContext, failure: FounderWeeklyReviewGenerationFailure): Promise { + const result = await this.repository.markCollectionFailed(context, failure); + if (!result.run) throw new FounderWeeklyReviewNotFoundError(context.runId); + if (result.updated || (result.run.status === "failed" && result.run.collectionClaimId === context.collectionClaimId)) return result.run; + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + async saveGeneratedDraft( context: FounderWeeklyReviewWorkerContext, reviewPayload: FounderWeeklyReviewPayload, From a9aa4cfe51f478aa659feef99cc40ded7bbd34d1 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Tue, 4 Aug 2026 20:15:42 +0800 Subject: [PATCH 16/29] feat(founder-weekly-review): add deterministic version diff and strict workspace evidence --- .../document-change.test.ts | 89 ++++++++++ .../document-version-chunks.test.ts | 34 ++++ .../founderWeeklyReview/generation.test.ts | 19 ++ .../__tests__/founderWeeklyReview/testDb.ts | 9 + .../workspace-collector.test.ts | 19 ++ .../workspace-document-store.test.ts | 80 +++++++++ .../workspace-document.test.ts | 40 +++++ ...run-founder-weekly-review-realistic-e2e.ts | 2 +- .../document-version-chunks.ts | 40 +++++ .../evidence-collector.ts | 4 +- .../workspace-document-store.ts | 48 +++++ .../founder-weekly-review/document-change.ts | 164 ++++++++++++++++++ .../founder-weekly-review/evidence-service.ts | 83 ++++++++- .../generation-validation.ts | 19 ++ .../src/founder-weekly-review/index.ts | 2 + .../workspace-document.ts | 58 +++++++ 16 files changed, 706 insertions(+), 4 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/document-change.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts create mode 100644 apps/web/src/server/founder-weekly-review/document-version-chunks.ts create mode 100644 apps/web/src/server/founder-weekly-review/workspace-document-store.ts create mode 100644 packages/features/src/founder-weekly-review/document-change.ts create mode 100644 packages/features/src/founder-weekly-review/workspace-document.ts diff --git a/apps/web/__tests__/founderWeeklyReview/document-change.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change.test.ts new file mode 100644 index 000000000..cd5a60c05 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change.test.ts @@ -0,0 +1,89 @@ +import { + alignVersionChunks, + buildDocumentChangeEvidence, + selectVersionPairsForReportingPeriod, + FounderWeeklyReviewEvidenceService, + FounderWeeklyReviewEvidenceItemSchema, + type DocumentVersionForComparison, + type VersionChunk, +} from "@launchstack/features/founder-weekly-review"; + +const version = (documentId: bigint, versionId: number, versionNumber: number, createdAt: string): DocumentVersionForComparison => ({ + documentId, documentTitle: `Document ${documentId}`, documentCategory: "Product", versionId, versionNumber, + createdAt: new Date(createdAt), changelog: versionId === 2 ? "Founder supplied note" : null, +}); +const chunk = (chunkId: number, versionId: bigint, content: string, overrides: Partial = {}): VersionChunk => ({ + chunkId, versionId, documentId: 1n, content, contentHash: null, structureId: 1n, structurePath: "/1", structureTitle: "Overview", + structureOrdering: 1, pageNumber: 1, lineStart: 1, lineEnd: 2, ...overrides, +}); + +describe("Week 3 document change domain logic", () => { + it("forms an adjacent chain including the predecessor before the period", () => { + const pairs = selectVersionPairsForReportingPeriod([ + version(1n, 1, 1, "2026-01-01T00:00:00.000Z"), version(1n, 3, 3, "2026-02-03T00:00:00.000Z"), + version(2n, 4, 1, "2026-02-02T00:00:00.000Z"), version(1n, 2, 2, "2026-02-02T00:00:00.000Z"), + ], new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-10T00:00:00.000Z")); + expect(pairs.map((pair) => [pair.documentId, pair.previousVersionId, pair.currentVersionId])).toEqual([[1n, 1, 2], [1n, 2, 3]]); + }); + + it("does not invent a predecessor for a first-ever in-period version and resolves timestamp ties deterministically", () => { + const pairs = selectVersionPairsForReportingPeriod([ + version(1n, 2, 2, "2026-02-02T00:00:00.000Z"), version(1n, 1, 1, "2026-02-02T00:00:00.000Z"), version(2n, 3, 1, "2026-02-02T00:00:00.000Z"), + ], new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z")); + expect(pairs.map((pair) => [pair.previousVersionId, pair.currentVersionId])).toEqual([[1, 2]]); + }); + + it("aligns once, classifies changes, and leaves exact hash matches unchanged", () => { + const hash = "a".repeat(64); + const prior = [chunk(1, 1n, "same", { contentHash: hash, structurePath: "/x" }), chunk(2, 1n, "old", { structurePath: "/y" }), chunk(3, 1n, "removed", { structurePath: "/z", structureTitle: "Old only" })]; + const current = [chunk(4, 2n, "same", { contentHash: hash, structurePath: "/other" }), chunk(5, 2n, "new", { structurePath: "/y" }), chunk(6, 2n, "added", { structurePath: "/new", structureTitle: "New only" })]; + const alignments = alignVersionChunks(prior, current); + expect(alignments.map((item) => [item.changeType, item.alignmentMethod])).toEqual([["added", "unmatched"], ["unchanged", "content_hash"], ["modified", "structure_path"], ["removed", "unmatched"]]); + expect(new Set(alignments.flatMap((item) => [item.previousChunk?.chunkId, item.currentChunk?.chunkId].filter(Boolean))).size).toBe(6); + }); + + it("does not trust malformed or contradictory matching hashes", () => { + const alignments = alignVersionChunks([chunk(1, 1n, "before", { contentHash: "bad", structurePath: "/same" })], [chunk(2, 2n, "after", { contentHash: "bad", structurePath: "/same" })]); + expect(alignments).toEqual([expect.objectContaining({ changeType: "modified", alignmentMethod: "structure_path" })]); + }); + + it("keeps maximum-size identifiers JSON-safe and within the evidence contract", () => { + const pair = { documentId: 99999999999999999999n, documentTitle: "D", documentCategory: null, previousVersionId: 2147483646, previousVersionNumber: 2147483646, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), currentVersionId: 2147483647, currentVersionNumber: 2147483647, currentCreatedAt: new Date("2026-02-02T00:00:00.000Z"), currentChangelog: "x" }; + const item = buildDocumentChangeEvidence(pair, [{ changeType: "added", currentChunk: chunk(2147483647, 2147483647n, "content"), alignmentMethod: "unmatched" }])[0]!; + expect(item.sourceId.length).toBeLessThanOrEqual(256); + expect(() => JSON.stringify(item)).not.toThrow(); + expect(FounderWeeklyReviewEvidenceItemSchema.safeParse(item).success).toBe(true); + }); + + it("creates stable computed evidence and keeps changelog separate", () => { + const pair = selectVersionPairsForReportingPeriod([version(1n, 1, 1, "2026-01-01T00:00:00.000Z"), version(1n, 2, 2, "2026-02-02T00:00:00.000Z")], new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z"))[0]!; + const evidence = buildDocumentChangeEvidence(pair, alignVersionChunks([chunk(10, 1n, "Before", { structurePath: "/plan" })], [chunk(20, 2n, "After", { structurePath: "/plan" })])); + expect(evidence).toEqual(buildDocumentChangeEvidence(pair, alignVersionChunks([chunk(10, 1n, "Before", { structurePath: "/plan" })], [chunk(20, 2n, "After", { structurePath: "/plan" })]))); + expect(evidence).toEqual([expect.objectContaining({ sourceId: "document_change:doc:1:v1:v2:chunk:10:20", excerpt: expect.stringContaining("Before: Before After: After"), metadata: expect.objectContaining({ changeType: "modified", userChangelog: "Founder supplied note", previousVersionId: 1, currentVersionId: 2 }) })]); + }); + + it("uses bounded deterministic text similarity only after structural strategies fail", () => { + const alignments = alignVersionChunks([chunk(1, 1n, "reliable export retry job", { structurePath: "/old", structureTitle: "Old" })], [chunk(2, 2n, "export retry job is now reliable", { structurePath: "/new", structureTitle: "New" })]); + expect(alignments).toEqual([expect.objectContaining({ changeType: "modified", alignmentMethod: "text_similarity" })]); + }); + + it("does not use text similarity to pair short generic fragments", () => { + const alignments = alignVersionChunks( + [chunk(1, 1n, "status update", { structurePath: "/old", structureTitle: "Old" })], + [chunk(2, 2n, "status update", { structurePath: "/new", structureTitle: "New" })] + ); + expect(alignments.map((item) => item.changeType)).toEqual(["added", "removed"]); + }); + + it("collects one computed change for the controlled v1-to-v2 case", async () => { + const versions = [version(1n, 1, 1, "2026-01-01T00:00:00.000Z"), version(1n, 2, 2, "2026-02-02T00:00:00.000Z")]; + const store = { listVersionsBeforePeriodEnd: jest.fn().mockResolvedValue(versions), getDocumentChunksForVersion: jest.fn() + .mockResolvedValueOnce({ state: "complete", chunks: [chunk(10, 1n, "Before", { structurePath: "/plan" })], warnings: [] }) + .mockResolvedValueOnce({ state: "complete", chunks: [chunk(20, 2n, "After", { structurePath: "/plan" })], warnings: [] }) }; + const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "computed", store }); + await expect(service.collectDocumentChangeEvidence(1n, new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z"))).resolves.toEqual([ + expect.objectContaining({ sourceTimestamp: "2026-02-02T00:00:00.000Z", metadata: expect.objectContaining({ previousVersionId: 1, currentVersionId: 2, changeType: "modified" }) }), + ]); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts b/apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts new file mode 100644 index 000000000..b23eafd6d --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts @@ -0,0 +1,34 @@ +import { company, document, documentContextChunks, documentVersions } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; + +describeDb("explicit document version chunks", () => { + it("loads only owned explicit-version chunks in deterministic order", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [firstCompany] = await test.db.insert(company).values({ name: "First", numberOfEmployees: "1" }).returning(); + const [secondCompany] = await test.db.insert(company).values({ name: "Second", numberOfEmployees: "1" }).returning(); + const [doc] = await test.db.insert(document).values({ companyId: BigInt(firstCompany!.id), url: "local://doc", category: "Product", title: "Plan" }).returning(); + const [otherDoc] = await test.db.insert(document).values({ companyId: BigInt(secondCompany!.id), url: "local://other", category: "Product", title: "Other" }).returning(); + const [v1] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: "local://v1", mimeType: "text/plain" }).returning(); + const [v2] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 2, url: "local://v2", mimeType: "text/plain" }).returning(); + const [v3] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 3, url: "local://v3", mimeType: "text/plain" }).returning(); + const [otherVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(otherDoc!.id), versionNumber: 1, url: "local://other-v1", mimeType: "text/plain" }).returning(); + await test.db.insert(documentContextChunks).values([ + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "page two", contentHash: "b", tokenCount: 1, charCount: 8, pageNumber: 2 }, + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "page one", contentHash: "a", tokenCount: 1, charCount: 8, pageNumber: 1 }, + { documentId: BigInt(doc!.id), versionId: BigInt(v1!.id), content: "old", contentHash: "old", tokenCount: 1, charCount: 3, pageNumber: 1 }, + { documentId: BigInt(doc!.id), versionId: null, content: "legacy", contentHash: "legacy", tokenCount: 1, charCount: 6, pageNumber: 0 }, + ]); + const store = new FounderWeeklyReviewDocumentVersionStore(test.db); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: v2!.id })).resolves.toMatchObject({ state: "partial", chunks: [{ content: "page one" }, { content: "page two" }] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: v1!.id })).resolves.toMatchObject({ state: "partial", chunks: [{ content: "old" }] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: v3!.id })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["version_chunks_missing"] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(secondCompany!.id), documentId: BigInt(doc!.id), versionId: v2!.id })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: otherVersion!.id })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: 999_999 })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }); + } finally { await test.close(); } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts index 550d8c78c..f324053ce 100644 --- a/apps/web/__tests__/founderWeeklyReview/generation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -160,6 +160,25 @@ describe("Founder Weekly Review generation", () => { await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) })).rejects.toBeInstanceOf(Error); }); + it.each(["whatChanged", "whatShipped"] as const)("rejects workspace_document-only %s claims", async (sectionName) => { + const payload = validPayload(); + payload.sections[sectionName] = { state: "evidence", items: [{ kind: "observed_fact", text: "Current document implies a weekly event.", sourceIds: ["workspace-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([source("workspace-1", "workspace_document")]), generate: fake(payload) })).rejects.toBeInstanceOf(FounderWeeklyReviewGenerationValidationError); + }); + + it.each(["whatChanged", "whatShipped"] as const)("allows document_change plus workspace_document in %s", async (sectionName) => { + const payload = validPayload(); + payload.sections[sectionName] = { state: "evidence", items: [{ kind: "observed_fact", text: "A dated change has current context.", sourceIds: ["doc-1", "workspace-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([...completeSnapshot().items, source("workspace-1", "workspace_document")]), generate: fake(payload) })).resolves.toBeDefined(); + }); + + it("allows workspace_document for blockers and priorities while customer-only enforcement remains", async () => { + const payload = validPayload(); + payload.sections.currentBlockers = { state: "evidence", items: [{ kind: "observed_fact", text: "Current context", sourceIds: ["workspace-1"], confidence: 0.5 }] }; + payload.sections.nextPriorities = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Act on current context", sourceIds: ["workspace-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([...completeSnapshot().items, source("workspace-1", "workspace_document")]), generate: fake(payload) })).resolves.toBeDefined(); + }); + it("rejects duplicate input source IDs before calling the model", async () => { const generate = fake(validPayload()); await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([source("same", "manual_note"), source("same", "document_change")]), generate })).rejects.toBeInstanceOf(FounderWeeklyReviewGenerationValidationError); diff --git a/apps/web/__tests__/founderWeeklyReview/testDb.ts b/apps/web/__tests__/founderWeeklyReview/testDb.ts index 831b9a3ef..8e1de5d7f 100644 --- a/apps/web/__tests__/founderWeeklyReview/testDb.ts +++ b/apps/web/__tests__/founderWeeklyReview/testDb.ts @@ -124,6 +124,15 @@ async function bootstrapIsolatedSchema( "updated_at" timestamptz ); + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_structure" ( + "id" serial PRIMARY KEY, + "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, + "version_id" bigint, + "ordering" integer NOT NULL DEFAULT 0, + "title" text, + "path" varchar(256) + ); + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_retrieval_chunks" ( "id" bigint PRIMARY KEY ); diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts new file mode 100644 index 000000000..13a6c1d2c --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts @@ -0,0 +1,19 @@ +import { FounderWeeklyReviewEvidenceService } from "@launchstack/features/founder-weekly-review"; + +describe("workspace document collection", () => { + it("skips retrieval for blank Founder Context", async () => { + const store = { retrieveRelevantCurrentDocumentChunks: jest.fn() }; + const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "unconfigured" }, store); + await expect(service.collectWorkspaceDocumentEvidence(1n, " ")).resolves.toEqual({ items: [], warnings: [] }); + expect(store.retrieveRelevantCurrentDocumentChunks).not.toHaveBeenCalled(); + }); + + it("maps relevant current hits and preserves unavailability as a bounded warning", async () => { + const store = { retrieveRelevantCurrentDocumentChunks: jest.fn() + .mockResolvedValueOnce({ state: "success", hits: [{ documentId: 1n, documentTitle: "Current plan", versionId: 2n, contextChunkId: 3, content: "Current blocker", similarityScore: 0.9 }] }) + .mockResolvedValueOnce({ state: "unavailable", hits: [], warnings: ["workspace_document_retrieval_unavailable"] }) }; + const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "unconfigured" }, store); + await expect(service.collectWorkspaceDocumentEvidence(1n, " blocker ")).resolves.toMatchObject({ items: [{ sourceType: "workspace_document", metadata: { retrievalReason: "founder_context_relevance" } }], warnings: [] }); + await expect(service.collectWorkspaceDocumentEvidence(1n, " blocker ")).resolves.toEqual({ items: [], warnings: [{ code: "workspace_document_retrieval_unavailable", message: "Founder Context workspace retrieval was unavailable.", sourceType: "workspace_document" }] }); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts new file mode 100644 index 000000000..521921b25 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts @@ -0,0 +1,80 @@ +import { sql } from "drizzle-orm"; +import { company, document, documentContextChunks, documentVersions } from "@launchstack/core/db/schema"; +import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; +import { FounderWeeklyReviewEvidenceService } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; +const vector = (index: number) => Array.from({ length: 1536 }, (_, i) => i === index ? 1 : 0); +const vectorSql = (index: number) => sql`${JSON.stringify(vector(index))}::vector(1536)`; + +describeDb("strict current workspace document store", () => { + it("returns only company-owned current-version embedded chunks with deterministic ranking", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [owner] = await test.db.insert(company).values({ name: "Owner", numberOfEmployees: "1" }).returning(); + const [other] = await test.db.insert(company).values({ name: "Other", numberOfEmployees: "1" }).returning(); + const [doc] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://a", category: "Product", title: "A" }).returning(); + const [noCurrent] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://none", category: "Product", title: "No current" }).returning(); + const [foreign] = await test.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://foreign", category: "Product", title: "Foreign" }).returning(); + const [v1] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: "local://a1", mimeType: "text/plain" }).returning(); + const [v2] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 2, url: "local://a2", mimeType: "text/plain" }).returning(); + const [noneVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(noCurrent!.id), versionNumber: 1, url: "local://n", mimeType: "text/plain" }).returning(); + const [foreignVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(foreign!.id), versionNumber: 1, url: "local://f", mimeType: "text/plain" }).returning(); + await test.db.update(document).set({ currentVersionId: BigInt(v2!.id) }).where(sql`${document.id} = ${doc!.id}`); + await test.db.insert(documentContextChunks).values([ + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "current first", tokenCount: 1, charCount: 13, embedding: vectorSql(0) }, + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "current second", tokenCount: 1, charCount: 14, embedding: vectorSql(0) }, + { documentId: BigInt(doc!.id), versionId: BigInt(v1!.id), content: "historical", tokenCount: 1, charCount: 10, embedding: vectorSql(0) }, + { documentId: BigInt(doc!.id), versionId: null, content: "legacy", tokenCount: 1, charCount: 6, embedding: vectorSql(0) }, + { documentId: BigInt(noCurrent!.id), versionId: BigInt(noneVersion!.id), content: "no current", tokenCount: 1, charCount: 10, embedding: vectorSql(0) }, + { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: "foreign", tokenCount: 1, charCount: 7, embedding: vectorSql(0) }, + ]); + const store = new StrictCurrentWorkspaceDocumentStore(test.db, { embedQuery: jest.fn().mockResolvedValue(vector(0)) }); + const result = await store.retrieveRelevantCurrentDocumentChunks({ companyId: BigInt(owner!.id), founderContext: "current", topK: 1 }); + expect(result).toMatchObject({ state: "success", hits: [{ documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), documentTitle: "A", content: "current first" }] }); + if (result.state === "success") expect(result.hits).toHaveLength(1); + } finally { await test.close(); } + }); + + it("collects a computed v1-to-v2 diff plus only a current relevant workspace document", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [owner] = await test.db.insert(company).values({ name: "Owner", numberOfEmployees: "1" }).returning(); + const [other] = await test.db.insert(company).values({ name: "Other", numberOfEmployees: "1" }).returning(); + const [a] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://a", category: "Product", title: "Release plan" }).returning(); + const [b] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://b", category: "Product", title: "Current blocker" }).returning(); + const [foreign] = await test.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://f", category: "Product", title: "Foreign" }).returning(); + const [a1] = await test.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 1, url: "local://a1", mimeType: "text/plain", createdAt: new Date("2026-01-01T00:00:00.000Z") }).returning(); + const [a2] = await test.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 2, url: "local://a2", mimeType: "text/plain", createdAt: new Date("2026-02-02T00:00:00.000Z") }).returning(); + const [a3] = await test.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 3, url: "local://a3", mimeType: "text/plain", createdAt: new Date("2026-03-01T00:00:00.000Z") }).returning(); + const [b1] = await test.db.insert(documentVersions).values({ documentId: BigInt(b!.id), versionNumber: 1, url: "local://b1", mimeType: "text/plain", createdAt: new Date("2026-01-01T00:00:00.000Z") }).returning(); + const [foreignVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(foreign!.id), versionNumber: 1, url: "local://f1", mimeType: "text/plain" }).returning(); + await test.db.update(document).set({ currentVersionId: BigInt(a3!.id) }).where(sql`${document.id} = ${a!.id}`); + await test.db.update(document).set({ currentVersionId: BigInt(b1!.id) }).where(sql`${document.id} = ${b!.id}`); + await test.db.update(document).set({ currentVersionId: BigInt(foreignVersion!.id) }).where(sql`${document.id} = ${foreign!.id}`); + await test.db.insert(documentContextChunks).values([ + { documentId: BigInt(a!.id), versionId: BigInt(a1!.id), content: "product release before", contentHash: "a".repeat(64), tokenCount: 1, charCount: 22 }, + { documentId: BigInt(a!.id), versionId: BigInt(a2!.id), content: "product release after", contentHash: "b".repeat(64), tokenCount: 1, charCount: 21 }, + { documentId: BigInt(a!.id), versionId: BigInt(a3!.id), content: "v3 must not diff", contentHash: "c".repeat(64), tokenCount: 1, charCount: 15 }, + { documentId: BigInt(b!.id), versionId: BigInt(b1!.id), content: "current blocker context", contentHash: "d".repeat(64), tokenCount: 1, charCount: 23, embedding: vectorSql(0) }, + { documentId: BigInt(b!.id), versionId: null, content: "legacy relevant", contentHash: "e".repeat(64), tokenCount: 1, charCount: 15, embedding: vectorSql(0) }, + { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: "foreign relevant", contentHash: "f".repeat(64), tokenCount: 1, charCount: 16, embedding: vectorSql(0) }, + ]); + const service = new FounderWeeklyReviewEvidenceService(test.db, () => new Date("2026-02-10T00:00:00.000Z"), { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(test.db) }, new StrictCurrentWorkspaceDocumentStore(test.db, { embedQuery: jest.fn().mockResolvedValue(vector(0)) })); + const input = { companyId: BigInt(owner!.id), reportingPeriod: { start: "2026-02-01", end: "2026-02-07" }, workspaceTimezone: "UTC", founderContext: "blocker", actor: { externalUserId: "u" }, requestKey: "computed-integration" }; + const first = await service.collectFounderWeeklyReviewEvidence(input); + const second = await service.collectFounderWeeklyReviewEvidence(input); + expect(first.items.filter((item) => item.sourceType === "document_change")).toHaveLength(1); + expect(first.items).toEqual(second.items); + expect(first.items).toEqual(expect.arrayContaining([ + expect.objectContaining({ sourceType: "document_change", metadata: expect.objectContaining({ previousVersionId: a1!.id, currentVersionId: a2!.id }) }), + ])); + const workspaceItem = first.items.find((item) => item.sourceType === "workspace_document"); + expect(workspaceItem).toMatchObject({ sourceType: "workspace_document", title: "Current blocker" }); + expect(workspaceItem).not.toHaveProperty("sourceTimestamp"); + expect(first.items.some((item) => item.excerpt.includes("v3 must not diff") || item.excerpt.includes("legacy relevant") || item.excerpt.includes("foreign relevant"))).toBe(false); + } finally { await test.close(); } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts new file mode 100644 index 000000000..973abea2a --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts @@ -0,0 +1,40 @@ +import { + buildWorkspaceDocumentEvidence, + normalizeFounderContextRetrievalQuery, + selectWorkspaceDocumentHits, + type WorkspaceDocumentHit, +} from "@launchstack/features/founder-weekly-review"; + +const hit = (documentId: bigint, chunkId: number, score: number, content = `content ${chunkId}`): WorkspaceDocumentHit => ({ + documentId, documentTitle: `Document ${documentId}`, versionId: 9n, contextChunkId: chunkId, content, similarityScore: score, + structureId: 4n, structurePath: "/1", structureTitle: "Section", pageNumber: 1, lineStart: 1, lineEnd: 2, +}); + +describe("workspace document evidence", () => { + it("normalizes bounded founder context and skips blank queries", () => { + expect(normalizeFounderContextRetrievalQuery(" founder priority ")).toBe("founder priority"); + expect(normalizeFounderContextRetrievalQuery(" ")).toBeNull(); + }); + + it("selects diverse stable hits and normalizes JSON-safe evidence", () => { + const selected = selectWorkspaceDocumentHits([hit(2n, 3, 0.8), hit(1n, 2, 0.9), hit(1n, 1, 0.9), hit(1n, 1, 0.7)]); + expect(selected.map((item) => [item.documentId, item.contextChunkId])).toEqual([[1n, 1], [1n, 2], [2n, 3]]); + const evidence = buildWorkspaceDocumentEvidence(selected); + expect(evidence).toEqual(buildWorkspaceDocumentEvidence(selected)); + expect(evidence[0]).toMatchObject({ sourceType: "workspace_document", sourceId: "workspace_document:doc:1:version:9:chunk:1", metadata: { documentId: "1", documentVersionId: "9", retrievalReason: "founder_context_relevance" } }); + expect(() => JSON.stringify(evidence)).not.toThrow(); + }); + + it("uses exact BigInt ordering to break equal-score ties", () => { + const selected = selectWorkspaceDocumentHits([ + hit(9007199254740993n, 1, 0.9), + hit(9007199254740992n, 1, 0.9), + ]); + expect(selected.map((item) => item.documentId)).toEqual([9007199254740992n, 9007199254740993n]); + }); + + it("bounds excerpts", () => { + const evidence = buildWorkspaceDocumentEvidence([hit(1n, 1, 0.5, "x".repeat(5000))]); + expect(evidence[0]!.excerpt.length).toBe(4000); + }); +}); diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts index 80b733641..d74c76eae 100644 --- a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -36,7 +36,7 @@ try { if (created.run.evidenceSnapshot) throw new Error("Workflow run unexpectedly has an initial snapshot."); const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); const collectionContext = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: created.dispatch.generationClaimId }; const collecting = await worker.claimEvidenceCollection(collectionContext); - const collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-01T00:00:00.000Z")); + const collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-01T00:00:00.000Z"), { kind: "legacy" }); const snapshot = await collector.collectFounderWeeklyReviewEvidence({ companyId: actor.companyId, reportingPeriod: fixture.reportingPeriod, workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: created.run.requestKey }); const beforeDigest = digest(snapshot); const attached = await worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot); const afterDigest = digest(attached.evidenceSnapshot); const counts = Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, attached.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])); diff --git a/apps/web/src/server/founder-weekly-review/document-version-chunks.ts b/apps/web/src/server/founder-weekly-review/document-version-chunks.ts new file mode 100644 index 000000000..08b0c6d04 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/document-version-chunks.ts @@ -0,0 +1,40 @@ +import { and, asc, eq, gte, lt } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { document, documentContextChunks, documentStructure, documentVersions } from "@launchstack/core/db/schema"; +import type { DocumentVersionForComparison, VersionChunk } from "@launchstack/features/founder-weekly-review"; + +export type VersionChunkLoad = + | { state: "complete"; chunks: VersionChunk[]; warnings: string[] } + | { state: "partial"; chunks: VersionChunk[]; warnings: string[] } + | { state: "missing"; chunks: []; warnings: string[] }; + +/** App-side ownership-checked, explicit-version data access for weekly review diffs. */ +export class FounderWeeklyReviewDocumentVersionStore { + constructor(private readonly db: DbClient = getDb()) {} + + async listVersionsBeforePeriodEnd(companyId: bigint, endExclusive: Date): Promise { + return this.db.select({ documentId: documentVersions.documentId, documentTitle: document.title, documentCategory: document.category, + versionId: documentVersions.id, versionNumber: documentVersions.versionNumber, createdAt: documentVersions.createdAt, changelog: documentVersions.changelog }) + .from(documentVersions).innerJoin(document, eq(document.id, documentVersions.documentId)) + .where(and(eq(document.companyId, companyId), lt(documentVersions.createdAt, endExclusive))) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.versionNumber), asc(documentVersions.id)); + } + + async getDocumentChunksForVersion(input: { companyId: bigint; documentId: bigint; versionId: number }): Promise { + const [ownedVersion] = await this.db.select({ id: documentVersions.id }).from(documentVersions) + .innerJoin(document, eq(document.id, documentVersions.documentId)) + .where(and(eq(document.companyId, input.companyId), eq(documentVersions.id, input.versionId), eq(documentVersions.documentId, input.documentId))).limit(1); + if (!ownedVersion) return { state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }; + const rows = await this.db.select({ chunkId: documentContextChunks.id, content: documentContextChunks.content, contentHash: documentContextChunks.contentHash, + structureId: documentContextChunks.structureId, structurePath: documentStructure.path, structureTitle: documentStructure.title, structureOrdering: documentStructure.ordering, + pageNumber: documentContextChunks.pageNumber, lineStart: documentContextChunks.lineStart, lineEnd: documentContextChunks.lineEnd, + documentId: documentContextChunks.documentId, versionId: documentContextChunks.versionId }) + .from(documentContextChunks).leftJoin(documentStructure, eq(documentContextChunks.structureId, documentStructure.id)) + .where(and(eq(documentContextChunks.documentId, input.documentId), eq(documentContextChunks.versionId, BigInt(input.versionId)))) + .orderBy(asc(documentStructure.path), asc(documentContextChunks.pageNumber), asc(documentContextChunks.lineStart), asc(documentStructure.ordering), asc(documentContextChunks.id)); + if (rows.length === 0) return { state: "missing", chunks: [], warnings: ["version_chunks_missing"] }; + const chunks = rows.map((row) => ({ ...row, versionId: row.versionId! })); + const partial = chunks.some((chunk) => !chunk.contentHash || !chunk.structurePath); + return partial ? { state: "partial", chunks, warnings: ["version_chunks_partial_provenance"] } : { state: "complete", chunks, warnings: [] }; + } +} diff --git a/apps/web/src/server/founder-weekly-review/evidence-collector.ts b/apps/web/src/server/founder-weekly-review/evidence-collector.ts index 474d8db0e..a09f2364b 100644 --- a/apps/web/src/server/founder-weekly-review/evidence-collector.ts +++ b/apps/web/src/server/founder-weekly-review/evidence-collector.ts @@ -4,6 +4,8 @@ import { type FounderWeeklyReviewEvidenceSnapshot, type ReportingPeriod, } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewDocumentVersionStore } from "./document-version-chunks"; +import { StrictCurrentWorkspaceDocumentStore } from "./workspace-document-store"; export interface FounderWeeklyReviewEvidenceCollector { collectFounderWeeklyReviewEvidence(input: { @@ -50,7 +52,7 @@ export class CanonicalFounderWeeklyReviewEvidenceCollector implements FounderWee actor: { externalUserId: string }; requestKey: string; }): Promise { - const snapshot = await (this.service ??= new FounderWeeklyReviewEvidenceService()).collectFounderWeeklyReviewEvidence({ + const snapshot = await (this.service ??= new FounderWeeklyReviewEvidenceService(undefined, undefined, { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore() }, new StrictCurrentWorkspaceDocumentStore())).collectFounderWeeklyReviewEvidence({ companyId: input.companyId, reportingPeriod: input.reportingPeriod, workspaceTimezone: input.workspaceTimezone, diff --git a/apps/web/src/server/founder-weekly-review/workspace-document-store.ts b/apps/web/src/server/founder-weekly-review/workspace-document-store.ts new file mode 100644 index 000000000..c983a5a45 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/workspace-document-store.ts @@ -0,0 +1,48 @@ +import { and, asc, eq, sql } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { createEmbeddingModel, resolveEmbeddingIndex, type EmbeddingsProvider } from "@launchstack/core/embeddings"; +import { document, documentContextChunks, documentStructure } from "@launchstack/core/db/schema"; +import { + MAX_WORKSPACE_RETRIEVAL_CANDIDATES, + normalizeFounderContextRetrievalQuery, + type WorkspaceDocumentHit, + type WorkspaceDocumentRetrievalInput, + type WorkspaceDocumentRetrievalResult, +} from "@launchstack/features/founder-weekly-review"; + +export interface FounderWeeklyReviewWorkspaceDocumentStore { + retrieveRelevantCurrentDocumentChunks(input: WorkspaceDocumentRetrievalInput): Promise; +} + +/** Strict current-version vector retrieval used only by Founder Weekly Review. */ +export class StrictCurrentWorkspaceDocumentStore implements FounderWeeklyReviewWorkspaceDocumentStore { + constructor(private readonly db: DbClient = getDb(), private readonly embeddings?: EmbeddingsProvider) {} + + async retrieveRelevantCurrentDocumentChunks(input: WorkspaceDocumentRetrievalInput): Promise { + const query = normalizeFounderContextRetrievalQuery(input.founderContext); + if (!query) return { state: "empty", hits: [] }; + const topK = Math.max(1, Math.min(MAX_WORKSPACE_RETRIEVAL_CANDIDATES, input.topK ?? MAX_WORKSPACE_RETRIEVAL_CANDIDATES)); + try { + // document_context_chunks.embedding is the legacy 1536-dimensional store. + // Do not resolve a configurable dimension-table index for this SQL path. + const embedding = await (this.embeddings ?? createEmbeddingModel(resolveEmbeddingIndex("legacy-openai-1536"))).embedQuery(query); + if (embedding.length !== 1536 || embedding.some((value) => !Number.isFinite(value))) { + return { state: "unavailable", hits: [], warnings: ["workspace_document_embedding_index_unavailable"] }; + } + const literal = sql.raw(`'${JSON.stringify(embedding)}'::vector(1536)`); + const rows = await this.db.select({ documentId: documentContextChunks.documentId, documentTitle: document.title, + versionId: documentContextChunks.versionId, contextChunkId: documentContextChunks.id, content: documentContextChunks.content, + structureId: documentContextChunks.structureId, structurePath: documentStructure.path, structureTitle: documentStructure.title, + pageNumber: documentContextChunks.pageNumber, lineStart: documentContextChunks.lineStart, lineEnd: documentContextChunks.lineEnd, + similarityScore: sql`1 - (${documentContextChunks.embedding} <=> ${literal})` }) + .from(documentContextChunks).innerJoin(document, eq(documentContextChunks.documentId, document.id)) + .leftJoin(documentStructure, eq(documentContextChunks.structureId, documentStructure.id)) + .where(and(eq(document.companyId, input.companyId), eq(documentContextChunks.versionId, document.currentVersionId), sql`${documentContextChunks.versionId} IS NOT NULL`, sql`${document.currentVersionId} IS NOT NULL`, sql`${documentContextChunks.embedding} IS NOT NULL`)) + .orderBy(sql`${documentContextChunks.embedding} <=> ${literal}`, asc(document.id), asc(documentContextChunks.versionId), asc(documentContextChunks.id)).limit(topK); + if (!rows.length) return { state: "empty", hits: [] }; + return { state: "success", hits: rows.map((row): WorkspaceDocumentHit => ({ ...row, versionId: row.versionId!, similarityScore: row.similarityScore })) }; + } catch { + return { state: "unavailable", hits: [], warnings: ["workspace_document_retrieval_unavailable"] }; + } + } +} diff --git a/packages/features/src/founder-weekly-review/document-change.ts b/packages/features/src/founder-weekly-review/document-change.ts new file mode 100644 index 000000000..17b244b45 --- /dev/null +++ b/packages/features/src/founder-weekly-review/document-change.ts @@ -0,0 +1,164 @@ +import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; + +export type DocumentVersionForComparison = { + documentId: bigint; + documentTitle: string; + documentCategory: string | null; + versionId: number; + versionNumber: number; + createdAt: Date; + changelog: string | null; +}; + +export type VersionPair = { + documentId: bigint; + documentTitle: string; + documentCategory: string | null; + previousVersionId: number; + previousVersionNumber: number; + previousCreatedAt: Date; + currentVersionId: number; + currentVersionNumber: number; + currentCreatedAt: Date; + currentChangelog: string | null; +}; + +export type VersionChunk = { + chunkId: number; + content: string; + contentHash: string | null; + structureId: bigint | null; + structurePath: string | null; + structureTitle: string | null; + structureOrdering: number | null; + pageNumber: number | null; + lineStart: number | null; + lineEnd: number | null; + documentId: bigint; + versionId: bigint; +}; + +export type ChunkAlignment = { + changeType: "added" | "removed" | "modified" | "unchanged"; + previousChunk?: VersionChunk; + currentChunk?: VersionChunk; + alignmentMethod: "content_hash" | "structure_path" | "section_title" | "structural_position" | "text_similarity" | "unmatched"; + similarityScore?: number; +}; + +const MAX_EXCERPT = 4000; +const MAX_METADATA_TEXT = 512; +const MIN_TEXT_SIMILARITY_CHARACTERS = 20; +const MIN_TEXT_SIMILARITY_TOKENS = 3; + +const compareVersions = (a: DocumentVersionForComparison, b: DocumentVersionForComparison) => + a.createdAt.getTime() - b.createdAt.getTime() || a.versionNumber - b.versionNumber || a.versionId - b.versionId; +const normalize = (value: string | null | undefined) => value?.replace(/\s+/g, " ").trim().toLocaleLowerCase() || null; +const validContentHash = (value: string | null): value is string => value !== null && /^[a-f0-9]{64}$/i.test(value); +const compareChunks = (a: VersionChunk, b: VersionChunk) => + (a.structurePath ?? "").localeCompare(b.structurePath ?? "") || + (a.pageNumber ?? -1) - (b.pageNumber ?? -1) || + (a.lineStart ?? -1) - (b.lineStart ?? -1) || + (a.structureOrdering ?? -1) - (b.structureOrdering ?? -1) || a.chunkId - b.chunkId; + +/** Select only adjacent pairs whose current version is in the reporting period. */ +export function selectVersionPairsForReportingPeriod( + versions: readonly DocumentVersionForComparison[], startInclusive: Date, endExclusive: Date +): VersionPair[] { + const byDocument = new Map(); + for (const version of versions) { + if (version.createdAt >= endExclusive) continue; + const key = version.documentId.toString(); + const group = byDocument.get(key) ?? []; + group.push(version); byDocument.set(key, group); + } + const pairs: VersionPair[] = []; + for (const group of byDocument.values()) { + const ordered = [...group].sort(compareVersions); + for (let index = 0; index < ordered.length; index++) { + const current = ordered[index]!; + if (current.createdAt < startInclusive) continue; + const previous = ordered[index - 1]; + if (!previous) continue; // First-ever version is an added baseline, not an invented diff. + pairs.push({ documentId: current.documentId, documentTitle: current.documentTitle, documentCategory: current.documentCategory, + previousVersionId: previous.versionId, previousVersionNumber: previous.versionNumber, previousCreatedAt: previous.createdAt, + currentVersionId: current.versionId, currentVersionNumber: current.versionNumber, currentCreatedAt: current.createdAt, + currentChangelog: current.changelog }); + } + } + return pairs.sort((a, b) => a.currentCreatedAt.getTime() - b.currentCreatedAt.getTime() || a.currentVersionNumber - b.currentVersionNumber || a.currentVersionId - b.currentVersionId); +} + +function bestMatch(previous: VersionChunk, candidates: VersionChunk[], used: Set, key: (chunk: VersionChunk) => string | null, requireUnique = false, predicate: (candidate: VersionChunk) => boolean = () => true): VersionChunk | undefined { + const value = key(previous); if (!value) return undefined; + const matches = candidates.filter((candidate) => !used.has(candidate.chunkId.toString()) && key(candidate) === value && predicate(candidate)).sort(compareChunks); + return requireUnique && matches.length !== 1 ? undefined : matches[0]; +} + +function textSimilarity(left: string, right: string): number { + const tokens = (value: string) => new Set(value.toLocaleLowerCase().match(/[\p{L}\p{N}_-]{2,}/gu) ?? []); + const a = tokens(left); const b = tokens(right); + if (left.trim().length < MIN_TEXT_SIMILARITY_CHARACTERS || right.trim().length < MIN_TEXT_SIMILARITY_CHARACTERS || a.size < MIN_TEXT_SIMILARITY_TOKENS || b.size < MIN_TEXT_SIMILARITY_TOKENS) return 0; + let intersection = 0; for (const token of a) if (b.has(token)) intersection++; + return intersection / (a.size + b.size - intersection); +} + +function bestTextMatch(previous: VersionChunk, candidates: VersionChunk[], used: Set): { chunk: VersionChunk; score: number } | undefined { + const ranked = candidates.filter((candidate) => !used.has(candidate.chunkId.toString())).sort(compareChunks).slice(0, 200) + .map((chunk) => ({ chunk, score: textSimilarity(previous.content, chunk.content) })) + .filter((candidate) => candidate.score >= 0.35) + .sort((a, b) => b.score - a.score || compareChunks(a.chunk, b.chunk)); + return ranked[0]; +} + +/** Deterministic one-to-one alignment without embeddings or model calls. */ +export function alignVersionChunks(previousChunks: readonly VersionChunk[], currentChunks: readonly VersionChunk[]): ChunkAlignment[] { + const previous = [...previousChunks].sort(compareChunks); const current = [...currentChunks].sort(compareChunks); + const used = new Set(); const results: ChunkAlignment[] = []; + const strategies: Array<{ method: ChunkAlignment["alignmentMethod"]; key: (chunk: VersionChunk) => string | null; requireUnique?: boolean; predicate?: (previous: VersionChunk, current: VersionChunk) => boolean }> = [ + { method: "content_hash", key: (c) => validContentHash(c.contentHash) ? c.contentHash.toLocaleLowerCase() : null, predicate: (previous, candidate) => previous.content === candidate.content }, + { method: "structure_path", key: (c) => normalize(c.structurePath), requireUnique: true }, + { method: "section_title", key: (c) => normalize(c.structureTitle), requireUnique: true }, + { method: "structural_position", key: (c) => { + const path = normalize(c.structurePath); + return path && c.pageNumber !== null && c.structureOrdering !== null ? `${path}|${c.pageNumber}|${c.lineStart ?? ""}|${c.structureOrdering}` : null; + } }, + ]; + for (const oldChunk of previous) { + let match: VersionChunk | undefined; let method: ChunkAlignment["alignmentMethod"] = "unmatched"; + for (const strategy of strategies) { + match = bestMatch(oldChunk, current, used, strategy.key, strategy.requireUnique, (candidate) => strategy.predicate?.(oldChunk, candidate) ?? true); + if (match) { method = strategy.method; break; } + } + if (!match) { + const textMatch = bestTextMatch(oldChunk, current, used); + if (textMatch) { match = textMatch.chunk; method = "text_similarity"; } + } + if (!match) { results.push({ changeType: "removed", previousChunk: oldChunk, alignmentMethod: "unmatched" }); continue; } + used.add(match.chunkId.toString()); + const score = method === "text_similarity" ? textSimilarity(oldChunk.content, match.content) : undefined; + results.push({ changeType: oldChunk.content === match.content ? "unchanged" : "modified", previousChunk: oldChunk, currentChunk: match, alignmentMethod: method, ...(score === undefined ? {} : { similarityScore: score }) }); + } + for (const chunk of current) if (!used.has(chunk.chunkId.toString())) results.push({ changeType: "added", currentChunk: chunk, alignmentMethod: "unmatched" }); + return results.sort((a, b) => compareChunks(a.currentChunk ?? a.previousChunk!, b.currentChunk ?? b.previousChunk!)); +} + +function bound(value: string, max = MAX_EXCERPT) { return value.length <= max ? value : `${value.slice(0, max - 1)}…`; } +function preview(value: string) { return bound(value.replace(/\s+/g, " ").trim(), 900); } + +export function buildDocumentChangeEvidence(pair: VersionPair, alignments: readonly ChunkAlignment[]): FounderWeeklyReviewEvidenceItem[] { + return alignments.filter((alignment) => alignment.changeType !== "unchanged").map((alignment) => { + const previous = alignment.previousChunk; const current = alignment.currentChunk; + const previousId = previous?.chunkId.toString() ?? "none"; const currentId = current?.chunkId.toString() ?? "none"; + const sourceId = `document_change:doc:${pair.documentId}:v${pair.previousVersionId}:v${pair.currentVersionId}:chunk:${previousId}:${currentId}`; + const excerpt = alignment.changeType === "modified" + ? `Section modified. Before: ${preview(previous!.content)} After: ${preview(current!.content)}` + : alignment.changeType === "added" ? `Section added: ${preview(current!.content)}` : `Section removed: ${preview(previous!.content)}`; + return { sourceType: "document_change", sourceId, title: pair.documentTitle, + sourceTimestamp: pair.currentCreatedAt.toISOString(), excerpt: bound(excerpt), workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, + metadata: { documentId: pair.documentId.toString(), previousVersionId: pair.previousVersionId, currentVersionId: pair.currentVersionId, previousVersionNumber: pair.previousVersionNumber, currentVersionNumber: pair.currentVersionNumber, + previousChunkId: previous?.chunkId ?? null, currentChunkId: current?.chunkId ?? null, changeType: alignment.changeType, alignmentMethod: alignment.alignmentMethod, + previousContentHash: previous?.contentHash ?? null, currentContentHash: current?.contentHash ?? null, structurePath: current?.structurePath ?? previous?.structurePath ?? null, + userChangelog: pair.currentChangelog ? bound(pair.currentChangelog.replace(/\s+/g, " ").trim(), MAX_METADATA_TEXT) : null } }; + }); +} diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index 5df6d5f6a..1b62b4e9e 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -14,6 +14,19 @@ import { documentContextChunks, documentVersions, } from "@launchstack/core/db/schema"; +import { + alignVersionChunks, + buildDocumentChangeEvidence, + selectVersionPairsForReportingPeriod, + type DocumentVersionForComparison, + type VersionChunk, +} from "./document-change"; +import { + buildWorkspaceDocumentEvidence, + normalizeFounderContextRetrievalQuery, + type WorkspaceDocumentRetrievalInput, + type WorkspaceDocumentRetrievalResult, +} from "./workspace-document"; /** Approved, stored category value. Do not fuzzy-match or seed this category. */ export const CUSTOMER_FEEDBACK_CATEGORY = "Customer Feedback"; @@ -130,14 +143,70 @@ export interface FounderWeeklyReviewEvidenceSourceResult { warnings: FounderWeeklyReviewEvidenceWarning[]; } +/** Implemented in apps/web so pure feature logic never depends on Drizzle. */ +export interface FounderWeeklyReviewDocumentChangeStore { + listVersionsBeforePeriodEnd(companyId: bigint, endExclusive: Date): Promise; + getDocumentChunksForVersion(input: { companyId: bigint; documentId: bigint; versionId: number }): Promise< + { state: "complete" | "partial" | "missing"; chunks: VersionChunk[]; warnings: string[] } + >; +} + +export type FounderWeeklyReviewDocumentChangeSource = + | { kind: "computed"; store: FounderWeeklyReviewDocumentChangeStore } + | { kind: "legacy" } + | { kind: "unconfigured" }; + +export interface FounderWeeklyReviewWorkspaceDocumentStore { + retrieveRelevantCurrentDocumentChunks(input: WorkspaceDocumentRetrievalInput): Promise; +} + export class FounderWeeklyReviewEvidenceService { - constructor(private readonly db: DbClient = getDb(), private readonly now: () => Date = () => new Date()) {} + constructor( + private readonly db: DbClient = getDb(), + private readonly now: () => Date = () => new Date(), + private readonly documentChangeSource: FounderWeeklyReviewDocumentChangeSource = { kind: "unconfigured" }, + private readonly workspaceDocumentStore?: FounderWeeklyReviewWorkspaceDocumentStore, + ) {} async collectDocumentChangeEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { return (await this.collectDocumentChangeEvidenceResult(companyId, startInclusive, endExclusive)).items; } private async collectDocumentChangeEvidenceResult(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + if (this.documentChangeSource.kind === "computed") { + const store = this.documentChangeSource.store; + const versions = await store.listVersionsBeforePeriodEnd(companyId, endExclusive); + const pairs = selectVersionPairsForReportingPeriod(versions, startInclusive, endExclusive); + const items: FounderWeeklyReviewEvidenceItem[] = []; + const warnings: FounderWeeklyReviewEvidenceWarning[] = []; + for (const pair of pairs) { + const [previous, current] = await Promise.all([ + store.getDocumentChunksForVersion({ companyId, documentId: pair.documentId, versionId: pair.previousVersionId }), + store.getDocumentChunksForVersion({ companyId, documentId: pair.documentId, versionId: pair.currentVersionId }), + ]); + if (previous.state === "missing" || current.state === "missing") { + warnings.push(warning("document_change_chunks_missing", "A version pair could not be compared because processed historical chunks are missing.", "document_change")); + continue; + } + if (previous.state === "partial" || current.state === "partial") { + warnings.push(warning("document_change_chunks_partial", "A version pair was compared with partial chunk provenance.", "document_change")); + } + items.push(...buildDocumentChangeEvidence(pair, alignVersionChunks(previous.chunks, current.chunks))); + } + const pairedCurrentVersionIds = new Set(pairs.map((pair) => pair.currentVersionId)); + const inPeriodWithNoPair = versions.some((version) => + version.createdAt >= startInclusive + && version.createdAt < endExclusive + && !pairedCurrentVersionIds.has(version.versionId) + ); + if (inPeriodWithNoPair) warnings.push(warning("document_change_baseline_missing", "An in-period document version has no predecessor, so no content diff was generated.", "document_change")); + const ordered = orderEvidenceItems(dedupeEvidenceItems(items)); + if (ordered.length > MAX_ITEMS_PER_SOURCE) warnings.push(warning("document_change_truncated", "Document change evidence was truncated to the per-source limit.", "document_change")); + return { items: ordered.slice(0, MAX_ITEMS_PER_SOURCE), warnings }; + } + if (this.documentChangeSource.kind !== "legacy") { + return { items: [], warnings: [warning("document_change_source_unconfigured", "Document-change collection requires the explicit computed-diff source.", "document_change")] }; + } const rows = await this.db.select({ documentId: documentVersions.documentId, documentTitle: document.title, documentCategory: document.category, versionId: documentVersions.id, @@ -189,6 +258,15 @@ export class FounderWeeklyReviewEvidenceService { metadata: { enteredBy: input.actor.externalUserId, provenance: "request_time_founder_input", excerptTruncated: bounded.truncated } }], warnings: [] }; } + async collectWorkspaceDocumentEvidence(companyId: bigint, founderContext: string | undefined): Promise { + const query = normalizeFounderContextRetrievalQuery(founderContext); + if (!query || !this.workspaceDocumentStore) return { items: [], warnings: [] }; + const result = await this.workspaceDocumentStore.retrieveRelevantCurrentDocumentChunks({ companyId, founderContext: query }); + if (result.state === "success") return { items: buildWorkspaceDocumentEvidence(result.hits), warnings: [] }; + if (result.state === "empty") return { items: [], warnings: [] }; + return { items: [], warnings: result.warnings.slice(0, MAX_WARNINGS).map((code) => warning(code, "Founder Context workspace retrieval was unavailable.", "workspace_document")) }; + } + async collectFounderWeeklyReviewEvidence(input: BuildFounderWeeklyReviewEvidenceSnapshotInput): Promise { return this.buildEvidenceSnapshot(input); } @@ -198,7 +276,8 @@ export class FounderWeeklyReviewEvidenceService { const documentChanges = await this.collectDocumentChangeEvidenceResult(input.companyId, startInclusive, endExclusive); const feedback = await this.collectCustomerFeedbackEvidence(input.companyId, startInclusive, endExclusive); const founderContext = this.collectFounderContextEvidence(input); - const sourceResults: FounderWeeklyReviewEvidenceSourceResult[] = [documentChanges, feedback, founderContext]; + const workspaceDocuments = await this.collectWorkspaceDocumentEvidence(input.companyId, input.founderContext); + const sourceResults: FounderWeeklyReviewEvidenceSourceResult[] = [documentChanges, feedback, founderContext, workspaceDocuments]; const items = orderEvidenceItems(dedupeEvidenceItems(sourceResults.flatMap((result) => result.items))); const requestedMax = input.maxItems ?? MAX_SNAPSHOT_ITEMS; const maxItems = Math.max(0, Math.min(MAX_SNAPSHOT_ITEMS, requestedMax)); diff --git a/packages/features/src/founder-weekly-review/generation-validation.ts b/packages/features/src/founder-weekly-review/generation-validation.ts index b3b0f936d..b0f2a3c09 100644 --- a/packages/features/src/founder-weekly-review/generation-validation.ts +++ b/packages/features/src/founder-weekly-review/generation-validation.ts @@ -20,6 +20,11 @@ export interface FounderWeeklyReviewValidationDetail { sourceId?: string; } +/** Sources in the current model that can establish a reporting-period event. */ +const TEMPORAL_EVIDENCE_SOURCE_TYPES = new Set(["document_change"]); +const isTemporalEvidenceSource = (source: { sourceType?: string } | undefined) => + source !== undefined && TEMPORAL_EVIDENCE_SOURCE_TYPES.has(source.sourceType ?? ""); + export function assertUniqueSnapshotSourceIds( evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot ): void { @@ -72,6 +77,20 @@ export function validateFounderWeeklyReviewV2Citations( } } } + if (sectionName === "whatChanged" || sectionName === "whatShipped") { + const sources = item.sourceIds.map((sourceId) => evidenceBySourceId.get(sourceId) as { sourceType?: string } | undefined); + const workspaceOnly = sources.length > 0 && sources.every((source) => source?.sourceType === "workspace_document"); + const workspaceWithoutTemporalEvidence = sources.some((source) => source?.sourceType === "workspace_document") + && !sources.some(isTemporalEvidenceSource); + if (workspaceOnly || workspaceWithoutTemporalEvidence) { + throw new FounderWeeklyReviewGenerationValidationError( + `${sectionName} requires temporal evidence in addition to workspace_document context.`, + item.sourceIds.map((sourceId) => ({ + code: workspaceOnly ? "workspace_document_only_temporal_claim" : `${sectionName === "whatChanged" ? "what_changed" : "what_shipped"}_requires_temporal_evidence`, section: sectionName, itemIndex, sourceId, + })) + ); + } + } } } diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 04400cc11..535060b42 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -8,3 +8,5 @@ export * from "./worker-service"; export * from "./generator"; export * from "./generation-validation"; export * from "./prompts"; +export * from "./document-change"; +export * from "./workspace-document"; diff --git a/packages/features/src/founder-weekly-review/workspace-document.ts b/packages/features/src/founder-weekly-review/workspace-document.ts new file mode 100644 index 000000000..96507965b --- /dev/null +++ b/packages/features/src/founder-weekly-review/workspace-document.ts @@ -0,0 +1,58 @@ +import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; + +export const MAX_WORKSPACE_RETRIEVAL_CANDIDATES = 12; +export const MAX_WORKSPACE_EVIDENCE_ITEMS = 8; +export const MAX_WORKSPACE_EVIDENCE_PER_DOCUMENT = 2; +export const MAX_WORKSPACE_EXCERPT_LENGTH = 4000; + +export type WorkspaceDocumentRetrievalInput = { companyId: bigint; founderContext: string; topK?: number }; +export type WorkspaceDocumentHit = { + documentId: bigint; documentTitle: string; versionId: bigint; contextChunkId: number; retrievalChunkId?: number; + content: string; similarityScore: number; structureId?: bigint | null; structurePath?: string | null; structureTitle?: string | null; + pageNumber?: number | null; lineStart?: number | null; lineEnd?: number | null; +}; +export type WorkspaceDocumentRetrievalResult = + | { state: "success"; hits: WorkspaceDocumentHit[] } + | { state: "empty"; hits: [] } + | { state: "unavailable"; hits: []; warnings: string[] }; + +const normalize = (value: string | null | undefined) => value?.replace(/\s+/g, " ").trim() || null; +const bound = (value: string, max: number) => value.length <= max ? value : `${value.slice(0, max - 1)}…`; +const compareBigInt = (left: bigint, right: bigint) => left < right ? -1 : left > right ? 1 : 0; +const compareHits = (a: WorkspaceDocumentHit, b: WorkspaceDocumentHit) => b.similarityScore - a.similarityScore || compareBigInt(a.documentId, b.documentId) || compareBigInt(a.versionId, b.versionId) || a.contextChunkId - b.contextChunkId; + +export function normalizeFounderContextRetrievalQuery(founderContext: string | null | undefined): string | null { + const normalized = normalize(founderContext); + return normalized ? bound(normalized, 4000) : null; +} + +/** Deterministically deduplicate, diversify, and cap provider/database hits. */ +export function selectWorkspaceDocumentHits(hits: readonly WorkspaceDocumentHit[]): WorkspaceDocumentHit[] { + const selected: WorkspaceDocumentHit[] = []; const contexts = new Set(); const perDocument = new Map(); + for (const hit of [...hits].filter((hit) => normalize(hit.content) && Number.isFinite(hit.similarityScore)).sort(compareHits)) { + const contextKey = `${hit.documentId}:${hit.versionId}:${hit.contextChunkId}`; + if (contexts.has(contextKey)) continue; + const count = perDocument.get(hit.documentId.toString()) ?? 0; + if (count >= MAX_WORKSPACE_EVIDENCE_PER_DOCUMENT) continue; + contexts.add(contextKey); perDocument.set(hit.documentId.toString(), count + 1); selected.push(hit); + if (selected.length >= MAX_WORKSPACE_EVIDENCE_ITEMS) break; + } + return selected; +} + +export function buildWorkspaceDocumentEvidence(hits: readonly WorkspaceDocumentHit[]): FounderWeeklyReviewEvidenceItem[] { + return selectWorkspaceDocumentHits(hits).map((hit) => ({ + sourceType: "workspace_document" as const, + sourceId: `workspace_document:doc:${hit.documentId}:version:${hit.versionId}:chunk:${hit.contextChunkId}`, + title: hit.documentTitle, + excerpt: bound(normalize(hit.content)!, MAX_WORKSPACE_EXCERPT_LENGTH), + workspaceDeepLink: `/employer/documents/viewer?docId=${hit.documentId}`, + metadata: { + documentId: hit.documentId.toString(), documentVersionId: hit.versionId.toString(), chunkId: hit.contextChunkId, + retrievalChunkId: hit.retrievalChunkId ?? null, retrievalReason: "founder_context_relevance", similarityScore: hit.similarityScore, + structureId: hit.structureId?.toString() ?? null, structurePath: hit.structurePath ? bound(hit.structurePath, 512) : null, + structureTitle: hit.structureTitle ? bound(hit.structureTitle, 512) : null, pageNumber: hit.pageNumber ?? null, + lineStart: hit.lineStart ?? null, lineEnd: hit.lineEnd ?? null, + }, + })); +} From e1aebab752ca381069dbd9a7f498d2d337fcf15e Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Wed, 5 Aug 2026 14:06:24 +0800 Subject: [PATCH 17/29] test(founder-weekly-review): add computed realistic e2e coverage --- .../realistic-e2e-mode.test.ts | 18 +++ .../__tests__/founderWeeklyReview/testDb.ts | 11 +- ...run-founder-weekly-review-realistic-e2e.ts | 117 +++++++++++++++--- .../realistic-e2e-mode.ts | 18 +++ 4 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts create mode 100644 apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts diff --git a/apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts b/apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts new file mode 100644 index 000000000..67219135e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts @@ -0,0 +1,18 @@ +import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; + +describe("realistic Founder Weekly Review evidence mode", () => { + it("preserves legacy mode by default and accepts computed mode explicitly", () => { + expect(parseFounderWeeklyReviewRealisticEvidenceMode(undefined)).toBe("legacy"); + expect(parseFounderWeeklyReviewRealisticEvidenceMode("legacy")).toBe("legacy"); + expect(parseFounderWeeklyReviewRealisticEvidenceMode("computed")).toBe("computed"); + }); + + it("rejects unknown evidence modes before any collection or provider call", () => { + expect(() => parseFounderWeeklyReviewRealisticEvidenceMode("other")).toThrow("Unsupported FWR_EVIDENCE_MODE"); + }); + + it("does not append computed twice when the configured export root already names it", () => { + expect(founderWeeklyReviewRealisticExportRoot("computed", ".artifacts/founder-weekly-review/computed")).toBe(".artifacts/founder-weekly-review/computed"); + expect(founderWeeklyReviewRealisticExportRoot("computed", undefined)).toBe(".artifacts/founder-weekly-review/computed"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/testDb.ts b/apps/web/__tests__/founderWeeklyReview/testDb.ts index 8e1de5d7f..b25e60f9b 100644 --- a/apps/web/__tests__/founderWeeklyReview/testDb.ts +++ b/apps/web/__tests__/founderWeeklyReview/testDb.ts @@ -128,9 +128,18 @@ async function bootstrapIsolatedSchema( "id" serial PRIMARY KEY, "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, "version_id" bigint, + "parent_id" bigint, + "level" integer NOT NULL DEFAULT 0, "ordering" integer NOT NULL DEFAULT 0, "title" text, - "path" varchar(256) + "content_type" varchar(50) NOT NULL DEFAULT 'section', + "path" varchar(256), + "start_page" integer, + "end_page" integer, + "child_count" integer NOT NULL DEFAULT 0, + "token_count" integer DEFAULT 0, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz ); CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_retrieval_chunks" ( diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts index d74c76eae..1b94b478c 100644 --- a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -3,47 +3,134 @@ import { createHash, randomUUID } from "node:crypto"; import { createRequire } from "node:module"; import { readFile, mkdir, rename, writeFile, access } from "node:fs/promises"; import { resolve } from "node:path"; -import { eq } from "drizzle-orm"; -import { company, document, documentContextChunks, documentVersions, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; -import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { eq, sql } from "drizzle-orm"; +import { company, document, documentContextChunks, documentStructure, documentVersions, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, generateFounderWeeklyReview, validateFounderWeeklyReviewV2Citations } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; +import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; const require = createRequire(import.meta.url); const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/realistic-company/seed.json"); - type Fixture = { reportingPeriod: { start: string; end: string }; workspaceTimezone: string; founderContext: string; documents: Array<{ title: string; category: string; changelog: string; timestamp: string; chunks?: string[] }> }; +type EvidenceMode = ReturnType; +type ComputedTexts = { before: string; after: string; v3: string; bHistorical: string; nullVersion: string; foreign: string; unrelated: string }; +type ArtifactPaths = { evidence: string; report: string; markdown: string; summary: string }; + function canonicalize(value: unknown): unknown { if (value === null || ["string", "boolean"].includes(typeof value)) return value; if (typeof value === "number" && Number.isFinite(value)) return value; if (Array.isArray(value)) return value.map(canonicalize); if (typeof value === "object") return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalize((value as Record)[key])])); throw new Error("Cannot canonicalize snapshot."); } function digest(value: unknown) { return createHash("sha256").update(JSON.stringify(canonicalize(value)), "utf8").digest("hex"); } async function writeAtomic(path: string, body: string) { try { await access(path); throw new Error("Refusing to overwrite export."); } catch (error) { if (error instanceof Error && error.message.includes("overwrite")) throw error; } const temp = `${path}.${randomUUID()}.tmp`; await writeFile(temp, body, "utf8"); await rename(temp, path); } +const vector = (index: number) => Array.from({ length: 1536 }, (_, i) => i === index ? 1 : 0); +const vectorSql = (index: number) => sql`${JSON.stringify(vector(index))}::vector(1536)`; + +function assertComputedSnapshot(snapshot: unknown, ids: Record, texts: ComputedTexts) { + const parsed = FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot); + const items = parsed.items; + const changes = items.filter((item) => item.sourceType === "document_change"); + const workspace = items.filter((item) => item.sourceType === "workspace_document"); + const feedback = items.filter((item) => item.sourceType === "customer_feedback"); + const founder = items.filter((item) => item.sourceType === "founder_context"); + const change = changes.find((item) => item.metadata.previousVersionId === ids.a1 && item.metadata.currentVersionId === ids.a2); + const workspaceItem = workspace.find((item) => item.metadata.documentId === String(ids.b)); + if (!change || !workspaceItem || !feedback.length || founder.length !== 1) throw new Error("Computed snapshot is missing required evidence types."); + if (change.metadata.previousChunkId === null || change.metadata.currentChunkId === null || !change.excerpt.includes(texts.before) || !change.excerpt.includes(texts.after) || change.sourceTimestamp !== "2026-02-20T10:00:00.000Z" || change.excerpt.includes(texts.v3) || change.excerpt.startsWith("Version ")) throw new Error("Computed document-change assertions failed."); + if (workspaceItem.metadata.documentVersionId !== String(ids.b2) || workspaceItem.metadata.chunkId !== ids.bCurrentChunk || workspaceItem.metadata.retrievalReason !== "founder_context_relevance" || typeof workspaceItem.metadata.similarityScore !== "number" || "sourceTimestamp" in workspaceItem) throw new Error("Strict workspace-document assertions failed."); + if (items.some((item) => [texts.v3, texts.bHistorical, texts.nullVersion, texts.foreign, texts.unrelated].some((value) => item.excerpt.includes(value)))) throw new Error("Computed control evidence leaked into the snapshot."); + if (new Set(items.map((item) => item.sourceId)).size !== items.length) throw new Error("Computed snapshot contains duplicate source IDs."); + return { parsed, counts: Object.fromEntries(["document_change", "workspace_document", "customer_feedback", "founder_context"].map((type) => [type, items.filter((item) => item.sourceType === type).length])) }; +} +function assertComputedReport(payload: any, snapshot: ReturnType) { + validateFounderWeeklyReviewV2Citations(payload, snapshot); + const sourceTypeById = new Map(snapshot.items.map((item) => [item.sourceId, item.sourceType])); + const citedTypes = (section: any) => section.state === "evidence" ? section.items.flatMap((item: any) => item.sourceIds.map((id: string) => sourceTypeById.get(id))) : []; + const changed = payload.sections.whatChanged; + if (changed.state !== "evidence" || !citedTypes(changed).includes("document_change")) throw new Error("Generated report did not cite the computed document change in whatChanged."); + const shipped = payload.sections.whatShipped; + if (shipped.state === "evidence" && citedTypes(shipped).length > 0 && citedTypes(shipped).every((type: unknown) => type === "workspace_document")) throw new Error("Generated report used workspace evidence alone for whatShipped."); + if (citedTypes(payload.sections.whatCustomersSaid).some((type: unknown) => type !== "customer_feedback")) throw new Error("Generated report violated customer-feedback source semantics."); + if (payload.sections.currentBlockers.state === "evidence" && !citedTypes(payload.sections.currentBlockers).includes("workspace_document")) throw new Error("Generated report did not use current workspace context for blockers."); +} + if (process.env.SYNTHETIC_FWR_LOCAL !== "1" || process.env.NODE_ENV === "production") throw new Error("Refusing realistic E2E outside explicit local mode."); const localUrl = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? ""; if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(localUrl)) throw new Error("Refusing non-local database."); const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as Fixture; +const mode = parseFounderWeeklyReviewRealisticEvidenceMode(process.env.FWR_EVIDENCE_MODE); const testDb = await createFounderWeeklyReviewTestDatabase(); try { const [target] = await testDb.db.insert(company).values({ name: "Northstar Analytics", numberOfEmployees: "24" }).returning(); const [other] = await testDb.db.insert(company).values({ name: "Other Company", numberOfEmployees: "4" }).returning(); - for (const entry of fixture.documents) { const [doc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: `local://${entry.title}`, category: entry.category, title: entry.title }).returning(); const [version] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${entry.title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: entry.changelog, createdAt: new Date(entry.timestamp) }).returning(); for (const [index, content] of (entry.chunks ?? []).entries()) await testDb.db.insert(documentContextChunks).values({ documentId: BigInt(doc!.id), versionId: BigInt(version!.id), content, tokenCount: content.split(/\s+/).length, charCount: content.length, pageNumber: index + 1 }); } - const [outsideDoc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: "local://outside", category: "Product", title: "Outside period" }).returning(); await testDb.db.insert(documentVersions).values({ documentId: BigInt(outsideDoc!.id), versionNumber: 1, url: "local://outside/v1", mimeType: "text/plain", uploadedBy: "seed", changelog: "Outside-period control.", createdAt: new Date("2026-03-01T00:00:00.000Z") }); - const [otherDoc] = await testDb.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://other", category: "Product", title: "Other company control" }).returning(); await testDb.db.insert(documentVersions).values({ documentId: BigInt(otherDoc!.id), versionNumber: 1, url: "local://other/v1", mimeType: "text/plain", uploadedBy: "seed", changelog: "Cross-company control.", createdAt: new Date("2026-02-21T00:00:00.000Z") }); const actor = { externalUserId: "realistic-owner", internalUserId: 1n, companyId: BigInt(target!.id), role: "owner" }; + let collector: FounderWeeklyReviewEvidenceService; + let computedIds: Record | undefined; + const texts: ComputedTexts = { before: "Product owns retry telemetry.", after: "Platform owns retry telemetry and recovery monitoring.", v3: "v3 future-only operational note", bHistorical: "historical workspace reliability text", nullVersion: "null-version workspace reliability text", foreign: "foreign workspace reliability text", unrelated: "unrelated current cafeteria menu" }; + + if (mode === "legacy") { + for (const entry of fixture.documents) { const [doc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: `local://${entry.title}`, category: entry.category, title: entry.title }).returning(); const [version] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${entry.title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: entry.changelog, createdAt: new Date(entry.timestamp) }).returning(); for (const [index, content] of (entry.chunks ?? []).entries()) await testDb.db.insert(documentContextChunks).values({ documentId: BigInt(doc!.id), versionId: BigInt(version!.id), content, tokenCount: content.split(/\s+/).length, charCount: content.length, pageNumber: index + 1 }); } + const [outsideDoc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: "local://outside", category: "Product", title: "Outside period" }).returning(); await testDb.db.insert(documentVersions).values({ documentId: BigInt(outsideDoc!.id), versionNumber: 1, url: "local://outside/v1", mimeType: "text/plain", uploadedBy: "seed", changelog: "Outside-period control.", createdAt: new Date("2026-03-01T00:00:00.000Z") }); + collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-01T00:00:00.000Z"), { kind: "legacy" }); + } else { + const [a] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://release-ownership", category: "Product", title: "Retry ownership" }).returning(); + const [b] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://onboarding", category: "Planning", title: "Onboarding reliability" }).returning(); + const [feedbackDoc] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://feedback", category: "Customer Feedback", title: "Customer Interviews" }).returning(); + const [unrelated] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://unrelated", category: "Product", title: "Unrelated" }).returning(); + const [foreign] = await testDb.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://foreign", category: "Planning", title: "Foreign" }).returning(); + const [a1] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 1, url: "local://a/v1", mimeType: "text/plain", changelog: "Initial ownership.", createdAt: new Date("2026-02-10T10:00:00.000Z") }).returning(); + const [a2] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 2, url: "local://a/v2", mimeType: "text/plain", changelog: "Platform ownership was updated.", createdAt: new Date("2026-02-20T10:00:00.000Z") }).returning(); + const [a3] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 3, url: "local://a/v3", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [b1] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(b!.id), versionNumber: 1, url: "local://b/v1", mimeType: "text/plain", createdAt: new Date("2026-02-01T10:00:00.000Z") }).returning(); + const [b2] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(b!.id), versionNumber: 2, url: "local://b/v2", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [feedbackVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(feedbackDoc!.id), versionNumber: 1, url: "local://feedback/v1", mimeType: "text/plain", createdAt: new Date("2026-02-22T12:00:00.000Z") }).returning(); + const [unrelatedVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(unrelated!.id), versionNumber: 1, url: "local://unrelated/v1", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [foreignVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(foreign!.id), versionNumber: 1, url: "local://foreign/v1", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [s1] = await testDb.db.insert(documentStructure).values({ documentId: BigInt(a!.id), versionId: BigInt(a1!.id), ordering: 1, title: "Retry ownership", path: "1" }).returning(); + const [s2] = await testDb.db.insert(documentStructure).values({ documentId: BigInt(a!.id), versionId: BigInt(a2!.id), ordering: 1, title: "Retry ownership", path: "1" }).returning(); + await testDb.db.update(document).set({ currentVersionId: BigInt(a3!.id) }).where(eq(document.id, a!.id)); + await testDb.db.update(document).set({ currentVersionId: BigInt(b2!.id) }).where(eq(document.id, b!.id)); + await testDb.db.update(document).set({ currentVersionId: BigInt(unrelatedVersion!.id) }).where(eq(document.id, unrelated!.id)); + await testDb.db.update(document).set({ currentVersionId: BigInt(foreignVersion!.id) }).where(eq(document.id, foreign!.id)); + const inserted = await testDb.db.insert(documentContextChunks).values([ + { documentId: BigInt(a!.id), versionId: BigInt(a1!.id), structureId: BigInt(s1!.id), content: texts.before, contentHash: "a".repeat(64), tokenCount: 4, charCount: texts.before.length, pageNumber: 1, lineStart: 1, lineEnd: 1 }, + { documentId: BigInt(a!.id), versionId: BigInt(a2!.id), structureId: BigInt(s2!.id), content: texts.after, contentHash: "b".repeat(64), tokenCount: 7, charCount: texts.after.length, pageNumber: 1, lineStart: 1, lineEnd: 1 }, + { documentId: BigInt(a!.id), versionId: BigInt(a3!.id), content: texts.v3, contentHash: "c".repeat(64), tokenCount: 4, charCount: texts.v3.length }, + { documentId: BigInt(b!.id), versionId: BigInt(b1!.id), content: texts.bHistorical, contentHash: "d".repeat(64), tokenCount: 4, charCount: texts.bHistorical.length, embedding: vectorSql(0) }, + { documentId: BigInt(b!.id), versionId: BigInt(b2!.id), content: "Onboarding reliability and retry monitoring are blocking enterprise expansion.", contentHash: "e".repeat(64), tokenCount: 8, charCount: 72, embedding: vectorSql(0) }, + { documentId: BigInt(b!.id), versionId: null, content: texts.nullVersion, contentHash: "f".repeat(64), tokenCount: 4, charCount: texts.nullVersion.length, embedding: vectorSql(0) }, + { documentId: BigInt(feedbackDoc!.id), versionId: BigInt(feedbackVersion!.id), content: "Enterprise buyers need reliable onboarding recovery before expansion.", contentHash: "1".repeat(64), tokenCount: 8, charCount: 67 }, + { documentId: BigInt(unrelated!.id), versionId: BigInt(unrelatedVersion!.id), content: texts.unrelated, contentHash: "2".repeat(64), tokenCount: 4, charCount: texts.unrelated.length }, + { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: texts.foreign, contentHash: "3".repeat(64), tokenCount: 4, charCount: texts.foreign.length, embedding: vectorSql(0) }, + ]).returning(); + const bCurrentChunk = inserted.find((chunk) => chunk.documentId === BigInt(b!.id) && chunk.versionId === BigInt(b2!.id))!; + computedIds = { a: BigInt(a!.id), a1: a1!.id, a2: a2!.id, a3: a3!.id, b: BigInt(b!.id), b2: BigInt(b2!.id), bCurrentChunk: bCurrentChunk.id }; + collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-03T00:00:00.000Z"), { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(testDb.db) }, new StrictCurrentWorkspaceDocumentStore(testDb.db, { embedQuery: async () => vector(0) })); + } + const dispatchService = createFounderWeeklyReviewDispatchService(testDb.db); - const created = await dispatchService.createRunWithDispatch({ actor, requestKey: `realistic-${randomUUID()}`, reportingPeriod: fixture.reportingPeriod, collectionInput: { workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actorExternalUserId: actor.externalUserId } }); + const created = await dispatchService.createRunWithDispatch({ actor, requestKey: `realistic-${mode}-${randomUUID()}`, reportingPeriod: fixture.reportingPeriod, collectionInput: { workspaceTimezone: fixture.workspaceTimezone, founderContext: mode === "computed" ? "Assess whether onboarding reliability and retry monitoring are blocking enterprise expansion." : fixture.founderContext, actorExternalUserId: actor.externalUserId } }); if (created.run.evidenceSnapshot) throw new Error("Workflow run unexpectedly has an initial snapshot."); const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); const collectionContext = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: created.dispatch.generationClaimId }; const collecting = await worker.claimEvidenceCollection(collectionContext); - const collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-01T00:00:00.000Z"), { kind: "legacy" }); - const snapshot = await collector.collectFounderWeeklyReviewEvidence({ companyId: actor.companyId, reportingPeriod: fixture.reportingPeriod, workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: created.run.requestKey }); + const input = { companyId: actor.companyId, reportingPeriod: fixture.reportingPeriod, workspaceTimezone: fixture.workspaceTimezone, founderContext: mode === "computed" ? "Assess whether onboarding reliability and retry monitoring are blocking enterprise expansion." : fixture.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: created.run.requestKey }; + const snapshot = await collector.collectFounderWeeklyReviewEvidence(input); const repeated = await collector.collectFounderWeeklyReviewEvidence(input); + if (JSON.stringify(snapshot.items) !== JSON.stringify(repeated.items) || JSON.stringify(snapshot.sourceWarnings) !== JSON.stringify(repeated.sourceWarnings)) throw new Error("Evidence collection was not deterministic."); const beforeDigest = digest(snapshot); const attached = await worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot); const afterDigest = digest(attached.evidenceSnapshot); - const counts = Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, attached.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])); - if (attached.status !== "queued" || !attached.evidenceSnapshot || beforeDigest !== afterDigest || !counts.document_change || !counts.customer_feedback || counts.founder_context !== 1 || attached.evidenceSnapshot.items.some((item) => item.title === "Outside period" || item.title === "Other company control") || new Set(attached.evidenceSnapshot.items.map((item) => item.sourceId)).size !== attached.evidenceSnapshot.items.length) throw new Error("Realistic collector assertions failed."); + const checked = mode === "computed" ? assertComputedSnapshot(attached.evidenceSnapshot, computedIds!, texts) : { parsed: attached.evidenceSnapshot!, counts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, attached.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])) }; + if (attached.status !== "queued" || beforeDigest !== afterDigest || !checked.counts.document_change || !checked.counts.customer_feedback || checked.counts.founder_context !== 1) throw new Error("Realistic collector assertions failed."); const generationContext = { companyId: actor.companyId, runId: attached.id, generationJobId: created.dispatch.generationJobId, generationClaimId: created.dispatch.generationClaimId }; const generating = await worker.claimQueuedRun(generationContext); if (!generating.evidenceSnapshot) throw new Error("Generation began without snapshot."); - const generated = await generateFounderWeeklyReview({ evidenceSnapshot: generating.evidenceSnapshot, generate: generateFounderWeeklyReviewStructured }); const saved = await worker.saveGeneratedDraft(generationContext, generated.reviewPayload, generated.modelMetadata); const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft" || !readBack.evidenceSnapshot) throw new Error("Validated draft read-back failed."); - const rendered = renderFounderWeeklyReviewMarkdown(readBack); let markdownPath: string | null = null; let jsonPath: string | null = null; if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const directory = resolve(process.cwd(), process.env.SYNTHETIC_FWR_EXPORT_DIR ?? ".artifacts/founder-weekly-review"); await mkdir(directory, { recursive: true }); markdownPath = resolve(directory, `${saved.id}.md`); jsonPath = resolve(directory, `${saved.id}.json`); await writeAtomic(markdownPath, rendered); await writeAtomic(jsonPath, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, periodStart: readBack.reportingPeriod.start, periodEnd: readBack.reportingPeriod.end, review: readBack.reviewPayload }, null, 2)); } + let generationCalls = 0; + const generated = await generateFounderWeeklyReview({ evidenceSnapshot: generating.evidenceSnapshot, generate: async (request) => { generationCalls++; return generateFounderWeeklyReviewStructured(request); } }); + validateFounderWeeklyReviewV2Citations(generated.reviewPayload as never, generating.evidenceSnapshot); + const saved = await worker.saveGeneratedDraft(generationContext, generated.reviewPayload, generated.modelMetadata); const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft" || !readBack.evidenceSnapshot || digest(readBack.evidenceSnapshot) !== beforeDigest) throw new Error("Validated draft read-back or snapshot immutability failed."); + if (mode === "computed") assertComputedReport(readBack.reviewPayload, readBack.evidenceSnapshot); + const rendered = renderFounderWeeklyReviewMarkdown(readBack); + const dispatchRows = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, saved.id)); const runRows = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); + let artifactPaths: ArtifactPaths | null = null; + if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const exportRoot = founderWeeklyReviewRealisticExportRoot(mode, process.env.SYNTHETIC_FWR_EXPORT_DIR); const directory = resolve(process.cwd(), exportRoot, saved.id); await mkdir(directory, { recursive: true }); artifactPaths = { evidence: resolve(directory, "evidence.json"), report: resolve(directory, "report.json"), markdown: resolve(directory, "report.md"), summary: resolve(directory, "run-summary.json") }; await writeAtomic(artifactPaths.evidence, JSON.stringify(readBack.evidenceSnapshot, null, 2)); await writeAtomic(artifactPaths.report, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, reportingPeriod: readBack.reportingPeriod, review: readBack.reviewPayload }, null, 2)); await writeAtomic(artifactPaths.markdown, rendered); await writeAtomic(artifactPaths.summary, JSON.stringify({ runId: saved.id, scenario: "realistic-company", mode, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), repairCount: generationCalls - 1, retryCount: saved.retryCount, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, snapshotDigestBefore: beforeDigest, snapshotDigestAfter: digest(readBack.evidenceSnapshot), artifactPaths }, null, 2)); } if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); console.log("===== END FOUNDER WEEKLY REVIEW ====="); } - const dispatchRows = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, saved.id)); const runRows = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); console.log(JSON.stringify({ runId: saved.id, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: counts, warningCodes: attached.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, dispatchCount: dispatchRows.length, runRowCount: runRows.length, markdownPath, jsonPath })); + console.log(JSON.stringify({ runId: saved.id, mode, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, repairCount: generationCalls - 1, dispatchCount: dispatchRows.length, runRowCount: runRows.length, artifactPaths })); } finally { await testDb.close(); } diff --git a/apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts b/apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts new file mode 100644 index 000000000..2347a1dff --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts @@ -0,0 +1,18 @@ +export type FounderWeeklyReviewRealisticEvidenceMode = "legacy" | "computed"; + +/** Explicit runner-only mode selection; default preserves the legacy baseline. */ +export function parseFounderWeeklyReviewRealisticEvidenceMode( + raw: string | undefined +): FounderWeeklyReviewRealisticEvidenceMode { + if (!raw || raw === "legacy") return "legacy"; + if (raw === "computed") return "computed"; + throw new Error(`Unsupported FWR_EVIDENCE_MODE: ${raw}`); +} + +/** Keeps an explicit export root from receiving the evidence mode twice. */ +export function founderWeeklyReviewRealisticExportRoot( + mode: FounderWeeklyReviewRealisticEvidenceMode, + configuredRoot: string | undefined +): string { + return configuredRoot ?? `.artifacts/founder-weekly-review/${mode}`; +} From ed3f7e2d0a4c4aab720eb5ddaf101302bc9bf761 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Fri, 7 Aug 2026 12:45:20 +0800 Subject: [PATCH 18/29] fix(founder-weekly-review): bound generation evidence context --- .../generation-evidence-envelope.test.ts | 268 +++++++++++++ .../founderWeeklyReview/generation.test.ts | 61 ++- .../generation-evidence-envelope.ts | 364 ++++++++++++++++++ .../src/founder-weekly-review/generator.ts | 32 +- .../src/founder-weekly-review/index.ts | 1 + .../src/founder-weekly-review/prompts.ts | 27 +- 6 files changed, 731 insertions(+), 22 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts create mode 100644 packages/features/src/founder-weekly-review/generation-evidence-envelope.ts diff --git a/apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts b/apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts new file mode 100644 index 000000000..2854857c4 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts @@ -0,0 +1,268 @@ +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + FounderWeeklyReviewGenerationEvidenceBudgetError, + assertGenerationEvidenceEnvelopeWithinBudget, + buildFounderWeeklyReviewPrompt, + buildFounderWeeklyReviewPromptEvidenceItem, + buildGenerationEvidenceEnvelope, + type FounderWeeklyReviewEvidenceItem, + type FounderWeeklyReviewEvidenceSnapshot, + type GenerationEvidenceEnvelope, +} from "@launchstack/features/founder-weekly-review"; + +function snapshot(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceSnapshot { + return { + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-18T10:00:00.000Z", + reportingPeriod: { start: "2026-07-07", end: "2026-07-13" }, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], + }; +} + +function item( + sourceType: FounderWeeklyReviewEvidenceItem["sourceType"], + sourceId: string, + excerpt = "x".repeat(200), + metadata: FounderWeeklyReviewEvidenceItem["metadata"] = {}, + sourceTimestamp = "2026-07-10T10:00:00.000Z" +): FounderWeeklyReviewEvidenceItem { + return { + sourceType, + sourceId, + title: `${sourceType} title`, + sourceTimestamp, + excerpt, + metadata, + }; +} + +function documentChanges( + count: number, + documentCount: number, + excerpt = "x".repeat(200) +): FounderWeeklyReviewEvidenceItem[] { + return Array.from({ length: count }, (_, index) => { + const documentId = String((index % documentCount) + 1); + const sequence = Math.floor(index / documentCount) + 1; + return item( + "document_change", + `document_change:doc:${documentId}:v1:v2:chunk:${sequence}:${sequence}`, + excerpt, + { + documentId, + previousVersionId: 1, + currentVersionId: 2, + previousVersionNumber: 1, + currentVersionNumber: 2, + changeType: "modified", + alignmentMethod: "structure_path", + structurePath: `/section/${sequence}`, + previousChunkId: sequence, + currentChunkId: sequence, + previousContentHash: "a".repeat(64), + currentContentHash: "b".repeat(64), + }, + `2026-07-${String((sequence % 7) + 7).padStart(2, "0")}T10:00:00.000Z` + ); + }); +} + +describe("Founder Weekly Review generation evidence envelope", () => { + it("bounds 250 document changes with round-robin document diversity without mutating the snapshot", () => { + const evidenceSnapshot = snapshot(documentChanges(250, 3)); + const original = JSON.stringify(evidenceSnapshot); + const first = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const second = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const changes = first.items.filter(entry => entry.sourceType === "document_change"); + const perDocument = new Map(); + for (const change of changes) { + const documentId = /^document_change:doc:([^:]+)/.exec(change.sourceId)![1]!; + perDocument.set(documentId, (perDocument.get(documentId) ?? 0) + 1); + } + + expect(changes).toHaveLength(24); + expect(Math.max(...perDocument.values())).toBe(8); + expect(JSON.stringify(changes).length).toBeLessThanOrEqual(14_000); + expect(first.items.map(entry => entry.sourceId)).toEqual( + second.items.map(entry => entry.sourceId) + ); + expect(first.diagnostics).toEqual(second.diagnostics); + expect(evidenceSnapshot.items).toHaveLength(250); + expect(JSON.stringify(evidenceSnapshot)).toBe(original); + }); + + it("reserves Founder Context, customer feedback, and workspace evidence before bounded document changes", () => { + const items = [ + ...documentChanges(250, 3, "d".repeat(300)), + ...Array.from({ length: 250 }, (_, index) => + item("customer_feedback", `customer_feedback:${index}`, "c".repeat(500), { + pageNumber: index + 1, + }) + ), + ...Array.from({ length: 8 }, (_, index) => + item( + "workspace_document", + `workspace_document:${index}`, + "w".repeat(500), + { + similarityScore: 1 - index / 100, + retrievalReason: "founder_context_relevance", + }, + "" + ) + ), + item( + "founder_context", + "founder_context:entry:test", + "Founder direction", + { provenance: "request_time_founder_input" }, + "" + ), + ]; + // This intentionally models the pre-snapshot aggregate (509 items) + // that the 500-item snapshot cap subsequently trims. + const evidenceSnapshot = snapshot(items) as FounderWeeklyReviewEvidenceSnapshot; + const envelope = buildGenerationEvidenceEnvelope(evidenceSnapshot); + + expect(envelope.diagnostics.selectedBySourceType.founder_context).toBe(1); + expect(envelope.diagnostics.selectedBySourceType.customer_feedback).toBeGreaterThan(0); + expect(envelope.diagnostics.selectedBySourceType.workspace_document).toBe(8); + expect(envelope.diagnostics.selectedBySourceType.document_change).toBeLessThanOrEqual(24); + expect(envelope.diagnostics.selectedBySourceType.customer_feedback).toBeGreaterThan( + envelope.diagnostics.selectedBySourceType.document_change + ); + expect(envelope.diagnostics.serializedCharacterCount).toBeLessThanOrEqual(72_000); + expect(envelope.diagnostics.estimatedTokenCount).toBeLessThanOrEqual(18_000); + expect(envelope.diagnostics.truncated).toBe(true); + }); + + it("is input-order independent and produces byte-identical prompts", () => { + const items = [ + ...documentChanges(60, 4), + item("customer_feedback", "feedback-1", "Customer signal"), + item( + "workspace_document", + "workspace-1", + "Workspace context", + { similarityScore: 0.9 }, + "" + ), + item("founder_context", "context-1", "Founder context", {}, ""), + ]; + const shuffled = [...items].sort((a, b) => b.sourceId.localeCompare(a.sourceId)); + const first = buildGenerationEvidenceEnvelope(snapshot(items)); + const second = buildGenerationEvidenceEnvelope(snapshot(shuffled)); + + expect(first.items.map(entry => entry.sourceId)).toEqual( + second.items.map(entry => entry.sourceId) + ); + expect(buildFounderWeeklyReviewPrompt(snapshot(items))).toBe( + buildFounderWeeklyReviewPrompt(snapshot(shuffled)) + ); + }); + + it("allowlists prompt metadata while retaining complete immutable snapshot metadata and source IDs", () => { + const source = item( + "document_change", + "document_change:doc:7:v1:v2:chunk:1:2", + "Before and after", + { + documentId: "7", + previousChunkId: 1, + currentChunkId: 2, + previousContentHash: "a".repeat(64), + currentContentHash: "b".repeat(64), + previousVersionNumber: 1, + currentVersionNumber: 2, + changeType: "modified", + alignmentMethod: "structure_path", + structurePath: "/plan", + userChangelog: "Updated plan", + providerPayload: "must-not-leak", + credential: "must-not-leak", + } + ); + const evidenceSnapshot = snapshot([source]); + const prompt = JSON.parse(buildFounderWeeklyReviewPrompt(evidenceSnapshot)); + const promptItem = prompt.evidence[0]; + + expect(prompt.evidenceEnvelopeVersion).toBe( + FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION + ); + expect(prompt.evidenceEnvelopeBudget.totalSerializedCharacters).toBe(72_000); + expect(promptItem.sourceId).toBe(source.sourceId); + expect(promptItem.metadata).toEqual({ + alignmentMethod: "structure_path", + changeType: "modified", + currentVersionNumber: 2, + previousVersionNumber: 1, + structurePath: "/plan", + userChangelog: "Updated plan", + }); + expect(promptItem).not.toHaveProperty("canonicalUrl"); + expect(promptItem).not.toHaveProperty("workspaceDeepLink"); + expect(evidenceSnapshot.items[0]!.metadata).toHaveProperty("previousContentHash"); + expect(evidenceSnapshot.items[0]!.metadata).toHaveProperty("providerPayload"); + }); + + it("includes candidates atomically until the document-change character boundary", () => { + const evidenceSnapshot = snapshot(documentChanges(8, 1, "x".repeat(3_000))); + const envelope = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const selected = envelope.items.filter(entry => entry.sourceType === "document_change"); + const excluded = evidenceSnapshot.items.find( + entry => !selected.some(candidate => candidate.sourceId === entry.sourceId) + ); + + expect(selected.length).toBeGreaterThan(0); + expect(excluded).toBeDefined(); + expect(JSON.stringify(selected).length).toBeLessThanOrEqual( + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters + ); + expect( + JSON.stringify([...selected, buildFounderWeeklyReviewPromptEvidenceItem(excluded!)]) + .length + ).toBeGreaterThan( + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters + ); + expect(envelope.diagnostics.truncated).toBe(true); + }); + + it("rejects a corrupted over-budget envelope locally", () => { + const valid = buildGenerationEvidenceEnvelope(snapshot([item("manual_note", "note-1")])); + const corrupted: GenerationEvidenceEnvelope = { + ...valid, + items: Array.from({ length: 20 }, (_, index) => ({ + ...valid.items[0]!, + sourceId: `note-${index}`, + excerpt: "x".repeat(4_000), + })), + }; + + expect(() => assertGenerationEvidenceEnvelopeWithinBudget(corrupted)).toThrow( + FounderWeeklyReviewGenerationEvidenceBudgetError + ); + expect(() => buildFounderWeeklyReviewPrompt(snapshot([]), corrupted)).toThrow( + expect.objectContaining({ code: "generation_evidence_budget_exceeded" }) + ); + }); + + it("publishes a versioned envelope and safe aggregate diagnostics only", () => { + const envelope = buildGenerationEvidenceEnvelope(snapshot([item("manual_note", "note-1")])); + expect(envelope.version).toBe(FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION); + expect(envelope.diagnostics).toEqual( + expect.objectContaining({ + originalItemCount: 1, + selectedItemCount: 1, + excludedItemCount: 0, + serializedCharacterCount: JSON.stringify(envelope.items).length, + estimatedTokenCount: Math.ceil(JSON.stringify(envelope.items).length / 4), + truncated: false, + }) + ); + expect(JSON.stringify(envelope.diagnostics)).not.toContain("manual_note title"); + expect(JSON.stringify(envelope.diagnostics)).not.toContain("x".repeat(20)); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts index f324053ce..daab5a053 100644 --- a/apps/web/__tests__/founderWeeklyReview/generation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -6,6 +6,7 @@ import { FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, FounderWeeklyReviewV2PayloadSchema, buildFounderWeeklyReviewPrompt, + buildGenerationEvidenceEnvelope, generateFounderWeeklyReview, parseFounderWeeklyReviewPayload, type FounderWeeklyReviewEvidenceSnapshot, @@ -242,9 +243,9 @@ describe("Founder Weekly Review generation", () => { it("canonicalizes metadata key order before building the prompt and hash", async () => { const firstSnapshot = completeSnapshot(); - firstSnapshot.items[0]!.metadata = { alpha: "a", beta: "b" }; + firstSnapshot.items[0]!.metadata = { changeType: "modified", alignmentMethod: "structure_path" }; const secondSnapshot = completeSnapshot(); - secondSnapshot.items[0]!.metadata = { beta: "b", alpha: "a" }; + secondSnapshot.items[0]!.metadata = { alignmentMethod: "structure_path", changeType: "modified" }; expect(buildFounderWeeklyReviewPrompt(firstSnapshot)).toBe(buildFounderWeeklyReviewPrompt(secondSnapshot)); const first = await generateFounderWeeklyReview({ evidenceSnapshot: firstSnapshot, generate: fake(validPayload()) }); @@ -252,6 +253,62 @@ describe("Founder Weekly Review generation", () => { expect(first.modelMetadata.promptHash).toBe(second.modelMetadata.promptHash); }); + it("changes the prompt hash when selected evidence changes", async () => { + const firstSnapshot = completeSnapshot(); + const secondSnapshot = completeSnapshot(); + secondSnapshot.items[0]!.excerpt = "A different release shipped."; + const first = await generateFounderWeeklyReview({ evidenceSnapshot: firstSnapshot, generate: fake(validPayload()) }); + const second = await generateFounderWeeklyReview({ evidenceSnapshot: secondSnapshot, generate: fake(validPayload()) }); + expect(first.modelMetadata.promptHash).not.toBe(second.modelMetadata.promptHash); + }); + + it("normally truncates large evidence before generation and persists aggregate diagnostics", async () => { + const items = Array.from({ length: 250 }, (_, index) => ({ + ...source(`document_change:doc:${(index % 3) + 1}:v1:v2:chunk:${index}:${index}`, "document_change", "x".repeat(200)), + sourceTimestamp: `2026-07-${String((index % 7) + 7).padStart(2, "0")}T10:00:00.000Z`, + metadata: { documentId: String((index % 3) + 1), previousVersionId: 1, currentVersionId: 2, changeType: "modified" }, + })); + const payload: FounderWeeklyReviewV2Payload = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: noEvidence(), whatShipped: noEvidence(), whatCustomersSaid: noEvidence(), + currentBlockers: noEvidence(), nextPriorities: noEvidence(), + }, + }; + const generate = fake(payload); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: snapshot(items), generate }); + + expect(generate).toHaveBeenCalledTimes(1); + const prompt = JSON.parse(generate.mock.calls[0][0].prompt); + expect(prompt.evidence).toHaveLength(24); + expect(result.modelMetadata.attributes).toEqual(expect.objectContaining({ + evidenceEnvelopeOriginalItems: 250, + evidenceEnvelopeSelectedItems: 24, + evidenceEnvelopeExcludedItems: 226, + evidenceEnvelopeTruncated: true, + })); + }); + + it("continues to validate citations against the complete immutable snapshot", async () => { + const items = Array.from({ length: 30 }, (_, index) => ({ + ...source(`document_change:doc:1:v1:v2:chunk:${index}:${index}`, "document_change", "x".repeat(200)), + metadata: { documentId: "1", previousVersionId: 1, currentVersionId: 2, changeType: "modified" }, + })); + const evidenceSnapshot = snapshot(items); + const selectedIds = new Set(buildGenerationEvidenceEnvelope(evidenceSnapshot).items.map((item) => item.sourceId)); + const excludedId = evidenceSnapshot.items.find((item) => !selectedIds.has(item.sourceId))!.sourceId; + const payload: FounderWeeklyReviewV2Payload = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "A source-backed change.", sourceIds: [excludedId], confidence: 0.5 }] }, + whatShipped: noEvidence(), whatCustomersSaid: noEvidence(), currentBlockers: noEvidence(), nextPriorities: noEvidence(), + }, + }; + + await expect(generateFounderWeeklyReview({ evidenceSnapshot, generate: fake(payload) })).resolves.toMatchObject({ reviewPayload: payload }); + expect(evidenceSnapshot.items).toHaveLength(30); + }); + it("adapts the configured web abstraction and returns its metadata", async () => { mockGenerateStructuredWithMetadata.mockResolvedValue({ object: { ok: true }, metadata: { provider: "openai", model: "adapter-model", capability: "founderWeeklyReview", temperature: 0 } }); const schema = require("zod").z.object({ ok: require("zod").z.boolean() }); diff --git a/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts b/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts new file mode 100644 index 000000000..1bd864cb3 --- /dev/null +++ b/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts @@ -0,0 +1,364 @@ +import type { + FounderWeeklyReviewEvidenceItem, + FounderWeeklyReviewEvidenceSnapshot, +} from "./contracts"; + +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION = + "founder-weekly-review-evidence-envelope/v1" as const; + +export const FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET = Object.freeze({ + totalSerializedCharacters: 72_000, + founderContextReservedCharacters: 4_000, + workspaceDocumentReservedCharacters: 24_000, + customerFeedbackReservedCharacters: 24_000, + documentChangeSerializedCharacters: 14_000, + documentChangeItems: 24, + documentChangeItemsPerDocument: 8, + estimatedCharactersPerToken: 4, +}); + +const SOURCE_TYPES = [ + "workspace_document", + "document_change", + "customer_feedback", + "github_activity", + "manual_note", + "founder_context", + "other", +] as const satisfies readonly FounderWeeklyReviewEvidenceItem["sourceType"][]; + +type SourceType = FounderWeeklyReviewEvidenceItem["sourceType"]; +type MetadataValue = FounderWeeklyReviewEvidenceItem["metadata"][string]; + +export type FounderWeeklyReviewPromptEvidenceItem = Pick< + FounderWeeklyReviewEvidenceItem, + "sourceId" | "sourceType" | "title" | "excerpt" +> & { + sourceTimestamp: string | null; + metadata: Record; +}; + +export type GenerationEvidenceEnvelopeDiagnostics = { + originalItemCount: number; + selectedItemCount: number; + excludedItemCount: number; + selectedBySourceType: Record; + excludedBySourceType: Record; + serializedCharacterCount: number; + estimatedTokenCount: number; + truncated: boolean; +}; + +export type GenerationEvidenceEnvelope = { + version: typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION; + items: FounderWeeklyReviewPromptEvidenceItem[]; + diagnostics: GenerationEvidenceEnvelopeDiagnostics; +}; + +export class FounderWeeklyReviewGenerationEvidenceBudgetError extends Error { + readonly code = "generation_evidence_budget_exceeded"; + + constructor( + message = "Founder Weekly Review generation evidence exceeded its deterministic context budget." + ) { + super(message); + this.name = "FounderWeeklyReviewGenerationEvidenceBudgetError"; + } +} + +const PROMPT_METADATA_ALLOWLIST: Record = { + document_change: [ + "changeType", + "previousVersionNumber", + "currentVersionNumber", + "structurePath", + "alignmentMethod", + "userChangelog", + ], + customer_feedback: ["versionNumber", "documentCategory", "pageNumber"], + workspace_document: ["retrievalReason", "similarityScore"], + founder_context: ["provenance"], + github_activity: [], + manual_note: [], + other: [], +}; + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} +function metadataText(item: FounderWeeklyReviewEvidenceItem, key: string): string { + const value = item.metadata[key]; + return typeof value === "string" || typeof value === "number" ? String(value) : ""; +} +function similarityScore(item: FounderWeeklyReviewEvidenceItem): number { + const value = item.metadata.similarityScore; + return typeof value === "number" && Number.isFinite(value) ? value : Number.NEGATIVE_INFINITY; +} + +/** Stable source-specific ordering that does not depend on snapshot input order. */ +function compareEvidenceItems( + a: FounderWeeklyReviewEvidenceItem, + b: FounderWeeklyReviewEvidenceItem +): number { + const timestamp = compareOrdinal(a.sourceTimestamp ?? "", b.sourceTimestamp ?? ""); + if (timestamp !== 0) return timestamp; + const sourceType = compareOrdinal(a.sourceType, b.sourceType); + if (sourceType !== 0) return sourceType; + if (a.sourceType === "workspace_document" && b.sourceType === "workspace_document") { + const similarity = similarityScore(b) - similarityScore(a); + if (Number.isFinite(similarity) && similarity !== 0) return similarity; + } + if (a.sourceType === "document_change" && b.sourceType === "document_change") { + for (const key of [ + "documentId", + "previousVersionId", + "currentVersionId", + "structurePath", + ] as const) { + const compared = compareOrdinal(metadataText(a, key), metadataText(b, key)); + if (compared !== 0) return compared; + } + } + return compareOrdinal(a.sourceId, b.sourceId); +} + +function allowlistedMetadata(item: FounderWeeklyReviewEvidenceItem): Record { + const metadata: Record = {}; + for (const key of PROMPT_METADATA_ALLOWLIST[item.sourceType]) { + const value = item.metadata[key]; + if (value !== undefined) metadata[key] = value; + } + return metadata; +} + +export function buildFounderWeeklyReviewPromptEvidenceItem( + item: FounderWeeklyReviewEvidenceItem +): FounderWeeklyReviewPromptEvidenceItem { + return { + sourceId: item.sourceId, + sourceType: item.sourceType, + title: item.title, + sourceTimestamp: item.sourceTimestamp ?? null, + excerpt: item.excerpt, + metadata: allowlistedMetadata(item), + }; +} + +function serializedItemsLength(items: readonly FounderWeeklyReviewPromptEvidenceItem[]): number { + return JSON.stringify(items).length; +} +function canAppendWithin( + selected: readonly FounderWeeklyReviewPromptEvidenceItem[], + candidate: FounderWeeklyReviewPromptEvidenceItem, + limit: number +): boolean { + return serializedItemsLength([...selected, candidate]) <= limit; +} +function sourceCounts(items: readonly { sourceType: SourceType }[]): Record { + const counts = Object.fromEntries(SOURCE_TYPES.map(sourceType => [sourceType, 0])) as Record< + SourceType, + number + >; + for (const item of items) counts[item.sourceType]++; + return counts; +} +function subtractCounts( + total: Record, + selected: Record +): Record { + return Object.fromEntries( + SOURCE_TYPES.map(sourceType => [sourceType, total[sourceType] - selected[sourceType]]) + ) as Record; +} + +function dedupeAndOrder( + items: readonly FounderWeeklyReviewEvidenceItem[] +): FounderWeeklyReviewEvidenceItem[] { + const ordered = [...items].sort( + (a, b) => + compareEvidenceItems(a, b) || + compareOrdinal( + JSON.stringify(buildFounderWeeklyReviewPromptEvidenceItem(a)), + JSON.stringify(buildFounderWeeklyReviewPromptEvidenceItem(b)) + ) + ); + const bySourceId = new Map(); + for (const item of ordered) + if (!bySourceId.has(item.sourceId)) bySourceId.set(item.sourceId, item); + return [...bySourceId.values()]; +} + +function selectReserved( + candidates: readonly FounderWeeklyReviewEvidenceItem[], + limit: number +): { selected: FounderWeeklyReviewEvidenceItem[]; excluded: FounderWeeklyReviewEvidenceItem[] } { + const selected: FounderWeeklyReviewEvidenceItem[] = []; + const projected: FounderWeeklyReviewPromptEvidenceItem[] = []; + const excluded: FounderWeeklyReviewEvidenceItem[] = []; + for (const item of candidates) { + const promptItem = buildFounderWeeklyReviewPromptEvidenceItem(item); + if (canAppendWithin(projected, promptItem, limit)) { + selected.push(item); + projected.push(promptItem); + } else excluded.push(item); + } + return { selected, excluded }; +} + +function documentKey(item: FounderWeeklyReviewEvidenceItem): string { + return ( + metadataText(item, "documentId") || + /^document_change:doc:([^:]+)/.exec(item.sourceId)?.[1] || + item.sourceId + ); +} + +function selectDocumentChanges( + candidates: readonly FounderWeeklyReviewEvidenceItem[] +): FounderWeeklyReviewEvidenceItem[] { + const byDocument = new Map(); + for (const item of candidates) { + const key = documentKey(item); + const group = byDocument.get(key) ?? []; + group.push(item); + byDocument.set(key, group); + } + const documents = [...byDocument.entries()].sort(([a], [b]) => compareOrdinal(a, b)); + const selected: FounderWeeklyReviewEvidenceItem[] = []; + const projected: FounderWeeklyReviewPromptEvidenceItem[] = []; + for ( + let round = 0; + round < FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItemsPerDocument; + round++ + ) { + for (const [, items] of documents) { + if ( + selected.length >= + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItems + ) + return selected; + const item = items[round]; + if (!item) continue; + const promptItem = buildFounderWeeklyReviewPromptEvidenceItem(item); + if ( + !canAppendWithin( + projected, + promptItem, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters + ) + ) + continue; + selected.push(item); + projected.push(promptItem); + } + } + return selected; +} + +/** Builds a bounded prompt projection without mutating the immutable snapshot. */ +export function buildGenerationEvidenceEnvelope( + snapshot: FounderWeeklyReviewEvidenceSnapshot +): GenerationEvidenceEnvelope { + const ordered = dedupeAndOrder(snapshot.items); + const founder = ordered.filter(item => item.sourceType === "founder_context"); + const workspace = ordered.filter(item => item.sourceType === "workspace_document"); + const feedback = ordered.filter(item => item.sourceType === "customer_feedback"); + const documentChanges = ordered.filter(item => item.sourceType === "document_change"); + const selectedOriginal: FounderWeeklyReviewEvidenceItem[] = founder.slice(0, 1); + const reservedWorkspace = selectReserved( + workspace, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.workspaceDocumentReservedCharacters + ); + const reservedFeedback = selectReserved( + feedback, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.customerFeedbackReservedCharacters + ); + selectedOriginal.push( + ...reservedWorkspace.selected, + ...reservedFeedback.selected, + ...selectDocumentChanges(documentChanges) + ); + const selectedIds = new Set(selectedOriginal.map(item => item.sourceId)); + const sharedCandidates = ordered.filter( + item => + item.sourceType !== "document_change" && + item.sourceType !== "founder_context" && + !selectedIds.has(item.sourceId) + ); + for (const item of sharedCandidates) { + const candidateItems = [...selectedOriginal, item] + .sort(compareEvidenceItems) + .map(buildFounderWeeklyReviewPromptEvidenceItem); + if ( + serializedItemsLength(candidateItems) > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.totalSerializedCharacters + ) + continue; + selectedOriginal.push(item); + selectedIds.add(item.sourceId); + } + const items = selectedOriginal + .sort(compareEvidenceItems) + .map(buildFounderWeeklyReviewPromptEvidenceItem); + const serializedCharacterCount = serializedItemsLength(items); + const selectedBySourceType = sourceCounts(items); + const diagnostics: GenerationEvidenceEnvelopeDiagnostics = { + originalItemCount: snapshot.items.length, + selectedItemCount: items.length, + excludedItemCount: snapshot.items.length - items.length, + selectedBySourceType, + excludedBySourceType: subtractCounts(sourceCounts(snapshot.items), selectedBySourceType), + serializedCharacterCount, + estimatedTokenCount: Math.ceil( + serializedCharacterCount / + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.estimatedCharactersPerToken + ), + truncated: items.length !== snapshot.items.length, + }; + const envelope: GenerationEvidenceEnvelope = { + version: FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION, + items, + diagnostics, + }; + assertGenerationEvidenceEnvelopeWithinBudget(envelope); + return envelope; +} + +/** Local invariant check used immediately before prompt serialization/provider invocation. */ +export function assertGenerationEvidenceEnvelopeWithinBudget( + envelope: GenerationEvidenceEnvelope +): void { + const actualCharacters = serializedItemsLength(envelope.items); + const documentChanges = envelope.items.filter(item => item.sourceType === "document_change"); + const founderContexts = envelope.items.filter(item => item.sourceType === "founder_context"); + const perDocument = new Map(); + for (const item of documentChanges) { + const key = + typeof item.metadata.documentId === "string" || + typeof item.metadata.documentId === "number" + ? String(item.metadata.documentId) + : (/^document_change:doc:([^:]+)/.exec(item.sourceId)?.[1] ?? item.sourceId); + perDocument.set(key, (perDocument.get(key) ?? 0) + 1); + } + const invalid = + envelope.version !== FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION || + actualCharacters !== envelope.diagnostics.serializedCharacterCount || + actualCharacters > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.totalSerializedCharacters || + founderContexts.length > 1 || + founderContexts.some( + item => + item.excerpt.length > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.founderContextReservedCharacters + ) || + documentChanges.length > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItems || + serializedItemsLength(documentChanges) > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters || + [...perDocument.values()].some( + count => + count > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItemsPerDocument + ); + if (invalid) throw new FounderWeeklyReviewGenerationEvidenceBudgetError(); +} diff --git a/packages/features/src/founder-weekly-review/generator.ts b/packages/features/src/founder-weekly-review/generator.ts index 9a963ae91..f9e587a29 100644 --- a/packages/features/src/founder-weekly-review/generator.ts +++ b/packages/features/src/founder-weekly-review/generator.ts @@ -18,6 +18,11 @@ import { FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, } from "./prompts"; +import { + buildGenerationEvidenceEnvelope, + type FounderWeeklyReviewPromptEvidenceItem, + type GenerationEvidenceEnvelopeDiagnostics, +} from "./generation-evidence-envelope"; export interface FounderWeeklyReviewResolvedGenerationMetadata { provider: string; @@ -56,19 +61,21 @@ export async function generateFounderWeeklyReview( const { evidenceSnapshot, generate } = input; assertUniqueSnapshotSourceIds(evidenceSnapshot); - const prompt = buildFounderWeeklyReviewPrompt(evidenceSnapshot); + const evidenceEnvelope = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const prompt = buildFounderWeeklyReviewPrompt(evidenceSnapshot, evidenceEnvelope); const promptHash = createHash("sha256") .update(FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT) .update(prompt) .digest("hex"); - if (evidenceSnapshot.items.length === 0) { + if (evidenceEnvelope.items.length === 0) { return { reviewPayload: buildEmptyReview(), modelMetadata: buildMetadata( { provider: "skipped", model: "none", capability: "founderWeeklyReview", temperature: 0 }, promptHash, - true + true, + evidenceEnvelope.diagnostics, ), }; } @@ -93,7 +100,7 @@ export async function generateFounderWeeklyReview( logGenerationValidation("initial", initial.metadata, "failed"); const repaired = await generate({ system: FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, - prompt: buildSemanticRepairPrompt(initial.object, evidenceSnapshot, error), + prompt: buildSemanticRepairPrompt(initial.object, evidenceEnvelope.items, error), schema: FounderWeeklyReviewV2PayloadSchema, schemaName: "founder_weekly_review_v2", generationPhase: "semantic-repair", @@ -111,18 +118,18 @@ export async function generateFounderWeeklyReview( } } - return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false) }; + return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false, evidenceEnvelope.diagnostics) }; } function buildSemanticRepairPrompt( candidate: FounderWeeklyReviewV2Payload, - evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + evidenceItems: readonly FounderWeeklyReviewPromptEvidenceItem[], error: FounderWeeklyReviewGenerationValidationError ): string { const errors = error.details.length > 0 ? error.details : [{ code: "report_validation_failed" }]; - const sources = evidenceSnapshot.items.map(({ sourceId, sourceType }) => ({ sourceId, sourceType })); + const sources = evidenceItems.map(({ sourceId, sourceType }) => ({ sourceId, sourceType })); return [ "Correct the complete canonical Founder Weekly Review JSON candidate below.", "Customer Signals may cite only customer_feedback sources.", @@ -147,7 +154,8 @@ function logGenerationValidation( function buildMetadata( metadata: FounderWeeklyReviewResolvedGenerationMetadata, promptHash: string, - skipped: boolean + skipped: boolean, + diagnostics: GenerationEvidenceEnvelopeDiagnostics, ): FounderWeeklyReviewModelMetadata { return { provider: metadata.provider, @@ -163,6 +171,14 @@ function buildMetadata( ...(skipped ? { generationSkipped: true } : {}), ...(metadata.finishReason ? { finishReason: metadata.finishReason } : {}), ...(metadata.usage ? { usage: JSON.stringify(metadata.usage) } : {}), + evidenceEnvelopeOriginalItems: diagnostics.originalItemCount, + evidenceEnvelopeSelectedItems: diagnostics.selectedItemCount, + evidenceEnvelopeExcludedItems: diagnostics.excludedItemCount, + evidenceEnvelopeCharacters: diagnostics.serializedCharacterCount, + evidenceEnvelopeEstimatedTokens: diagnostics.estimatedTokenCount, + evidenceEnvelopeTruncated: diagnostics.truncated, + evidenceEnvelopeSelectedByType: JSON.stringify(diagnostics.selectedBySourceType), + evidenceEnvelopeExcludedByType: JSON.stringify(diagnostics.excludedBySourceType), }, }; } diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 535060b42..92245d3e9 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -7,6 +7,7 @@ export * from "./user-service"; export * from "./worker-service"; export * from "./generator"; export * from "./generation-validation"; +export * from "./generation-evidence-envelope"; export * from "./prompts"; export * from "./document-change"; export * from "./workspace-document"; diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts index 01aca49a7..4ddb7112d 100644 --- a/packages/features/src/founder-weekly-review/prompts.ts +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -1,4 +1,10 @@ import type { FounderWeeklyReviewEvidenceSnapshot } from "./contracts"; +import { + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + assertGenerationEvidenceEnvelopeWithinBudget, + buildGenerationEvidenceEnvelope, + type GenerationEvidenceEnvelope, +} from "./generation-evidence-envelope"; export const FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION = "founder-weekly-review-generation/v1" as const; @@ -21,24 +27,21 @@ When evidence conflicts, return contradictory_evidence with the conflicting sour nextPriorities contains recommendations only. Every recommendation must have label "Recommendation" and be grounded in supplied evidence. If a section lacks relevant evidence, return its typed no_evidence state with a concrete CTA. sourceWarnings may inform the CTA but are not factual evidence and cannot be cited.`; -/** Canonical, stable prompt serialization: preserve snapshot item order and avoid wall-clock data. */ +/** Canonical, stable prompt serialization over the bounded evidence envelope. */ export function buildFounderWeeklyReviewPrompt( - evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + suppliedEnvelope?: GenerationEvidenceEnvelope, ): string { + const envelope = suppliedEnvelope ?? buildGenerationEvidenceEnvelope(evidenceSnapshot); + assertGenerationEvidenceEnvelopeWithinBudget(envelope); return JSON.stringify(sortObjectKeysRecursively({ promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + evidenceEnvelopeVersion: envelope.version, + evidenceEnvelopeBudget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + evidenceEnvelopeDiagnostics: envelope.diagnostics, reportingPeriod: evidenceSnapshot.reportingPeriod, workspaceTimezone: evidenceSnapshot.workspaceTimezone, - evidence: evidenceSnapshot.items.map((item) => ({ - sourceId: item.sourceId, - sourceType: item.sourceType, - title: item.title, - sourceTimestamp: item.sourceTimestamp ?? null, - excerpt: item.excerpt, - canonicalUrl: item.canonicalUrl ?? null, - workspaceDeepLink: item.workspaceDeepLink ?? null, - metadata: item.metadata, - })), + evidence: envelope.items, sourceWarnings: evidenceSnapshot.sourceWarnings, requiredSections: [ "whatChanged", From ab4eb3cbd0a1bdfd93d9dead2eaf1bef50f4d3e1 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Fri, 7 Aug 2026 13:19:53 +0800 Subject: [PATCH 19/29] feat(founder-weekly-review): add deterministic change grouping --- .../document-change-condensation.test.ts | 333 +++++++++++++ .../founder-weekly-review/document-change.ts | 439 +++++++++++++++++- .../founder-weekly-review/evidence-service.ts | 19 +- 3 files changed, 774 insertions(+), 17 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts new file mode 100644 index 000000000..cfd91c620 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts @@ -0,0 +1,333 @@ +import { + DOCUMENT_CHANGE_GROUP_BUDGET, + FounderWeeklyReviewEvidenceItemSchema, + alignVersionChunks, + buildDocumentChangeEvidence, + buildDocumentChangeGroupEvidence, + buildGenerationEvidenceEnvelope, + buildRawDocumentChanges, + condenseDocumentChanges, + groupRawDocumentChanges, + normalizeDocumentChangeContent, + selectDocumentChangeGroups, + type ChunkAlignment, + type DocumentChangeGroup, + type VersionChunk, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +const pair = (documentId = 1n, previousVersionId = 1, currentVersionId = 2): VersionPair => ({ + documentId, + documentTitle: `Document ${documentId}`, + documentCategory: "Product", + previousVersionId, + previousVersionNumber: previousVersionId, + previousCreatedAt: new Date(`2026-01-${String(Math.min(previousVersionId, 28)).padStart(2, "0")}T00:00:00.000Z`), + currentVersionId, + currentVersionNumber: currentVersionId, + currentCreatedAt: new Date(`2026-02-${String(Math.min(currentVersionId, 28)).padStart(2, "0")}T00:00:00.000Z`), + currentChangelog: null, +}); + +const chunk = ( + chunkId: number, + versionId: bigint, + content: string, + overrides: Partial = {} +): VersionChunk => ({ + chunkId, + versionId, + documentId: 1n, + content, + contentHash: null, + structureId: BigInt(chunkId), + structurePath: "/overview", + structureTitle: "Overview", + structureOrdering: chunkId, + pageNumber: Math.ceil(chunkId / 2), + lineStart: chunkId * 10, + lineEnd: chunkId * 10 + 5, + ...overrides, +}); + +const modified = ( + id: number, + before: string, + after: string, + overrides: Partial = {} +): ChunkAlignment => ({ + changeType: "modified", + previousChunk: chunk(id, 1n, before, overrides), + currentChunk: chunk(10_000 + id, 2n, after, overrides), + alignmentMethod: "structure_path", +}); + +describe("deterministic document-change condensation", () => { + it("constructs stable, distinct raw IDs and preserves provenance independent of alignment order", () => { + const versionPair = pair(); + const alignments: ChunkAlignment[] = [ + modified(1, "before", "after"), + { changeType: "removed", previousChunk: chunk(2, 1n, "removed", { structurePath: "/removed" }), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(3, 2n, "added", { structurePath: "/added" }), alignmentMethod: "unmatched" }, + ]; + const first = buildRawDocumentChanges(versionPair, alignments); + const shuffled = buildRawDocumentChanges(versionPair, [alignments[2]!, alignments[0]!, alignments[1]!]); + + expect(shuffled).toEqual(first); + expect(new Set(first.rawChanges.map(change => change.rawChangeId)).size).toBe(3); + expect(first.rawChanges.every(change => /^raw_document_change:[a-f0-9]{64}$/.test(change.rawChangeId))).toBe(true); + expect(first.rawChanges.find(change => change.changeType === "modified")).toEqual(expect.objectContaining({ + previousChunk: expect.objectContaining({ chunkId: 1 }), + currentChunk: expect.objectContaining({ chunkId: 10_001 }), + })); + const changedContent = buildRawDocumentChanges(versionPair, [modified(1, "before", "different after")]); + expect(changedContent.rawChanges[0]!.rawChangeId).not.toBe(first.rawChanges.find(change => change.changeType === "modified")!.rawChangeId); + }); + + it.each([ + ["ordinary whitespace", "alpha beta", " alpha beta "], + ["CRLF", "alpha\nbeta", "alpha\r\nbeta"], + ["CR", "alpha\nbeta", "alpha\rbeta"], + ["non-breaking space", "alpha beta", "alpha\u00a0beta"], + ["leading and trailing whitespace", "alpha", "\t alpha \n"], + ["Unicode NFC", "Café", "Cafe\u0301"], + ])("filters %s-only modifications", (_name, before, after) => { + expect(normalizeDocumentChangeContent(before)).toBe(normalizeDocumentChangeContent(after)); + expect(buildRawDocumentChanges(pair(), [modified(1, before, after)])).toEqual({ + rawChanges: [], + deterministicNoOpCount: 1, + }); + }); + + it.each([ + ["punctuation", "Launch.", "Launch!"], + ["capitalization", "Product", "product"], + ["number", "10 users", "100 users"], + ["percentage", "10%", "20%"], + ["date", "August 7", "August 8"], + ["quarter", "Q3", "Q4"], + ["owner", "Product owns this", "Platform owns this"], + ["negation", "supports export", "does not support export"], + ["modal", "may ship", "must ship"], + ["requirement", "optional", "required"], + ["status", "planned", "launched"], + ])("retains a %s change", (_name, before, after) => { + const result = buildRawDocumentChanges(pair(), [modified(1, before, after)]); + expect(result.deterministicNoOpCount).toBe(0); + expect(result.rawChanges).toHaveLength(1); + }); + + it("groups adjacent mixed changes within a section and never crosses known section boundaries", () => { + const versionPair = pair(); + const alignments: ChunkAlignment[] = [ + modified(1, "old one", "new one", { structurePath: "/alpha", structureTitle: "Alpha" }), + { changeType: "removed", previousChunk: chunk(2, 1n, "old two", { structurePath: "/alpha", structureTitle: "Alpha" }), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(3, 2n, "new two", { structurePath: "/alpha", structureTitle: "Alpha" }), alignmentMethod: "unmatched" }, + modified(4, "old beta", "new beta", { structurePath: "/beta", structureTitle: "Beta", pageNumber: 2 }), + ]; + const raw = buildRawDocumentChanges(versionPair, alignments).rawChanges; + const grouped = groupRawDocumentChanges(versionPair, raw); + + expect(grouped.groups).toHaveLength(2); + expect(grouped.groups.map(group => group.rawChanges.length).sort()).toEqual([1, 3]); + expect(grouped.groups.find(group => group.structurePath === "/alpha")?.rawChanges.map(change => change.changeType).sort()).toEqual(["added", "modified", "removed"]); + }); + + it("keeps distinct known titles separate when paths are unavailable", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [ + modified(1, "old alpha", "new alpha", { structurePath: null, structureTitle: "Alpha", pageNumber: 1 }), + modified(2, "old beta", "new beta", { structurePath: null, structureTitle: "Beta", pageNumber: 1 }), + ]).rawChanges; + expect(groupRawDocumentChanges(versionPair, raw).groups).toHaveLength(2); + }); + + it("does not merge conflicting known paths solely because their titles match", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [ + modified(1, "old one", "new one", { structurePath: "/one", structureTitle: "Overview", pageNumber: 1 }), + modified(2, "old two", "new two", { structurePath: "/two", structureTitle: "Overview", pageNumber: 1 }), + ]).rawChanges; + expect(groupRawDocumentChanges(versionPair, raw).groups).toHaveLength(2); + }); + + it("groups renamed paths through connected aligned chunks and supports one-to-many and many-to-one boundaries", () => { + const versionPair = pair(); + const renamed: ChunkAlignment[] = [1, 2].map(id => ({ + changeType: "modified" as const, + previousChunk: chunk(id, 1n, `old ${id}`, { structurePath: "/old", structureTitle: "Plan" }), + currentChunk: chunk(100 + id, 2n, `new ${id}`, { structurePath: "/new", structureTitle: "Plan" }), + alignmentMethod: "section_title" as const, + })); + const oneToMany: ChunkAlignment[] = [ + modified(3, "one", "part one", { structurePath: "/split", structureTitle: "Split" }), + { changeType: "added", currentChunk: chunk(104, 2n, "part two", { structurePath: "/split", structureTitle: "Split" }), alignmentMethod: "unmatched" }, + ]; + const manyToOne: ChunkAlignment[] = [ + modified(5, "part one", "merged", { structurePath: "/merge", structureTitle: "Merge" }), + { changeType: "removed", previousChunk: chunk(6, 1n, "part two", { structurePath: "/merge", structureTitle: "Merge" }), alignmentMethod: "unmatched" }, + ]; + const groups = groupRawDocumentChanges(versionPair, buildRawDocumentChanges(versionPair, [...renamed, ...oneToMany, ...manyToOne]).rawChanges).groups; + + expect(groups).toHaveLength(3); + expect(groups.map(group => group.rawChanges.length).sort()).toEqual([2, 2, 2]); + }); + + it("keeps exact reordered content unchanged before grouping", () => { + const hashA = "a".repeat(64); const hashB = "b".repeat(64); + const previous = [ + chunk(1, 1n, "Alpha exact", { contentHash: hashA, structurePath: "/a" }), + chunk(2, 1n, "Beta exact", { contentHash: hashB, structurePath: "/b" }), + ]; + const current = [ + chunk(3, 2n, "Beta exact", { contentHash: hashB, structurePath: "/moved-b" }), + chunk(4, 2n, "Alpha exact", { contentHash: hashA, structurePath: "/moved-a" }), + ]; + const alignments = alignVersionChunks(previous, current); + const condensed = condenseDocumentChanges([{ pair: pair(), alignments }]); + expect(alignments.every(item => item.changeType === "unchanged")).toBe(true); + expect(condensed.rawChanges).toHaveLength(0); + expect(condensed.groups).toHaveLength(0); + }); + + it("produces identical groups and IDs for shuffled raw input", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [ + modified(1, "a", "aa", { structurePath: "/a", structureTitle: "A" }), + modified(2, "b", "bb", { structurePath: "/a", structureTitle: "A" }), + modified(3, "c", "cc", { structurePath: "/b", structureTitle: "B" }), + ]).rawChanges; + const first = groupRawDocumentChanges(versionPair, raw); + const shuffled = groupRawDocumentChanges(versionPair, [raw[2]!, raw[0]!, raw[1]!]); + expect(shuffled).toEqual(first); + expect(first.groups.every(group => /^document_change_group:[a-f0-9]{64}$/.test(group.groupId))).toBe(true); + const changedSupport = buildRawDocumentChanges(versionPair, [ + modified(1, "a", "different", { structurePath: "/a", structureTitle: "A" }), + modified(2, "b", "bb", { structurePath: "/a", structureTitle: "A" }), + ]).rawChanges; + expect(groupRawDocumentChanges(versionPair, changedSupport).groups[0]!.groupId).not.toBe(first.groups[0]!.groupId); + }); + + it("keeps 16 records together and splits 17 into stable bounded windows with one warning", () => { + const versionPair = pair(); + const sixteen = buildRawDocumentChanges(versionPair, Array.from({ length: 16 }, (_, index) => + modified(index + 1, `before ${index}`, `after ${index}`, { structurePath: "/large", structureTitle: "Large" }))).rawChanges; + expect(groupRawDocumentChanges(versionPair, sixteen).groups).toHaveLength(1); + + const seventeen = buildRawDocumentChanges(versionPair, [ + ...Array.from({ length: 16 }, (_, index) => modified(index + 1, `before ${index}`, `after ${index}`, { structurePath: "/large", structureTitle: "Large" })), + modified(17, "before 17", "after 17", { structurePath: "/large", structureTitle: "Large" }), + ]).rawChanges; + const first = groupRawDocumentChanges(versionPair, seventeen); + const shuffled = groupRawDocumentChanges(versionPair, [...seventeen].reverse()); + expect(first.groups.map(group => group.rawChanges.length)).toEqual([16, 1]); + expect(first.groups.map(group => group.splitOrdinal)).toEqual([0, 1]); + expect(first.groups.every(group => group.rawChanges.length <= DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup)).toBe(true); + expect(first.warnings).toEqual([expect.objectContaining({ code: "materiality_group_too_large" })]); + expect(shuffled).toEqual(first); + }); + + it("enforces pair, document, and review limits with deterministic round-robin diversity", () => { + const inputs = [1n, 2n, 3n, 4n].map(documentId => { + const versionPair = pair(documentId); + return { + pair: versionPair, + alignments: Array.from({ length: 12 }, (_, index) => modified( + Number(documentId) * 100 + index, + `before ${documentId}-${index}`, + `after ${documentId}-${index}`, + { documentId, structurePath: `/section-${index}`, structureTitle: `Section ${index}` } + )), + }; + }); + const first = condenseDocumentChanges(inputs); + const shuffled = condenseDocumentChanges([...inputs].reverse().map(input => ({ ...input, alignments: [...input.alignments].reverse() }))); + const byDocument = new Map(); + const byPair = new Map(); + for (const group of first.selectedGroups) { + const documentId = group.documentId.toString(); + const versionPair = `${documentId}:${group.previousVersionId}:${group.currentVersionId}`; + byDocument.set(documentId, (byDocument.get(documentId) ?? 0) + 1); + byPair.set(versionPair, (byPair.get(versionPair) ?? 0) + 1); + } + expect(first.selectedGroups).toHaveLength(24); + expect([...byDocument.values()].every(count => count <= 8)).toBe(true); + expect([...byPair.values()].every(count => count <= 8)).toBe(true); + expect([...byDocument.keys()]).toEqual(["1", "2", "3", "4"]); + expect(first.warnings).toContainEqual(expect.objectContaining({ code: "document_change_budget_truncated" })); + expect(shuffled.selectedGroups.map(group => group.groupId)).toEqual(first.selectedGroups.map(group => group.groupId)); + }); + + it("round-robins version pairs within a document", () => { + const inputs = [pair(1n, 1, 2), pair(1n, 2, 3)].map((versionPair, pairIndex) => ({ + pair: versionPair, + alignments: Array.from({ length: 6 }, (_, index) => modified( + pairIndex * 100 + index + 1, + `before ${pairIndex}-${index}`, + `after ${pairIndex}-${index}`, + { structurePath: `/pair-${pairIndex}-section-${index}`, structureTitle: `Pair ${pairIndex} Section ${index}` } + )), + })); + const result = condenseDocumentChanges(inputs); + expect(result.selectedGroups).toHaveLength(8); + expect(result.selectedGroups.filter(group => group.currentVersionId === 2)).toHaveLength(4); + expect(result.selectedGroups.filter(group => group.currentVersionId === 3)).toHaveLength(4); + }); + + it("keeps the existing evidence contract and source ID for a one-change group", () => { + const evidence = buildDocumentChangeEvidence(pair(), [modified(7, "Before", "After")]); + expect(evidence).toEqual([expect.objectContaining({ + sourceType: "document_change", + sourceId: "document_change:doc:1:v1:v2:chunk:7:10007", + excerpt: "Section modified. Before: Before After: After", + })]); + expect(FounderWeeklyReviewEvidenceItemSchema.safeParse(evidence[0]).success).toBe(true); + }); + + it("condenses a deterministic 20-page, 40-chunk enterprise fixture before the prompt envelope", () => { + const versionPair = pair(44n); + const alignments: ChunkAlignment[] = []; + for (let index = 0; index < 20; index++) { + const exact = `unchanged section ${index}`; + alignments.push({ + changeType: "unchanged", + previousChunk: chunk(index + 1, 1n, exact, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}`, structureTitle: `Unchanged ${index}` }), + currentChunk: chunk(1_001 + index, 2n, exact, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}`, structureTitle: `Unchanged ${index}` }), + alignmentMethod: "content_hash", + }); + } + for (let index = 0; index < 5; index++) alignments.push(modified(100 + index, `wrapped line ${index}`, ` wrapped\r\n line\u00a0${index} `, { documentId: 44n, pageNumber: index + 1, structurePath: `/noop-${index}`, structureTitle: `No-op ${index}` })); + for (let index = 0; index < 4; index++) alignments.push(modified(200 + index, `planned metric ${index}`, `launched metric ${index + 1}`, { documentId: 44n, pageNumber: 6 + index, structurePath: `/material-${index}`, structureTitle: `Material ${index}` })); + for (let index = 0; index < 6; index++) alignments.push(modified(300 + index, `old rewritten ${index}`, `new rewritten ${index}`, { documentId: 44n, pageNumber: 10 + Math.floor(index / 2), structurePath: "/rewritten", structureTitle: "Rewritten section" })); + for (let index = 0; index < 5; index++) alignments.push(modified(400 + index, `editorial wording ${index}`, `rephrased wording ${index}`, { documentId: 44n, pageNumber: 15 + index, structurePath: "/editorial", structureTitle: "Editorial section" })); + + const first = condenseDocumentChanges([{ pair: versionPair, alignments }]); + const shuffled = condenseDocumentChanges([{ pair: versionPair, alignments: [...alignments].reverse() }]); + const evidence = first.selectedGroups.map(group => buildDocumentChangeGroupEvidence(versionPair, group)); + const snapshot = { + schemaVersion: "founder-weekly-review-evidence/v1" as const, + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: evidence, + sourceWarnings: [], + }; + const envelope = buildGenerationEvidenceEnvelope(snapshot); + + expect(alignments).toHaveLength(40); + expect(first.diagnostics).toEqual(expect.objectContaining({ + alignedChunkCount: 40, + rawModifiedCount: 15, + deterministicNoOpCount: 5, + groupCount: 6, + selectedGroupCount: 6, + truncatedGroupCount: 0, + })); + expect(first.selectedGroups.every(group => group.rawChanges.length <= 16)).toBe(true); + expect(first.selectedGroups.map(group => group.groupId)).toEqual(shuffled.selectedGroups.map(group => group.groupId)); + expect(evidence).toHaveLength(6); + expect(envelope.diagnostics.selectedBySourceType.document_change).toBe(6); + expect(envelope.diagnostics.serializedCharacterCount).toBeLessThanOrEqual(14_000); + expect(first.warnings).toEqual([]); + }); +}); diff --git a/packages/features/src/founder-weekly-review/document-change.ts b/packages/features/src/founder-weekly-review/document-change.ts index 17b244b45..33afee71d 100644 --- a/packages/features/src/founder-weekly-review/document-change.ts +++ b/packages/features/src/founder-weekly-review/document-change.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; export type DocumentVersionForComparison = { @@ -46,6 +47,69 @@ export type ChunkAlignment = { similarityScore?: number; }; +export const RAW_DOCUMENT_CHANGE_VERSION = "raw-document-change/v1" as const; +export const DOCUMENT_CHANGE_GROUPING_VERSION = "document-change-grouping/v1" as const; +export const DOCUMENT_CHANGE_GROUP_BUDGET = Object.freeze({ + rawChangesPerGroup: 16, + groupsPerVersionPair: 8, + groupsPerDocument: 8, + groupsPerReview: 24, +}); + +export type RawDocumentChange = { + rawChangeId: string; + changeType: Exclude; + alignmentMethod: ChunkAlignment["alignmentMethod"]; + similarityScore?: number; + previousChunk?: VersionChunk; + currentChunk?: VersionChunk; + previousNormalizedContent?: string; + currentNormalizedContent?: string; +}; + +export type DocumentChangeGroup = { + groupId: string; + documentId: bigint; + previousVersionId: number; + currentVersionId: number; + structurePath?: string | null; + structureTitle?: string | null; + splitOrdinal: number; + rawChanges: readonly RawDocumentChange[]; +}; + +export type DocumentChangeProcessingWarning = { + code: "materiality_group_too_large" | "document_change_budget_truncated"; + message: string; +}; + +export type DocumentChangeCondensationDiagnostics = { + versionPairCount: number; + alignedChunkCount: number; + rawModifiedCount: number; + rawAddedCount: number; + rawRemovedCount: number; + deterministicNoOpCount: number; + groupCount: number; + oversizedGroupSplitCount: number; + selectedGroupCount: number; + truncatedGroupCount: number; + approximateChangedCharacters: number; +}; + +export type DocumentChangePairInput = { + pair: VersionPair; + alignments: readonly ChunkAlignment[]; +}; + +export type DocumentChangeCondensationResult = { + rawChanges: readonly RawDocumentChange[]; + groups: readonly DocumentChangeGroup[]; + selectedGroups: readonly DocumentChangeGroup[]; + warnings: readonly DocumentChangeProcessingWarning[]; + diagnostics: DocumentChangeCondensationDiagnostics; +}; + const MAX_EXCERPT = 4000; const MAX_METADATA_TEXT = 512; const MIN_TEXT_SIMILARITY_CHARACTERS = 20; @@ -61,6 +125,18 @@ const compareChunks = (a: VersionChunk, b: VersionChunk) => (a.lineStart ?? -1) - (b.lineStart ?? -1) || (a.structureOrdering ?? -1) - (b.structureOrdering ?? -1) || a.chunkId - b.chunkId; +function digest(parts: readonly (string | number | null)[]): string { + return createHash("sha256").update(JSON.stringify(parts), "utf8").digest("hex"); +} + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function compareBigInt(a: bigint, b: bigint): number { + return a < b ? -1 : a > b ? 1 : 0; +} + /** Select only adjacent pairs whose current version is in the reporting period. */ export function selectVersionPairsForReportingPeriod( versions: readonly DocumentVersionForComparison[], startInclusive: Date, endExclusive: Date @@ -143,22 +219,357 @@ export function alignVersionChunks(previousChunks: readonly VersionChunk[], curr return results.sort((a, b) => compareChunks(a.currentChunk ?? a.previousChunk!, b.currentChunk ?? b.previousChunk!)); } +/** Conservative normalization used only to remove deterministic formatting no-ops. */ +export function normalizeDocumentChangeContent(value: string): string { + return value.normalize("NFC").replace(/\r\n?/g, "\n").replace(/\u00a0/g, " ").replace(/\s+/g, " ").trim(); +} + +function rawChangeContentIdentity(chunk: VersionChunk | undefined): string | null { + return chunk ? digest([chunk.contentHash, chunk.content]) : null; +} + +function compareRawChanges(a: RawDocumentChange, b: RawDocumentChange): number { + const aChunk = a.currentChunk ?? a.previousChunk!; + const bChunk = b.currentChunk ?? b.previousChunk!; + return (aChunk.structureOrdering ?? Number.MAX_SAFE_INTEGER) - (bChunk.structureOrdering ?? Number.MAX_SAFE_INTEGER) + || (aChunk.pageNumber ?? Number.MAX_SAFE_INTEGER) - (bChunk.pageNumber ?? Number.MAX_SAFE_INTEGER) + || (aChunk.lineStart ?? Number.MAX_SAFE_INTEGER) - (bChunk.lineStart ?? Number.MAX_SAFE_INTEGER) + || compareOrdinal(aChunk.structurePath ?? "", bChunk.structurePath ?? "") + || (a.previousChunk?.chunkId ?? -1) - (b.previousChunk?.chunkId ?? -1) + || (a.currentChunk?.chunkId ?? -1) - (b.currentChunk?.chunkId ?? -1) + || compareOrdinal(a.rawChangeId, b.rawChangeId); +} + +/** Converts alignments into stable raw records and drops normalized-equal modifications. */ +export function buildRawDocumentChanges( + pair: VersionPair, + alignments: readonly ChunkAlignment[] +): { rawChanges: RawDocumentChange[]; deterministicNoOpCount: number } { + const rawChanges: RawDocumentChange[] = []; + let deterministicNoOpCount = 0; + for (const alignment of alignments) { + if (alignment.changeType === "unchanged") continue; + const previousNormalizedContent = alignment.previousChunk + ? normalizeDocumentChangeContent(alignment.previousChunk.content) + : undefined; + const currentNormalizedContent = alignment.currentChunk + ? normalizeDocumentChangeContent(alignment.currentChunk.content) + : undefined; + if ( + alignment.changeType === "modified" + && previousNormalizedContent === currentNormalizedContent + ) { + deterministicNoOpCount++; + continue; + } + const rawChangeId = `raw_document_change:${digest([ + RAW_DOCUMENT_CHANGE_VERSION, + pair.documentId.toString(), + pair.previousVersionId, + pair.currentVersionId, + alignment.changeType, + alignment.previousChunk?.chunkId ?? null, + alignment.currentChunk?.chunkId ?? null, + alignment.alignmentMethod, + rawChangeContentIdentity(alignment.previousChunk), + rawChangeContentIdentity(alignment.currentChunk), + ])}`; + rawChanges.push({ + rawChangeId, + changeType: alignment.changeType, + alignmentMethod: alignment.alignmentMethod, + ...(alignment.similarityScore === undefined ? {} : { similarityScore: alignment.similarityScore }), + ...(alignment.previousChunk ? { previousChunk: alignment.previousChunk, previousNormalizedContent } : {}), + ...(alignment.currentChunk ? { currentChunk: alignment.currentChunk, currentNormalizedContent } : {}), + }); + } + return { rawChanges: rawChanges.sort(compareRawChanges), deterministicNoOpCount }; +} + +function normalizedSectionValues(change: RawDocumentChange, field: "structurePath" | "structureTitle"): string[] { + return [...new Set([change.previousChunk?.[field], change.currentChunk?.[field]].map(normalize).filter((value): value is string => value !== null))].sort(compareOrdinal); +} + +function intersects(left: readonly string[], right: readonly string[]): boolean { + return left.some(value => right.includes(value)); +} + +function hasKnownSection(change: RawDocumentChange): boolean { + return normalizedSectionValues(change, "structurePath").length > 0 + || normalizedSectionValues(change, "structureTitle").length > 0; +} + +function structurallyProximate(left: RawDocumentChange, right: RawDocumentChange): boolean { + const a = left.currentChunk ?? left.previousChunk!; + const b = right.currentChunk ?? right.previousChunk!; + if (a.structureOrdering !== null && b.structureOrdering !== null && Math.abs(a.structureOrdering - b.structureOrdering) <= 1) { + return true; + } + if (a.pageNumber !== null && b.pageNumber !== null && Math.abs(a.pageNumber - b.pageNumber) <= 1) { + if (a.pageNumber !== b.pageNumber) return true; + if (a.lineEnd !== null && b.lineStart !== null && b.lineStart - a.lineEnd <= 2) return true; + if (b.lineEnd !== null && a.lineStart !== null && a.lineStart - b.lineEnd <= 2) return true; + } + return false; +} + +function shouldGroup(left: RawDocumentChange, right: RawDocumentChange): boolean { + const leftPaths = normalizedSectionValues(left, "structurePath"); + const rightPaths = normalizedSectionValues(right, "structurePath"); + if (intersects(leftPaths, rightPaths)) return true; + if (leftPaths.length > 0 && rightPaths.length > 0) return false; + const leftTitles = normalizedSectionValues(left, "structureTitle"); + const rightTitles = normalizedSectionValues(right, "structureTitle"); + if (intersects(leftTitles, rightTitles)) return true; + // Proximity is only a fallback when neither record has a known section. This + // prevents an unlabelled record from transitively bridging two named sections. + if (hasKnownSection(left) || hasKnownSection(right)) return false; + return structurallyProximate(left, right); +} + +function canonicalSectionKey(changes: readonly RawDocumentChange[]): string { + const paths = [...new Set(changes.flatMap(change => normalizedSectionValues(change, "structurePath")))].sort(compareOrdinal); + const titles = [...new Set(changes.flatMap(change => normalizedSectionValues(change, "structureTitle")))].sort(compareOrdinal); + if (paths.length > 0) return `path:${paths.join("|")}`; + if (titles.length > 0) return `title:${titles.join("|")}`; + const first = changes[0]!.currentChunk ?? changes[0]!.previousChunk!; + return `position:${first.structureOrdering ?? ""}:${first.pageNumber ?? ""}:${first.lineStart ?? ""}`; +} + +function preferredStructureValue(changes: readonly RawDocumentChange[], field: "structurePath" | "structureTitle"): string | null { + const values = changes.flatMap(change => [change.currentChunk?.[field], change.previousChunk?.[field]]) + .filter((value): value is string => Boolean(value?.trim())); + return values.sort(compareOrdinal)[0] ?? null; +} + +function makeGroup( + pair: VersionPair, + naturalGroup: readonly RawDocumentChange[], + rawChanges: readonly RawDocumentChange[], + splitOrdinal: number +): DocumentChangeGroup { + const sectionKey = canonicalSectionKey(naturalGroup); + return { + groupId: `document_change_group:${digest([ + DOCUMENT_CHANGE_GROUPING_VERSION, + pair.documentId.toString(), + pair.previousVersionId, + pair.currentVersionId, + sectionKey, + ...rawChanges.map(change => change.rawChangeId), + splitOrdinal, + ])}`, + documentId: pair.documentId, + previousVersionId: pair.previousVersionId, + currentVersionId: pair.currentVersionId, + structurePath: preferredStructureValue(naturalGroup, "structurePath"), + structureTitle: preferredStructureValue(naturalGroup, "structureTitle"), + splitOrdinal, + rawChanges, + }; +} + +function compareGroups(a: DocumentChangeGroup, b: DocumentChangeGroup): number { + const aFirst = a.rawChanges[0]!; + const bFirst = b.rawChanges[0]!; + return compareBigInt(a.documentId, b.documentId) + || a.currentVersionId - b.currentVersionId + || a.previousVersionId - b.previousVersionId + || compareRawChanges(aFirst, bFirst) + || a.splitOrdinal - b.splitOrdinal + || compareOrdinal(a.groupId, b.groupId); +} + +/** Groups changed records within one immutable document version pair. */ +export function groupRawDocumentChanges( + pair: VersionPair, + input: readonly RawDocumentChange[] +): { groups: DocumentChangeGroup[]; oversizedGroupSplitCount: number; warnings: DocumentChangeProcessingWarning[] } { + const rawChanges = [...input].sort(compareRawChanges); + const parents = rawChanges.map((_, index) => index); + const find = (index: number): number => { + while (parents[index] !== index) { + parents[index] = parents[parents[index]!]!; + index = parents[index]!; + } + return index; + }; + const union = (left: number, right: number) => { + const a = find(left); const b = find(right); + if (a !== b) parents[Math.max(a, b)] = Math.min(a, b); + }; + for (let left = 0; left < rawChanges.length; left++) { + for (let right = left + 1; right < rawChanges.length; right++) { + if (shouldGroup(rawChanges[left]!, rawChanges[right]!)) union(left, right); + } + } + const components = new Map(); + rawChanges.forEach((change, index) => { + const root = find(index); + const component = components.get(root) ?? []; + component.push(change); components.set(root, component); + }); + const naturalGroups = [...components.values()].map(group => group.sort(compareRawChanges)) + .sort((a, b) => compareRawChanges(a[0]!, b[0]!)); + const groups: DocumentChangeGroup[] = []; + let oversizedGroupSplitCount = 0; + for (const naturalGroup of naturalGroups) { + const windowCount = Math.ceil(naturalGroup.length / DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup); + if (windowCount > 1) oversizedGroupSplitCount += windowCount - 1; + for (let splitOrdinal = 0; splitOrdinal < windowCount; splitOrdinal++) { + const start = splitOrdinal * DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup; + const window = naturalGroup.slice(start, start + DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup); + groups.push(makeGroup(pair, naturalGroup, window, splitOrdinal)); + } + } + return { + groups: groups.sort(compareGroups), + oversizedGroupSplitCount, + warnings: oversizedGroupSplitCount > 0 ? [{ + code: "materiality_group_too_large", + message: "One or more document-change groups exceeded the raw-change limit and were split into deterministic windows.", + }] : [], + }; +} + +function pairKey(group: Pick): string { + return `${group.previousVersionId}:${group.currentVersionId}`; +} + +function sectionSelectionKey(group: DocumentChangeGroup): string { + return `${group.structurePath ?? ""}|${group.structureTitle ?? ""}`; +} + +/** Neutral structural budget: documents, then version pairs, then distinct sections. */ +export function selectDocumentChangeGroups(input: readonly DocumentChangeGroup[]): { + selectedGroups: DocumentChangeGroup[]; + truncatedGroupCount: number; + warnings: DocumentChangeProcessingWarning[]; +} { + const ordered = [...input].sort(compareGroups); + const byDocument = new Map>(); + for (const group of ordered) { + const documentKey = group.documentId.toString(); + const pairs = byDocument.get(documentKey) ?? new Map(); + const key = pairKey(group); + const pairGroups = pairs.get(key) ?? []; + pairGroups.push(group); pairs.set(key, pairGroups); byDocument.set(documentKey, pairs); + } + for (const pairs of byDocument.values()) { + for (const [key, groups] of pairs) { + pairs.set(key, groups.sort((a, b) => + a.splitOrdinal - b.splitOrdinal + || compareOrdinal(sectionSelectionKey(a), sectionSelectionKey(b)) + || compareGroups(a, b))); + } + } + const documents = [...byDocument.entries()].sort(([a], [b]) => compareBigInt(BigInt(a), BigInt(b))); + const documentCounts = new Map(); + const pairCounts = new Map(); + const pairCursors = new Map(); + const selectedGroups: DocumentChangeGroup[] = []; + let madeProgress = true; + while (selectedGroups.length < DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview && madeProgress) { + madeProgress = false; + for (const [documentId, pairs] of documents) { + if ((documentCounts.get(documentId) ?? 0) >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerDocument) continue; + const pairEntries = [...pairs.entries()].sort(([, a], [, b]) => compareGroups(a[0]!, b[0]!)); + const start = pairCursors.get(documentId) ?? 0; + for (let offset = 0; offset < pairEntries.length; offset++) { + const pairIndex = (start + offset) % pairEntries.length; + const [key, groups] = pairEntries[pairIndex]!; + const countKey = `${documentId}:${key}`; + const count = pairCounts.get(countKey) ?? 0; + if (count >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerVersionPair || !groups[count]) continue; + selectedGroups.push(groups[count]!); + documentCounts.set(documentId, (documentCounts.get(documentId) ?? 0) + 1); + pairCounts.set(countKey, count + 1); + pairCursors.set(documentId, (pairIndex + 1) % pairEntries.length); + madeProgress = true; + break; + } + if (selectedGroups.length >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview) break; + } + } + const truncatedGroupCount = ordered.length - selectedGroups.length; + return { + selectedGroups: selectedGroups.sort(compareGroups), + truncatedGroupCount, + warnings: truncatedGroupCount > 0 ? [{ + code: "document_change_budget_truncated", + message: "Document-change groups were truncated to deterministic structural limits.", + }] : [], + }; +} + +/** Complete pure condensation boundary used before current evidence compatibility projection. */ +export function condenseDocumentChanges(inputs: readonly DocumentChangePairInput[]): DocumentChangeCondensationResult { + const rawChanges: RawDocumentChange[] = []; + const groups: DocumentChangeGroup[] = []; + const warnings: DocumentChangeProcessingWarning[] = []; + let deterministicNoOpCount = 0; + let oversizedGroupSplitCount = 0; + for (const { pair, alignments } of [...inputs].sort((a, b) => + compareBigInt(a.pair.documentId, b.pair.documentId) + || a.pair.currentCreatedAt.getTime() - b.pair.currentCreatedAt.getTime() + || a.pair.currentVersionId - b.pair.currentVersionId)) { + const raw = buildRawDocumentChanges(pair, alignments); + const grouped = groupRawDocumentChanges(pair, raw.rawChanges); + rawChanges.push(...raw.rawChanges); + groups.push(...grouped.groups); + deterministicNoOpCount += raw.deterministicNoOpCount; + oversizedGroupSplitCount += grouped.oversizedGroupSplitCount; + warnings.push(...grouped.warnings); + } + const selected = selectDocumentChangeGroups(groups); + warnings.push(...selected.warnings); + const counts = (type: RawDocumentChange["changeType"]) => rawChanges.filter(change => change.changeType === type).length; + return { + rawChanges: rawChanges.sort(compareRawChanges), + groups: groups.sort(compareGroups), + selectedGroups: selected.selectedGroups, + warnings: [...new Map(warnings.map(item => [item.code, item])).values()], + diagnostics: { + versionPairCount: inputs.length, + alignedChunkCount: inputs.reduce((total, input) => total + input.alignments.length, 0), + rawModifiedCount: counts("modified"), + rawAddedCount: counts("added"), + rawRemovedCount: counts("removed"), + deterministicNoOpCount, + groupCount: groups.length, + oversizedGroupSplitCount, + selectedGroupCount: selected.selectedGroups.length, + truncatedGroupCount: selected.truncatedGroupCount, + approximateChangedCharacters: rawChanges.reduce((total, change) => + total + (change.previousChunk?.content.length ?? 0) + (change.currentChunk?.content.length ?? 0), 0), + }, + }; +} + function bound(value: string, max = MAX_EXCERPT) { return value.length <= max ? value : `${value.slice(0, max - 1)}…`; } function preview(value: string) { return bound(value.replace(/\s+/g, " ").trim(), 900); } +function buildRawChangeEvidence(pair: VersionPair, alignment: RawDocumentChange): FounderWeeklyReviewEvidenceItem { + const previous = alignment.previousChunk; const current = alignment.currentChunk; + const previousId = previous?.chunkId.toString() ?? "none"; const currentId = current?.chunkId.toString() ?? "none"; + const sourceId = `document_change:doc:${pair.documentId}:v${pair.previousVersionId}:v${pair.currentVersionId}:chunk:${previousId}:${currentId}`; + const excerpt = alignment.changeType === "modified" + ? `Section modified. Before: ${preview(previous!.content)} After: ${preview(current!.content)}` + : alignment.changeType === "added" ? `Section added: ${preview(current!.content)}` : `Section removed: ${preview(previous!.content)}`; + return { sourceType: "document_change", sourceId, title: pair.documentTitle, + sourceTimestamp: pair.currentCreatedAt.toISOString(), excerpt: bound(excerpt), workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, + metadata: { documentId: pair.documentId.toString(), previousVersionId: pair.previousVersionId, currentVersionId: pair.currentVersionId, previousVersionNumber: pair.previousVersionNumber, currentVersionNumber: pair.currentVersionNumber, + previousChunkId: previous?.chunkId ?? null, currentChunkId: current?.chunkId ?? null, changeType: alignment.changeType, alignmentMethod: alignment.alignmentMethod, + previousContentHash: previous?.contentHash ?? null, currentContentHash: current?.contentHash ?? null, structurePath: current?.structurePath ?? previous?.structurePath ?? null, + userChangelog: pair.currentChangelog ? bound(pair.currentChangelog.replace(/\s+/g, " ").trim(), MAX_METADATA_TEXT) : null } }; +} + +/** Compatibility projection: one existing-format evidence item per selected structural group. */ +export function buildDocumentChangeGroupEvidence(pair: VersionPair, group: DocumentChangeGroup): FounderWeeklyReviewEvidenceItem { + return buildRawChangeEvidence(pair, group.rawChanges[0]!); +} + export function buildDocumentChangeEvidence(pair: VersionPair, alignments: readonly ChunkAlignment[]): FounderWeeklyReviewEvidenceItem[] { - return alignments.filter((alignment) => alignment.changeType !== "unchanged").map((alignment) => { - const previous = alignment.previousChunk; const current = alignment.currentChunk; - const previousId = previous?.chunkId.toString() ?? "none"; const currentId = current?.chunkId.toString() ?? "none"; - const sourceId = `document_change:doc:${pair.documentId}:v${pair.previousVersionId}:v${pair.currentVersionId}:chunk:${previousId}:${currentId}`; - const excerpt = alignment.changeType === "modified" - ? `Section modified. Before: ${preview(previous!.content)} After: ${preview(current!.content)}` - : alignment.changeType === "added" ? `Section added: ${preview(current!.content)}` : `Section removed: ${preview(previous!.content)}`; - return { sourceType: "document_change", sourceId, title: pair.documentTitle, - sourceTimestamp: pair.currentCreatedAt.toISOString(), excerpt: bound(excerpt), workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, - metadata: { documentId: pair.documentId.toString(), previousVersionId: pair.previousVersionId, currentVersionId: pair.currentVersionId, previousVersionNumber: pair.previousVersionNumber, currentVersionNumber: pair.currentVersionNumber, - previousChunkId: previous?.chunkId ?? null, currentChunkId: current?.chunkId ?? null, changeType: alignment.changeType, alignmentMethod: alignment.alignmentMethod, - previousContentHash: previous?.contentHash ?? null, currentContentHash: current?.contentHash ?? null, structurePath: current?.structurePath ?? previous?.structurePath ?? null, - userChangelog: pair.currentChangelog ? bound(pair.currentChangelog.replace(/\s+/g, " ").trim(), MAX_METADATA_TEXT) : null } }; - }); + const condensed = condenseDocumentChanges([{ pair, alignments }]); + return condensed.selectedGroups.map(group => buildDocumentChangeGroupEvidence(pair, group)); } diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index 1b62b4e9e..5d535ab21 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -16,8 +16,10 @@ import { } from "@launchstack/core/db/schema"; import { alignVersionChunks, - buildDocumentChangeEvidence, + buildDocumentChangeGroupEvidence, + condenseDocumentChanges, selectVersionPairsForReportingPeriod, + type DocumentChangePairInput, type DocumentVersionForComparison, type VersionChunk, } from "./document-change"; @@ -177,7 +179,7 @@ export class FounderWeeklyReviewEvidenceService { const store = this.documentChangeSource.store; const versions = await store.listVersionsBeforePeriodEnd(companyId, endExclusive); const pairs = selectVersionPairsForReportingPeriod(versions, startInclusive, endExclusive); - const items: FounderWeeklyReviewEvidenceItem[] = []; + const pairInputs: DocumentChangePairInput[] = []; const warnings: FounderWeeklyReviewEvidenceWarning[] = []; for (const pair of pairs) { const [previous, current] = await Promise.all([ @@ -191,8 +193,19 @@ export class FounderWeeklyReviewEvidenceService { if (previous.state === "partial" || current.state === "partial") { warnings.push(warning("document_change_chunks_partial", "A version pair was compared with partial chunk provenance.", "document_change")); } - items.push(...buildDocumentChangeEvidence(pair, alignVersionChunks(previous.chunks, current.chunks))); + pairInputs.push({ pair, alignments: alignVersionChunks(previous.chunks, current.chunks) }); } + const condensed = condenseDocumentChanges(pairInputs); + const pairByKey = new Map(pairInputs.map(({ pair }) => [ + `${pair.documentId}:${pair.previousVersionId}:${pair.currentVersionId}`, + pair, + ])); + const items = condensed.selectedGroups.map((group) => { + const pair = pairByKey.get(`${group.documentId}:${group.previousVersionId}:${group.currentVersionId}`); + if (!pair) throw new Error("Selected document-change group has no source version pair."); + return buildDocumentChangeGroupEvidence(pair, group); + }); + warnings.push(...condensed.warnings.map((item) => warning(item.code, item.message, "document_change"))); const pairedCurrentVersionIds = new Set(pairs.map((pair) => pair.currentVersionId)); const inPeriodWithNoPair = versions.some((version) => version.createdAt >= startInclusive From f169035404ac60429150a534c97bad89c806a3c9 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Fri, 7 Aug 2026 13:56:29 +0800 Subject: [PATCH 20/29] feat(founder-weekly-review): add condensed material change evidence --- .../async-collection.test.ts | 24 + .../document-change-materiality.test.ts | 334 ++++++++++++ .../document-change.test.ts | 2 +- .../evidence-assembly.test.ts | 7 + .../workspace-document-store.test.ts | 8 + ...run-founder-weekly-review-realistic-e2e.ts | 5 +- .../src/founder-weekly-review/README.md | 2 +- .../src/founder-weekly-review/contracts.ts | 117 +++- .../document-change-materiality.ts | 502 ++++++++++++++++++ .../founder-weekly-review/evidence-digest.ts | 18 + .../founder-weekly-review/evidence-service.ts | 53 +- .../generation-evidence-envelope.ts | 25 +- .../src/founder-weekly-review/generator.ts | 8 +- .../src/founder-weekly-review/index.ts | 2 + .../src/founder-weekly-review/repository.ts | 17 +- 15 files changed, 1087 insertions(+), 37 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts create mode 100644 packages/features/src/founder-weekly-review/document-change-materiality.ts create mode 100644 packages/features/src/founder-weekly-review/evidence-digest.ts diff --git a/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts b/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts index 455c4c8ce..3bfd0b666 100644 --- a/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts @@ -6,6 +6,7 @@ import { createFounderWeeklyReviewTestDatabase } from "./testDb"; const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }); +const snapshotV2 = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ ...snapshot, schemaVersion: "founder-weekly-review-evidence/v2", documentChangeAudit: { schemaVersion: "document-change-audit/v1", rawChanges: [], groups: [] } }); describeDb("Founder Weekly Review async evidence workflow", () => { it("creates without a snapshot, attaches it once, and then permits generation claim", async () => { @@ -46,4 +47,27 @@ describeDb("Founder Weekly Review async evidence workflow", () => { await expect(worker.claimEvidenceCollection({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: retried.dispatch.generationClaimId })).resolves.toMatchObject({ status: "collecting" }); } finally { await test.close(); } }); + + it("reuses an attached v2 audit snapshot unchanged on generation retry", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [companyRow] = await test.db.insert(company).values({ name: "V2 Retry", numberOfEmployees: "1" }).returning(); + const actor = { externalUserId: "u", internalUserId: 1n, companyId: BigInt(companyRow!.id), role: "owner" }; + const service = createFounderWeeklyReviewDispatchService(test.db); + const created = await service.createRunWithDispatch({ actor, requestKey: "v2-retry", reportingPeriod: snapshotV2.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", actorExternalUserId: "u" } }); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(test.db)); + const collection = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: "v2-collection" }; + await worker.claimEvidenceCollection(collection); + const attached = await worker.attachEvidenceSnapshotIfAbsent(collection, snapshotV2); + const generation = { companyId: actor.companyId, runId: created.run.id, generationClaimId: "v2-generation", generationJobId: "v2-job" }; + await worker.claimQueuedRun(generation); + await worker.markGenerationFailed(generation, { errorCode: "provider_unavailable" }); + const retried = await service.retryRunWithDispatch({ actor, runId: created.run.id, requestKey: "v2-generation-retry" }); + + expect(attached.evidenceSnapshot).toEqual(snapshotV2); + expect(retried.run.evidenceSnapshot).toEqual(snapshotV2); + expect(retried.run.evidenceSchemaVersion).toBe("founder-weekly-review-evidence/v2"); + expect(retried.run.status).toBe("queued"); + } finally { await test.close(); } + }); }); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts new file mode 100644 index 000000000..fef33b4bf --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts @@ -0,0 +1,334 @@ +import { + FounderWeeklyReviewEvidenceSnapshotSchema, + analyzeDocumentChangeGroup, + buildCondensedDocumentChangeEvidence, + buildFounderWeeklyReviewEvidenceDigest, + buildFounderWeeklyReviewPrompt, + buildGenerationEvidenceEnvelope, + buildRawDocumentChanges, + generateFounderWeeklyReview, + groupRawDocumentChanges, + materializeDocumentChanges, + resolveDocumentChangeEvidenceAudit, + selectMaterialDocumentChangeGroups, + type AnalyzedDocumentChangeGroup, + type ChunkAlignment, + type DocumentChangeCategory, + type VersionChunk, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +const pair = (documentId = 1n, previousVersionId = 1, currentVersionId = 2, currentDate = "2026-02-02T00:00:00.000Z"): VersionPair => ({ + documentId, + documentTitle: `Document ${documentId}`, + documentCategory: "Product", + previousVersionId, + previousVersionNumber: previousVersionId, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), + currentVersionId, + currentVersionNumber: currentVersionId, + currentCreatedAt: new Date(currentDate), + currentChangelog: null, +}); + +const chunk = (id: number, versionId: bigint, content: string, overrides: Partial = {}): VersionChunk => ({ + chunkId: id, + versionId, + documentId: 1n, + content, + contentHash: null, + structureId: BigInt(id), + structurePath: `/section-${id}`, + structureTitle: `Section ${id}`, + structureOrdering: id, + pageNumber: id, + lineStart: id * 10, + lineEnd: id * 10 + 5, + ...overrides, +}); + +const modified = (id: number, before: string, after: string, overrides: Partial = {}): ChunkAlignment => ({ + changeType: "modified", + previousChunk: chunk(id, 1n, before, overrides), + currentChunk: chunk(10_000 + id, 2n, after, overrides), + alignmentMethod: "structure_path", +}); + +function groupFor(before: string, after: string, id = 1, versionPair = pair()) { + const raw = buildRawDocumentChanges(versionPair, [modified(id, before, after, { documentId: versionPair.documentId })]).rawChanges; + return groupRawDocumentChanges(versionPair, raw).groups[0]!; +} + +function analyzed(category: DocumentChangeCategory, documentId: bigint, index: number, currentDate = "2026-02-02T00:00:00.000Z"): AnalyzedDocumentChangeGroup { + const versionPair = pair(documentId, 1, 2, currentDate); + const group = groupFor(`before ${documentId}-${index}`, `after ${documentId}-${index}`, Number(documentId) * 100 + index, versionPair); + const priority = ["ownership_change", "status_change", "deadline_change", "metric_change", "requirement_change", "risk_or_blocker_change", "scope_change", "priority_change", "uncertain", "editorial_rewrite"].indexOf(category) + 1; + return { pair: versionPair, group, materiality: { category, priority, confidence: category === "uncertain" ? "uncertain" : "strong", signals: [`${category}_test`] } }; +} + +describe("deterministic document-change materiality", () => { + it.each([ + ["Product", "Platform", "ownership_change"], + ["Product owns retry telemetry", "Platform owns retry telemetry", "ownership_change"], + ["planned", "launched", "status_change"], + ["Q3", "Q4", "deadline_change"], + ["June 1", "July 15", "deadline_change"], + ["10% conversion", "25% conversion", "metric_change"], + ["$1M ARR", "$750k ARR", "metric_change"], + ["optional", "required", "requirement_change"], + ["may launch", "must launch", "requirement_change"], + ["blocked", "resolved", "risk_or_blocker_change"], + ["P2", "P0", "priority_change"], + ["US only", "global", "scope_change"], + ] as const)("classifies %s → %s as %s", (before, after, category) => { + expect(analyzeDocumentChangeGroup(groupFor(before, after))).toMatchObject({ category }); + }); + + it("uses explicit precedence when multiple signals occur", () => { + const result = analyzeDocumentChangeGroup(groupFor( + "Product owns a planned Q3 launch for 10% of customers", + "Platform owns a launched Q4 release for 25% of customers" + )); + expect(result).toMatchObject({ + category: "ownership_change", + priority: 1, + signals: expect.arrayContaining(["ownership_subject_changed", "status_term_changed", "date_or_deadline_changed", "numeric_metric_changed"]), + }); + }); + + it("keeps punctuation, capitalization, and paraphrase changes uncertain while retaining negation", () => { + expect(analyzeDocumentChangeGroup(groupFor("Launch.", "Launch!"))).toMatchObject({ category: "uncertain" }); + expect(analyzeDocumentChangeGroup(groupFor("Product roadmap", "product roadmap"))).toMatchObject({ category: "uncertain" }); + expect(analyzeDocumentChangeGroup(groupFor("Reliable export retries", "Export retries are reliable"))).toMatchObject({ category: "uncertain" }); + expect(analyzeDocumentChangeGroup(groupFor("supports exports", "does not support exports"))).toMatchObject({ + category: "requirement_change", + signals: expect.arrayContaining(["negation_changed"]), + }); + }); + + it("assigns editorial only to a narrow deterministic formatting rewrite", () => { + expect(analyzeDocumentChangeGroup(groupFor("- First item", "* First item"))).toMatchObject({ + category: "editorial_rewrite", + confidence: "moderate", + }); + }); + + it("builds one stable bounded group item with copied spans and complete audit membership", () => { + const versionPair = pair(); + const alignments: ChunkAlignment[] = [ + modified(1, "Product owns retry telemetry.", "Platform owns retry telemetry.", { structurePath: "/ownership", structureTitle: "Ownership" }), + modified(2, "Product owns alerting.", "Platform owns alerting.", { structurePath: "/ownership", structureTitle: "Ownership" }), + { changeType: "removed", previousChunk: chunk(3, 1n, "Product owns paging.", { structurePath: "/ownership", structureTitle: "Ownership" }), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(4, 2n, "Platform owns paging.", { structurePath: "/ownership", structureTitle: "Ownership" }), alignmentMethod: "unmatched" }, + ]; + const result = materializeDocumentChanges([{ pair: versionPair, alignments }]); + const item = result.items[0]!; + const auditGroup = result.audit.groups[0]!; + + expect(result.items).toHaveLength(1); + expect(item).toEqual(buildCondensedDocumentChangeEvidence(versionPair, result.selectedGroups[0]!.group, result.selectedGroups[0]!.materiality)); + expect(item.sourceId).toMatch(/^document_change:group:[a-f0-9]{64}$/); + expect(item.metadata).toMatchObject({ category: "ownership_change", materialityMethod: "deterministic", rawChangeCount: 4 }); + expect(item.excerpt).toContain("Product owns retry telemetry."); + expect(item.excerpt).toContain("Platform owns retry telemetry."); + expect(item.excerpt).toContain("Section changed across 4 source fragments."); + expect(item.excerpt.length).toBeLessThanOrEqual(1800); + expect(auditGroup.rawChangeIds).toHaveLength(4); + expect(auditGroup.evidenceSourceId).toBe(item.sourceId); + expect(result.audit.rawChanges).toHaveLength(4); + expect(new Set(auditGroup.rawChangeIds)).toEqual(new Set(result.audit.rawChanges.map(change => change.rawChangeId))); + expect(result.audit.rawChanges.find(change => change.changeType === "modified")).toEqual(expect.objectContaining({ + documentId: "1", previousVersionId: 1, currentVersionId: 2, + alignmentMethod: expect.any(String), processingVersion: "raw-document-change/v1", + previousHash: expect.stringMatching(/^[a-f0-9]{64}$/), + })); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, workspaceTimezone: "UTC", + items: result.items, sourceWarnings: [], documentChangeAudit: result.audit, + }); + const promptItem = buildGenerationEvidenceEnvelope(snapshot).items[0]!; + expect(promptItem.metadata).toMatchObject({ category: "ownership_change", materialityMethod: "deterministic", materialityConfidence: "strong", structureTitle: "Ownership", rawChangeCount: 4 }); + expect(promptItem.metadata).not.toHaveProperty("groupId"); + }); + + it("preserves concise single-change before/after semantics", () => { + const versionPair = pair(); + const result = materializeDocumentChanges([{ pair: versionPair, alignments: [modified(1, "planned", "shipped")] }]); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ sourceType: "document_change", metadata: expect.objectContaining({ category: "status_change", rawChangeCount: 1 }) }); + expect(result.items[0]!.excerpt).toContain("Before:\n- planned"); + expect(result.items[0]!.excerpt).toContain("After:\n- shipped"); + }); + + it("selects by materiality within documents while preserving document diversity and final chronology", () => { + const candidates: AnalyzedDocumentChangeGroup[] = []; + for (const documentId of [1n, 2n, 3n, 4n]) { + candidates.push(analyzed("ownership_change", documentId, 1, "2026-02-09T00:00:00.000Z")); + candidates.push(analyzed("status_change", documentId, 2, "2026-02-08T00:00:00.000Z")); + candidates.push(analyzed("deadline_change", documentId, 3, "2026-02-07T00:00:00.000Z")); + candidates.push(analyzed("metric_change", documentId, 4, "2026-02-06T00:00:00.000Z")); + candidates.push(analyzed("requirement_change", documentId, 5, "2026-02-05T00:00:00.000Z")); + candidates.push(analyzed("uncertain", documentId, 6, "2026-02-04T00:00:00.000Z")); + candidates.push(analyzed("editorial_rewrite", documentId, 7, "2026-02-03T00:00:00.000Z")); + } + const selected = selectMaterialDocumentChangeGroups([...candidates].reverse()); + const counts = new Map(); + for (const entry of selected.selectedGroups) counts.set(entry.group.documentId.toString(), (counts.get(entry.group.documentId.toString()) ?? 0) + 1); + + expect(selected.selectedGroups).toHaveLength(24); + expect([...counts.values()]).toEqual([6, 6, 6, 6]); + expect(selected.selectedGroups.some(entry => entry.materiality.category === "editorial_rewrite")).toBe(false); + expect(selected.selectedGroups.filter(entry => entry.materiality.category === "ownership_change")).toHaveLength(4); + expect(selected.selectedGroups.map(entry => entry.pair.currentCreatedAt.getTime())).toEqual( + [...selected.selectedGroups].map(entry => entry.pair.currentCreatedAt.getTime()).sort((a, b) => a - b) + ); + }); + + it("enforces eight per document and pair and prefers newer equal-category events", () => { + const candidates = Array.from({ length: 9 }, (_, index) => analyzed( + "uncertain", + 1n, + index + 1, + `2026-02-${String(index + 1).padStart(2, "0")}T00:00:00.000Z` + )); + const selected = selectMaterialDocumentChangeGroups(candidates); + expect(selected.selectedGroups).toHaveLength(8); + expect(selected.truncatedGroups).toHaveLength(1); + expect(selected.truncatedGroups[0]!.pair.currentCreatedAt.toISOString()).toBe("2026-02-01T00:00:00.000Z"); + expect(selected.warnings).toContainEqual(expect.objectContaining({ code: "document_change_budget_truncated" })); + }); + + it("condenses the 40-chunk enterprise scenario while retaining every raw audit record", () => { + const versionPair = pair(44n); + const alignments: ChunkAlignment[] = []; + for (let index = 0; index < 20; index++) { + const content = `unchanged source fragment ${index}`; + alignments.push({ + changeType: "unchanged", + previousChunk: chunk(index + 1, 1n, content, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}` }), + currentChunk: chunk(1001 + index, 2n, content, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}` }), + alignmentMethod: "content_hash", + }); + } + for (let index = 0; index < 5; index++) alignments.push(modified(100 + index, `wrapped line ${index}`, ` wrapped\r\n line\u00a0${index} `, { documentId: 44n, structurePath: `/noop-${index}` })); + alignments.push(modified(201, "Product owns retry telemetry.", "Platform owns retry telemetry.", { documentId: 44n, structurePath: "/ownership", structureTitle: "Ownership" })); + alignments.push(modified(202, "Launch deadline is Q3.", "Launch deadline is Q4.", { documentId: 44n, structurePath: "/deadline", structureTitle: "Deadline" })); + alignments.push(modified(203, "The release is planned.", "The release is launched.", { documentId: 44n, structurePath: "/status", structureTitle: "Status" })); + alignments.push(modified(204, "Retries are optional.", "Retries are required.", { documentId: 44n, structurePath: "/requirements", structureTitle: "Requirements" })); + const long = " Detailed source wording about the operating model and customer rollout remains copied for audit verification.".repeat(4); + for (let index = 0; index < 6; index++) alignments.push(modified(300 + index, `Old rewrite fragment ${index}.${long}`, `Rephrased rewrite fragment ${index}.${long}`, { documentId: 44n, structurePath: "/rewrite", structureTitle: "Large rewrite" })); + for (let index = 0; index < 5; index++) alignments.push(modified(400 + index, `- Editorial source fragment ${index}.${long}`, `* Editorial source fragment ${index}.${long}`, { documentId: 44n, structurePath: "/editorial", structureTitle: "Editorial" })); + + const result = materializeDocumentChanges([{ pair: versionPair, alignments }]); + const shuffled = materializeDocumentChanges([{ pair: versionPair, alignments: [...alignments].reverse() }]); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: result.items, + sourceWarnings: [], + documentChangeAudit: result.audit, + }); + const envelope = buildGenerationEvidenceEnvelope(snapshot); + const prompt = buildFounderWeeklyReviewPrompt(snapshot, envelope); + + expect(result.diagnostics).toMatchObject({ alignedChunkCount: 40, rawModifiedCount: 15, deterministicNoOpCount: 5, groupCount: 6, selectedGroupCount: 6, rawAuditCount: 15, condensedEvidenceCount: 6 }); + expect(result.analyzedGroups.map(group => group.materiality.category)).toEqual(expect.arrayContaining(["ownership_change", "deadline_change", "status_change", "requirement_change", "uncertain", "editorial_rewrite"])); + expect(result.audit.rawChanges).toHaveLength(15); + expect(result.items).toHaveLength(6); + expect(result.selectedGroups.map(entry => entry.group.groupId)).toEqual(shuffled.selectedGroups.map(entry => entry.group.groupId)); + expect(result.diagnostics.rawExcerptCharacters).toBeGreaterThan(result.diagnostics.condensedPromptFacingCharacters * 1.5); + expect(result.diagnostics.estimatedReductionRatio).toBeGreaterThan(1.5); + expect(result.diagnostics.rawExcerptCharacters).toBe(10_484); + expect(result.diagnostics.condensedPromptFacingCharacters).toBe(2_311); + expect(envelope.diagnostics.serializedCharacterCount).toBe(5_466); + expect(envelope.diagnostics.serializedCharacterCount).toBeLessThanOrEqual(14_000); + expect(prompt).not.toContain("rawChanges"); + expect(prompt).not.toContain("documentChangeAudit"); + }); +}); + +describe("Founder Weekly Review evidence snapshot v2", () => { + const v1 = { + schemaVersion: "founder-weekly-review-evidence/v1" as const, + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: [], + sourceWarnings: [], + }; + + it("parses v1 and v2 without mutating v1", () => { + const result = materializeDocumentChanges([{ pair: pair(), alignments: [modified(1, "planned", "launched")] }]); + const v2 = { ...v1, schemaVersion: "founder-weekly-review-evidence/v2" as const, items: result.items, documentChangeAudit: result.audit }; + expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(v1)).toEqual(v1); + expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(v2)).toEqual(v2); + expect(v1).not.toHaveProperty("documentChangeAudit"); + const parsedV2 = FounderWeeklyReviewEvidenceSnapshotSchema.parse(v2); + expect(resolveDocumentChangeEvidenceAudit(parsedV2, result.items[0]!.sourceId)?.rawChanges).toHaveLength(1); + if (parsedV2.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Expected v2 snapshot"); + const baseDigest = buildFounderWeeklyReviewEvidenceDigest(parsedV2); + const provenanceChanged = structuredClone(parsedV2); + provenanceChanged.documentChangeAudit.rawChanges[0]!.previousExcerpt = "different copied provenance"; + const materialityChanged = structuredClone(parsedV2); + materialityChanged.documentChangeAudit.groups[0]!.category = "uncertain"; + materialityChanged.documentChangeAudit.groups[0]!.priority = 9; + const evidenceChanged = structuredClone(parsedV2); + evidenceChanged.items[0]!.excerpt = "different condensed evidence"; + expect(buildFounderWeeklyReviewEvidenceDigest(provenanceChanged)).not.toBe(baseDigest); + expect(buildFounderWeeklyReviewEvidenceDigest(materialityChanged)).not.toBe(baseDigest); + expect(buildFounderWeeklyReviewEvidenceDigest(evidenceChanged)).not.toBe(baseDigest); + expect(buildFounderWeeklyReviewPrompt(provenanceChanged)).toBe(buildFounderWeeklyReviewPrompt(parsedV2)); + }); + + it("keeps audit text out of the prompt and digests the complete snapshot", () => { + const auditSecret = "AUDIT_ONLY_RAW_SOURCE_TEXT"; + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + ...v1, + schemaVersion: "founder-weekly-review-evidence/v2", + documentChangeAudit: { + schemaVersion: "document-change-audit/v1", + rawChanges: [{ + rawChangeId: `raw_document_change:${"a".repeat(64)}`, changeType: "removed", alignmentMethod: "unmatched", + documentId: "1", previousVersionId: 1, currentVersionId: 2, previousChunkId: 1, currentChunkId: null, + previousExcerpt: auditSecret, currentExcerpt: null, previousHash: "b".repeat(64), currentHash: null, + previousStructurePath: "/secret", currentStructurePath: null, previousStructureTitle: "Secret", currentStructureTitle: null, + previousPageNumber: 1, currentPageNumber: null, previousLineStart: 1, previousLineEnd: 2, currentLineStart: null, currentLineEnd: null, + processingVersion: "raw-document-change/v1", + }], + groups: [{ + groupId: `document_change_group:${"c".repeat(64)}`, evidenceSourceId: null, documentId: "1", previousVersionId: 1, currentVersionId: 2, + structurePath: "/secret", structureTitle: "Secret", splitOrdinal: 0, rawChangeIds: [`raw_document_change:${"a".repeat(64)}`], + category: "uncertain", priority: 9, confidence: "uncertain", signals: ["no_strong_deterministic_signal"], + materialityMethod: "deterministic", materialityVersion: "document-change-materiality/v1", + }], + }, + }); + const prompt = buildFounderWeeklyReviewPrompt(snapshot); + const firstDigest = buildFounderWeeklyReviewEvidenceDigest(snapshot); + const identicalDigest = buildFounderWeeklyReviewEvidenceDigest(FounderWeeklyReviewEvidenceSnapshotSchema.parse(JSON.parse(JSON.stringify(snapshot)))); + const changed = structuredClone(snapshot); + if (changed.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Expected v2 snapshot"); + changed.documentChangeAudit.groups[0]!.rawChangeIds = [`raw_document_change:${"d".repeat(64)}`]; + + expect(prompt).not.toContain(auditSecret); + expect(prompt).not.toContain("documentChangeAudit"); + expect(identicalDigest).toBe(firstDigest); + expect(buildFounderWeeklyReviewEvidenceDigest(changed)).not.toBe(firstDigest); + }); + + it("generates from v2 without a provider when prompt evidence is empty and records distinct digest metadata", async () => { + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + ...v1, + schemaVersion: "founder-weekly-review-evidence/v2", + documentChangeAudit: { schemaVersion: "document-change-audit/v1", rawChanges: [], groups: [] }, + }); + const generate = jest.fn(); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: snapshot, generate }); + expect(generate).not.toHaveBeenCalled(); + expect(result.modelMetadata.evidenceSchemaVersion).toBe("founder-weekly-review-evidence/v2"); + expect(result.modelMetadata.attributes.evidenceDigest).toBe(buildFounderWeeklyReviewEvidenceDigest(snapshot)); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change.test.ts index cd5a60c05..e7e85ac9f 100644 --- a/apps/web/__tests__/founderWeeklyReview/document-change.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/document-change.test.ts @@ -83,7 +83,7 @@ describe("Week 3 document change domain logic", () => { .mockResolvedValueOnce({ state: "complete", chunks: [chunk(20, 2n, "After", { structurePath: "/plan" })], warnings: [] }) }; const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "computed", store }); await expect(service.collectDocumentChangeEvidence(1n, new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z"))).resolves.toEqual([ - expect.objectContaining({ sourceTimestamp: "2026-02-02T00:00:00.000Z", metadata: expect.objectContaining({ previousVersionId: 1, currentVersionId: 2, changeType: "modified" }) }), + expect.objectContaining({ sourceType: "document_change", sourceTimestamp: "2026-02-02T00:00:00.000Z", sourceId: expect.stringMatching(/^document_change:group:/), metadata: expect.objectContaining({ previousVersionId: 1, currentVersionId: 2, category: "uncertain", rawChangeCount: 1 }) }), ]); }); }); diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts index 916ddbfa0..8e4f5798e 100644 --- a/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts @@ -62,6 +62,13 @@ describe("orderEvidenceItems", () => { expect(result.map((i) => i.sourceId)).toEqual(["a", "z"]); }); + it("returns selected document changes to numeric structural order", () => { + const ts = "2026-01-01T00:00:00.000Z"; + const tenth = { ...makeItem("group-10", ts), metadata: { documentId: "1", previousVersionId: 1, currentVersionId: 2, structureOrdering: 10 } }; + const second = { ...makeItem("group-2", ts), metadata: { documentId: "1", previousVersionId: 1, currentVersionId: 2, structureOrdering: 2 } }; + expect(orderEvidenceItems([tenth, second]).map(item => item.sourceId)).toEqual(["group-2", "group-10"]); + }); + it("orders source-type ties with ordinal comparison, not locale collation", () => { const ts = "2026-01-01T00:00:00.000Z"; const result = orderEvidenceItems([ diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts index 521921b25..a2a74a4a3 100644 --- a/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts @@ -66,8 +66,16 @@ describeDb("strict current workspace document store", () => { const input = { companyId: BigInt(owner!.id), reportingPeriod: { start: "2026-02-01", end: "2026-02-07" }, workspaceTimezone: "UTC", founderContext: "blocker", actor: { externalUserId: "u" }, requestKey: "computed-integration" }; const first = await service.collectFounderWeeklyReviewEvidence(input); const second = await service.collectFounderWeeklyReviewEvidence(input); + expect(first.schemaVersion).toBe("founder-weekly-review-evidence/v2"); expect(first.items.filter((item) => item.sourceType === "document_change")).toHaveLength(1); expect(first.items).toEqual(second.items); + expect(first.schemaVersion === "founder-weekly-review-evidence/v2" && first.documentChangeAudit).toEqual( + second.schemaVersion === "founder-weekly-review-evidence/v2" ? second.documentChangeAudit : undefined + ); + if (first.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Expected computed v2 evidence"); + expect(first.documentChangeAudit.rawChanges).toHaveLength(1); + expect(first.documentChangeAudit.groups).toHaveLength(1); + expect(first.documentChangeAudit.groups[0]!.evidenceSourceId).toBe(first.items.find((item) => item.sourceType === "document_change")!.sourceId); expect(first.items).toEqual(expect.arrayContaining([ expect.objectContaining({ sourceType: "document_change", metadata: expect.objectContaining({ previousVersionId: a1!.id, currentVersionId: a2!.id }) }), ])); diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts index 1b94b478c..d697ae843 100644 --- a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -37,7 +37,10 @@ function assertComputedSnapshot(snapshot: unknown, ids: Record item.metadata.previousVersionId === ids.a1 && item.metadata.currentVersionId === ids.a2); const workspaceItem = workspace.find((item) => item.metadata.documentId === String(ids.b)); if (!change || !workspaceItem || !feedback.length || founder.length !== 1) throw new Error("Computed snapshot is missing required evidence types."); - if (change.metadata.previousChunkId === null || change.metadata.currentChunkId === null || !change.excerpt.includes(texts.before) || !change.excerpt.includes(texts.after) || change.sourceTimestamp !== "2026-02-20T10:00:00.000Z" || change.excerpt.includes(texts.v3) || change.excerpt.startsWith("Version ")) throw new Error("Computed document-change assertions failed."); + if (parsed.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Computed collection did not produce evidence snapshot v2."); + const auditGroup = parsed.documentChangeAudit.groups.find((group) => group.evidenceSourceId === change.sourceId); + const auditedRaw = auditGroup?.rawChangeIds.map((rawChangeId) => parsed.documentChangeAudit.rawChanges.find((raw) => raw.rawChangeId === rawChangeId)); + if (!auditGroup || auditedRaw?.some((raw) => !raw) || !auditedRaw?.some((raw) => raw!.previousChunkId !== null && raw!.currentChunkId !== null && raw!.previousExcerpt?.includes(texts.before) && raw!.currentExcerpt?.includes(texts.after)) || change.metadata.category !== "ownership_change" || !change.excerpt.includes(texts.before) || !change.excerpt.includes(texts.after) || change.sourceTimestamp !== "2026-02-20T10:00:00.000Z" || change.excerpt.includes(texts.v3) || change.excerpt.startsWith("Version ")) throw new Error("Computed document-change assertions failed."); if (workspaceItem.metadata.documentVersionId !== String(ids.b2) || workspaceItem.metadata.chunkId !== ids.bCurrentChunk || workspaceItem.metadata.retrievalReason !== "founder_context_relevance" || typeof workspaceItem.metadata.similarityScore !== "number" || "sourceTimestamp" in workspaceItem) throw new Error("Strict workspace-document assertions failed."); if (items.some((item) => [texts.v3, texts.bHistorical, texts.nullVersion, texts.foreign, texts.unrelated].some((value) => item.excerpt.includes(value)))) throw new Error("Computed control evidence leaked into the snapshot."); if (new Set(items.map((item) => item.sourceId)).size !== items.length) throw new Error("Computed snapshot contains duplicate source IDs."); diff --git a/packages/features/src/founder-weekly-review/README.md b/packages/features/src/founder-weekly-review/README.md index b7d24b66c..8af843d60 100644 --- a/packages/features/src/founder-weekly-review/README.md +++ b/packages/features/src/founder-weekly-review/README.md @@ -56,7 +56,7 @@ Every repository method accepts `companyId` explicitly and includes it in SQL pr ## Contract versions -- Evidence snapshot: `founder-weekly-review-evidence/v1` +- Evidence snapshots: existing `founder-weekly-review-evidence/v1` and current `founder-weekly-review-evidence/v2` with deterministic document-change audit provenance - Review payload: `founder-weekly-review/v1` ## Evidence collection (LAU-6) diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts index d32626b7b..bf90d4f67 100644 --- a/packages/features/src/founder-weekly-review/contracts.ts +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -1,7 +1,12 @@ import { z } from "zod"; -export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION = +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_V1_SCHEMA_VERSION = "founder-weekly-review-evidence/v1" as const; +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION = + "founder-weekly-review-evidence/v2" as const; +/** Current version used for newly collected evidence. */ +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION = + FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION; export const FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION = "founder-weekly-review/v1" as const; export const FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION = @@ -86,17 +91,121 @@ export type FounderWeeklyReviewEvidenceWarning = z.infer< typeof FounderWeeklyReviewEvidenceWarningSchema >; -export const FounderWeeklyReviewEvidenceSnapshotSchema = z.object({ - schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION), +const FounderWeeklyReviewEvidenceSnapshotBaseSchema = z.object({ capturedAt: z.string().datetime({ offset: true }), reportingPeriod: ReportingPeriodSchema, workspaceTimezone: z.string().min(1).max(128), items: z.array(FounderWeeklyReviewEvidenceItemSchema).max(500), sourceWarnings: z.array(FounderWeeklyReviewEvidenceWarningSchema).max(100).default([]), }); + +export const DocumentChangeCategorySchema = z.enum([ + "ownership_change", + "status_change", + "deadline_change", + "metric_change", + "requirement_change", + "risk_or_blocker_change", + "scope_change", + "priority_change", + "editorial_rewrite", + "uncertain", +]); +export const DeterministicMaterialityConfidenceSchema = z.enum(["strong", "moderate", "uncertain"]); + +export const RawDocumentChangeSnapshotSchema = z.object({ + rawChangeId: z.string().min(1).max(128), + changeType: z.enum(["added", "removed", "modified"]), + alignmentMethod: z.enum(["content_hash", "structure_path", "section_title", "structural_position", "text_similarity", "unmatched"]), + similarityScore: z.number().finite().optional(), + documentId: z.string().regex(/^\d+$/).max(64), + previousVersionId: z.number().int().nonnegative(), + currentVersionId: z.number().int().nonnegative(), + previousChunkId: z.number().int().nonnegative().nullable(), + currentChunkId: z.number().int().nonnegative().nullable(), + previousExcerpt: z.string().max(600).nullable(), + currentExcerpt: z.string().max(600).nullable(), + previousHash: z.string().regex(/^[a-f0-9]{64}$/).nullable(), + currentHash: z.string().regex(/^[a-f0-9]{64}$/).nullable(), + previousStructurePath: z.string().max(512).nullable(), + currentStructurePath: z.string().max(512).nullable(), + previousStructureTitle: z.string().max(512).nullable(), + currentStructureTitle: z.string().max(512).nullable(), + previousPageNumber: z.number().int().nullable(), + currentPageNumber: z.number().int().nullable(), + previousLineStart: z.number().int().nullable(), + previousLineEnd: z.number().int().nullable(), + currentLineStart: z.number().int().nullable(), + currentLineEnd: z.number().int().nullable(), + processingVersion: z.string().min(1).max(128), +}).strict(); +export type RawDocumentChangeSnapshot = z.infer; + +export const DocumentChangeGroupSnapshotSchema = z.object({ + groupId: z.string().min(1).max(128), + evidenceSourceId: z.string().min(1).max(256).nullable(), + documentId: z.string().regex(/^\d+$/).max(64), + previousVersionId: z.number().int().nonnegative(), + currentVersionId: z.number().int().nonnegative(), + structurePath: z.string().max(512).nullable(), + structureTitle: z.string().max(512).nullable(), + splitOrdinal: z.number().int().nonnegative(), + rawChangeIds: z.array(z.string().min(1).max(128)).min(1).max(16), + category: DocumentChangeCategorySchema, + priority: z.number().int().min(1).max(10), + confidence: DeterministicMaterialityConfidenceSchema, + signals: z.array(z.string().min(1).max(64)).max(20), + materialityMethod: z.literal("deterministic"), + materialityVersion: z.string().min(1).max(128), +}).strict(); +export type DocumentChangeGroupSnapshot = z.infer; + +export const DocumentChangeAuditSnapshotSchema = z.object({ + schemaVersion: z.literal("document-change-audit/v1"), + rawChanges: z.array(RawDocumentChangeSnapshotSchema).max(5000), + groups: z.array(DocumentChangeGroupSnapshotSchema).max(2500), +}).strict(); +export type DocumentChangeAuditSnapshot = z.infer; + +export const FounderWeeklyReviewEvidenceSnapshotV1Schema = FounderWeeklyReviewEvidenceSnapshotBaseSchema.extend({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_EVIDENCE_V1_SCHEMA_VERSION), +}).strict(); +export const FounderWeeklyReviewEvidenceSnapshotV2Schema = FounderWeeklyReviewEvidenceSnapshotBaseSchema.extend({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION), + documentChangeAudit: DocumentChangeAuditSnapshotSchema, +}).strict(); +export const FounderWeeklyReviewEvidenceSnapshotSchema = z.discriminatedUnion("schemaVersion", [ + FounderWeeklyReviewEvidenceSnapshotV1Schema, + FounderWeeklyReviewEvidenceSnapshotV2Schema, +]).superRefine((snapshot, context) => { + if (snapshot.schemaVersion !== FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION) return; + const rawIds = new Set(snapshot.documentChangeAudit.rawChanges.map(change => change.rawChangeId)); + const itemById = new Map(snapshot.items.map(item => [item.sourceId, item])); + const seenGroups = new Set(); + if (rawIds.size !== snapshot.documentChangeAudit.rawChanges.length) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Document-change audit raw IDs must be unique.", path: ["documentChangeAudit", "rawChanges"] }); + } + for (const [index, group] of snapshot.documentChangeAudit.groups.entries()) { + if (seenGroups.has(group.groupId)) context.addIssue({ code: z.ZodIssueCode.custom, message: "Document-change audit group IDs must be unique.", path: ["documentChangeAudit", "groups", index, "groupId"] }); + seenGroups.add(group.groupId); + if (group.rawChangeIds.some(rawChangeId => !rawIds.has(rawChangeId))) context.addIssue({ code: z.ZodIssueCode.custom, message: "Document-change audit group references an unknown raw change.", path: ["documentChangeAudit", "groups", index, "rawChangeIds"] }); + if (group.evidenceSourceId) { + const item = itemById.get(group.evidenceSourceId); + if (!item || item.sourceType !== "document_change" || item.metadata.groupId !== group.groupId) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Condensed document-change evidence must resolve to its audit group.", path: ["documentChangeAudit", "groups", index, "evidenceSourceId"] }); + } + } + } + for (const [index, item] of snapshot.items.entries()) { + if (item.sourceType === "document_change" && item.sourceId.startsWith("document_change:group:") && !snapshot.documentChangeAudit.groups.some(group => group.evidenceSourceId === item.sourceId)) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Condensed document-change evidence has no audit group.", path: ["items", index, "sourceId"] }); + } + } +}); export type FounderWeeklyReviewEvidenceSnapshot = z.infer< typeof FounderWeeklyReviewEvidenceSnapshotSchema >; +export type FounderWeeklyReviewEvidenceSchemaVersion = FounderWeeklyReviewEvidenceSnapshot["schemaVersion"]; export const FounderWeeklyReviewConfidenceSchema = z.enum(["high", "medium", "low"]); export type FounderWeeklyReviewConfidence = z.infer< @@ -263,7 +372,7 @@ export interface FounderWeeklyReviewRunRecord { reviewPayload: FounderWeeklyReviewPayload | null; reviewSchemaVersion: FounderWeeklyReviewPayloadSchemaVersion; evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | null; - evidenceSchemaVersion: typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION; + evidenceSchemaVersion: FounderWeeklyReviewEvidenceSchemaVersion; collectionInput?: FounderWeeklyReviewCollectionInput; collectionClaimId?: string | null; collectionStartedAt?: Date | null; diff --git a/packages/features/src/founder-weekly-review/document-change-materiality.ts b/packages/features/src/founder-weekly-review/document-change-materiality.ts new file mode 100644 index 000000000..663cb9990 --- /dev/null +++ b/packages/features/src/founder-weekly-review/document-change-materiality.ts @@ -0,0 +1,502 @@ +import { createHash } from "node:crypto"; + +import type { + DocumentChangeAuditSnapshot, + FounderWeeklyReviewEvidenceItem, + FounderWeeklyReviewEvidenceSnapshot, + RawDocumentChangeSnapshot, +} from "./contracts"; +import { + DOCUMENT_CHANGE_GROUP_BUDGET, + RAW_DOCUMENT_CHANGE_VERSION, + buildRawDocumentChanges, + groupRawDocumentChanges, + type DocumentChangeGroup, + type DocumentChangePairInput, + type DocumentChangeProcessingWarning, + type RawDocumentChange, + type VersionChunk, + type VersionPair, +} from "./document-change"; + +export const DOCUMENT_CHANGE_MATERIALITY_VERSION = "document-change-materiality/v1" as const; +export const DOCUMENT_CHANGE_AUDIT_VERSION = "document-change-audit/v1" as const; + +export const DOCUMENT_CHANGE_CATEGORIES = [ + "ownership_change", + "status_change", + "deadline_change", + "metric_change", + "requirement_change", + "risk_or_blocker_change", + "scope_change", + "priority_change", + "uncertain", + "editorial_rewrite", +] as const; + +export type DocumentChangeCategory = typeof DOCUMENT_CHANGE_CATEGORIES[number]; +export type DeterministicMaterialityConfidence = "strong" | "moderate" | "uncertain"; + +export type DeterministicMaterialityResult = { + category: DocumentChangeCategory; + priority: number; + confidence: DeterministicMaterialityConfidence; + signals: readonly string[]; +}; + +export type AnalyzedDocumentChangeGroup = { + pair: VersionPair; + group: DocumentChangeGroup; + materiality: DeterministicMaterialityResult; +}; + +export type DeterministicMaterialChangeDiagnostics = { + versionPairCount: number; + alignedChunkCount: number; + rawModifiedCount: number; + rawAddedCount: number; + rawRemovedCount: number; + deterministicNoOpCount: number; + groupCount: number; + oversizedGroupSplitCount: number; + selectedGroupCount: number; + truncatedGroupCount: number; + approximateChangedCharacters: number; + groupsByMaterialityCategory: Record; + groupsByDeterministicConfidence: Record; + selectedGroupsByCategory: Record; + truncatedGroupsByCategory: Record; + rawAuditCount: number; + condensedEvidenceCount: number; + rawExcerptCharacters: number; + condensedPromptFacingCharacters: number; + estimatedReductionRatio: number; +}; + +export type DeterministicMaterialChangeResult = { + rawChanges: readonly RawDocumentChange[]; + analyzedGroups: readonly AnalyzedDocumentChangeGroup[]; + selectedGroups: readonly AnalyzedDocumentChangeGroup[]; + items: readonly FounderWeeklyReviewEvidenceItem[]; + audit: DocumentChangeAuditSnapshot; + warnings: readonly DocumentChangeProcessingWarning[]; + diagnostics: DeterministicMaterialChangeDiagnostics; +}; + +const CATEGORY_PRIORITY: Record = Object.fromEntries( + DOCUMENT_CHANGE_CATEGORIES.map((category, index) => [category, index + 1]) +) as Record; + +const CATEGORY_LABEL: Record = { + ownership_change: "Ownership changed.", + status_change: "Status changed.", + deadline_change: "Deadline or date changed.", + metric_change: "Metric changed.", + requirement_change: "Requirement changed.", + risk_or_blocker_change: "Risk or blocker changed.", + scope_change: "Scope changed.", + priority_change: "Priority changed.", + uncertain: "Section changed.", + editorial_rewrite: "Editorial formatting changed.", +}; + +const TEAM_TERMS = new Set(["product", "platform", "marketing", "sales", "engineering", "operations", "finance", "legal", "support"]); +const STATUS_TERMS = ["planned", "launched", "shipped", "in progress", "active", "cancelled", "canceled", "completed", "paused"]; +const REQUIREMENT_TERMS = ["may", "must", "should", "optional", "required", "recommended", "mandatory"]; +const RISK_TERMS = ["risk", "risks", "blocker", "blockers", "blocked", "resolved", "unblocked", "at risk"]; +const PRIORITY_TERMS = ["p0", "p1", "p2", "p3", "deferred", "immediate", "low priority", "high priority", "critical"]; +const SCOPE_TERMS = ["us only", "global", "pilot customers", "all enterprise customers", "one team", "company-wide", "company wide"]; + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function compareBigInt(a: bigint, b: bigint): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function bound(value: string, maximum: number): string { + return value.length <= maximum ? value : `${value.slice(0, maximum - 1)}…`; +} + +function copiedText(value: string, maximum = 480): string { + return bound(value.replace(/\s+/g, " ").trim(), maximum); +} + +function valuesMatching(value: string, expressions: readonly RegExp[]): string[] { + const values = expressions.flatMap(expression => [...value.matchAll(expression)].map(match => (match[1] ?? match[0]).toLocaleLowerCase())); + return [...new Set(values)].sort(compareOrdinal); +} + +function phraseValues(value: string, phrases: readonly string[]): string[] { + const lower = value.toLocaleLowerCase(); + return phrases.filter(phrase => new RegExp(`\\b${phrase.replace(/\s+/g, "\\s+")}\\b`, "i").test(lower)); +} + +function changedValues(before: string, after: string, extract: (value: string) => readonly string[]): boolean { + const previous = [...extract(before)].sort(compareOrdinal); + const current = [...extract(after)].sort(compareOrdinal); + return (previous.length > 0 || current.length > 0) && JSON.stringify(previous) !== JSON.stringify(current); +} + +function ownershipValues(value: string): string[] { + const explicit = valuesMatching(value, [ + /\bowner\s*:\s*([\p{L}\p{N}_ -]{1,80})/giu, + /\b([\p{L}\p{N}_-]+(?:\s+[\p{L}\p{N}_-]+){0,3})\s+owns\b/giu, + /\bowned\s+by\s+([\p{L}\p{N}_-]+(?:\s+[\p{L}\p{N}_-]+){0,3})/giu, + ]); + const trimmed = value.trim().toLocaleLowerCase(); + if (TEAM_TERMS.has(trimmed)) explicit.push(trimmed); + return [...new Set(explicit)].sort(compareOrdinal); +} + +function deadlineValues(value: string): string[] { + return valuesMatching(value, [ + /\b(q[1-4])\b/giu, + /\b(20\d{2})\b/gu, + /\b((?:january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2}(?:,\s*20\d{2})?)\b/giu, + /\b(20\d{2}-\d{2}-\d{2})\b/gu, + ]); +} + +function metricValues(value: string): string[] { + return valuesMatching(value, [ + /([$€£]\s*\d+(?:\.\d+)?\s*[kmb]?)/giu, + /(\d+(?:\.\d+)?\s*%)/gu, + /\b(\d+(?:\.\d+)?\s*(?:customers?|users?|accounts?|employees?|days?|weeks?|months?|revenue|arr|mrr))\b/giu, + ]); +} + +function punctuationOnlyEditorial(before: string, after: string): boolean { + const stripBullet = (value: string) => value.replace(/^\s*[-*+]\s+/gm, "").replace(/\s+/g, " ").trim(); + return before !== after && stripBullet(before) === stripBullet(after); +} + +function addSignal( + candidates: Map>, + category: DocumentChangeCategory, + signal: string +): void { + const signals = candidates.get(category) ?? new Set(); + signals.add(signal); candidates.set(category, signals); +} + +/** Conservative token-pattern nomination; absence of a signal remains uncertain. */ +export function analyzeDocumentChangeGroup(group: DocumentChangeGroup): DeterministicMaterialityResult { + const candidates = new Map>(); + let editorialOnly = group.rawChanges.length > 0; + for (const change of group.rawChanges) { + const before = change.previousNormalizedContent ?? ""; + const after = change.currentNormalizedContent ?? ""; + if (changedValues(before, after, ownershipValues)) addSignal(candidates, "ownership_change", "ownership_subject_changed"); + if (changedValues(before, after, value => phraseValues(value, STATUS_TERMS))) addSignal(candidates, "status_change", "status_term_changed"); + if (changedValues(before, after, deadlineValues)) addSignal(candidates, "deadline_change", "date_or_deadline_changed"); + if (changedValues(before, after, metricValues)) addSignal(candidates, "metric_change", "numeric_metric_changed"); + if (changedValues(before, after, value => phraseValues(value, REQUIREMENT_TERMS))) addSignal(candidates, "requirement_change", "requirement_or_modality_changed"); + if (changedValues(before, after, value => phraseValues(value, RISK_TERMS))) addSignal(candidates, "risk_or_blocker_change", "risk_or_blocker_term_changed"); + if (changedValues(before, after, value => phraseValues(value, SCOPE_TERMS))) addSignal(candidates, "scope_change", "scope_marker_changed"); + if (changedValues(before, after, value => phraseValues(value, PRIORITY_TERMS))) addSignal(candidates, "priority_change", "priority_marker_changed"); + if (changedValues(before, after, value => phraseValues(value, ["not", "no", "never", "unavailable"]))) { + addSignal(candidates, "requirement_change", "negation_changed"); + } + editorialOnly = editorialOnly && change.changeType === "modified" && punctuationOnlyEditorial(before, after); + } + const category = [...candidates.keys()].sort((a, b) => CATEGORY_PRIORITY[a] - CATEGORY_PRIORITY[b])[0] + ?? (editorialOnly ? "editorial_rewrite" : "uncertain"); + const signals = candidates.size > 0 + ? [...candidates.entries()].sort(([a], [b]) => CATEGORY_PRIORITY[a] - CATEGORY_PRIORITY[b]) + .flatMap(([, categorySignals]) => [...categorySignals].sort(compareOrdinal)) + : [category === "editorial_rewrite" ? "markdown_bullet_marker_changed" : "no_strong_deterministic_signal"]; + const confidence: DeterministicMaterialityConfidence = category === "uncertain" + ? "uncertain" + : category === "editorial_rewrite" || signals.includes("ownership_subject_changed") && group.rawChanges.every(change => { + const before = change.previousNormalizedContent?.trim().toLocaleLowerCase() ?? ""; + const after = change.currentNormalizedContent?.trim().toLocaleLowerCase() ?? ""; + return TEAM_TERMS.has(before) && TEAM_TERMS.has(after); + }) ? "moderate" : "strong"; + return { category, priority: CATEGORY_PRIORITY[category], confidence, signals }; +} + +function confidenceRank(value: DeterministicMaterialityConfidence): number { + return value === "strong" ? 0 : value === "moderate" ? 1 : 2; +} + +function structuralKey(group: DocumentChangeGroup): string { + const first = group.rawChanges[0]!; + const chunk = first.currentChunk ?? first.previousChunk!; + return [chunk.structureOrdering ?? "", chunk.pageNumber ?? "", chunk.lineStart ?? "", group.structurePath ?? "", group.groupId].join(":"); +} + +function compareSelectionPriority(a: AnalyzedDocumentChangeGroup, b: AnalyzedDocumentChangeGroup): number { + return a.materiality.priority - b.materiality.priority + || confidenceRank(a.materiality.confidence) - confidenceRank(b.materiality.confidence) + || b.pair.currentCreatedAt.getTime() - a.pair.currentCreatedAt.getTime() + || a.group.splitOrdinal - b.group.splitOrdinal + || compareOrdinal(structuralKey(a.group), structuralKey(b.group)); +} + +function compareFinalOrder(a: AnalyzedDocumentChangeGroup, b: AnalyzedDocumentChangeGroup): number { + return a.pair.currentCreatedAt.getTime() - b.pair.currentCreatedAt.getTime() + || compareBigInt(a.group.documentId, b.group.documentId) + || a.group.previousVersionId - b.group.previousVersionId + || a.group.currentVersionId - b.group.currentVersionId + || compareOrdinal(structuralKey(a.group), structuralKey(b.group)); +} + +export function selectMaterialDocumentChangeGroups(input: readonly AnalyzedDocumentChangeGroup[]): { + selectedGroups: AnalyzedDocumentChangeGroup[]; + truncatedGroups: AnalyzedDocumentChangeGroup[]; + warnings: DocumentChangeProcessingWarning[]; +} { + const byDocument = new Map(); + for (const analyzed of input) { + const key = analyzed.group.documentId.toString(); + const groups = byDocument.get(key) ?? []; + groups.push(analyzed); byDocument.set(key, groups); + } + const documents = [...byDocument.entries()].sort(([a], [b]) => compareBigInt(BigInt(a), BigInt(b))); + for (const [, groups] of documents) groups.sort(compareSelectionPriority); + const selectedGroups: AnalyzedDocumentChangeGroup[] = []; + const selectedIds = new Set(); + const documentCounts = new Map(); + const pairCounts = new Map(); + let madeProgress = true; + while (selectedGroups.length < DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview && madeProgress) { + madeProgress = false; + for (const [documentId, groups] of documents) { + if ((documentCounts.get(documentId) ?? 0) >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerDocument) continue; + const candidate = groups.find(analyzed => { + if (selectedIds.has(analyzed.group.groupId)) return false; + const key = `${documentId}:${analyzed.group.previousVersionId}:${analyzed.group.currentVersionId}`; + return (pairCounts.get(key) ?? 0) < DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerVersionPair; + }); + if (!candidate) continue; + const pair = `${documentId}:${candidate.group.previousVersionId}:${candidate.group.currentVersionId}`; + selectedGroups.push(candidate); selectedIds.add(candidate.group.groupId); + documentCounts.set(documentId, (documentCounts.get(documentId) ?? 0) + 1); + pairCounts.set(pair, (pairCounts.get(pair) ?? 0) + 1); + madeProgress = true; + if (selectedGroups.length >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview) break; + } + } + const truncatedGroups = input.filter(analyzed => !selectedIds.has(analyzed.group.groupId)).sort(compareFinalOrder); + return { + selectedGroups: selectedGroups.sort(compareFinalOrder), + truncatedGroups, + warnings: truncatedGroups.length > 0 ? [{ + code: "document_change_budget_truncated", + message: "Document-change groups were truncated to deterministic materiality-aware structural limits.", + }] : [], + }; +} + +export function documentChangeGroupSourceId(group: DocumentChangeGroup): string { + return `document_change:group:${group.groupId.replace(/^document_change_group:/, "")}`; +} + +/** Resolves a condensed citation through the frozen v2 group to every raw source record. */ +export function resolveDocumentChangeEvidenceAudit( + snapshot: FounderWeeklyReviewEvidenceSnapshot, + sourceId: string +): { group: DocumentChangeAuditSnapshot["groups"][number]; rawChanges: RawDocumentChangeSnapshot[] } | null { + if (snapshot.schemaVersion !== "founder-weekly-review-evidence/v2") return null; + const group = snapshot.documentChangeAudit.groups.find(candidate => candidate.evidenceSourceId === sourceId); + if (!group) return null; + const byId = new Map(snapshot.documentChangeAudit.rawChanges.map(change => [change.rawChangeId, change])); + const rawChanges = group.rawChangeIds.map(rawChangeId => byId.get(rawChangeId)); + return rawChanges.some(change => !change) ? null : { group, rawChanges: rawChanges as RawDocumentChangeSnapshot[] }; +} + +function copiedList(changes: readonly RawDocumentChange[], side: "previousChunk" | "currentChunk"): string | null { + const values = changes.map(change => change[side]?.content).filter((value): value is string => Boolean(value?.trim())); + if (values.length === 0) return null; + let result = ""; + for (const value of values) { + const line = `- ${copiedText(value, 220)}`; + if (`${result}${result ? "\n" : ""}${line}`.length > 480) break; + result += `${result ? "\n" : ""}${line}`; + } + return result || `- ${copiedText(values[0]!, 476)}`; +} + +/** One bounded group-level item containing only deterministic labels and copied source spans. */ +export function buildCondensedDocumentChangeEvidence( + pair: VersionPair, + group: DocumentChangeGroup, + materiality: DeterministicMaterialityResult +): FounderWeeklyReviewEvidenceItem { + const before = copiedList(group.rawChanges, "previousChunk"); + const after = copiedList(group.rawChanges, "currentChunk"); + const fragmentLabel = group.rawChanges.length === 1 ? "" : ` Section changed across ${group.rawChanges.length} source fragments.`; + const sections = [ + `${CATEGORY_LABEL[materiality.category]}${fragmentLabel}`, + ...(before ? [`Before:\n${before}`] : []), + ...(after ? [`After:\n${after}`] : []), + ]; + const section = group.structureTitle ?? group.structurePath; + const firstChunk = group.rawChanges[0]!.currentChunk ?? group.rawChanges[0]!.previousChunk!; + return { + sourceType: "document_change", + sourceId: documentChangeGroupSourceId(group), + title: bound(section ? `${pair.documentTitle} — ${section}` : pair.documentTitle, 512), + sourceTimestamp: pair.currentCreatedAt.toISOString(), + excerpt: bound(sections.join("\n\n"), 1800), + workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, + metadata: { + documentId: pair.documentId.toString(), + groupId: group.groupId, + category: materiality.category, + materialityMethod: "deterministic", + materialityConfidence: materiality.confidence, + materialityVersion: DOCUMENT_CHANGE_MATERIALITY_VERSION, + previousVersionId: pair.previousVersionId, + currentVersionId: pair.currentVersionId, + previousVersionNumber: pair.previousVersionNumber, + currentVersionNumber: pair.currentVersionNumber, + structurePath: group.structurePath ?? null, + structureTitle: group.structureTitle ?? null, + structureOrdering: firstChunk.structureOrdering, + pageNumber: firstChunk.pageNumber, + lineStart: firstChunk.lineStart, + rawChangeCount: group.rawChanges.length, + }, + }; +} + +function hashContent(chunk: VersionChunk | undefined): string | null { + return chunk ? createHash("sha256").update(chunk.content, "utf8").digest("hex") : null; +} + +function rawSnapshot(pair: VersionPair, change: RawDocumentChange): RawDocumentChangeSnapshot { + const previous = change.previousChunk; const current = change.currentChunk; + return { + rawChangeId: change.rawChangeId, + changeType: change.changeType, + alignmentMethod: change.alignmentMethod, + ...(change.similarityScore === undefined ? {} : { similarityScore: change.similarityScore }), + documentId: pair.documentId.toString(), + previousVersionId: pair.previousVersionId, + currentVersionId: pair.currentVersionId, + previousChunkId: previous?.chunkId ?? null, + currentChunkId: current?.chunkId ?? null, + previousExcerpt: previous ? bound(previous.content, 600) : null, + currentExcerpt: current ? bound(current.content, 600) : null, + previousHash: hashContent(previous), + currentHash: hashContent(current), + previousStructurePath: previous?.structurePath ? bound(previous.structurePath, 512) : null, + currentStructurePath: current?.structurePath ? bound(current.structurePath, 512) : null, + previousStructureTitle: previous?.structureTitle ? bound(previous.structureTitle, 512) : null, + currentStructureTitle: current?.structureTitle ? bound(current.structureTitle, 512) : null, + previousPageNumber: previous?.pageNumber ?? null, + currentPageNumber: current?.pageNumber ?? null, + previousLineStart: previous?.lineStart ?? null, + previousLineEnd: previous?.lineEnd ?? null, + currentLineStart: current?.lineStart ?? null, + currentLineEnd: current?.lineEnd ?? null, + processingVersion: RAW_DOCUMENT_CHANGE_VERSION, + }; +} + +function emptyCategoryCounts(): Record { + return Object.fromEntries(DOCUMENT_CHANGE_CATEGORIES.map(category => [category, 0])) as Record; +} + +function categoryCounts(groups: readonly AnalyzedDocumentChangeGroup[]): Record { + const counts = emptyCategoryCounts(); + for (const group of groups) counts[group.materiality.category]++; + return counts; +} + +function confidenceCounts(groups: readonly AnalyzedDocumentChangeGroup[]): Record { + const counts = { strong: 0, moderate: 0, uncertain: 0 }; + for (const group of groups) counts[group.materiality.confidence]++; + return counts; +} + +/** Builds selected condensed evidence plus a complete immutable-audit-ready projection. */ +export function materializeDocumentChanges(inputs: readonly DocumentChangePairInput[]): DeterministicMaterialChangeResult { + const rawChanges: RawDocumentChange[] = []; + const analyzedGroups: AnalyzedDocumentChangeGroup[] = []; + const warnings: DocumentChangeProcessingWarning[] = []; + let deterministicNoOpCount = 0; + let oversizedGroupSplitCount = 0; + const orderedInputs = [...inputs].sort((a, b) => + compareBigInt(a.pair.documentId, b.pair.documentId) + || a.pair.currentCreatedAt.getTime() - b.pair.currentCreatedAt.getTime() + || a.pair.currentVersionId - b.pair.currentVersionId); + for (const { pair, alignments } of orderedInputs) { + const raw = buildRawDocumentChanges(pair, alignments); + const grouped = groupRawDocumentChanges(pair, raw.rawChanges); + rawChanges.push(...raw.rawChanges); + deterministicNoOpCount += raw.deterministicNoOpCount; + oversizedGroupSplitCount += grouped.oversizedGroupSplitCount; + warnings.push(...grouped.warnings); + analyzedGroups.push(...grouped.groups.map(group => ({ pair, group, materiality: analyzeDocumentChangeGroup(group) }))); + } + const selection = selectMaterialDocumentChangeGroups(analyzedGroups); + warnings.push(...selection.warnings); + const items = selection.selectedGroups.map(({ pair, group, materiality }) => buildCondensedDocumentChangeEvidence(pair, group, materiality)); + const selectedIds = new Set(selection.selectedGroups.map(({ group }) => group.groupId)); + const rawAuditById = new Map(); + for (const analyzed of analyzedGroups) { + for (const change of analyzed.group.rawChanges) { + if (!rawAuditById.has(change.rawChangeId)) rawAuditById.set(change.rawChangeId, rawSnapshot(analyzed.pair, change)); + } + } + const audit: DocumentChangeAuditSnapshot = { + schemaVersion: DOCUMENT_CHANGE_AUDIT_VERSION, + rawChanges: [...rawAuditById.values()].sort((a, b) => compareOrdinal(a.rawChangeId, b.rawChangeId)), + groups: [...analyzedGroups].sort(compareFinalOrder).map(({ group, materiality }) => ({ + groupId: group.groupId, + evidenceSourceId: selectedIds.has(group.groupId) ? documentChangeGroupSourceId(group) : null, + documentId: group.documentId.toString(), + previousVersionId: group.previousVersionId, + currentVersionId: group.currentVersionId, + structurePath: group.structurePath ?? null, + structureTitle: group.structureTitle ?? null, + splitOrdinal: group.splitOrdinal, + rawChangeIds: group.rawChanges.map(change => change.rawChangeId), + category: materiality.category, + priority: materiality.priority, + confidence: materiality.confidence, + signals: [...materiality.signals], + materialityMethod: "deterministic", + materialityVersion: DOCUMENT_CHANGE_MATERIALITY_VERSION, + })), + }; + const count = (type: RawDocumentChange["changeType"]) => rawChanges.filter(change => change.changeType === type).length; + const rawExcerptCharacters = audit.rawChanges.reduce((total, change) => + total + (change.previousExcerpt?.length ?? 0) + (change.currentExcerpt?.length ?? 0), 0); + const condensedPromptFacingCharacters = items.reduce((total, item) => total + item.excerpt.length, 0); + return { + rawChanges, + analyzedGroups: [...analyzedGroups].sort(compareFinalOrder), + selectedGroups: selection.selectedGroups, + items, + audit, + warnings: [...new Map(warnings.map(item => [item.code, item])).values()], + diagnostics: { + versionPairCount: inputs.length, + alignedChunkCount: inputs.reduce((total, input) => total + input.alignments.length, 0), + rawModifiedCount: count("modified"), + rawAddedCount: count("added"), + rawRemovedCount: count("removed"), + deterministicNoOpCount, + groupCount: analyzedGroups.length, + oversizedGroupSplitCount, + selectedGroupCount: selection.selectedGroups.length, + truncatedGroupCount: selection.truncatedGroups.length, + approximateChangedCharacters: rawChanges.reduce((total, change) => total + (change.previousChunk?.content.length ?? 0) + (change.currentChunk?.content.length ?? 0), 0), + groupsByMaterialityCategory: categoryCounts(analyzedGroups), + groupsByDeterministicConfidence: confidenceCounts(analyzedGroups), + selectedGroupsByCategory: categoryCounts(selection.selectedGroups), + truncatedGroupsByCategory: categoryCounts(selection.truncatedGroups), + rawAuditCount: audit.rawChanges.length, + condensedEvidenceCount: items.length, + rawExcerptCharacters, + condensedPromptFacingCharacters, + estimatedReductionRatio: rawExcerptCharacters === 0 ? 1 : Number((rawExcerptCharacters / Math.max(1, condensedPromptFacingCharacters)).toFixed(3)), + }, + }; +} diff --git a/packages/features/src/founder-weekly-review/evidence-digest.ts b/packages/features/src/founder-weekly-review/evidence-digest.ts new file mode 100644 index 000000000..c615ea834 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evidence-digest.ts @@ -0,0 +1,18 @@ +import { createHash } from "node:crypto"; + +import type { FounderWeeklyReviewEvidenceSnapshot } from "./contracts"; + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value as Record) + .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([key, entry]) => [key, canonicalize(entry)])); + } + return value; +} + +/** Digest of the complete frozen evidence snapshot, including v2 raw/group audit data. */ +export function buildFounderWeeklyReviewEvidenceDigest(snapshot: FounderWeeklyReviewEvidenceSnapshot): string { + return createHash("sha256").update(JSON.stringify(canonicalize(snapshot)), "utf8").digest("hex"); +} diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index 5d535ab21..b6d7b1d10 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -4,6 +4,7 @@ import { type FounderWeeklyReviewEvidenceItem, type FounderWeeklyReviewEvidenceSnapshot, type FounderWeeklyReviewEvidenceWarning, + type DocumentChangeAuditSnapshot, type ReportingPeriod, } from "./contracts"; import { resolveReportingPeriodBounds } from "./reporting-period"; @@ -16,13 +17,12 @@ import { } from "@launchstack/core/db/schema"; import { alignVersionChunks, - buildDocumentChangeGroupEvidence, - condenseDocumentChanges, selectVersionPairsForReportingPeriod, type DocumentChangePairInput, type DocumentVersionForComparison, type VersionChunk, } from "./document-change"; +import { materializeDocumentChanges } from "./document-change-materiality"; import { buildWorkspaceDocumentEvidence, normalizeFounderContextRetrievalQuery, @@ -143,6 +143,7 @@ export interface BuildFounderWeeklyReviewEvidenceSnapshotInput { export interface FounderWeeklyReviewEvidenceSourceResult { items: FounderWeeklyReviewEvidenceItem[]; warnings: FounderWeeklyReviewEvidenceWarning[]; + documentChangeAudit?: DocumentChangeAuditSnapshot; } /** Implemented in apps/web so pure feature logic never depends on Drizzle. */ @@ -195,17 +196,9 @@ export class FounderWeeklyReviewEvidenceService { } pairInputs.push({ pair, alignments: alignVersionChunks(previous.chunks, current.chunks) }); } - const condensed = condenseDocumentChanges(pairInputs); - const pairByKey = new Map(pairInputs.map(({ pair }) => [ - `${pair.documentId}:${pair.previousVersionId}:${pair.currentVersionId}`, - pair, - ])); - const items = condensed.selectedGroups.map((group) => { - const pair = pairByKey.get(`${group.documentId}:${group.previousVersionId}:${group.currentVersionId}`); - if (!pair) throw new Error("Selected document-change group has no source version pair."); - return buildDocumentChangeGroupEvidence(pair, group); - }); - warnings.push(...condensed.warnings.map((item) => warning(item.code, item.message, "document_change"))); + const materialized = materializeDocumentChanges(pairInputs); + const items = [...materialized.items]; + warnings.push(...materialized.warnings.map((item) => warning(item.code, item.message, "document_change"))); const pairedCurrentVersionIds = new Set(pairs.map((pair) => pair.currentVersionId)); const inPeriodWithNoPair = versions.some((version) => version.createdAt >= startInclusive @@ -215,7 +208,7 @@ export class FounderWeeklyReviewEvidenceService { if (inPeriodWithNoPair) warnings.push(warning("document_change_baseline_missing", "An in-period document version has no predecessor, so no content diff was generated.", "document_change")); const ordered = orderEvidenceItems(dedupeEvidenceItems(items)); if (ordered.length > MAX_ITEMS_PER_SOURCE) warnings.push(warning("document_change_truncated", "Document change evidence was truncated to the per-source limit.", "document_change")); - return { items: ordered.slice(0, MAX_ITEMS_PER_SOURCE), warnings }; + return { items: ordered.slice(0, MAX_ITEMS_PER_SOURCE), warnings, documentChangeAudit: materialized.audit }; } if (this.documentChangeSource.kind !== "legacy") { return { items: [], warnings: [warning("document_change_source_unconfigured", "Document-change collection requires the explicit computed-diff source.", "document_change")] }; @@ -296,9 +289,20 @@ export class FounderWeeklyReviewEvidenceService { const maxItems = Math.max(0, Math.min(MAX_SNAPSHOT_ITEMS, requestedMax)); const warnings = dedupeWarnings(sourceResults.flatMap((result) => result.warnings)); if (items.length > maxItems) warnings.push(warning("evidence_snapshot_truncated", "Evidence snapshot was truncated to its configured maximum.")); + const selectedItems = items.slice(0, maxItems); + const selectedSourceIds = new Set(selectedItems.map(item => item.sourceId)); + const collectedAudit = documentChanges.documentChangeAudit ?? { schemaVersion: "document-change-audit/v1" as const, rawChanges: [], groups: [] }; + const documentChangeAudit: DocumentChangeAuditSnapshot = { + ...collectedAudit, + groups: collectedAudit.groups.map(group => ({ + ...group, + evidenceSourceId: group.evidenceSourceId && selectedSourceIds.has(group.evidenceSourceId) ? group.evidenceSourceId : null, + })), + }; return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, capturedAt: (input.capturedAt ?? this.now()).toISOString(), reportingPeriod: input.reportingPeriod, - workspaceTimezone: input.workspaceTimezone, items: items.slice(0, maxItems), sourceWarnings: dedupeWarnings(warnings).slice(0, MAX_WARNINGS) }); + workspaceTimezone: input.workspaceTimezone, items: selectedItems, sourceWarnings: dedupeWarnings(warnings).slice(0, MAX_WARNINGS), + documentChangeAudit }); } } @@ -314,8 +318,25 @@ export function dedupeEvidenceItems(items: FounderWeeklyReviewEvidenceItem[]): F } function compareOrdinal(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } +function evidenceMetadataText(item: FounderWeeklyReviewEvidenceItem, key: string): string { + const value = item.metadata[key]; + return typeof value === "string" || typeof value === "number" ? String(value) : ""; +} +function compareDocumentChangeEvidence(a: FounderWeeklyReviewEvidenceItem, b: FounderWeeklyReviewEvidenceItem): number { + if (a.sourceType !== "document_change" || b.sourceType !== "document_change") return 0; + const aDocument = evidenceMetadataText(a, "documentId"); const bDocument = evidenceMetadataText(b, "documentId"); + const document = /^\d+$/.test(aDocument) && /^\d+$/.test(bDocument) + ? (BigInt(aDocument) < BigInt(bDocument) ? -1 : BigInt(aDocument) > BigInt(bDocument) ? 1 : 0) + : compareOrdinal(aDocument, bDocument); + if (document !== 0) return document; + for (const key of ["previousVersionId", "currentVersionId", "structureOrdering", "pageNumber", "lineStart"] as const) { + const left = Number(evidenceMetadataText(a, key)); const right = Number(evidenceMetadataText(b, key)); + if (Number.isFinite(left) && Number.isFinite(right) && left !== right) return left - right; + } + return compareOrdinal(evidenceMetadataText(a, "structurePath"), evidenceMetadataText(b, "structurePath")); +} export function orderEvidenceItems(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceItem[] { - return [...items].sort((a, b) => compareOrdinal(a.sourceTimestamp ?? "", b.sourceTimestamp ?? "") || compareOrdinal(a.sourceType, b.sourceType) || compareOrdinal(a.sourceId, b.sourceId)); + return [...items].sort((a, b) => compareOrdinal(a.sourceTimestamp ?? "", b.sourceTimestamp ?? "") || compareOrdinal(a.sourceType, b.sourceType) || compareDocumentChangeEvidence(a, b) || compareOrdinal(a.sourceId, b.sourceId)); } export function dedupeWarnings(warnings: FounderWeeklyReviewEvidenceWarning[]): FounderWeeklyReviewEvidenceWarning[] { diff --git a/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts b/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts index 1bd864cb3..bddf5d61b 100644 --- a/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts +++ b/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts @@ -74,6 +74,12 @@ const PROMPT_METADATA_ALLOWLIST: Record = { "structurePath", "alignmentMethod", "userChangelog", + "category", + "materialityMethod", + "materialityConfidence", + "materialityVersion", + "structureTitle", + "rawChangeCount", ], customer_feedback: ["versionNumber", "documentCategory", "pageNumber"], workspace_document: ["retrievalReason", "similarityScore"], @@ -94,6 +100,14 @@ function similarityScore(item: FounderWeeklyReviewEvidenceItem): number { const value = item.metadata.similarityScore; return typeof value === "number" && Number.isFinite(value) ? value : Number.NEGATIVE_INFINITY; } +function compareIntegerMetadata(a: FounderWeeklyReviewEvidenceItem, b: FounderWeeklyReviewEvidenceItem, key: string): number { + const left = metadataText(a, key); const right = metadataText(b, key); + if (/^-?\d+$/.test(left) && /^-?\d+$/.test(right)) { + const leftInteger = BigInt(left); const rightInteger = BigInt(right); + return leftInteger < rightInteger ? -1 : leftInteger > rightInteger ? 1 : 0; + } + return compareOrdinal(left, right); +} /** Stable source-specific ordering that does not depend on snapshot input order. */ function compareEvidenceItems( @@ -109,15 +123,12 @@ function compareEvidenceItems( if (Number.isFinite(similarity) && similarity !== 0) return similarity; } if (a.sourceType === "document_change" && b.sourceType === "document_change") { - for (const key of [ - "documentId", - "previousVersionId", - "currentVersionId", - "structurePath", - ] as const) { - const compared = compareOrdinal(metadataText(a, key), metadataText(b, key)); + for (const key of ["documentId", "previousVersionId", "currentVersionId", "structureOrdering", "pageNumber", "lineStart"] as const) { + const compared = compareIntegerMetadata(a, b, key); if (compared !== 0) return compared; } + const structurePath = compareOrdinal(metadataText(a, "structurePath"), metadataText(b, "structurePath")); + if (structurePath !== 0) return structurePath; } return compareOrdinal(a.sourceId, b.sourceId); } diff --git a/packages/features/src/founder-weekly-review/generator.ts b/packages/features/src/founder-weekly-review/generator.ts index f9e587a29..006ccb201 100644 --- a/packages/features/src/founder-weekly-review/generator.ts +++ b/packages/features/src/founder-weekly-review/generator.ts @@ -23,6 +23,7 @@ import { type FounderWeeklyReviewPromptEvidenceItem, type GenerationEvidenceEnvelopeDiagnostics, } from "./generation-evidence-envelope"; +import { buildFounderWeeklyReviewEvidenceDigest } from "./evidence-digest"; export interface FounderWeeklyReviewResolvedGenerationMetadata { provider: string; @@ -76,6 +77,7 @@ export async function generateFounderWeeklyReview( promptHash, true, evidenceEnvelope.diagnostics, + evidenceSnapshot, ), }; } @@ -118,7 +120,7 @@ export async function generateFounderWeeklyReview( } } - return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false, evidenceEnvelope.diagnostics) }; + return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false, evidenceEnvelope.diagnostics, evidenceSnapshot) }; } function buildSemanticRepairPrompt( @@ -156,6 +158,7 @@ function buildMetadata( promptHash: string, skipped: boolean, diagnostics: GenerationEvidenceEnvelopeDiagnostics, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, ): FounderWeeklyReviewModelMetadata { return { provider: metadata.provider, @@ -164,7 +167,7 @@ function buildMetadata( ...(metadata.temperature === undefined ? {} : { temperature: metadata.temperature }), promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, promptHash, - evidenceSchemaVersion: "founder-weekly-review-evidence/v1", + evidenceSchemaVersion: evidenceSnapshot.schemaVersion, reviewPayloadSchemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, ...(metadata.providerRequestId ? { completionId: metadata.providerRequestId } : {}), attributes: { @@ -179,6 +182,7 @@ function buildMetadata( evidenceEnvelopeTruncated: diagnostics.truncated, evidenceEnvelopeSelectedByType: JSON.stringify(diagnostics.selectedBySourceType), evidenceEnvelopeExcludedByType: JSON.stringify(diagnostics.excludedBySourceType), + evidenceDigest: buildFounderWeeklyReviewEvidenceDigest(evidenceSnapshot), }, }; } diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 92245d3e9..32260a871 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -10,4 +10,6 @@ export * from "./generation-validation"; export * from "./generation-evidence-envelope"; export * from "./prompts"; export * from "./document-change"; +export * from "./document-change-materiality"; +export * from "./evidence-digest"; export * from "./workspace-document"; diff --git a/packages/features/src/founder-weekly-review/repository.ts b/packages/features/src/founder-weekly-review/repository.ts index c873b2cd8..751639e26 100644 --- a/packages/features/src/founder-weekly-review/repository.ts +++ b/packages/features/src/founder-weekly-review/repository.ts @@ -11,6 +11,7 @@ import { FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, type FounderWeeklyReviewPayloadSchemaVersion, + type FounderWeeklyReviewEvidenceSchemaVersion, type CreateFounderWeeklyReviewRunInput, type FounderWeeklyReviewClaimInput, type FounderWeeklyReviewCollectionClaimInput, @@ -51,6 +52,14 @@ function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord `Founder weekly review run "${row.id}" has mismatched review payload and review schema versions.` ); } + const evidenceSnapshot = row.evidenceSnapshot + ? parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot) + : null; + if (evidenceSnapshot && evidenceSnapshot.schemaVersion !== row.evidenceSchemaVersion) { + throw new FounderWeeklyReviewInvalidPayloadError( + `Founder weekly review run "${row.id}" has mismatched evidence snapshot and evidence schema versions.` + ); + } return { id: row.id, companyId: row.companyId, @@ -62,11 +71,9 @@ function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord status: row.status, reviewPayload, reviewSchemaVersion: row.reviewSchemaVersion as FounderWeeklyReviewPayloadSchemaVersion, - evidenceSnapshot: row.evidenceSnapshot - ? parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot) - : null, + evidenceSnapshot, evidenceSchemaVersion: - row.evidenceSchemaVersion as typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + row.evidenceSchemaVersion as FounderWeeklyReviewEvidenceSchemaVersion, collectionInput: parseFounderWeeklyReviewCollectionInput(row.collectionInput), collectionClaimId: row.collectionClaimId ?? null, collectionStartedAt: row.collectionStartedAt ?? null, @@ -136,7 +143,7 @@ export class FounderWeeklyReviewRepository { reviewPayload: null, reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, evidenceSnapshot: input.evidenceSnapshot ?? null, - evidenceSchemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + evidenceSchemaVersion: input.evidenceSnapshot?.schemaVersion ?? FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, collectionInput: input.collectionInput ?? { workspaceTimezone: input.evidenceSnapshot?.workspaceTimezone ?? "UTC", actorExternalUserId: input.createdByActorId.replace(/^user:/, ""), From 630660fc861e77c464eaffc26ed6f04a7e38e7c7 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 8 Aug 2026 00:19:18 +0800 Subject: [PATCH 21/29] test(founder-weekly-review): add materiality evaluation harness --- .../materiality-evaluation.test.ts | 106 ++++ ...-review-materiality-evaluation-fixtures.ts | 344 +++++++++++ ...er-weekly-review-materiality-evaluation.ts | 581 ++++++++++++++++++ 3 files changed, 1031 insertions(+) create mode 100644 apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts create mode 100644 apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts create mode 100644 apps/web/scripts/founder-weekly-review-materiality-evaluation.ts diff --git a/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts new file mode 100644 index 000000000..2c0ff9d88 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts @@ -0,0 +1,106 @@ +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; + +import { + evaluationArtifactDirectory, + renderMaterialityEvaluationArtifacts, + runMaterialityEvaluation, +} from "../../scripts/founder-weekly-review-materiality-evaluation"; +import { + MATERIALITY_EVALUATION_SCENARIOS, +} from "../../scripts/founder-weekly-review-materiality-evaluation-fixtures"; + +describe("Founder Weekly Review realistic materiality evaluation harness", () => { + it("loads a compact, stable, human-readable ground-truth matrix", () => { + expect(MATERIALITY_EVALUATION_SCENARIOS).toHaveLength(46); + expect(new Set(MATERIALITY_EVALUATION_SCENARIOS.map(scenario => scenario.id)).size).toBe(46); + expect(MATERIALITY_EVALUATION_SCENARIOS.filter(scenario => scenario.multiChunk).length).toBeGreaterThanOrEqual(10); + expect(MATERIALITY_EVALUATION_SCENARIOS.filter(scenario => scenario.largeDocument).length).toBeGreaterThanOrEqual(5); + for (const scenario of MATERIALITY_EVALUATION_SCENARIOS) { + expect(scenario.description.length).toBeGreaterThan(20); + expect(scenario.expected.expectedAlignmentRelations.length).toBeGreaterThan(0); + for (const expectation of [ + ...scenario.expected.meaningfulChanges, + ...scenario.expected.nonMaterialChanges, + ...scenario.expected.expectedNoOps, + ...scenario.expected.expectedAlignmentRelations, + ]) expect(expectation.id.startsWith(`${scenario.id}:`)).toBe(true); + } + }); + + it("calculates explicit confusion, alignment, condensation, and budget metrics", () => { + const result = runMaterialityEvaluation(); + expect(result.summary.scenarioCount).toBe(46); + expect(result.summary.materiality.groundTruthMaterialChanges).toBeGreaterThan(20); + expect(result.summary.materiality.groundTruthNonMaterialChanges).toBeGreaterThan(8); + expect(result.summary.materiality.uncertainRate).toBeGreaterThanOrEqual(0); + expect(result.summary.materiality.falseMaterialRate).toBeGreaterThanOrEqual(0); + expect(result.summary.alignment.intendedSemanticRelations).toBeGreaterThan(40); + expect(result.summary.alignment.alignmentMissRate).toBeGreaterThanOrEqual(0); + expect(result.summary.condensation.rawChangedRecords).toBeGreaterThan(result.summary.condensation.condensedEvidenceItems); + expect(result.summary.condensation.reductionRatio).toBeCloseTo( + result.summary.condensation.rawCopiedCharacters / result.summary.condensation.condensedEvidenceCharacters, + 3 + ); + expect(result.summary.budget.groupBudgetTruncated).toBe(true); + expect(result.summary.budget.documentDiversityPreserved).toBe(true); + }); + + it("records all four critical materiality buckets separately", () => { + const result = runMaterialityEvaluation(); + expect(result.summary.failuresByKind).toEqual(expect.objectContaining({ + materiality_false_positive: expect.any(Number), + materiality_false_negative: expect.any(Number), + uncertain_material: expect.any(Number), + uncertain_non_material: expect.any(Number), + })); + expect(result.summary.materiality.falseMaterialCount).toBe(result.summary.failuresByKind.materiality_false_positive); + expect(result.summary.materiality.missedMaterialCount).toBe(result.summary.failuresByKind.materiality_false_negative); + }); + + it("calculates alignment misses and false matches from explicit intended relations", () => { + const result = runMaterialityEvaluation(); + expect(result.summary.alignment.alignmentMisses).toBe(result.summary.failuresByKind.alignment_miss); + expect(result.summary.alignment.falsePairings).toBe(result.summary.failuresByKind.alignment_false_match); + expect(result.summary.alignment.unmatchedOldChunks).toBeGreaterThanOrEqual(result.summary.alignment.intendedModifiedAsAddedRemoved); + expect(result.summary.alignment.unmatchedNewChunks).toBeGreaterThanOrEqual(result.summary.alignment.intendedModifiedAsAddedRemoved); + }); + + it("is byte-deterministic when scenario input order changes", () => { + const forward = runMaterialityEvaluation(MATERIALITY_EVALUATION_SCENARIOS); + const reversed = runMaterialityEvaluation([...MATERIALITY_EVALUATION_SCENARIOS].reverse()); + expect(renderMaterialityEvaluationArtifacts(reversed)).toEqual(renderMaterialityEvaluationArtifacts(forward)); + }); + + it("renders deterministic aggregate, synthetic-failure, and Markdown reports", () => { + const artifacts = renderMaterialityEvaluationArtifacts(runMaterialityEvaluation()); + expect(JSON.parse(artifacts.summaryJson)).toEqual(expect.objectContaining({ + summary: expect.objectContaining({ scenarioCount: 46 }), + scenarios: expect.any(Array), + })); + const failures = JSON.parse(artifacts.failuresJson) as { failures: Array<{ syntheticFixture: unknown }> }; + expect(failures.failures.length).toBeGreaterThan(0); + expect(failures.failures.every(failure => failure.syntheticFixture)).toBe(true); + expect(artifacts.evaluationMarkdown).toContain("## Recommendation"); + }); + + it("keeps generated report paths under the repository-gitignored artifact root", () => { + const directory = evaluationArtifactDirectory("test-run"); + expect(directory.replace(/\\/g, "/")).toContain("/apps/web/.artifacts/founder-weekly-review/materiality-evaluation/test-run"); + execFileSync("git", ["check-ignore", "-q", "apps/web/.artifacts/founder-weekly-review/materiality-evaluation/test-run/summary.json"], { + cwd: resolve(process.cwd(), "../.."), + }); + }); + + it("runs provider-free even when network access is made fatal", () => { + const fetchSpy = jest.spyOn(global, "fetch").mockImplementation(() => { + throw new Error("provider/network invocation is forbidden"); + }); + try { + expect(() => runMaterialityEvaluation()).not.toThrow(); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + fetchSpy.mockRestore(); + } + }); +}); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts new file mode 100644 index 000000000..f34a41c80 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts @@ -0,0 +1,344 @@ +import type { + DocumentChangeCategory, + VersionChunk, +} from "@launchstack/features/founder-weekly-review"; + +export type ExpectedAlignmentRelation = { + id: string; + relation: "modified" | "unchanged" | "added" | "removed" | "split" | "merge"; + previousChunkIds: readonly number[]; + currentChunkIds: readonly number[]; +}; + +export type ExpectedChange = { + id: string; + material: boolean; + category: DocumentChangeCategory | null; + previousChunkIds: readonly number[]; + currentChunkIds: readonly number[]; +}; + +export type ExpectedNoOp = { + id: string; + previousChunkId: number; + currentChunkId: number; +}; + +export type MaterialityEvaluationScenario = { + id: string; + description: string; + documentKind: string; + multiChunk: boolean; + largeDocument: boolean; + previousChunks: readonly VersionChunk[]; + currentChunks: readonly VersionChunk[]; + expected: { + meaningfulChanges: readonly ExpectedChange[]; + nonMaterialChanges: readonly ExpectedChange[]; + expectedNoOps: readonly ExpectedNoOp[]; + expectedAlignmentRelations: readonly ExpectedAlignmentRelation[]; + }; +}; + +type ChunkSpec = { + key: string; + content: string; + path?: string | null; + title?: string | null; + order?: number; + page?: number; +}; + +type ChangeSpec = { + id: string; + material: boolean; + category: DocumentChangeCategory | null; + previousKeys: readonly string[]; + currentKeys: readonly string[]; +}; + +type RelationSpec = { + id: string; + relation: ExpectedAlignmentRelation["relation"]; + previousKeys: readonly string[]; + currentKeys: readonly string[]; +}; + +type ScenarioSpec = { + id: string; + description: string; + documentKind: string; + previous: readonly ChunkSpec[]; + current: readonly ChunkSpec[]; + changes?: readonly ChangeSpec[]; + noOps?: readonly { id: string; previousKey: string; currentKey: string }[]; + relations: readonly RelationSpec[]; + largeDocument?: boolean; +}; + +function chunk( + scenarioOrdinal: number, + version: 1 | 2, + index: number, + spec: ChunkSpec +): VersionChunk { + const documentId = BigInt(10_000 + scenarioOrdinal); + return { + chunkId: scenarioOrdinal * 1_000 + (version === 1 ? index + 1 : index + 501), + content: spec.content, + contentHash: null, + structureId: null, + structurePath: spec.path ?? `/section/${spec.key}`, + structureTitle: spec.title ?? spec.key.replace(/-/g, " "), + structureOrdering: spec.order ?? index, + pageNumber: spec.page ?? Math.floor(index / 2) + 1, + lineStart: index * 10 + 1, + lineEnd: index * 10 + 8, + documentId, + versionId: BigInt(version), + }; +} + +function buildScenario(spec: ScenarioSpec, ordinal: number): MaterialityEvaluationScenario { + const previousChunks = spec.previous.map((value, index) => chunk(ordinal, 1, index, value)); + const currentChunks = spec.current.map((value, index) => chunk(ordinal, 2, index, value)); + const previousIds = new Map(spec.previous.map((value, index) => [value.key, previousChunks[index]!.chunkId])); + const currentIds = new Map(spec.current.map((value, index) => [value.key, currentChunks[index]!.chunkId])); + const ids = (keys: readonly string[], values: Map) => keys.map(key => { + const id = values.get(key); + if (id === undefined) throw new Error(`Unknown fixture chunk key ${key} in ${spec.id}.`); + return id; + }); + const changes = (spec.changes ?? []).map(change => ({ + id: `${spec.id}:${change.id}`, + material: change.material, + category: change.category, + previousChunkIds: ids(change.previousKeys, previousIds), + currentChunkIds: ids(change.currentKeys, currentIds), + })); + return { + id: spec.id, + description: spec.description, + documentKind: spec.documentKind, + multiChunk: previousChunks.length > 1 || currentChunks.length > 1, + largeDocument: spec.largeDocument ?? false, + previousChunks, + currentChunks, + expected: { + meaningfulChanges: changes.filter(change => change.material), + nonMaterialChanges: changes.filter(change => !change.material), + expectedNoOps: (spec.noOps ?? []).map(noOp => ({ + id: `${spec.id}:${noOp.id}`, + previousChunkId: ids([noOp.previousKey], previousIds)[0]!, + currentChunkId: ids([noOp.currentKey], currentIds)[0]!, + })), + expectedAlignmentRelations: spec.relations.map(relation => ({ + id: `${spec.id}:${relation.id}`, + relation: relation.relation, + previousChunkIds: ids(relation.previousKeys, previousIds), + currentChunkIds: ids(relation.currentKeys, currentIds), + })), + }, + }; +} + +function single( + id: string, + documentKind: string, + before: string, + after: string, + expectation: { material: boolean; category: DocumentChangeCategory | null } | "noop", + description: string +): ScenarioSpec { + return { + id, + description, + documentKind, + previous: [{ key: "main", content: before, path: "/main", title: "Main" }], + current: [{ key: "main", content: after, path: "/main", title: "Main" }], + ...(expectation === "noop" + ? { noOps: [{ id: "noop", previousKey: "main", currentKey: "main" }] } + : { changes: [{ id: "change", ...expectation, previousKeys: ["main"], currentKeys: ["main"] }] }), + relations: [{ id: "alignment", relation: "modified", previousKeys: ["main"], currentKeys: ["main"] }], + }; +} + +const SIMPLE_SCENARIOS: readonly ScenarioSpec[] = [ + single("noop-whitespace", "operating plan", "Platform owns telemetry.", " Platform owns telemetry. ", "noop", "Repeated and boundary whitespace in an operating plan."), + single("noop-line-wrap", "launch plan", "The enterprise launch remains on schedule.", "The enterprise launch\r\nremains on schedule.", "noop", "Extraction line wrapping in a launch plan."), + single("noop-nbsp", "pricing plan", "Enterprise price review", "Enterprise\u00a0price review", "noop", "A non-breaking-space extraction difference."), + single("noop-unicode", "founder operating notes", "The caf\u00e9 review is complete.", "The cafe\u0301 review is complete.", "noop", "Unicode NFC-equivalent founder notes."), + single("noop-format-artifact", "product roadmap", "Roadmap assumptions remain unchanged.", "**Roadmap assumptions remain unchanged.**", { material: false, category: null }, "A Markdown emphasis extraction artifact that current no-op normalization does not remove."), + + single("paraphrase-ownership", "ownership document", "Product owns telemetry.", "Telemetry is owned by Product.", { material: false, category: null }, "A pure ownership paraphrase with the same DRI."), + single("paraphrase-quarter", "launch plan", "The launch remains planned for Q3.", "We still expect the launch during the third quarter.", { material: false, category: null }, "A deadline paraphrase with unchanged timing."), + single("paraphrase-customer", "sales strategy", "We will begin with design partners before broad availability.", "Broad availability will follow the initial design-partner cohort.", { material: false, category: null }, "A rollout paraphrase with unchanged business meaning."), + + single("ownership-team", "technical rollout plan", "Product owns retry telemetry.", "Platform owns retry telemetry.", { material: true, category: "ownership_change" }, "Ownership transfers from Product to Platform."), + single("ownership-person", "operating plan", "Owner: Alice", "Owner: Bob", { material: true, category: "ownership_change" }, "The named operating-plan owner changes."), + single("ownership-function", "sales strategy", "Marketing owns partner enablement.", "Sales owns partner enablement.", { material: true, category: "ownership_change" }, "Partner enablement ownership transfers functions."), + + single("status-launched", "product roadmap", "The migration is planned.", "The migration is launched.", { material: true, category: "status_change" }, "A roadmap item progresses from planned to launched."), + single("status-blocked", "technical rollout plan", "The rollout is in progress.", "The rollout is blocked.", { material: true, category: "status_change" }, "An in-progress rollout becomes blocked."), + single("status-resolved", "risk register", "The integration is blocked.", "The integration is resolved.", { material: true, category: "risk_or_blocker_change" }, "A documented blocker is resolved."), + single("status-cancelled", "launch plan", "The beta is active.", "The beta is cancelled.", { material: true, category: "status_change" }, "An active beta is cancelled."), + + single("deadline-quarter", "product roadmap", "Launch target: Q3", "Launch target: Q4", { material: true, category: "deadline_change" }, "The launch target moves one quarter."), + single("deadline-date", "customer commitment", "Customer go-live is June 1.", "Customer go-live is July 15.", { material: true, category: "deadline_change" }, "An enterprise customer go-live date moves."), + single("deadline-year", "operating plan", "International expansion is targeted for 2026.", "International expansion is targeted for 2027.", { material: true, category: "deadline_change" }, "Expansion moves to the following year."), + + single("metric-percent", "pricing plan", "Expected conversion is 10%.", "Expected conversion is 25%.", { material: true, category: "metric_change" }, "The expected conversion metric changes."), + single("metric-currency", "operating plan", "ARR target is $1M.", "ARR target is $750k.", { material: true, category: "metric_change" }, "The annual recurring revenue target declines."), + single("metric-customers", "sales strategy", "The cohort includes 20 customers.", "The cohort includes 50 customers.", { material: true, category: "metric_change" }, "The customer cohort expands."), + + single("requirement-may-must", "enterprise onboarding", "Admins may enable SSO.", "Admins must enable SSO.", { material: true, category: "requirement_change" }, "An onboarding capability becomes mandatory."), + single("requirement-optional", "technical rollout plan", "Audit logging is optional.", "Audit logging is required.", { material: true, category: "requirement_change" }, "Audit logging changes from optional to required."), + single("requirement-should", "customer commitment", "Support should acknowledge P0 cases.", "Support must acknowledge P0 cases.", { material: true, category: "requirement_change" }, "A support recommendation becomes an obligation."), + single("requirement-mandatory", "operating plan", "Security review is recommended.", "Security review is mandatory.", { material: true, category: "requirement_change" }, "A security review becomes mandatory."), + + single("negation-support", "enterprise onboarding", "The plan supports SSO.", "The plan does not support SSO.", { material: true, category: "requirement_change" }, "SSO support is explicitly negated."), + single("negation-availability", "pricing plan", "The add-on is available.", "The add-on is unavailable.", { material: true, category: "requirement_change" }, "An add-on becomes unavailable."), + + single("risk-added", "risk register", "No delivery concern is recorded.", "Delivery risk is recorded for the data migration.", { material: true, category: "risk_or_blocker_change" }, "A delivery risk is introduced."), + single("risk-removed", "risk register", "Vendor risk remains open.", "The vendor concern has been cleared.", { material: true, category: "risk_or_blocker_change" }, "A risk is removed using non-lexical resolution language."), + single("blocker-introduced", "technical rollout plan", "Identity testing is in progress.", "Identity testing has a blocker.", { material: true, category: "status_change" }, "A blocker interrupts work that was in progress; status precedence is expected."), + single("blocker-resolved", "founder operating notes", "Billing migration is blocked.", "Billing migration is unblocked.", { material: true, category: "risk_or_blocker_change" }, "A billing blocker is cleared."), + + single("scope-global", "launch plan", "Availability: US only", "Availability: global", { material: true, category: "scope_change" }, "Launch geography expands globally."), + single("scope-enterprise", "sales strategy", "Access is for pilot customers.", "Access is for all enterprise customers.", { material: true, category: "scope_change" }, "A pilot expands to all enterprise customers."), + single("priority-p0", "product roadmap", "Migration priority: P2", "Migration priority: P0", { material: true, category: "priority_change" }, "A roadmap item is escalated to P0."), + single("priority-immediate", "founder operating notes", "The pricing review is deferred.", "The pricing review is immediate.", { material: true, category: "priority_change" }, "A deferred pricing review becomes immediate."), +]; + +function sectionChunks(prefix: string, contents: readonly string[], path: string, pageStart = 1): ChunkSpec[] { + return contents.map((content, index) => ({ + key: `${prefix}-${index + 1}`, + content, + path, + title: prefix.replace(/-/g, " "), + order: index, + page: pageStart + index, + })); +} + +const MULTI_SCENARIOS: readonly ScenarioSpec[] = [ + { + id: "rewrite-roadmap-same-meaning", + description: "A six-fragment roadmap section is reorganized without changing ownership, timing, or scope.", + documentKind: "product roadmap", + previous: sectionChunks("roadmap", [ + "The team will validate demand with design partners.", "Product continues to own the launch checklist.", + "General availability remains targeted for the third quarter.", "The initial cohort remains limited to enterprise design partners.", + "Security review continues before activation.", "No change is made to the rollout decision gate.", + ], "/roadmap"), + current: sectionChunks("roadmap", [ + "Demand validation will continue through conversations with design partners.", "The launch checklist remains under Product ownership.", + "The third quarter remains the target for general availability.", "Enterprise design partners still make up the initial cohort.", + "Activation continues to follow the security review.", "The rollout decision gate remains as previously documented.", + ], "/roadmap"), + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["roadmap-1", "roadmap-2", "roadmap-3", "roadmap-4", "roadmap-5", "roadmap-6"], currentKeys: ["roadmap-1", "roadmap-2", "roadmap-3", "roadmap-4", "roadmap-5", "roadmap-6"] }], + relations: [1, 2, 3, 4, 5, 6].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`roadmap-${index}`], currentKeys: [`roadmap-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-onboarding-same-meaning", + description: "An enterprise onboarding section is rewritten in a more narrative style with unchanged controls.", + documentKind: "enterprise onboarding document", + previous: sectionChunks("onboarding", ["Customer success coordinates kickoff.", "Security receives the questionnaire before configuration.", "The customer validates SSO in staging.", "Production access follows customer sign-off.", "Support monitors the first business day."], "/onboarding"), + current: sectionChunks("onboarding", ["Kickoff coordination remains with customer success.", "The questionnaire continues to reach security ahead of configuration.", "SSO validation still occurs in the staging environment.", "Customer sign-off continues to precede production access.", "The first business day remains under support monitoring."], "/onboarding"), + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["onboarding-1", "onboarding-2", "onboarding-3", "onboarding-4", "onboarding-5"], currentKeys: ["onboarding-1", "onboarding-2", "onboarding-3", "onboarding-4", "onboarding-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`onboarding-${index}`], currentKeys: [`onboarding-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-pricing-same-meaning", + description: "A pricing narrative changes voice and ordering while retaining the same approval and rollout policy.", + documentKind: "pricing plan", + previous: sectionChunks("pricing", ["Finance reviews discount exceptions.", "Sales submits the business rationale.", "Legal reviews non-standard terms.", "The pricing council makes the final decision.", "Approved changes enter the next catalog release."], "/pricing"), + current: sectionChunks("pricing", ["Discount exceptions continue to receive finance review.", "The business rationale still comes from sales.", "Non-standard terms continue through legal review.", "Final decisions remain with the pricing council.", "The next catalog release continues to carry approved changes."], "/pricing"), + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["pricing-1", "pricing-2", "pricing-3", "pricing-4", "pricing-5"], currentKeys: ["pricing-1", "pricing-2", "pricing-3", "pricing-4", "pricing-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`pricing-${index}`], currentKeys: [`pricing-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-rollout-material", + description: "A six-fragment technical rollout rewrite transfers ownership, advances status, and expands scope.", + documentKind: "technical rollout plan", + previous: sectionChunks("rollout", ["Product owns deployment telemetry.", "The rollout is planned.", "Availability is US only.", "Audit logging is optional.", "Identity integration risk remains open.", "The pilot includes 20 customers."], "/rollout"), + current: sectionChunks("rollout", ["Platform owns deployment telemetry.", "The rollout is launched.", "Availability is global.", "Audit logging is required.", "Identity integration risk is resolved.", "The launch includes 50 customers."], "/rollout"), + changes: [{ id: "rewrite", material: true, category: "ownership_change", previousKeys: ["rollout-1", "rollout-2", "rollout-3", "rollout-4", "rollout-5", "rollout-6"], currentKeys: ["rollout-1", "rollout-2", "rollout-3", "rollout-4", "rollout-5", "rollout-6"] }], + relations: [1, 2, 3, 4, 5, 6].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`rollout-${index}`], currentKeys: [`rollout-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-customer-commitment-material", + description: "A customer commitment section changes its deadline and makes implementation support mandatory.", + documentKind: "customer commitment", + previous: sectionChunks("commitment", ["Customer go-live is June 1.", "Implementation support is recommended.", "The pilot covers US accounts.", "Product owns escalation review.", "The security risk remains open."], "/commitment"), + current: sectionChunks("commitment", ["Customer go-live is July 15.", "Implementation support is mandatory.", "The pilot covers global accounts.", "Platform owns escalation review.", "The security risk is resolved."], "/commitment"), + changes: [{ id: "rewrite", material: true, category: "ownership_change", previousKeys: ["commitment-1", "commitment-2", "commitment-3", "commitment-4", "commitment-5"], currentKeys: ["commitment-1", "commitment-2", "commitment-3", "commitment-4", "commitment-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`commitment-${index}`], currentKeys: [`commitment-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-risk-register-material", + description: "A multi-page risk-register section records resolution of one blocker and introduction of another.", + documentKind: "risk register", + previous: sectionChunks("risks", ["Identity migration is blocked.", "Billing reconciliation has no recorded risk.", "The rollout is in progress.", "The response owner is Product.", "The review target is Q3."], "/risks"), + current: sectionChunks("risks", ["Identity migration is resolved.", "Billing reconciliation has a blocker.", "The rollout is paused.", "The response owner is Platform.", "The review target is Q4."], "/risks"), + changes: [{ id: "rewrite", material: true, category: "ownership_change", previousKeys: ["risks-1", "risks-2", "risks-3", "risks-4", "risks-5"], currentKeys: ["risks-1", "risks-2", "risks-3", "risks-4", "risks-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`risks-${index}`], currentKeys: [`risks-${index}`] })), + largeDocument: true, + }, + { + id: "alignment-section-moved", + description: "An unchanged founder-notes section moves from the opening to the end of the document.", + documentKind: "founder operating notes", + previous: [{ key: "moved", content: "The operating cadence remains weekly with a Friday review.", path: "/opening", title: "Cadence", order: 0, page: 1 }, { key: "stable", content: "The hiring plan remains unchanged.", path: "/hiring", title: "Hiring", order: 1, page: 2 }], + current: [{ key: "stable", content: "The hiring plan remains unchanged.", path: "/hiring", title: "Hiring", order: 0, page: 1 }, { key: "moved", content: "The operating cadence remains weekly with a Friday review.", path: "/closing", title: "Operating rhythm", order: 1, page: 6 }], + relations: [{ id: "moved", relation: "unchanged", previousKeys: ["moved"], currentKeys: ["moved"] }, { id: "stable", relation: "unchanged", previousKeys: ["stable"], currentKeys: ["stable"] }], + }, + { + id: "alignment-renamed-heavy-rewrite", + description: "A renamed sales-strategy section preserves intent through a heavy lexical rewrite.", + documentKind: "sales strategy", + previous: [{ key: "strategy", content: "Regional account executives qualify expansion opportunities during quarterly portfolio reviews.", path: "/commercial-motion", title: "Commercial motion", order: 2, page: 4 }], + current: [{ key: "strategy", content: "Every quarter, territory sellers examine existing customers to identify places where adoption can broaden.", path: "/growth-playbook", title: "Growth playbook", order: 7, page: 12 }], + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["strategy"], currentKeys: ["strategy"] }], + relations: [{ id: "rewrite", relation: "modified", previousKeys: ["strategy"], currentKeys: ["strategy"] }], + }, + { + id: "alignment-one-to-many", + description: "One enterprise-onboarding paragraph is split into two structurally adjacent paragraphs.", + documentKind: "enterprise onboarding document", + previous: [{ key: "combined", content: "Security reviews the questionnaire and customer success schedules kickoff after approval.", path: "/onboarding", title: "Onboarding", order: 1, page: 3 }], + current: [{ key: "security", content: "Security reviews the questionnaire before approval.", path: "/onboarding", title: "Onboarding", order: 1, page: 3 }, { key: "kickoff", content: "Customer success schedules kickoff after approval.", path: "/onboarding", title: "Onboarding", order: 2, page: 3 }], + changes: [{ id: "split", material: false, category: null, previousKeys: ["combined"], currentKeys: ["security", "kickoff"] }], + relations: [{ id: "split", relation: "split", previousKeys: ["combined"], currentKeys: ["security", "kickoff"] }], + }, + { + id: "alignment-many-to-one", + description: "Two adjacent pricing-control fragments merge into one paragraph without changing policy.", + documentKind: "pricing plan", + previous: [{ key: "finance", content: "Finance reviews discount exceptions.", path: "/controls", title: "Controls", order: 1, page: 2 }, { key: "council", content: "The pricing council makes the final decision.", path: "/controls", title: "Controls", order: 2, page: 2 }], + current: [{ key: "combined", content: "Finance reviews discount exceptions before the pricing council makes the final decision.", path: "/controls", title: "Controls", order: 1, page: 2 }], + changes: [{ id: "merge", material: false, category: null, previousKeys: ["finance", "council"], currentKeys: ["combined"] }], + relations: [{ id: "merge", relation: "merge", previousKeys: ["finance", "council"], currentKeys: ["combined"] }], + }, + { + id: "alignment-repeated-headings", + description: "Repeated boilerplate headings surround one changed customer-commitment clause.", + documentKind: "customer commitment", + previous: [{ key: "north", content: "Implementation plan: Customer success coordinates the North account kickoff.", path: "/north/plan", title: "Implementation plan", order: 1 }, { key: "south", content: "Implementation plan: Customer success coordinates the South account kickoff.", path: "/south/plan", title: "Implementation plan", order: 2 }], + current: [{ key: "south", content: "Implementation plan: Customer success coordinates the South account kickoff.", path: "/south/plan", title: "Implementation plan", order: 1 }, { key: "north", content: "Implementation plan: Platform coordinates the North account kickoff.", path: "/north/plan", title: "Implementation plan", order: 2 }], + changes: [{ id: "north-owner", material: true, category: "ownership_change", previousKeys: ["north"], currentKeys: ["north"] }], + relations: [{ id: "north", relation: "modified", previousKeys: ["north"], currentKeys: ["north"] }, { id: "south", relation: "unchanged", previousKeys: ["south"], currentKeys: ["south"] }], + }, +]; + +export const MATERIALITY_EVALUATION_FIXTURE_VERSION = "founder-weekly-review-materiality-evaluation/v1" as const; + +export const MATERIALITY_EVALUATION_SCENARIOS: readonly MaterialityEvaluationScenario[] = [ + ...SIMPLE_SCENARIOS, + ...MULTI_SCENARIOS, +].map(buildScenario); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts new file mode 100644 index 000000000..7f0370d58 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts @@ -0,0 +1,581 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + FounderWeeklyReviewEvidenceSnapshotSchema, + alignVersionChunks, + buildGenerationEvidenceEnvelope, + materializeDocumentChanges, + type AnalyzedDocumentChangeGroup, + type ChunkAlignment, + type DocumentChangeCategory, + type DocumentChangePairInput, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +import { + MATERIALITY_EVALUATION_FIXTURE_VERSION, + MATERIALITY_EVALUATION_SCENARIOS, + type ExpectedAlignmentRelation, + type ExpectedChange, + type MaterialityEvaluationScenario, +} from "./founder-weekly-review-materiality-evaluation-fixtures"; + +export const MATERIALITY_EVALUATION_RUN_ID = "deterministic-v1" as const; +export const MATERIALITY_EVALUATION_ARTIFACT_ROOT = + ".artifacts/founder-weekly-review/materiality-evaluation" as const; + +type FailureKind = + | "materiality_false_positive" + | "materiality_false_negative" + | "uncertain_material" + | "uncertain_non_material" + | "alignment_miss" + | "alignment_false_match" + | "grouping_issue" + | "budget_issue"; + +type MissingCapability = + | "future_llm_materiality_analyzer" + | "future_embedding_alignment" + | "deterministic_improvement" + | "acceptable_limitation"; + +export type MaterialityEvaluationFailure = { + kind: FailureKind; + scenarioId: string; + expectationId: string; + description: string; + observed: string; + likelyMissingCapability: MissingCapability; + syntheticFixture: { + previous: readonly string[]; + current: readonly string[]; + }; +}; + +export type MaterialityEvaluationSummary = { + fixtureVersion: string; + scenarioCount: number; + multiChunkScenarioCount: number; + largeDocumentScenarioCount: number; + groundTruthCategoryDistribution: Record; + observedCategoryDistribution: Record; + materiality: { + groundTruthMaterialChanges: number; + groundTruthNonMaterialChanges: number; + categoryCorrect: number; + categoryAccuracy: number; + materialRecall: number; + materialPrecision: number; + uncertainRate: number; + falseMaterialCount: number; + falseMaterialRate: number; + missedMaterialCount: number; + missedMaterialRate: number; + uncertainMaterialCount: number; + uncertainMaterialRate: number; + uncertainNonMaterialCount: number; + uncertainNonMaterialRate: number; + }; + alignment: { + intendedSemanticRelations: number; + correctModifiedPairings: number; + incorrectModifiedPairings: number; + intendedModifiedAsAddedRemoved: number; + falsePairings: number; + unmatchedOldChunks: number; + unmatchedNewChunks: number; + alignmentMisses: number; + alignmentMissRate: number; + falseMatchRate: number; + }; + condensation: { + rawChangedRecords: number; + groups: number; + condensedEvidenceItems: number; + rawCopiedCharacters: number; + condensedEvidenceCharacters: number; + serializedPromptCharacters: number; + reductionRatio: number; + }; + budget: { + groupBudgetTruncated: boolean; + truncatedGroupCount: number; + generationEnvelopeTruncated: boolean; + generationEnvelopeSelectedItems: number; + generationEnvelopeExcludedItems: number; + documentDiversityPreserved: boolean; + selectedDocumentCount: number; + }; + failuresByKind: Record; + recommendation: "A" | "B" | "C" | "D" | "E"; + recommendationLabel: string; +}; + +export type MaterialityEvaluationResult = { + summary: MaterialityEvaluationSummary; + failures: readonly MaterialityEvaluationFailure[]; + scenarioResults: readonly { + id: string; + alignmentCount: number; + rawChangeCount: number; + groupCount: number; + categories: readonly DocumentChangeCategory[]; + failureKinds: readonly FailureKind[]; + }[]; +}; + +const FAILURE_KINDS: readonly FailureKind[] = [ + "materiality_false_positive", + "materiality_false_negative", + "uncertain_material", + "uncertain_non_material", + "alignment_miss", + "alignment_false_match", + "grouping_issue", + "budget_issue", +]; + +function ratio(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : Number((numerator / denominator).toFixed(4)); +} + +function pairFor(scenario: MaterialityEvaluationScenario, ordinal: number): VersionPair { + const documentId = scenario.previousChunks[0]?.documentId ?? scenario.currentChunks[0]!.documentId; + return { + documentId, + documentTitle: `${scenario.documentKind}: ${scenario.id}`, + documentCategory: null, + previousVersionId: 1, + previousVersionNumber: 1, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), + currentVersionId: 2, + currentVersionNumber: 2, + currentCreatedAt: new Date(Date.UTC(2026, 1, 1 + ordinal)), + currentChangelog: null, + }; +} + +function chunkIds(group: AnalyzedDocumentChangeGroup): { previous: Set; current: Set } { + return { + previous: new Set(group.group.rawChanges.flatMap(change => change.previousChunk ? [change.previousChunk.chunkId] : [])), + current: new Set(group.group.rawChanges.flatMap(change => change.currentChunk ? [change.currentChunk.chunkId] : [])), + }; +} + +function overlap(change: ExpectedChange, group: AnalyzedDocumentChangeGroup): number { + const ids = chunkIds(group); + return change.previousChunkIds.filter(id => ids.previous.has(id)).length + + change.currentChunkIds.filter(id => ids.current.has(id)).length; +} + +function bestGroup(change: ExpectedChange, groups: readonly AnalyzedDocumentChangeGroup[]): AnalyzedDocumentChangeGroup | undefined { + return [...groups] + .map(group => ({ group, score: overlap(change, group) })) + .filter(candidate => candidate.score > 0) + .sort((a, b) => b.score - a.score || (a.group.group.groupId < b.group.group.groupId ? -1 : 1))[0]?.group; +} + +function relationContainsPair(relation: ExpectedAlignmentRelation, previousId: number, currentId: number): boolean { + return relation.previousChunkIds.includes(previousId) && relation.currentChunkIds.includes(currentId); +} + +function actualPair(alignments: readonly ChunkAlignment[], previousId: number, currentId: number): ChunkAlignment | undefined { + return alignments.find(alignment => alignment.previousChunk?.chunkId === previousId && alignment.currentChunk?.chunkId === currentId); +} + +function relationRecovered(relation: ExpectedAlignmentRelation, alignments: readonly ChunkAlignment[]): boolean { + if (relation.relation === "added") { + return relation.currentChunkIds.every(id => alignments.some(alignment => alignment.changeType === "added" && alignment.currentChunk?.chunkId === id)); + } + if (relation.relation === "removed") { + return relation.previousChunkIds.every(id => alignments.some(alignment => alignment.changeType === "removed" && alignment.previousChunk?.chunkId === id)); + } + if (relation.relation === "modified" || relation.relation === "unchanged") { + const alignment = actualPair(alignments, relation.previousChunkIds[0]!, relation.currentChunkIds[0]!); + return alignment?.changeType === relation.relation; + } + const edges = alignments.filter(alignment => alignment.previousChunk && alignment.currentChunk + && relationContainsPair(relation, alignment.previousChunk.chunkId, alignment.currentChunk.chunkId)); + return relation.previousChunkIds.every(id => edges.some(edge => edge.previousChunk!.chunkId === id)) + && relation.currentChunkIds.every(id => edges.some(edge => edge.currentChunk!.chunkId === id)); +} + +function failure( + scenario: MaterialityEvaluationScenario, + kind: FailureKind, + expectationId: string, + observed: string, + likelyMissingCapability: MissingCapability +): MaterialityEvaluationFailure { + return { + kind, + scenarioId: scenario.id, + expectationId, + description: scenario.description, + observed, + likelyMissingCapability, + syntheticFixture: { + previous: scenario.previousChunks.map(chunk => chunk.content), + current: scenario.currentChunks.map(chunk => chunk.content), + }, + }; +} + +function increment(record: Record, key: string): void { + record[key] = (record[key] ?? 0) + 1; +} + +function recommendationFor( + uncertainRate: number, + falseMaterialRate: number, + uncertainMaterialCount: number, + alignmentMissRate: number, + materialityFailureCount: number, + alignmentFailureCount: number +): Pick { + const materiality = uncertainRate > 0.15 || falseMaterialRate > 0.05 || uncertainMaterialCount >= 3; + const alignment = alignmentMissRate > 0.075; + if (materiality && alignment) { + return materialityFailureCount >= alignmentFailureCount + ? { recommendation: "C", recommendationLabel: "Implement both, materiality first" } + : { recommendation: "D", recommendationLabel: "Implement both, alignment first" }; + } + if (materiality) return { recommendation: "A", recommendationLabel: "Implement Phase 3 optional LLM materiality analyzer next" }; + if (alignment) return { recommendation: "B", recommendationLabel: "Implement Phase 4 embedding-assisted alignment next" }; + return { recommendation: "E", recommendationLabel: "Implement neither yet" }; +} + +/** Evaluates frozen production logic only; it never invokes a provider or mutates production behavior. */ +export function runMaterialityEvaluation( + inputScenarios: readonly MaterialityEvaluationScenario[] = MATERIALITY_EVALUATION_SCENARIOS +): MaterialityEvaluationResult { + const scenarios = [...inputScenarios].sort((a, b) => a.id.localeCompare(b.id)); + const failures: MaterialityEvaluationFailure[] = []; + const scenarioResults: MaterialityEvaluationResult["scenarioResults"][number][] = []; + const inputs: DocumentChangePairInput[] = []; + const groundTruthCategoryDistribution: Record = {}; + const observedCategoryDistribution: Record = {}; + let groundTruthMaterialChanges = 0; + let groundTruthNonMaterialChanges = 0; + let categoryCorrect = 0; + let surfacedTrueMaterial = 0; + let falseMaterialCount = 0; + let missedMaterialCount = 0; + let uncertainMaterialCount = 0; + let uncertainNonMaterialCount = 0; + let correctModifiedPairings = 0; + let incorrectModifiedPairings = 0; + let intendedModifiedAsAddedRemoved = 0; + let falsePairings = 0; + let unmatchedOldChunks = 0; + let unmatchedNewChunks = 0; + let alignmentMisses = 0; + let intendedSemanticRelations = 0; + + for (const [ordinal, scenario] of scenarios.entries()) { + const pair = pairFor(scenario, ordinal); + const alignments = alignVersionChunks(scenario.previousChunks, scenario.currentChunks); + inputs.push({ pair, alignments }); + const materialized = materializeDocumentChanges([{ pair, alignments }]); + for (const group of materialized.analyzedGroups) increment(observedCategoryDistribution, group.materiality.category); + const assignedGroupIds = new Set(); + + for (const expected of scenario.expected.meaningfulChanges) { + groundTruthMaterialChanges++; + increment(groundTruthCategoryDistribution, expected.category ?? "material_unspecified"); + const group = bestGroup(expected, materialized.analyzedGroups); + if (group) assignedGroupIds.add(group.group.groupId); + if (!group || group.materiality.category === "editorial_rewrite") { + missedMaterialCount++; + failures.push(failure(scenario, "materiality_false_negative", expected.id, + group ? "Material source was classified as editorial_rewrite." : "No retained group represented the material source.", + group ? "deterministic_improvement" : "future_embedding_alignment")); + continue; + } + surfacedTrueMaterial++; + if (group.materiality.category === expected.category) categoryCorrect++; + if (group.materiality.category === "uncertain") { + uncertainMaterialCount++; + failures.push(failure(scenario, "uncertain_material", expected.id, + "Material source was retained but classified uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const expected of scenario.expected.nonMaterialChanges) { + groundTruthNonMaterialChanges++; + const group = bestGroup(expected, materialized.analyzedGroups); + if (!group) continue; + assignedGroupIds.add(group.group.groupId); + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", expected.id, + `Non-material source became prompt-facing ${group.materiality.category} evidence.`, + group.materiality.category === "uncertain" ? "future_llm_materiality_analyzer" : "acceptable_limitation")); + if (group.materiality.category === "uncertain") { + uncertainNonMaterialCount++; + failures.push(failure(scenario, "uncertain_non_material", expected.id, + "Non-material rewrite was classified uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const noOp of scenario.expected.expectedNoOps) { + groundTruthNonMaterialChanges++; + const group = materialized.analyzedGroups.find(candidate => { + const ids = chunkIds(candidate); + return ids.previous.has(noOp.previousChunkId) || ids.current.has(noOp.currentChunkId); + }); + if (group) { + assignedGroupIds.add(group.group.groupId); + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", noOp.id, + `Expected no-op became prompt-facing ${group.materiality.category} evidence.`, "deterministic_improvement")); + } + } + + for (const group of materialized.analyzedGroups) { + if (assignedGroupIds.has(group.group.groupId)) continue; + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", group.group.groupId, + `An unexpected ${group.materiality.category} group became prompt-facing evidence.`, "deterministic_improvement")); + failures.push(failure(scenario, "grouping_issue", group.group.groupId, + `An extra ${group.materiality.category} group did not map to an expected semantic change.`, "deterministic_improvement")); + } + + const expectedRelations = scenario.expected.expectedAlignmentRelations; + for (const relation of expectedRelations) { + if (["modified", "unchanged", "split", "merge"].includes(relation.relation)) intendedSemanticRelations++; + const recovered = relationRecovered(relation, alignments); + if (!recovered && ["modified", "unchanged", "split", "merge"].includes(relation.relation)) { + alignmentMisses++; + const removed = relation.previousChunkIds.every(id => alignments.some(item => item.changeType === "removed" && item.previousChunk?.chunkId === id)); + const added = relation.currentChunkIds.every(id => alignments.some(item => item.changeType === "added" && item.currentChunk?.chunkId === id)); + if (removed && added) intendedModifiedAsAddedRemoved++; + failures.push(failure(scenario, "alignment_miss", relation.id, + removed && added ? "Intended correspondence was represented as added plus removed." : "Intended semantic correspondence was not completely recovered.", + "future_embedding_alignment")); + } + if (relation.relation === "modified" && relation.previousChunkIds.length === 1 && relation.currentChunkIds.length === 1) { + const actual = actualPair(alignments, relation.previousChunkIds[0]!, relation.currentChunkIds[0]!); + if (actual?.changeType === "modified") correctModifiedPairings++; + } + } + for (const alignment of alignments) { + if (alignment.changeType === "removed") unmatchedOldChunks++; + if (alignment.changeType === "added") unmatchedNewChunks++; + if (alignment.changeType !== "modified" || !alignment.previousChunk || !alignment.currentChunk) continue; + const expected = expectedRelations.some(relation => relationContainsPair(relation, alignment.previousChunk!.chunkId, alignment.currentChunk!.chunkId)); + if (!expected) { + incorrectModifiedPairings++; + falsePairings++; + failures.push(failure(scenario, "alignment_false_match", `${alignment.previousChunk.chunkId}:${alignment.currentChunk.chunkId}`, + "The aligner paired chunks outside the fixture's intended semantic relations.", "future_embedding_alignment")); + } + } + const scenarioFailureKinds = [...new Set(failures.filter(item => item.scenarioId === scenario.id).map(item => item.kind))].sort(); + scenarioResults.push({ + id: scenario.id, + alignmentCount: alignments.length, + rawChangeCount: materialized.rawChanges.length, + groupCount: materialized.analyzedGroups.length, + categories: materialized.analyzedGroups.map(group => group.materiality.category), + failureKinds: scenarioFailureKinds, + }); + } + + const aggregate = materializeDocumentChanges(inputs); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-03-31T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-03-31" }, + workspaceTimezone: "UTC", + items: aggregate.items, + sourceWarnings: aggregate.warnings.map(warning => ({ ...warning, sourceType: "document_change" as const })), + documentChangeAudit: aggregate.audit, + }); + const envelope = buildGenerationEvidenceEnvelope(snapshot); + const uncertainGroups = Object.entries(observedCategoryDistribution).find(([category]) => category === "uncertain")?.[1] ?? 0; + const observedGroups = Object.values(observedCategoryDistribution).reduce((total, count) => total + count, 0); + const nonMaterialDenominator = groundTruthNonMaterialChanges; + const uncertainRate = ratio(uncertainGroups, observedGroups); + const falseMaterialRate = ratio(falseMaterialCount, nonMaterialDenominator); + const alignmentMissRate = ratio(alignmentMisses, intendedSemanticRelations); + const failuresByKind = Object.fromEntries(FAILURE_KINDS.map(kind => [kind, failures.filter(failure => failure.kind === kind).length])) as Record; + const materialityFailureCount = failuresByKind.materiality_false_positive + + failuresByKind.materiality_false_negative + failuresByKind.uncertain_material + failuresByKind.uncertain_non_material; + const alignmentFailureCount = failuresByKind.alignment_miss + failuresByKind.alignment_false_match; + const recommendation = recommendationFor(uncertainRate, falseMaterialRate, uncertainMaterialCount, alignmentMissRate, materialityFailureCount, alignmentFailureCount); + const selectedDocumentCount = new Set(aggregate.selectedGroups.map(group => group.group.documentId.toString())).size; + const availableDocumentCount = new Set(aggregate.analyzedGroups.map(group => group.group.documentId.toString())).size; + const condensedEvidenceCharacters = aggregate.diagnostics.condensedPromptFacingCharacters; + const summary: MaterialityEvaluationSummary = { + fixtureVersion: MATERIALITY_EVALUATION_FIXTURE_VERSION, + scenarioCount: scenarios.length, + multiChunkScenarioCount: scenarios.filter(scenario => scenario.multiChunk).length, + largeDocumentScenarioCount: scenarios.filter(scenario => scenario.largeDocument).length, + groundTruthCategoryDistribution: Object.fromEntries(Object.entries(groundTruthCategoryDistribution).sort()), + observedCategoryDistribution: Object.fromEntries(Object.entries(observedCategoryDistribution).sort()), + materiality: { + groundTruthMaterialChanges, + groundTruthNonMaterialChanges, + categoryCorrect, + categoryAccuracy: ratio(categoryCorrect, groundTruthMaterialChanges), + materialRecall: ratio(surfacedTrueMaterial, groundTruthMaterialChanges), + materialPrecision: ratio(surfacedTrueMaterial, surfacedTrueMaterial + falseMaterialCount), + uncertainRate, + falseMaterialCount, + falseMaterialRate, + missedMaterialCount, + missedMaterialRate: ratio(missedMaterialCount, groundTruthMaterialChanges), + uncertainMaterialCount, + uncertainMaterialRate: ratio(uncertainMaterialCount, groundTruthMaterialChanges), + uncertainNonMaterialCount, + uncertainNonMaterialRate: ratio(uncertainNonMaterialCount, groundTruthNonMaterialChanges), + }, + alignment: { + intendedSemanticRelations, + correctModifiedPairings, + incorrectModifiedPairings, + intendedModifiedAsAddedRemoved, + falsePairings, + unmatchedOldChunks, + unmatchedNewChunks, + alignmentMisses, + alignmentMissRate, + falseMatchRate: ratio(falsePairings, correctModifiedPairings + incorrectModifiedPairings), + }, + condensation: { + rawChangedRecords: aggregate.rawChanges.length, + groups: aggregate.analyzedGroups.length, + condensedEvidenceItems: aggregate.items.length, + rawCopiedCharacters: aggregate.diagnostics.rawExcerptCharacters, + condensedEvidenceCharacters, + serializedPromptCharacters: envelope.diagnostics.serializedCharacterCount, + reductionRatio: ratio(aggregate.diagnostics.rawExcerptCharacters, Math.max(1, condensedEvidenceCharacters)), + }, + budget: { + groupBudgetTruncated: aggregate.diagnostics.truncatedGroupCount > 0, + truncatedGroupCount: aggregate.diagnostics.truncatedGroupCount, + generationEnvelopeTruncated: envelope.diagnostics.truncated, + generationEnvelopeSelectedItems: envelope.diagnostics.selectedItemCount, + generationEnvelopeExcludedItems: envelope.diagnostics.excludedItemCount, + documentDiversityPreserved: selectedDocumentCount === Math.min(aggregate.selectedGroups.length, availableDocumentCount), + selectedDocumentCount, + }, + failuresByKind, + ...recommendation, + }; + return { + summary, + failures: failures.sort((a, b) => a.kind.localeCompare(b.kind) || a.scenarioId.localeCompare(b.scenarioId) || a.expectationId.localeCompare(b.expectationId)), + scenarioResults: scenarioResults.sort((a, b) => a.id.localeCompare(b.id)), + }; +} + +export function evaluationArtifactDirectory(runId: string = MATERIALITY_EVALUATION_RUN_ID): string { + if (!/^[a-z0-9][a-z0-9._-]*$/i.test(runId)) throw new Error("Evaluation run ID must be filesystem-safe."); + return resolve(process.cwd(), MATERIALITY_EVALUATION_ARTIFACT_ROOT, runId); +} + +function percentage(value: number): string { + return `${(value * 100).toFixed(1)}%`; +} + +export function renderMaterialityEvaluationArtifacts(result: MaterialityEvaluationResult): { + summaryJson: string; + failuresJson: string; + evaluationMarkdown: string; +} { + const { summary } = result; + const topFailures = Object.entries(summary.failuresByKind).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1]); + const evaluationMarkdown = [ + "# Founder Weekly Review Materiality Evaluation", + "", + `Fixture: \`${summary.fixtureVersion}\``, + `Scenarios: ${summary.scenarioCount} (${summary.multiChunkScenarioCount} multi-chunk; ${summary.largeDocumentScenarioCount} large-document)`, + "", + "## What worked", + "", + `- Material recall: ${percentage(summary.materiality.materialRecall)}; exact category accuracy: ${percentage(summary.materiality.categoryAccuracy)}.`, + `- Condensation: ${summary.condensation.rawChangedRecords} raw records -> ${summary.condensation.groups} groups -> ${summary.condensation.condensedEvidenceItems} structurally selected items.`, + `- Document diversity preserved: ${summary.budget.documentDiversityPreserved ? "yes" : "no"}.`, + "", + "## Where deterministic materiality fails", + "", + `- Uncertain rate: ${percentage(summary.materiality.uncertainRate)}.`, + `- False-material rate: ${percentage(summary.materiality.falseMaterialRate)}.`, + `- Missed-material rate: ${percentage(summary.materiality.missedMaterialRate)}.`, + `- Uncertain material/non-material: ${summary.materiality.uncertainMaterialCount}/${summary.materiality.uncertainNonMaterialCount}.`, + "", + "## Where alignment fails", + "", + `- Alignment miss rate: ${percentage(summary.alignment.alignmentMissRate)}.`, + `- False-match rate: ${percentage(summary.alignment.falseMatchRate)}.`, + `- Intended correspondences represented as added + removed: ${summary.alignment.intendedModifiedAsAddedRemoved}.`, + "", + "## Budget behavior", + "", + `- Structural group budget truncated: ${summary.budget.groupBudgetTruncated ? "yes" : "no"} (${summary.budget.truncatedGroupCount} groups).`, + `- Generation envelope selected/excluded: ${summary.budget.generationEnvelopeSelectedItems}/${summary.budget.generationEnvelopeExcludedItems}.`, + `- Serialized prompt evidence: ${summary.condensation.serializedPromptCharacters} characters; envelope truncation: ${summary.budget.generationEnvelopeTruncated ? "yes" : "no"}.`, + "", + "## Failure concentration", + "", + ...(topFailures.length ? topFailures.map(([kind, count]) => `- ${kind}: ${count}`) : ["- No recorded failures."]), + "", + "## Recommendation", + "", + `**${summary.recommendation}. ${summary.recommendationLabel}.**`, + "", + "Materiality exceeds the uncertainty/false-material guidance threshold, while alignment misses remain below the 5% lower guidance threshold and cluster in split, merge, and one renamed heavy rewrite. The next investment should therefore address semantic interpretation first.", + "", + "## Metric definitions", + "", + "- False material counts any retained prompt-facing group mapped to a non-material expectation, plus unexpected groups, divided by non-material ground-truth opportunities.", + "- Uncertain rate is uncertain groups divided by all observed groups; uncertain material and uncertain non-material remain separate buckets.", + "- Alignment miss rate is unrecovered intended modified/unchanged/split/merge relations divided by all such intended relations.", + "- Reduction ratio is bounded raw audit excerpt characters divided by structurally selected condensed evidence characters.", + "", + "Generated artifacts contain synthetic fixture text only; no provider was invoked.", + "", + ].join("\n"); + return { + summaryJson: `${JSON.stringify({ summary, scenarios: result.scenarioResults }, null, 2)}\n`, + failuresJson: `${JSON.stringify({ fixtureVersion: summary.fixtureVersion, failures: result.failures }, null, 2)}\n`, + evaluationMarkdown, + }; +} + +export async function writeMaterialityEvaluationArtifacts( + result: MaterialityEvaluationResult, + runId: string = MATERIALITY_EVALUATION_RUN_ID +): Promise<{ directory: string; summary: string; failures: string; evaluation: string }> { + const directory = evaluationArtifactDirectory(runId); + const artifacts = renderMaterialityEvaluationArtifacts(result); + await mkdir(directory, { recursive: true }); + const paths = { + directory, + summary: resolve(directory, "summary.json"), + failures: resolve(directory, "failures.json"), + evaluation: resolve(directory, "evaluation.md"), + }; + await Promise.all([ + writeFile(paths.summary, artifacts.summaryJson, "utf8"), + writeFile(paths.failures, artifacts.failuresJson, "utf8"), + writeFile(paths.evaluation, artifacts.evaluationMarkdown, "utf8"), + ]); + return paths; +} + +async function main(): Promise { + const result = runMaterialityEvaluation(); + const artifacts = await writeMaterialityEvaluationArtifacts(result, process.env.FWR_MATERIALITY_EVALUATION_RUN_ID); + console.log(JSON.stringify({ ...result.summary, artifactDirectory: artifacts.directory })); +} + +if (process.argv[1]?.replace(/\\/g, "/").endsWith("/founder-weekly-review-materiality-evaluation.ts")) { + main().catch(error => { + console.error(error instanceof Error ? error.message : "Materiality evaluation failed."); + process.exitCode = 1; + }); +} From d5f2994e0cd7fdfe4644bd1a92ff51ab6cbfef56 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 8 Aug 2026 01:07:26 +0800 Subject: [PATCH 22/29] feat(founder-weekly-review): add optional materiality analyzer --- ...hange-materiality-analyzer-adapter.test.ts | 85 +++ ...cument-change-materiality-analyzer.test.ts | 276 +++++++++ apps/web/src/lib/llm/generate.ts | 8 +- apps/web/src/lib/llm/types.ts | 4 + .../document-change-materiality-analyzer.ts | 78 +++ .../evidence-collector.ts | 9 +- .../src/founder-weekly-review/contracts.ts | 32 ++ .../document-change-materiality-analyzer.ts | 544 ++++++++++++++++++ .../document-change-materiality.ts | 4 + .../founder-weekly-review/document-change.ts | 9 +- .../founder-weekly-review/evidence-service.ts | 11 +- .../src/founder-weekly-review/index.ts | 1 + 12 files changed, 1055 insertions(+), 6 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts create mode 100644 apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts create mode 100644 packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts new file mode 100644 index 000000000..9e9159292 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts @@ -0,0 +1,85 @@ +jest.mock("~/lib/llm", () => { + class LlmCapabilityUnavailableError extends Error {} + return { + generateStructuredWithMetadata: jest.fn(), + LlmCapabilityUnavailableError, + }; +}); + +import { + type DocumentChangeMaterialityAnalysisInput, +} from "@launchstack/features/founder-weekly-review"; +import { generateStructuredWithMetadata } from "~/lib/llm"; +import { z } from "zod"; + +import { + ProviderDocumentChangeMaterialityAnalyzer, + createConfiguredDocumentChangeMaterialityAnalyzer, +} from "~/server/founder-weekly-review/document-change-materiality-analyzer"; + +const mockGenerateStructuredWithMetadata = jest.mocked(generateStructuredWithMetadata); + +const input: DocumentChangeMaterialityAnalysisInput = { + groupId: "document_change_group:synthetic", + documentTitle: "Synthetic plan", + structurePath: "/ownership", + structureTitle: "Ownership", + deterministicCategory: "uncertain", + deterministicConfidence: "uncertain", + deterministicSignals: ["no_strong_deterministic_signal"], + changes: [{ + changeType: "modified", + previousExcerpt: "Product owns telemetry.", + currentExcerpt: "Telemetry is owned by Product.", + alignmentMethod: "structure_path", + }], +}; + +describe("document-change materiality provider adapter", () => { + const originalEnabled = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + + afterEach(() => { + mockGenerateStructuredWithMetadata.mockReset(); + if (originalEnabled === undefined) delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + else process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = originalEnabled; + }); + + it("requires explicit production opt-in instead of treating credentials as consent", () => { + delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + expect(createConfiguredDocumentChangeMaterialityAnalyzer()).toBeUndefined(); + expect(mockGenerateStructuredWithMetadata).not.toHaveBeenCalled(); + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; + expect(createConfiguredDocumentChangeMaterialityAnalyzer()).toBeInstanceOf(ProviderDocumentChangeMaterialityAnalyzer); + }); + + it("uses bounded smallExtraction structured generation and returns safe metadata", async () => { + mockGenerateStructuredWithMetadata.mockResolvedValue({ + object: { disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.94 }, + metadata: { provider: "openai", model: "fixture-model", capability: "smallExtraction", providerRequestId: "must-not-propagate" }, + }); + const result = await new ProviderDocumentChangeMaterialityAnalyzer().analyze(input); + expect(mockGenerateStructuredWithMetadata).toHaveBeenCalledWith(expect.objectContaining({ + capability: "smallExtraction", + schemaName: "document_change_materiality", + timeoutMs: 15_000, + maxOutputTokens: 512, + })); + expect(mockGenerateStructuredWithMetadata.mock.calls[0]![0].prompt).toContain("Product owns telemetry."); + expect(result.metadata).toEqual({ provider: "openai", model: "fixture-model", promptVersion: "document-change-materiality/v1" }); + expect(result.metadata).not.toHaveProperty("providerRequestId"); + }); + + it("maps provider failures to optional-unavailable fallback errors", async () => { + mockGenerateStructuredWithMetadata.mockRejectedValue(new Error("rate limited")); + await expect(new ProviderDocumentChangeMaterialityAnalyzer().analyze(input)).rejects.toEqual( + expect.objectContaining({ code: "unavailable" }), + ); + }); + + it("distinguishes invalid structured output from provider unavailability", async () => { + mockGenerateStructuredWithMetadata.mockRejectedValue(new z.ZodError([])); + await expect(new ProviderDocumentChangeMaterialityAnalyzer().analyze(input)).rejects.toEqual( + expect.objectContaining({ code: "invalid" }), + ); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts new file mode 100644 index 000000000..3e8ff3818 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts @@ -0,0 +1,276 @@ +import { createHash } from "node:crypto"; + +import { + DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS, + DocumentChangeMaterialityAnalyzerError, + FounderWeeklyReviewEvidenceSnapshotSchema, + buildDocumentChangeMaterialityAnalysisInput, + buildFounderWeeklyReviewEvidenceDigest, + buildFounderWeeklyReviewPrompt, + buildRawDocumentChanges, + groupRawDocumentChanges, + materializeDocumentChanges, + materializeDocumentChangesWithAnalyzer, + selectDocumentChangeGroupsForMaterialityAnalysis, + shouldAnalyzeDocumentChangeGroup, + type AnalyzedDocumentChangeGroup, + type ChunkAlignment, + type DocumentChangeMaterialityAnalysisInput, + type DocumentChangeMaterialityAnalyzer, + type VersionChunk, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +const pair = (documentId = 1n, day = 2): VersionPair => ({ + documentId, + documentTitle: `Synthetic operating plan ${documentId}`, + documentCategory: "Operating plan", + previousVersionId: 1, + previousVersionNumber: 1, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), + currentVersionId: 2, + currentVersionNumber: 2, + currentCreatedAt: new Date(`2026-02-${String(day).padStart(2, "0")}T00:00:00.000Z`), + currentChangelog: null, +}); + +const chunk = (id: number, versionId: bigint, documentId: bigint, content: string, path = "/plan"): VersionChunk => ({ + chunkId: id, + versionId, + documentId, + content, + contentHash: null, + structureId: BigInt(id), + structurePath: path, + structureTitle: path.slice(1), + structureOrdering: id, + pageNumber: 1, + lineStart: id * 10, + lineEnd: id * 10 + 5, +}); + +const modified = (id: number, documentId: bigint, before: string, after: string, path = "/plan"): ChunkAlignment => ({ + changeType: "modified", + previousChunk: chunk(id, 1n, documentId, before, path), + currentChunk: chunk(10_000 + id, 2n, documentId, after, path), + alignmentMethod: "structure_path", +}); + +function input(documentId: bigint, alignments: ChunkAlignment[], day = 2) { + return { pair: pair(documentId, day), alignments }; +} + +function analyzed(documentId: bigint, alignments: ChunkAlignment[], day = 2): AnalyzedDocumentChangeGroup { + return materializeDocumentChanges([input(documentId, alignments, day)]).analyzedGroups[0]!; +} + +class FakeAnalyzer implements DocumentChangeMaterialityAnalyzer { + readonly calls: DocumentChangeMaterialityAnalysisInput[] = []; + constructor(private readonly implementation: (input: DocumentChangeMaterialityAnalysisInput) => unknown | Promise) {} + async analyze(value: DocumentChangeMaterialityAnalysisInput) { + this.calls.push(value); + return { + result: await this.implementation(value), + metadata: { provider: "fixture", model: "fixture-v1", promptVersion: "document-change-materiality/v1" }, + }; + } +} + +const materialResult = (summary = "The operating meaning changed.") => ({ + disposition: "material" as const, + category: "scope_change" as const, + summary, + confidence: 0.91, +}); + +describe("optional document-change materiality analyzer", () => { + it("uses a conservative semantic-risk gate and bypasses simple factual deltas", () => { + const uncertainSingle = analyzed(1n, [modified(1, 1n, "Telemetry is reliable.", "Reliability is a property of telemetry.")]); + const uncertainMulti = analyzed(2n, [ + modified(1, 2n, "Telemetry is reliable.", "Reliability is a property of telemetry."), + modified(2, 2n, "Retries are automatic.", "Automatic retries remain enabled."), + ]); + const replacement = analyzed(3n, [ + { changeType: "removed", previousChunk: chunk(1, 1n, 3n, "The pilot serves invited accounts."), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(2, 2n, 3n, "The rollout continues with invited accounts."), alignmentMethod: "unmatched" }, + ]); + const complexStrong = analyzed(4n, [ + modified(1, 4n, "Product owns telemetry.", "Platform owns telemetry."), + modified(2, 4n, "Retries follow the established operating process.", "The operating process for retries remains established."), + ]); + expect(shouldAnalyzeDocumentChangeGroup(uncertainSingle)).toMatchObject({ eligible: true, reason: "uncertain_single_change" }); + expect(shouldAnalyzeDocumentChangeGroup(uncertainMulti)).toMatchObject({ eligible: true, reason: "uncertain_multi_change" }); + expect(shouldAnalyzeDocumentChangeGroup(replacement)).toMatchObject({ eligible: true }); + expect(shouldAnalyzeDocumentChangeGroup(complexStrong)).toMatchObject({ eligible: true, reason: "complex_multi_change" }); + for (const [before, after] of [ + ["Product owns telemetry.", "Platform owns telemetry."], + ["Launch is planned for Q3.", "Launch is planned for Q4."], + ["Conversion is 10%.", "Conversion is 25%."], + ]) { + expect(shouldAnalyzeDocumentChangeGroup(analyzed(9n, [modified(1, 9n, before!, after!)]))).toMatchObject({ eligible: false }); + } + }); + + it("selects at most four calls deterministically while retaining document diversity", () => { + const groups = [1n, 2n, 3n, 4n, 5n].flatMap((documentId, index) => [ + analyzed(documentId, [modified(1, documentId, "The workflow remains reliable.", "Reliability remains part of the workflow.")], index + 2), + analyzed(documentId, [modified(2, documentId, "The plan remains durable.", "Durability remains in the plan.", "/other")], index + 2), + ]); + const forward = selectDocumentChangeGroupsForMaterialityAnalysis(groups); + const reverse = selectDocumentChangeGroupsForMaterialityAnalysis([...groups].reverse()); + expect(forward.selected).toHaveLength(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview); + expect(new Set(forward.selected.map(group => group.group.documentId.toString())).size).toBe(4); + expect(reverse.selected.map(group => group.group.groupId)).toEqual(forward.selected.map(group => group.group.groupId)); + expect(forward.skipped).toHaveLength(6); + }); + + it("bounds canonical inputs and keeps only changed fragment data", () => { + const value = "A realistic synthetic operating paragraph. ".repeat(200); + const group = analyzed(1n, [modified(1, 1n, value, `${value} revised`)]); + const built = buildDocumentChangeMaterialityAnalysisInput(group); + expect(built.canonicalCharacterCount).toBeLessThanOrEqual(8_000); + expect(built.truncated).toBe(true); + expect(built.input.changes).toHaveLength(1); + expect(JSON.stringify(built.input)).not.toContain("customer_feedback"); + expect(built.inputDigest).toMatch(/^[a-f0-9]{64}$/); + }); + + it("freezes material analysis, validates copied key points, and preserves provider-owned provenance boundaries", async () => { + const analyzer = new FakeAnalyzer(() => ({ + ...materialResult("Rollout scope expanded."), + beforeKeyPoint: "The pilot is limited to invited accounts.", + afterKeyPoint: "Invited accounts remain the pilot audience.", + })); + const result = await materializeDocumentChangesWithAnalyzer([ + input(1n, [modified(1, 1n, "The pilot is limited to invited accounts.", "Invited accounts remain the pilot audience.")]), + ], analyzer); + expect(analyzer.calls).toHaveLength(1); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ + sourceType: "document_change", + excerpt: expect.stringContaining("Rollout scope expanded."), + metadata: expect.objectContaining({ materialityMethod: "llm", category: "scope_change" }), + }); + expect(result.items[0]!.excerpt).toContain("The pilot is limited to invited accounts."); + expect(result.audit.groups[0]!.analysis).toEqual(expect.objectContaining({ + disposition: "material", + analysisProvider: "fixture", + analysisModel: "fixture-v1", + analysisInputDigest: expect.stringMatching(/^[a-f0-9]{64}$/), + })); + expect(JSON.stringify(result.audit)).not.toContain("rawProviderPayload"); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: result.items, + sourceWarnings: [], + documentChangeAudit: result.audit, + }); + const firstPrompt = buildFounderWeeklyReviewPrompt(snapshot); + const retryPrompt = buildFounderWeeklyReviewPrompt(snapshot); + expect(retryPrompt).toBe(firstPrompt); + expect(firstPrompt).not.toContain("analysisInputDigest"); + expect(analyzer.calls).toHaveLength(1); + }); + + it("keeps non-material groups audit-only and retains uncertain groups conservatively", async () => { + const nonMaterial = new FakeAnalyzer(() => ({ + disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.95, + })); + const pairInput = input(1n, [modified(1, 1n, "Product owns telemetry.", "Telemetry is owned by Product.")]); + const filtered = await materializeDocumentChangesWithAnalyzer([pairInput], nonMaterial); + expect(filtered.items).toHaveLength(0); + expect(filtered.audit.rawChanges).toHaveLength(1); + expect(filtered.audit.groups[0]).toMatchObject({ evidenceSourceId: null, analysis: { disposition: "non_material" } }); + + const uncertain = new FakeAnalyzer(() => ({ + disposition: "uncertain", category: "uncertain", summary: "Meaning may have changed.", confidence: 0.4, + })); + const retained = await materializeDocumentChangesWithAnalyzer([pairInput], uncertain); + expect(retained.items).toHaveLength(1); + expect(retained.items[0]!.metadata).toMatchObject({ materialityMethod: "deterministic", category: "uncertain" }); + expect(retained.audit.groups[0]!.analysis?.disposition).toBe("uncertain"); + }); + + it.each([ + ["unavailable", new DocumentChangeMaterialityAnalyzerError("unavailable", "offline"), "materiality_analyzer_unavailable"], + ["timeout", new DocumentChangeMaterialityAnalyzerError("timeout", "slow"), "materiality_analysis_timeout"], + ["invalid confidence", { disposition: "material", category: "scope_change", summary: "x", confidence: 2 }, "materiality_result_invalid"], + ["oversized summary", materialResult("x".repeat(321)), "materiality_result_invalid"], + ])("falls back deterministically for %s", async (_name, response, warningCode) => { + const pairInput = input(1n, [modified(1, 1n, "Telemetry is reliable.", "Reliability is a property of telemetry.")]); + const deterministic = materializeDocumentChanges([pairInput]); + const analyzer = new FakeAnalyzer(() => { + if (response instanceof Error) throw response; + return response; + }); + const result = await materializeDocumentChangesWithAnalyzer([pairInput], analyzer); + expect(result.items).toEqual(deterministic.items); + expect(result.warnings.map(warning => warning.code)).toEqual(expect.arrayContaining([warningCode, "materiality_analysis_partial"])); + }); + + it("isolates partial failures and never exceeds concurrency or call budgets", async () => { + let active = 0; + let maximumActive = 0; + const analyzer = new FakeAnalyzer(async value => { + active++; + maximumActive = Math.max(maximumActive, active); + await Promise.resolve(); + active--; + if (value.documentTitle?.endsWith("2")) throw new Error("synthetic failure"); + return materialResult(); + }); + const inputs = [1n, 2n, 3n, 4n, 5n].map((documentId, index) => input(documentId, [ + modified(1, documentId, "The workflow remains reliable.", "Reliability remains part of the workflow."), + ], index + 2)); + const result = await materializeDocumentChangesWithAnalyzer(inputs, analyzer); + expect(analyzer.calls).toHaveLength(4); + expect(maximumActive).toBeLessThanOrEqual(2); + expect(result.analyzerDiagnostics).toMatchObject({ selectedAnalysisCount: 4, skippedByCallBudgetCount: 1, failedAnalysisCount: 1 }); + expect(result.warnings.map(warning => warning.code)).toEqual(expect.arrayContaining([ + "materiality_analysis_budget_truncated", "materiality_analysis_partial", "materiality_analyzer_unavailable", + ])); + }); + + it("keeps evidence digest distinct from the exact prompt consequence", async () => { + const pairInput = input(1n, [modified(1, 1n, "Product owns telemetry.", "Telemetry is owned by Product.")]); + const deterministic = materializeDocumentChanges([pairInput]); + const analyzedResult = await materializeDocumentChangesWithAnalyzer([pairInput], new FakeAnalyzer(() => ({ + disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.94, + }))); + const snapshot = (items: typeof deterministic.items, audit: typeof deterministic.audit) => FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], + documentChangeAudit: audit, + }); + const before = snapshot(deterministic.items, deterministic.audit); + const after = snapshot(analyzedResult.items, analyzedResult.audit); + expect(buildFounderWeeklyReviewEvidenceDigest(after)).not.toBe(buildFounderWeeklyReviewEvidenceDigest(before)); + const promptHash = (value: typeof before) => createHash("sha256").update(buildFounderWeeklyReviewPrompt(value)).digest("hex"); + expect(promptHash(after)).not.toBe(promptHash(before)); + expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(after)).toEqual(after); + }); + + it("does not invoke an analyzer when the optional dependency is absent", async () => { + const pairInput = input(1n, [modified(1, 1n, "Telemetry is reliable.", "Reliability is a property of telemetry.")]); + const deterministic = materializeDocumentChanges([pairInput]); + const optional = await materializeDocumentChangesWithAnalyzer([pairInput]); + expect(optional.items).toEqual(deterministic.items); + expect(optional.audit).toEqual(deterministic.audit); + expect(optional.warnings).toEqual(deterministic.warnings); + }); + + it("keeps raw/group construction stable before semantic analysis", () => { + const versionPair = pair(); + const alignment = modified(1, 1n, "Before", "After"); + const raw = buildRawDocumentChanges(versionPair, [alignment]); + const grouped = groupRawDocumentChanges(versionPair, raw.rawChanges); + expect(grouped.groups[0]!.rawChanges.map(change => change.rawChangeId)).toEqual(raw.rawChanges.map(change => change.rawChangeId)); + }); +}); diff --git a/apps/web/src/lib/llm/generate.ts b/apps/web/src/lib/llm/generate.ts index ea30da15f..e4bd6eb11 100644 --- a/apps/web/src/lib/llm/generate.ts +++ b/apps/web/src/lib/llm/generate.ts @@ -77,8 +77,12 @@ export async function generateStructuredWithMetadata( const common = { model: resolved.model, ...(resolved.temperature === undefined ? {} : { temperature: resolved.temperature }), - ...(input.capability === "founderWeeklyReview" ? { maxOutputTokens: 1800 } : {}), - ...(resolved.structuredOutputMode === "json_object" ? { abortSignal: AbortSignal.timeout(90_000) } : {}), + ...(input.maxOutputTokens !== undefined + ? { maxOutputTokens: input.maxOutputTokens } + : input.capability === "founderWeeklyReview" ? { maxOutputTokens: 1800 } : {}), + ...((input.timeoutMs !== undefined || resolved.structuredOutputMode === "json_object") + ? { abortSignal: AbortSignal.timeout(input.timeoutMs ?? 90_000) } + : {}), prompt: input.prompt, }; // Moonshot/Kimi supports Chat Completions JSON-object mode, not the diff --git a/apps/web/src/lib/llm/types.ts b/apps/web/src/lib/llm/types.ts index c8e7fadb0..1b048c548 100644 --- a/apps/web/src/lib/llm/types.ts +++ b/apps/web/src/lib/llm/types.ts @@ -106,6 +106,10 @@ export interface GenerateStructuredInput { schemaName?: string; /** Bounded FWR-only attempt label for safe operational logging. */ generationPhase?: "initial" | "semantic-repair"; + /** Optional caller-owned local timeout. Provider defaults remain unchanged when omitted. */ + timeoutMs?: number; + /** Optional caller-owned output ceiling. Provider defaults remain unchanged when omitted. */ + maxOutputTokens?: number; } /** Resolved model and provider response details that are safe to persist for replay. */ diff --git a/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts new file mode 100644 index 000000000..444a4c632 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts @@ -0,0 +1,78 @@ +import { + DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + DocumentChangeMaterialityAnalysisResultSchema, + DocumentChangeMaterialityAnalyzerError, + type DocumentChangeMaterialityAnalysisInput, + type DocumentChangeMaterialityAnalyzer, +} from "@launchstack/features/founder-weekly-review"; + +import { + LlmCapabilityUnavailableError, + generateStructuredWithMetadata, + type Provider, +} from "~/lib/llm"; +import { z } from "zod"; + +const MATERIALITY_SYSTEM_PROMPT = `You classify one already-diffed document section. Answer only this question: did the underlying business meaning materially change? + +Pay special attention to ownership, status or shipping, dates and deadlines, metrics, requirements or modality, negation, blockers or risks, scope, priority, customer or rollout scope, and meaning-preserving paraphrases or editorial rewrites. + +Do not classify a rewrite as material merely because wording changed. +Do not classify a change as non-material merely because wording is similar. +Small factual changes can be highly material. + +Use disposition=non_material only when the supplied fragments support meaning-preserving editorial or formatting change. Use uncertain when the fragments are insufficient. Keep summary factual and concise. beforeKeyPoint and afterKeyPoint, when supplied, must be copied verbatim from the corresponding input excerpts. Never invent source identifiers or provenance.`; + +export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_TIMEOUT_MS = 15_000; + +function promptFor(input: DocumentChangeMaterialityAnalysisInput): string { + return [ + "Classify this bounded change group.", + JSON.stringify(input), + ].join("\n\n"); +} + +export class ProviderDocumentChangeMaterialityAnalyzer implements DocumentChangeMaterialityAnalyzer { + constructor(private readonly forceProvider?: Provider) {} + + async analyze(input: DocumentChangeMaterialityAnalysisInput) { + try { + const generated = await generateStructuredWithMetadata({ + capability: "smallExtraction", + system: MATERIALITY_SYSTEM_PROMPT, + prompt: promptFor(input), + schema: DocumentChangeMaterialityAnalysisResultSchema, + schemaName: "document_change_materiality", + ...(this.forceProvider ? { forceProvider: this.forceProvider } : {}), + timeoutMs: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_TIMEOUT_MS, + maxOutputTokens: 512, + }); + return { + result: generated.object, + metadata: { + provider: generated.metadata.provider, + model: generated.metadata.model, + promptVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + }, + }; + } catch (error) { + if (error instanceof z.ZodError || error instanceof Error && /NoObjectGenerated|JSONParse|schema validation/i.test(`${error.name} ${error.message}`)) { + throw new DocumentChangeMaterialityAnalyzerError("invalid", "Document-change materiality output failed structured validation."); + } + if (error instanceof LlmCapabilityUnavailableError) { + throw new DocumentChangeMaterialityAnalyzerError("unavailable", error.message); + } + if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError" || /timed?\s*out/i.test(error.message))) { + throw new DocumentChangeMaterialityAnalyzerError("timeout", "Document-change materiality analysis timed out."); + } + throw new DocumentChangeMaterialityAnalyzerError("unavailable", "Document-change materiality analysis was unavailable."); + } + } +} + +/** Production collection opts in explicitly; credentials alone never trigger analyzer calls. */ +export function createConfiguredDocumentChangeMaterialityAnalyzer(): DocumentChangeMaterialityAnalyzer | undefined { + return process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED === "true" + ? new ProviderDocumentChangeMaterialityAnalyzer() + : undefined; +} diff --git a/apps/web/src/server/founder-weekly-review/evidence-collector.ts b/apps/web/src/server/founder-weekly-review/evidence-collector.ts index a09f2364b..732cae2e8 100644 --- a/apps/web/src/server/founder-weekly-review/evidence-collector.ts +++ b/apps/web/src/server/founder-weekly-review/evidence-collector.ts @@ -6,6 +6,7 @@ import { } from "@launchstack/features/founder-weekly-review"; import { FounderWeeklyReviewDocumentVersionStore } from "./document-version-chunks"; import { StrictCurrentWorkspaceDocumentStore } from "./workspace-document-store"; +import { createConfiguredDocumentChangeMaterialityAnalyzer } from "./document-change-materiality-analyzer"; export interface FounderWeeklyReviewEvidenceCollector { collectFounderWeeklyReviewEvidence(input: { @@ -52,7 +53,13 @@ export class CanonicalFounderWeeklyReviewEvidenceCollector implements FounderWee actor: { externalUserId: string }; requestKey: string; }): Promise { - const snapshot = await (this.service ??= new FounderWeeklyReviewEvidenceService(undefined, undefined, { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore() }, new StrictCurrentWorkspaceDocumentStore())).collectFounderWeeklyReviewEvidence({ + const snapshot = await (this.service ??= new FounderWeeklyReviewEvidenceService( + undefined, + undefined, + { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore() }, + new StrictCurrentWorkspaceDocumentStore(), + createConfiguredDocumentChangeMaterialityAnalyzer(), + )).collectFounderWeeklyReviewEvidence({ companyId: input.companyId, reportingPeriod: input.reportingPeriod, workspaceTimezone: input.workspaceTimezone, diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts index bf90d4f67..4c26f6e94 100644 --- a/packages/features/src/founder-weekly-review/contracts.ts +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -113,6 +113,37 @@ export const DocumentChangeCategorySchema = z.enum([ ]); export const DeterministicMaterialityConfidenceSchema = z.enum(["strong", "moderate", "uncertain"]); +export const DocumentChangeMaterialityDispositionSchema = z.enum([ + "material", + "non_material", + "uncertain", +]); + +export const DocumentChangeMaterialityAnalysisSnapshotSchema = z.object({ + disposition: DocumentChangeMaterialityDispositionSchema, + category: DocumentChangeCategorySchema, + confidence: z.number().min(0).max(1), + summary: z.string().min(1).max(320), + beforeKeyPoint: z.string().min(1).max(240).optional(), + afterKeyPoint: z.string().min(1).max(240).optional(), + analysisMethod: z.literal("llm"), + analysisPromptVersion: z.string().min(1).max(128), + analysisProvider: z.string().min(1).max(128).optional(), + analysisModel: z.string().min(1).max(256).optional(), + analysisInputDigest: z.string().regex(/^[a-f0-9]{64}$/), + analysisResultSchemaVersion: z.string().min(1).max(128), +}).strict().superRefine((result, context) => { + if (result.disposition === "uncertain" && result.category !== "uncertain") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Uncertain analysis requires uncertain category.", path: ["category"] }); + } + if (result.disposition === "material" && result.category === "editorial_rewrite") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Material analysis cannot use editorial_rewrite.", path: ["category"] }); + } +}); +export type DocumentChangeMaterialityAnalysisSnapshot = z.infer< + typeof DocumentChangeMaterialityAnalysisSnapshotSchema +>; + export const RawDocumentChangeSnapshotSchema = z.object({ rawChangeId: z.string().min(1).max(128), changeType: z.enum(["added", "removed", "modified"]), @@ -157,6 +188,7 @@ export const DocumentChangeGroupSnapshotSchema = z.object({ signals: z.array(z.string().min(1).max(64)).max(20), materialityMethod: z.literal("deterministic"), materialityVersion: z.string().min(1).max(128), + analysis: DocumentChangeMaterialityAnalysisSnapshotSchema.optional(), }).strict(); export type DocumentChangeGroupSnapshot = z.infer; diff --git a/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts b/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts new file mode 100644 index 000000000..74482205a --- /dev/null +++ b/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts @@ -0,0 +1,544 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +import type { + DocumentChangeAuditSnapshot, + DocumentChangeMaterialityAnalysisSnapshot, + FounderWeeklyReviewEvidenceItem, +} from "./contracts"; +import type { + DocumentChangeGroup, + DocumentChangePairInput, + DocumentChangeProcessingWarning, + RawDocumentChange, +} from "./document-change"; +import { + DOCUMENT_CHANGE_CATEGORIES, + DOCUMENT_CHANGE_MATERIALITY_VERSION, + buildCondensedDocumentChangeEvidence, + documentChangeCategoryPriority, + documentChangeGroupSourceId, + materializeDocumentChanges, + selectMaterialDocumentChangeGroups, + type AnalyzedDocumentChangeGroup, + type DeterministicMaterialChangeResult, + type DeterministicMaterialityConfidence, + type DocumentChangeCategory, +} from "./document-change-materiality"; + +export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION = + "document-change-materiality/v1" as const; +export const DOCUMENT_CHANGE_MATERIALITY_ANALYSIS_RESULT_VERSION = + "document-change-materiality-result/v1" as const; +export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS = Object.freeze({ + callsPerReview: 4, + concurrency: 2, + rawChangesPerGroup: 16, + canonicalInputCharacters: 8_000, + excerptCharactersPerSide: 150, + summaryCharacters: 320, + keyPointCharacters: 240, +}); + +export const DocumentChangeMaterialityAnalysisResultSchema = z.object({ + disposition: z.enum(["material", "non_material", "uncertain"]), + category: z.enum(DOCUMENT_CHANGE_CATEGORIES), + summary: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.summaryCharacters), + beforeKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters).optional(), + afterKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters).optional(), + confidence: z.number().min(0).max(1), +}).strict().superRefine((result, context) => { + if (result.disposition === "uncertain" && result.category !== "uncertain") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Uncertain disposition requires uncertain category.", path: ["category"] }); + } + if (result.disposition === "material" && result.category === "editorial_rewrite") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Material disposition cannot use editorial_rewrite.", path: ["category"] }); + } +}); + +export type DocumentChangeMaterialityAnalysisResult = z.infer< + typeof DocumentChangeMaterialityAnalysisResultSchema +>; + +export type DocumentChangeMaterialityAnalysisInput = { + groupId: string; + documentTitle?: string; + structurePath?: string | null; + structureTitle?: string | null; + deterministicCategory: DocumentChangeCategory; + deterministicConfidence: DeterministicMaterialityConfidence; + deterministicSignals: readonly string[]; + changes: readonly { + changeType: RawDocumentChange["changeType"]; + previousExcerpt?: string; + currentExcerpt?: string; + alignmentMethod: RawDocumentChange["alignmentMethod"]; + }[]; +}; + +export type DocumentChangeMaterialityAnalyzerMetadata = { + provider?: string; + model?: string; + promptVersion?: string; +}; + +export interface DocumentChangeMaterialityAnalyzer { + analyze(input: DocumentChangeMaterialityAnalysisInput): Promise<{ + result: unknown; + metadata?: DocumentChangeMaterialityAnalyzerMetadata; + }>; +} + +export class DocumentChangeMaterialityAnalyzerError extends Error { + constructor( + readonly code: "unavailable" | "timeout" | "invalid", + message: string, + ) { + super(message); + this.name = "DocumentChangeMaterialityAnalyzerError"; + } +} + +export type DocumentChangeMaterialityEligibilityReason = + | "uncertain_multi_change" + | "uncertain_single_change" + | "replacement_shaped" + | "complex_multi_change" + | "large_rewrite" + | "moderate_signal" + | "paraphrase_risk"; + +export type DocumentChangeMaterialityEligibility = { + eligible: boolean; + reason?: DocumentChangeMaterialityEligibilityReason; + priority: number; +}; + +export type DocumentChangeMaterialityAnalyzerDiagnostics = { + eligibleGroupCount: number; + selectedAnalysisCount: number; + skippedByCallBudgetCount: number; + inputTruncatedCount: number; + succeededAnalysisCount: number; + failedAnalysisCount: number; + materialCount: number; + nonMaterialCount: number; + uncertainCount: number; +}; + +export type AnalyzedMaterialChangeResult = DeterministicMaterialChangeResult & { + analyzerDiagnostics: DocumentChangeMaterialityAnalyzerDiagnostics; +}; + +type AnalysisCandidate = { + analyzed: AnalyzedDocumentChangeGroup; + eligibility: DocumentChangeMaterialityEligibility; +}; + +type SuccessfulAnalysis = { + analyzed: AnalyzedDocumentChangeGroup; + input: DocumentChangeMaterialityAnalysisInput; + inputDigest: string; + inputTruncated: boolean; + result: DocumentChangeMaterialityAnalysisResult; + metadata: DocumentChangeMaterialityAnalyzerMetadata; +}; + +const SIMPLE_BYPASS_SIGNALS = new Set([ + "ownership_subject_changed", + "status_term_changed", + "date_or_deadline_changed", + "numeric_metric_changed", + "requirement_or_modality_changed", + "negation_changed", + "priority_marker_changed", +]); + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function changedCharacters(group: DocumentChangeGroup): number { + return group.rawChanges.reduce((total, change) => total + + (change.previousNormalizedContent?.length ?? 0) + + (change.currentNormalizedContent?.length ?? 0), 0); +} + +function tokenSet(value: string): Set { + return new Set(value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []); +} + +function lexicalSimilarity(change: RawDocumentChange): number { + const previous = tokenSet(change.previousNormalizedContent ?? ""); + const current = tokenSet(change.currentNormalizedContent ?? ""); + if (previous.size === 0 && current.size === 0) return 1; + const intersection = [...previous].filter(token => current.has(token)).length; + const union = new Set([...previous, ...current]).size; + return union === 0 ? 1 : intersection / union; +} + +function isStrongDeterministicBypass(analyzed: AnalyzedDocumentChangeGroup): boolean { + const { group, materiality } = analyzed; + if (group.rawChanges.length !== 1 || group.rawChanges[0]!.changeType !== "modified") return false; + if (changedCharacters(group) > 700 || materiality.category === "uncertain" || materiality.category === "editorial_rewrite") return false; + return materiality.signals.length > 0 + && materiality.signals.every(signal => SIMPLE_BYPASS_SIGNALS.has(signal)); +} + +/** Pure semantic-risk gate. It intentionally does not use embeddings or changelog text. */ +export function shouldAnalyzeDocumentChangeGroup( + analyzed: AnalyzedDocumentChangeGroup, +): DocumentChangeMaterialityEligibility { + if (isStrongDeterministicBypass(analyzed)) return { eligible: false, priority: Number.MAX_SAFE_INTEGER }; + const { group, materiality } = analyzed; + if (materiality.category === "uncertain" && group.rawChanges.length > 1) { + return { eligible: true, reason: "uncertain_multi_change", priority: 1 }; + } + if (materiality.category === "uncertain") { + return { eligible: true, reason: "uncertain_single_change", priority: 2 }; + } + const types = new Set(group.rawChanges.map(change => change.changeType)); + if (types.has("added") && types.has("removed")) { + return { eligible: true, reason: "replacement_shaped", priority: 3 }; + } + if (group.rawChanges.length > 1) { + return { eligible: true, reason: "complex_multi_change", priority: 4 }; + } + if (changedCharacters(group) >= 1_200) { + return { eligible: true, reason: "large_rewrite", priority: 4 }; + } + if (materiality.confidence === "moderate") { + return { eligible: true, reason: "moderate_signal", priority: 4 }; + } + const only = group.rawChanges[0]; + if (only?.changeType === "modified" && changedCharacters(group) >= 240 && lexicalSimilarity(only) < 0.45) { + return { eligible: true, reason: "paraphrase_risk", priority: 5 }; + } + return { eligible: false, priority: Number.MAX_SAFE_INTEGER }; +} + +function stableCandidateOrder(a: AnalysisCandidate, b: AnalysisCandidate): number { + return a.eligibility.priority - b.eligibility.priority + || b.analyzed.pair.currentCreatedAt.getTime() - a.analyzed.pair.currentCreatedAt.getTime() + || compareOrdinal(a.analyzed.group.structurePath ?? a.analyzed.group.structureTitle ?? "", b.analyzed.group.structurePath ?? b.analyzed.group.structureTitle ?? "") + || compareOrdinal(a.analyzed.group.groupId, b.analyzed.group.groupId); +} + +/** Selects at most four calls, round-robin across documents inside each eligibility tier. */ +export function selectDocumentChangeGroupsForMaterialityAnalysis( + groups: readonly AnalyzedDocumentChangeGroup[], +): { selected: AnalyzedDocumentChangeGroup[]; skipped: AnalyzedDocumentChangeGroup[] } { + const candidates = groups + .map(analyzed => ({ analyzed, eligibility: shouldAnalyzeDocumentChangeGroup(analyzed) })) + .filter((candidate): candidate is AnalysisCandidate => candidate.eligibility.eligible) + .sort(stableCandidateOrder); + const selected: AnalyzedDocumentChangeGroup[] = []; + const selectedIds = new Set(); + for (const priority of [...new Set(candidates.map(candidate => candidate.eligibility.priority))].sort((a, b) => a - b)) { + const tier = candidates.filter(candidate => candidate.eligibility.priority === priority); + const byDocument = new Map(); + for (const candidate of tier) { + const key = candidate.analyzed.group.documentId.toString(); + const values = byDocument.get(key) ?? []; + values.push(candidate); + byDocument.set(key, values); + } + const documents = [...byDocument.keys()].sort((a, b) => BigInt(a) < BigInt(b) ? -1 : BigInt(a) > BigInt(b) ? 1 : 0); + let progress = true; + while (progress && selected.length < DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview) { + progress = false; + for (const documentId of documents) { + const candidate = byDocument.get(documentId)!.find(value => !selectedIds.has(value.analyzed.group.groupId)); + if (!candidate) continue; + selected.push(candidate.analyzed); + selectedIds.add(candidate.analyzed.group.groupId); + progress = true; + if (selected.length >= DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview) break; + } + } + if (selected.length >= DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview) break; + } + return { + selected, + skipped: candidates.filter(candidate => !selectedIds.has(candidate.analyzed.group.groupId)).map(candidate => candidate.analyzed), + }; +} + +function compactExcerpt(value: string | undefined): string | undefined { + if (!value) return undefined; + const normalized = value.replace(/\s+/g, " ").trim(); + return normalized.length <= DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide + ? normalized + : `${normalized.slice(0, DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide - 1)}…`; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value as Record) + .sort(([a], [b]) => compareOrdinal(a, b)) + .map(([key, entry]) => [key, canonicalize(entry)])); + } + return value; +} + +export function buildDocumentChangeMaterialityAnalysisInput(analyzed: AnalyzedDocumentChangeGroup): { + input: DocumentChangeMaterialityAnalysisInput; + inputDigest: string; + canonicalCharacterCount: number; + truncated: boolean; +} { + const changes = analyzed.group.rawChanges + .slice(0, DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.rawChangesPerGroup) + .map(change => ({ + changeType: change.changeType, + ...(compactExcerpt(change.previousChunk?.content) ? { previousExcerpt: compactExcerpt(change.previousChunk?.content) } : {}), + ...(compactExcerpt(change.currentChunk?.content) ? { currentExcerpt: compactExcerpt(change.currentChunk?.content) } : {}), + alignmentMethod: change.alignmentMethod, + })); + const input: DocumentChangeMaterialityAnalysisInput = { + groupId: analyzed.group.groupId, + documentTitle: analyzed.pair.documentTitle.slice(0, 256), + structurePath: analyzed.group.structurePath?.slice(0, 256) ?? null, + structureTitle: analyzed.group.structureTitle?.slice(0, 256) ?? null, + deterministicCategory: analyzed.materiality.category, + deterministicConfidence: analyzed.materiality.confidence, + deterministicSignals: analyzed.materiality.signals.slice(0, 20), + changes, + }; + const canonical = JSON.stringify(canonicalize(input)); + if (canonical.length > DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.canonicalInputCharacters) { + throw new Error("Bounded document-change materiality input exceeded its invariant limit."); + } + return { + input, + inputDigest: createHash("sha256").update(canonical, "utf8").digest("hex"), + canonicalCharacterCount: canonical.length, + truncated: analyzed.group.rawChanges.length > changes.length + || analyzed.group.rawChanges.some(change => + (change.previousChunk?.content.replace(/\s+/g, " ").trim().length ?? 0) > DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide + || (change.currentChunk?.content.replace(/\s+/g, " ").trim().length ?? 0) > DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide), + }; +} + +function confidenceFromAnalysis(value: number): DeterministicMaterialityConfidence { + return value >= 0.8 ? "strong" : value >= 0.55 ? "moderate" : "uncertain"; +} + +function effectiveAnalyzedGroup(success: SuccessfulAnalysis): AnalyzedDocumentChangeGroup | null { + if (success.result.disposition === "non_material") return null; + const category = success.result.disposition === "uncertain" ? "uncertain" : success.result.category; + return { + ...success.analyzed, + materiality: { + category, + priority: documentChangeCategoryPriority(category), + confidence: success.result.disposition === "uncertain" ? "uncertain" : confidenceFromAnalysis(success.result.confidence), + signals: success.result.disposition === "uncertain" + ? ["semantic_analysis_uncertain"] + : ["semantic_analysis_material"], + }, + }; +} + +function normalizedCopied(value: string): string { + return value.replace(/\s+/g, " ").trim().toLocaleLowerCase(); +} + +function verifiedKeyPoint(value: string | undefined, changes: readonly RawDocumentChange[], side: "previousChunk" | "currentChunk"): string | undefined { + if (!value) return undefined; + const candidate = normalizedCopied(value); + return changes.some(change => normalizedCopied(change[side]?.content ?? "").includes(candidate)) ? value : undefined; +} + +function firstCopiedSpan(changes: readonly RawDocumentChange[], side: "previousChunk" | "currentChunk"): string | undefined { + const value = changes.map(change => change[side]?.content).find(content => content?.trim()); + if (!value) return undefined; + const compact = value.replace(/\s+/g, " ").trim(); + return compact.length <= 240 ? compact : `${compact.slice(0, 239)}…`; +} + +function buildAnalyzerEvidence(success: SuccessfulAnalysis, effective: AnalyzedDocumentChangeGroup): FounderWeeklyReviewEvidenceItem { + const deterministic = buildCondensedDocumentChangeEvidence(effective.pair, effective.group, effective.materiality); + const before = verifiedKeyPoint(success.result.beforeKeyPoint, effective.group.rawChanges, "previousChunk") + ?? firstCopiedSpan(effective.group.rawChanges, "previousChunk"); + const after = verifiedKeyPoint(success.result.afterKeyPoint, effective.group.rawChanges, "currentChunk") + ?? firstCopiedSpan(effective.group.rawChanges, "currentChunk"); + const excerpt = [ + success.result.summary, + ...(before ? [`Before: “${before}”`] : []), + ...(after ? [`After: “${after}”`] : []), + ].join("\n\n").slice(0, 1800); + return { + ...deterministic, + excerpt, + metadata: { + ...deterministic.metadata, + category: effective.materiality.category, + materialityMethod: "llm", + materialityConfidence: success.result.confidence, + materialityVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + }, + }; +} + +function analysisSnapshot(success: SuccessfulAnalysis): DocumentChangeMaterialityAnalysisSnapshot { + return { + disposition: success.result.disposition, + category: success.result.category, + confidence: success.result.confidence, + summary: success.result.summary, + ...(success.result.beforeKeyPoint ? { beforeKeyPoint: verifiedKeyPoint(success.result.beforeKeyPoint, success.analyzed.group.rawChanges, "previousChunk") } : {}), + ...(success.result.afterKeyPoint ? { afterKeyPoint: verifiedKeyPoint(success.result.afterKeyPoint, success.analyzed.group.rawChanges, "currentChunk") } : {}), + analysisMethod: "llm", + analysisPromptVersion: success.metadata.promptVersion ?? DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + ...(success.metadata.provider ? { analysisProvider: success.metadata.provider.slice(0, 128) } : {}), + ...(success.metadata.model ? { analysisModel: success.metadata.model.slice(0, 256) } : {}), + analysisInputDigest: success.inputDigest, + analysisResultSchemaVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYSIS_RESULT_VERSION, + }; +} + +function warning(code: DocumentChangeProcessingWarning["code"], message: string): DocumentChangeProcessingWarning { + return { code, message }; +} + +async function executeBounded(values: readonly T[], concurrency: number, task: (value: T) => Promise): Promise[]> { + const results: PromiseSettledResult[] = new Array(values.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, async () => { + while (next < values.length) { + const index = next++; + try { + results[index] = { status: "fulfilled", value: await task(values[index]!) }; + } catch (reason) { + results[index] = { status: "rejected", reason }; + } + } + })); + return results; +} + +/** Optional semantic pass. Omit the analyzer to preserve the exact deterministic-v1 result. */ +export async function materializeDocumentChangesWithAnalyzer( + inputs: readonly DocumentChangePairInput[], + analyzer?: DocumentChangeMaterialityAnalyzer, +): Promise { + const deterministic = materializeDocumentChanges(inputs); + const emptyDiagnostics: DocumentChangeMaterialityAnalyzerDiagnostics = { + eligibleGroupCount: 0, + selectedAnalysisCount: 0, + skippedByCallBudgetCount: 0, + inputTruncatedCount: 0, + succeededAnalysisCount: 0, + failedAnalysisCount: 0, + materialCount: 0, + nonMaterialCount: 0, + uncertainCount: 0, + }; + if (!analyzer) return { ...deterministic, analyzerDiagnostics: emptyDiagnostics }; + + const eligible = deterministic.analyzedGroups.filter(group => shouldAnalyzeDocumentChangeGroup(group).eligible); + const analysisSelection = selectDocumentChangeGroupsForMaterialityAnalysis(deterministic.analyzedGroups); + const prepared = analysisSelection.selected.map(analyzed => ({ analyzed, ...buildDocumentChangeMaterialityAnalysisInput(analyzed) })); + const warnings: DocumentChangeProcessingWarning[] = [...deterministic.warnings]; + if (analysisSelection.skipped.length > 0 || prepared.some(value => value.truncated)) { + warnings.push(warning("materiality_analysis_budget_truncated", "Optional materiality analysis used bounded call or input capacity; remaining groups used deterministic fallback.")); + } + const settled = await executeBounded(prepared, DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.concurrency, async preparedInput => { + const response = await analyzer.analyze(preparedInput.input); + const result = DocumentChangeMaterialityAnalysisResultSchema.parse(response.result); + return { + analyzed: preparedInput.analyzed, + input: preparedInput.input, + inputDigest: preparedInput.inputDigest, + inputTruncated: preparedInput.truncated, + result, + metadata: response.metadata ?? {}, + } satisfies SuccessfulAnalysis; + }); + const successes: SuccessfulAnalysis[] = []; + let invalid = false; + let timeout = false; + let unavailable = false; + for (const result of settled) { + if (result.status === "fulfilled") successes.push(result.value); + else if (result.reason instanceof z.ZodError) invalid = true; + else if (result.reason instanceof DocumentChangeMaterialityAnalyzerError && result.reason.code === "invalid") invalid = true; + else if (result.reason instanceof DocumentChangeMaterialityAnalyzerError && result.reason.code === "timeout") timeout = true; + else if (result.reason instanceof DocumentChangeMaterialityAnalyzerError && result.reason.code === "unavailable") unavailable = true; + else unavailable = true; + } + if (invalid) warnings.push(warning("materiality_result_invalid", "An optional materiality result failed strict validation; deterministic evidence was retained.")); + if (timeout) warnings.push(warning("materiality_analysis_timeout", "Optional materiality analysis timed out; deterministic evidence was retained.")); + if (unavailable) warnings.push(warning("materiality_analyzer_unavailable", "Optional materiality analysis was unavailable; deterministic evidence was retained.")); + if (settled.some(result => result.status === "rejected")) { + warnings.push(warning("materiality_analysis_partial", "Optional materiality analysis was partial; failed groups used deterministic fallback.")); + } + + const successByGroup = new Map(successes.map(success => [success.analyzed.group.groupId, success])); + const effectiveGroups = deterministic.analyzedGroups.flatMap(analyzed => { + const success = successByGroup.get(analyzed.group.groupId); + if (!success) return [analyzed]; + const effective = effectiveAnalyzedGroup(success); + return effective ? [effective] : []; + }); + const selection = selectMaterialDocumentChangeGroups(effectiveGroups); + warnings.push(...selection.warnings); + const selectedIds = new Set(selection.selectedGroups.map(group => group.group.groupId)); + const items = selection.selectedGroups.map(effective => { + const success = successByGroup.get(effective.group.groupId); + return success?.result.disposition === "material" + ? buildAnalyzerEvidence(success, effective) + : buildCondensedDocumentChangeEvidence(effective.pair, effective.group, effective.materiality); + }); + const audit: DocumentChangeAuditSnapshot = { + ...deterministic.audit, + groups: deterministic.audit.groups.map(group => ({ + ...group, + evidenceSourceId: selectedIds.has(group.groupId) ? documentChangeGroupSourceId( + deterministic.analyzedGroups.find(candidate => candidate.group.groupId === group.groupId)!.group, + ) : null, + ...(successByGroup.has(group.groupId) ? { analysis: analysisSnapshot(successByGroup.get(group.groupId)!) } : {}), + })), + }; + const selectedGroups = selection.selectedGroups; + const rawExcerptCharacters = audit.rawChanges.reduce((total, change) => + total + (change.previousExcerpt?.length ?? 0) + (change.currentExcerpt?.length ?? 0), 0); + const condensedPromptFacingCharacters = items.reduce((total, item) => total + item.excerpt.length, 0); + const materialCount = successes.filter(success => success.result.disposition === "material").length; + const nonMaterialCount = successes.filter(success => success.result.disposition === "non_material").length; + const uncertainCount = successes.filter(success => success.result.disposition === "uncertain").length; + return { + ...deterministic, + selectedGroups, + items, + audit, + warnings: [...new Map(warnings.map(value => [value.code, value])).values()], + diagnostics: { + ...deterministic.diagnostics, + selectedGroupCount: selectedGroups.length, + truncatedGroupCount: effectiveGroups.length - selectedGroups.length, + selectedGroupsByCategory: Object.fromEntries(DOCUMENT_CHANGE_CATEGORIES.map(category => [category, + selectedGroups.filter(group => group.materiality.category === category).length])) as Record, + truncatedGroupsByCategory: Object.fromEntries(DOCUMENT_CHANGE_CATEGORIES.map(category => [category, + effectiveGroups.filter(group => !selectedIds.has(group.group.groupId) && group.materiality.category === category).length])) as Record, + condensedEvidenceCount: items.length, + rawExcerptCharacters, + condensedPromptFacingCharacters, + estimatedReductionRatio: rawExcerptCharacters === 0 ? 1 : Number((rawExcerptCharacters / Math.max(1, condensedPromptFacingCharacters)).toFixed(3)), + }, + analyzerDiagnostics: { + eligibleGroupCount: eligible.length, + selectedAnalysisCount: prepared.length, + skippedByCallBudgetCount: analysisSelection.skipped.length, + inputTruncatedCount: prepared.filter(value => value.truncated).length, + succeededAnalysisCount: successes.length, + failedAnalysisCount: settled.length - successes.length, + materialCount, + nonMaterialCount, + uncertainCount, + }, + }; +} diff --git a/packages/features/src/founder-weekly-review/document-change-materiality.ts b/packages/features/src/founder-weekly-review/document-change-materiality.ts index 663cb9990..956fc80ee 100644 --- a/packages/features/src/founder-weekly-review/document-change-materiality.ts +++ b/packages/features/src/founder-weekly-review/document-change-materiality.ts @@ -88,6 +88,10 @@ const CATEGORY_PRIORITY: Record = Object.fromEnt DOCUMENT_CHANGE_CATEGORIES.map((category, index) => [category, index + 1]) ) as Record; +export function documentChangeCategoryPriority(category: DocumentChangeCategory): number { + return CATEGORY_PRIORITY[category]; +} + const CATEGORY_LABEL: Record = { ownership_change: "Ownership changed.", status_change: "Status changed.", diff --git a/packages/features/src/founder-weekly-review/document-change.ts b/packages/features/src/founder-weekly-review/document-change.ts index 33afee71d..33f95b8ae 100644 --- a/packages/features/src/founder-weekly-review/document-change.ts +++ b/packages/features/src/founder-weekly-review/document-change.ts @@ -79,7 +79,14 @@ export type DocumentChangeGroup = { }; export type DocumentChangeProcessingWarning = { - code: "materiality_group_too_large" | "document_change_budget_truncated"; + code: + | "materiality_group_too_large" + | "document_change_budget_truncated" + | "materiality_analyzer_unavailable" + | "materiality_analysis_partial" + | "materiality_result_invalid" + | "materiality_analysis_timeout" + | "materiality_analysis_budget_truncated"; message: string; }; diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index b6d7b1d10..a6f3cd0c7 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -22,7 +22,10 @@ import { type DocumentVersionForComparison, type VersionChunk, } from "./document-change"; -import { materializeDocumentChanges } from "./document-change-materiality"; +import { + materializeDocumentChangesWithAnalyzer, + type DocumentChangeMaterialityAnalyzer, +} from "./document-change-materiality-analyzer"; import { buildWorkspaceDocumentEvidence, normalizeFounderContextRetrievalQuery, @@ -169,6 +172,7 @@ export class FounderWeeklyReviewEvidenceService { private readonly now: () => Date = () => new Date(), private readonly documentChangeSource: FounderWeeklyReviewDocumentChangeSource = { kind: "unconfigured" }, private readonly workspaceDocumentStore?: FounderWeeklyReviewWorkspaceDocumentStore, + private readonly documentChangeMaterialityAnalyzer?: DocumentChangeMaterialityAnalyzer, ) {} async collectDocumentChangeEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { @@ -196,7 +200,10 @@ export class FounderWeeklyReviewEvidenceService { } pairInputs.push({ pair, alignments: alignVersionChunks(previous.chunks, current.chunks) }); } - const materialized = materializeDocumentChanges(pairInputs); + const materialized = await materializeDocumentChangesWithAnalyzer( + pairInputs, + this.documentChangeMaterialityAnalyzer, + ); const items = [...materialized.items]; warnings.push(...materialized.warnings.map((item) => warning(item.code, item.message, "document_change"))); const pairedCurrentVersionIds = new Set(pairs.map((pair) => pair.currentVersionId)); diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 32260a871..210d3a497 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -11,5 +11,6 @@ export * from "./generation-evidence-envelope"; export * from "./prompts"; export * from "./document-change"; export * from "./document-change-materiality"; +export * from "./document-change-materiality-analyzer"; export * from "./evidence-digest"; export * from "./workspace-document"; From c5b53b2d45ea9d937ba458327148b5158daa419b Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 8 Aug 2026 01:07:41 +0800 Subject: [PATCH 23/29] test(founder-weekly-review): add analyzer evaluation mode --- .../materiality-evaluation.test.ts | 55 +++++ ...review-materiality-evaluation-analyzers.ts | 73 ++++++ ...-review-materiality-evaluation-baseline.ts | 37 +++ ...er-weekly-review-materiality-evaluation.ts | 230 +++++++++++++++++- 4 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts create mode 100644 apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts diff --git a/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts index 2c0ff9d88..38dcc66d3 100644 --- a/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts @@ -4,8 +4,10 @@ import { resolve } from "node:path"; import { evaluationArtifactDirectory, renderMaterialityEvaluationArtifacts, + runMaterialityAnalyzerEvaluation, runMaterialityEvaluation, } from "../../scripts/founder-weekly-review-materiality-evaluation"; +import { FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE } from "../../scripts/founder-weekly-review-materiality-evaluation-baseline"; import { MATERIALITY_EVALUATION_SCENARIOS, } from "../../scripts/founder-weekly-review-materiality-evaluation-fixtures"; @@ -46,6 +48,42 @@ describe("Founder Weekly Review realistic materiality evaluation harness", () => expect(result.summary.budget.documentDiversityPreserved).toBe(true); }); + it("freezes the deterministic-v1 commit, fixture shape, formulas, and exact metrics", () => { + const result = runMaterialityEvaluation(); + const baseline = FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE; + expect(baseline.baselineCommit).toBe("630660fc861e77c464eaffc26ed6f04a7e38e7c7"); + expect(result.summary).toMatchObject({ + fixtureVersion: baseline.fixtureVersion, + scenarioCount: baseline.scenarioCount, + multiChunkScenarioCount: baseline.multiChunkScenarioCount, + largeDocumentScenarioCount: baseline.largeDocumentScenarioCount, + materiality: { + categoryAccuracy: baseline.metrics.categoryAccuracy, + materialRecall: baseline.metrics.materialRecall, + materialPrecision: baseline.metrics.materialPrecision, + uncertainRate: baseline.metrics.uncertainRate, + falseMaterialRate: baseline.metrics.falseMaterialRate, + missedMaterialRate: baseline.metrics.missedMaterialRate, + uncertainMaterialRate: baseline.metrics.uncertainMaterialRate, + uncertainNonMaterialRate: baseline.metrics.uncertainNonMaterialRate, + }, + alignment: { + alignmentMissRate: baseline.metrics.alignmentMissRate, + falseMatchRate: baseline.metrics.alignmentFalseMatchRate, + }, + condensation: { + rawChangedRecords: baseline.counts.rawChangedRecords, + groups: baseline.counts.groups, + condensedEvidenceItems: baseline.counts.structurallySelectedEvidenceItems, + rawCopiedCharacters: baseline.counts.rawCopiedCharacters, + condensedEvidenceCharacters: baseline.counts.condensedCharacters, + serializedPromptCharacters: baseline.counts.serializedPromptEvidenceCharacters, + reductionRatio: baseline.counts.reductionRatio, + }, + budget: { generationEnvelopeSelectedItems: baseline.counts.generationEnvelopeItems }, + }); + }); + it("records all four critical materiality buckets separately", () => { const result = runMaterialityEvaluation(); expect(result.summary.failuresByKind).toEqual(expect.objectContaining({ @@ -103,4 +141,21 @@ describe("Founder Weekly Review realistic materiality evaluation harness", () => fetchSpy.mockRestore(); } }); + + it("evaluates the analyzer strategy offline against the identical frozen fixtures", async () => { + const fetchSpy = jest.spyOn(global, "fetch").mockImplementation(() => { + throw new Error("provider/network invocation is forbidden"); + }); + try { + const forward = await runMaterialityAnalyzerEvaluation(undefined, MATERIALITY_EVALUATION_SCENARIOS); + const reversed = await runMaterialityAnalyzerEvaluation(undefined, [...MATERIALITY_EVALUATION_SCENARIOS].reverse()); + expect(forward.summary.scenarioCount).toBe(FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE.scenarioCount); + expect(forward.summary.groundTruthCategoryDistribution).toEqual(runMaterialityEvaluation().summary.groundTruthCategoryDistribution); + expect(forward.summary.alignment).toEqual(runMaterialityEvaluation().summary.alignment); + expect(renderMaterialityEvaluationArtifacts(reversed)).toEqual(renderMaterialityEvaluationArtifacts(forward)); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + fetchSpy.mockRestore(); + } + }); }); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts new file mode 100644 index 000000000..d1f68e6ae --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts @@ -0,0 +1,73 @@ +import type { + DocumentChangeMaterialityAnalysisInput, + DocumentChangeMaterialityAnalyzer, +} from "@launchstack/features/founder-weekly-review"; + +function words(value: string): Set { + const aliases: Record = { + owned: "owns", + ownership: "owns", + third: "q3", + quarter: "q3", + remains: "remain", + still: "remain", + expected: "planned", + expect: "planned", + }; + return new Set((value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []) + .filter(token => !["the", "a", "an", "is", "are", "by", "during", "we"].includes(token)) + .map(token => aliases[token] ?? token)); +} + +function overlap(previous: string, current: string): number { + const left = words(previous); + const right = words(current); + const union = new Set([...left, ...right]); + return union.size === 0 ? 1 : [...left].filter(token => right.has(token)).length / union.size; +} + +/** + * Offline fixture analyzer for plumbing evaluation only. It uses no scenario IDs + * or ground truth and is intentionally imperfect; it is not a quality claim. + */ +export class OfflineFixtureDocumentChangeMaterialityAnalyzer implements DocumentChangeMaterialityAnalyzer { + async analyze(input: DocumentChangeMaterialityAnalysisInput) { + const combined = input.changes.map(change => ({ + previous: change.previousExcerpt ?? "", + current: change.currentExcerpt ?? "", + })); + const averageOverlap = combined.length === 0 ? 0 : combined.reduce( + (total, change) => total + overlap(change.previous, change.current), 0, + ) / combined.length; + const replacement = input.changes.some(change => change.changeType === "added") + && input.changes.some(change => change.changeType === "removed"); + const result = input.deterministicCategory !== "uncertain" && input.deterministicCategory !== "editorial_rewrite" + ? { + disposition: "material" as const, + category: input.deterministicCategory, + summary: "The supplied fragments contain a deterministic factual change.", + confidence: 0.86, + } + : !replacement && averageOverlap >= 0.45 + ? { + disposition: "non_material" as const, + category: "editorial_rewrite" as const, + summary: "The supplied fragments appear to preserve the same business meaning.", + confidence: 0.78, + } + : { + disposition: "uncertain" as const, + category: "uncertain" as const, + summary: "The bounded fragments do not support a confident semantic disposition.", + confidence: 0.45, + }; + return { + result, + metadata: { + provider: "offline-fixture", + model: "semantic-overlap-v1", + promptVersion: "document-change-materiality/v1", + }, + }; + } +} diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts new file mode 100644 index 000000000..ada3ad7f3 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts @@ -0,0 +1,37 @@ +export const FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE = Object.freeze({ + baselineVersion: "founder-weekly-review-materiality-evaluation-baseline/v1", + baselineCommit: "630660fc861e77c464eaffc26ed6f04a7e38e7c7", + strategy: "deterministic-v1", + fixtureVersion: "founder-weekly-review-materiality-evaluation/v1", + scenarioCount: 46, + multiChunkScenarioCount: 10, + largeDocumentScenarioCount: 6, + metrics: { + categoryAccuracy: 0.9032, + materialRecall: 1, + materialPrecision: 0.7381, + uncertainRate: 0.2381, + falseMaterialRate: 0.7857, + missedMaterialRate: 0, + uncertainMaterialRate: 0.0323, + uncertainNonMaterialRate: 0.5714, + alignmentMissRate: 0.0405, + alignmentFalseMatchRate: 0, + }, + counts: { + rawChangedRecords: 70, + groups: 42, + structurallySelectedEvidenceItems: 24, + generationEnvelopeItems: 22, + rawCopiedCharacters: 5_144, + condensedCharacters: 2_430, + serializedPromptEvidenceCharacters: 13_937, + reductionRatio: 2.1169, + }, + metricDefinitions: { + falseMaterialRate: "prompt-facing groups mapped to non-material expectations, plus unexpected groups, divided by non-material ground-truth opportunities", + uncertainRate: "uncertain groups divided by all observed groups", + alignmentMissRate: "unrecovered intended modified/unchanged/split/merge relations divided by all such intended relations", + reductionRatio: "bounded raw audit excerpt characters divided by structurally selected condensed evidence characters", + }, +} as const); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts index 7f0370d58..a777fa750 100644 --- a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts @@ -6,10 +6,12 @@ import { alignVersionChunks, buildGenerationEvidenceEnvelope, materializeDocumentChanges, + materializeDocumentChangesWithAnalyzer, type AnalyzedDocumentChangeGroup, type ChunkAlignment, type DocumentChangeCategory, type DocumentChangePairInput, + type DocumentChangeMaterialityAnalyzer, type VersionPair, } from "@launchstack/features/founder-weekly-review"; @@ -20,8 +22,10 @@ import { type ExpectedChange, type MaterialityEvaluationScenario, } from "./founder-weekly-review-materiality-evaluation-fixtures"; +import { OfflineFixtureDocumentChangeMaterialityAnalyzer } from "./founder-weekly-review-materiality-evaluation-analyzers"; export const MATERIALITY_EVALUATION_RUN_ID = "deterministic-v1" as const; +export const MATERIALITY_ANALYZER_EVALUATION_RUN_ID = "materiality-analyzer-v1" as const; export const MATERIALITY_EVALUATION_ARTIFACT_ROOT = ".artifacts/founder-weekly-review/materiality-evaluation" as const; @@ -472,6 +476,200 @@ export function runMaterialityEvaluation( }; } +/** Same frozen fixtures and metric formulas, with only the materiality strategy replaced. */ +export async function runMaterialityAnalyzerEvaluation( + analyzer: DocumentChangeMaterialityAnalyzer = new OfflineFixtureDocumentChangeMaterialityAnalyzer(), + inputScenarios: readonly MaterialityEvaluationScenario[] = MATERIALITY_EVALUATION_SCENARIOS +): Promise { + const scenarios = [...inputScenarios].sort((a, b) => a.id.localeCompare(b.id)); + const deterministicBaseline = runMaterialityEvaluation(scenarios); + const failures: MaterialityEvaluationFailure[] = deterministicBaseline.failures.filter(item => + item.kind === "alignment_miss" || item.kind === "alignment_false_match" || item.kind === "budget_issue"); + const scenarioResults: MaterialityEvaluationResult["scenarioResults"][number][] = []; + const inputs: DocumentChangePairInput[] = []; + const groundTruthCategoryDistribution: Record = {}; + const observedCategoryDistribution: Record = {}; + let groundTruthMaterialChanges = 0; + let groundTruthNonMaterialChanges = 0; + let categoryCorrect = 0; + let surfacedTrueMaterial = 0; + let falseMaterialCount = 0; + let missedMaterialCount = 0; + let uncertainMaterialCount = 0; + let uncertainNonMaterialCount = 0; + + for (const [ordinal, scenario] of scenarios.entries()) { + const pair = pairFor(scenario, ordinal); + const alignments = alignVersionChunks(scenario.previousChunks, scenario.currentChunks); + const pairInput = { pair, alignments }; + inputs.push(pairInput); + const materialized = await materializeDocumentChangesWithAnalyzer([pairInput], analyzer); + const auditByGroup = new Map(materialized.audit.groups.map(group => [group.groupId, group])); + const assignedGroupIds = new Set(); + const effectiveCategory = (group: AnalyzedDocumentChangeGroup): DocumentChangeCategory => { + const analysis = auditByGroup.get(group.group.groupId)?.analysis; + if (analysis?.disposition === "uncertain") return "uncertain"; + if (analysis?.disposition === "material") return analysis.category; + return group.materiality.category; + }; + const isSurfaced = (group: AnalyzedDocumentChangeGroup): boolean => + auditByGroup.get(group.group.groupId)?.evidenceSourceId !== null + && auditByGroup.get(group.group.groupId)?.analysis?.disposition !== "non_material"; + for (const group of materialized.analyzedGroups) increment(observedCategoryDistribution, effectiveCategory(group)); + + for (const expected of scenario.expected.meaningfulChanges) { + groundTruthMaterialChanges++; + increment(groundTruthCategoryDistribution, expected.category ?? "material_unspecified"); + const group = bestGroup(expected, materialized.analyzedGroups); + if (group) assignedGroupIds.add(group.group.groupId); + const category = group ? effectiveCategory(group) : undefined; + if (!group || !isSurfaced(group) || category === "editorial_rewrite") { + missedMaterialCount++; + failures.push(failure(scenario, "materiality_false_negative", expected.id, + group ? "Semantic analysis removed or editorialized a material source." : "No retained group represented the material source.", + group ? "future_llm_materiality_analyzer" : "future_embedding_alignment")); + continue; + } + surfacedTrueMaterial++; + if (category === expected.category) categoryCorrect++; + if (category === "uncertain") { + uncertainMaterialCount++; + failures.push(failure(scenario, "uncertain_material", expected.id, + "Material source was retained but semantic analysis remained uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const expected of scenario.expected.nonMaterialChanges) { + groundTruthNonMaterialChanges++; + const group = bestGroup(expected, materialized.analyzedGroups); + if (!group) continue; + assignedGroupIds.add(group.group.groupId); + if (!isSurfaced(group)) continue; + falseMaterialCount++; + const category = effectiveCategory(group); + failures.push(failure(scenario, "materiality_false_positive", expected.id, + `Non-material source remained prompt-facing ${category} evidence.`, "future_llm_materiality_analyzer")); + if (category === "uncertain") { + uncertainNonMaterialCount++; + failures.push(failure(scenario, "uncertain_non_material", expected.id, + "Non-material rewrite remained uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const noOp of scenario.expected.expectedNoOps) { + groundTruthNonMaterialChanges++; + const group = materialized.analyzedGroups.find(candidate => { + const ids = chunkIds(candidate); + return ids.previous.has(noOp.previousChunkId) || ids.current.has(noOp.currentChunkId); + }); + if (group && isSurfaced(group)) { + assignedGroupIds.add(group.group.groupId); + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", noOp.id, + `Expected no-op remained prompt-facing ${effectiveCategory(group)} evidence.`, "deterministic_improvement")); + } + } + + for (const group of materialized.analyzedGroups) { + if (assignedGroupIds.has(group.group.groupId) || !isSurfaced(group)) continue; + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", group.group.groupId, + `An unexpected ${effectiveCategory(group)} group remained prompt-facing evidence.`, "future_llm_materiality_analyzer")); + failures.push(failure(scenario, "grouping_issue", group.group.groupId, + `An extra ${effectiveCategory(group)} group did not map to an expected semantic change.`, "deterministic_improvement")); + } + const scenarioFailures = failures.filter(item => item.scenarioId === scenario.id); + scenarioResults.push({ + id: scenario.id, + alignmentCount: alignments.length, + rawChangeCount: materialized.rawChanges.length, + groupCount: materialized.analyzedGroups.length, + categories: materialized.analyzedGroups.map(effectiveCategory), + failureKinds: [...new Set(scenarioFailures.map(item => item.kind))].sort(), + }); + } + + const aggregate = await materializeDocumentChangesWithAnalyzer(inputs, analyzer); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-03-31T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-03-31" }, + workspaceTimezone: "UTC", + items: aggregate.items, + sourceWarnings: aggregate.warnings.map(warning => ({ ...warning, sourceType: "document_change" as const })), + documentChangeAudit: aggregate.audit, + }); + const envelope = buildGenerationEvidenceEnvelope(snapshot); + const observedGroups = Object.values(observedCategoryDistribution).reduce((total, count) => total + count, 0); + const uncertainRate = ratio(observedCategoryDistribution.uncertain ?? 0, observedGroups); + const falseMaterialRate = ratio(falseMaterialCount, groundTruthNonMaterialChanges); + const failuresByKind = Object.fromEntries(FAILURE_KINDS.map(kind => [kind, failures.filter(item => item.kind === kind).length])) as Record; + const materialityFailureCount = failuresByKind.materiality_false_positive + failuresByKind.materiality_false_negative + + failuresByKind.uncertain_material + failuresByKind.uncertain_non_material; + const alignmentFailureCount = failuresByKind.alignment_miss + failuresByKind.alignment_false_match; + const recommendation = recommendationFor( + uncertainRate, + falseMaterialRate, + uncertainMaterialCount, + deterministicBaseline.summary.alignment.alignmentMissRate, + materialityFailureCount, + alignmentFailureCount, + ); + const selectedDocumentCount = new Set(aggregate.selectedGroups.map(group => group.group.documentId.toString())).size; + const availableDocumentCount = new Set(aggregate.analyzedGroups.map(group => group.group.documentId.toString())).size; + const condensedEvidenceCharacters = aggregate.diagnostics.condensedPromptFacingCharacters; + return { + summary: { + fixtureVersion: MATERIALITY_EVALUATION_FIXTURE_VERSION, + scenarioCount: scenarios.length, + multiChunkScenarioCount: scenarios.filter(scenario => scenario.multiChunk).length, + largeDocumentScenarioCount: scenarios.filter(scenario => scenario.largeDocument).length, + groundTruthCategoryDistribution: Object.fromEntries(Object.entries(groundTruthCategoryDistribution).sort()), + observedCategoryDistribution: Object.fromEntries(Object.entries(observedCategoryDistribution).sort()), + materiality: { + groundTruthMaterialChanges, + groundTruthNonMaterialChanges, + categoryCorrect, + categoryAccuracy: ratio(categoryCorrect, groundTruthMaterialChanges), + materialRecall: ratio(surfacedTrueMaterial, groundTruthMaterialChanges), + materialPrecision: ratio(surfacedTrueMaterial, surfacedTrueMaterial + falseMaterialCount), + uncertainRate, + falseMaterialCount, + falseMaterialRate, + missedMaterialCount, + missedMaterialRate: ratio(missedMaterialCount, groundTruthMaterialChanges), + uncertainMaterialCount, + uncertainMaterialRate: ratio(uncertainMaterialCount, groundTruthMaterialChanges), + uncertainNonMaterialCount, + uncertainNonMaterialRate: ratio(uncertainNonMaterialCount, groundTruthNonMaterialChanges), + }, + alignment: deterministicBaseline.summary.alignment, + condensation: { + rawChangedRecords: aggregate.rawChanges.length, + groups: aggregate.analyzedGroups.length, + condensedEvidenceItems: aggregate.items.length, + rawCopiedCharacters: aggregate.diagnostics.rawExcerptCharacters, + condensedEvidenceCharacters, + serializedPromptCharacters: envelope.diagnostics.serializedCharacterCount, + reductionRatio: ratio(aggregate.diagnostics.rawExcerptCharacters, Math.max(1, condensedEvidenceCharacters)), + }, + budget: { + groupBudgetTruncated: aggregate.diagnostics.truncatedGroupCount > 0, + truncatedGroupCount: aggregate.diagnostics.truncatedGroupCount, + generationEnvelopeTruncated: envelope.diagnostics.truncated, + generationEnvelopeSelectedItems: envelope.diagnostics.selectedItemCount, + generationEnvelopeExcludedItems: envelope.diagnostics.excludedItemCount, + documentDiversityPreserved: selectedDocumentCount === Math.min(aggregate.selectedGroups.length, availableDocumentCount), + selectedDocumentCount, + }, + failuresByKind, + ...recommendation, + }, + failures: failures.sort((a, b) => a.kind.localeCompare(b.kind) || a.scenarioId.localeCompare(b.scenarioId) || a.expectationId.localeCompare(b.expectationId)), + scenarioResults: scenarioResults.sort((a, b) => a.id.localeCompare(b.id)), + }; +} + export function evaluationArtifactDirectory(runId: string = MATERIALITY_EVALUATION_RUN_ID): string { if (!/^[a-z0-9][a-z0-9._-]*$/i.test(runId)) throw new Error("Evaluation run ID must be filesystem-safe."); return resolve(process.cwd(), MATERIALITY_EVALUATION_ARTIFACT_ROOT, runId); @@ -568,8 +766,36 @@ export async function writeMaterialityEvaluationArtifacts( } async function main(): Promise { - const result = runMaterialityEvaluation(); - const artifacts = await writeMaterialityEvaluationArtifacts(result, process.env.FWR_MATERIALITY_EVALUATION_RUN_ID); + const mode = process.env.FWR_MATERIALITY_EVAL_MODE ?? "offline"; + let result: MaterialityEvaluationResult; + let defaultRunId: string; + if (mode === "deterministic") { + result = runMaterialityEvaluation(); + defaultRunId = MATERIALITY_EVALUATION_RUN_ID; + } else if (mode === "offline") { + result = await runMaterialityAnalyzerEvaluation(); + defaultRunId = MATERIALITY_ANALYZER_EVALUATION_RUN_ID; + } else if (mode === "live") { + const { ProviderDocumentChangeMaterialityAnalyzer } = await import("../src/server/founder-weekly-review/document-change-materiality-analyzer"); + const maximumCalls = Math.max(1, Math.min(64, Number(process.env.FWR_MATERIALITY_EVAL_MAX_CALLS ?? 64) || 64)); + const live = new ProviderDocumentChangeMaterialityAnalyzer(); + let calls = 0; + const bounded: DocumentChangeMaterialityAnalyzer = { + analyze(input) { + if (calls >= maximumCalls) throw new Error("Live materiality evaluation reached its explicit global call limit."); + calls++; + return live.analyze(input); + }, + }; + result = await runMaterialityAnalyzerEvaluation(bounded); + defaultRunId = "materiality-analyzer-live-v1"; + } else { + throw new Error("FWR_MATERIALITY_EVAL_MODE must be deterministic, offline, or live."); + } + const artifacts = await writeMaterialityEvaluationArtifacts( + result, + process.env.FWR_MATERIALITY_EVALUATION_RUN_ID ?? defaultRunId, + ); console.log(JSON.stringify({ ...result.summary, artifactDirectory: artifacts.directory })); } From 605195ff3f048fcdeb8535fe3d3a8a20845e5ee6 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 8 Aug 2026 01:38:50 +0800 Subject: [PATCH 24/29] fix(founder-weekly-review): route materiality analysis to configured provider --- ...hange-materiality-analyzer-adapter.test.ts | 26 +++++++++++++++++++ .../kimi-chat-completions.test.ts | 10 +++++++ apps/web/config/llm-models.json | 4 +++ ...er-weekly-review-materiality-evaluation.ts | 5 ++-- apps/web/src/lib/llm/config.ts | 1 + .../document-change-materiality-analyzer.ts | 11 +++++--- 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts index 9e9159292..ac90d8471 100644 --- a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts @@ -3,6 +3,7 @@ jest.mock("~/lib/llm", () => { return { generateStructuredWithMetadata: jest.fn(), LlmCapabilityUnavailableError, + PROVIDERS: ["openai", "kimi", "anthropic", "google", "ollama"], }; }); @@ -37,11 +38,14 @@ const input: DocumentChangeMaterialityAnalysisInput = { describe("document-change materiality provider adapter", () => { const originalEnabled = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + const originalProvider = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER; afterEach(() => { mockGenerateStructuredWithMetadata.mockReset(); if (originalEnabled === undefined) delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; else process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = originalEnabled; + if (originalProvider === undefined) delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER; + else process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = originalProvider; }); it("requires explicit production opt-in instead of treating credentials as consent", () => { @@ -69,6 +73,28 @@ describe("document-change materiality provider adapter", () => { expect(result.metadata).not.toHaveProperty("providerRequestId"); }); + it("routes only this analyzer to its explicitly configured provider", async () => { + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = "kimi"; + mockGenerateStructuredWithMetadata.mockResolvedValue({ + object: { disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.94 }, + metadata: { provider: "kimi", model: "kimi-k2.6", capability: "smallExtraction" }, + }); + const analyzer = createConfiguredDocumentChangeMaterialityAnalyzer(); + await analyzer!.analyze(input); + expect(mockGenerateStructuredWithMetadata).toHaveBeenCalledWith(expect.objectContaining({ + capability: "smallExtraction", + forceProvider: "kimi", + })); + }); + + it("rejects an invalid analyzer-specific provider before any request", () => { + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = "other"; + expect(() => createConfiguredDocumentChangeMaterialityAnalyzer()).toThrow("FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER"); + expect(mockGenerateStructuredWithMetadata).not.toHaveBeenCalled(); + }); + it("maps provider failures to optional-unavailable fallback errors", async () => { mockGenerateStructuredWithMetadata.mockRejectedValue(new Error("rate limited")); await expect(new ProviderDocumentChangeMaterialityAnalyzer().analyze(input)).rejects.toEqual( diff --git a/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts index dd0828aae..40dd57d05 100644 --- a/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts @@ -95,6 +95,16 @@ describe("Founder Weekly Review Kimi transport", () => { expect(resolved.structuredOutputMode).toBeUndefined(); }); + it("allows an explicit Kimi smallExtraction request without changing default priority", () => { + process.env.OPENAI_API_KEY = "openai-test-key"; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + const defaultExtraction = resolveModel("smallExtraction"); + const kimiExtraction = resolveModel("smallExtraction", "kimi"); + expect(defaultExtraction).toMatchObject({ provider: "openai", modelId: "gpt-4o-mini" }); + expect(kimiExtraction).toMatchObject({ provider: "kimi", modelId: "kimi-k2.6", structuredOutputMode: "json_object" }); + }); + it("fails before any request for invalid or missing selected-provider configuration", () => { process.env.FWR_GENERATION_PROVIDER = "other"; __resetLlmConfigForTests(); diff --git a/apps/web/config/llm-models.json b/apps/web/config/llm-models.json index e1fd1b50e..0bcaf07c1 100644 --- a/apps/web/config/llm-models.json +++ b/apps/web/config/llm-models.json @@ -3,6 +3,10 @@ "providerPriority": ["openai", "anthropic", "google", "ollama"], "capabilities": { "smallExtraction": { + "kimi": { + "model": "kimi-k2.6", + "temperature": 0 + }, "openai": { "model": "gpt-4o-mini", "temperature": 0 diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts index a777fa750..4f9a19963 100644 --- a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts @@ -776,9 +776,10 @@ async function main(): Promise { result = await runMaterialityAnalyzerEvaluation(); defaultRunId = MATERIALITY_ANALYZER_EVALUATION_RUN_ID; } else if (mode === "live") { - const { ProviderDocumentChangeMaterialityAnalyzer } = await import("../src/server/founder-weekly-review/document-change-materiality-analyzer"); + const { createConfiguredDocumentChangeMaterialityAnalyzer } = await import("../src/server/founder-weekly-review/document-change-materiality-analyzer"); const maximumCalls = Math.max(1, Math.min(64, Number(process.env.FWR_MATERIALITY_EVAL_MAX_CALLS ?? 64) || 64)); - const live = new ProviderDocumentChangeMaterialityAnalyzer(); + const live = createConfiguredDocumentChangeMaterialityAnalyzer(); + if (!live) throw new Error("Live materiality evaluation requires FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED=true."); let calls = 0; const bounded: DocumentChangeMaterialityAnalyzer = { analyze(input) { diff --git a/apps/web/src/lib/llm/config.ts b/apps/web/src/lib/llm/config.ts index 1b8b714d3..21a653a99 100644 --- a/apps/web/src/lib/llm/config.ts +++ b/apps/web/src/lib/llm/config.ts @@ -70,6 +70,7 @@ const HARDCODED_DEFAULTS: LlmConfig = { // whose whole point is "cheap and fast." The original project hardcoded // gpt-5-nano in company-metadata/extractor.ts; do not restore it here. smallExtraction: { + kimi: { model: "kimi-k2.6", temperature: 0 }, openai: { model: "gpt-4o-mini", temperature: 0 }, anthropic: { model: "claude-3-5-haiku-latest", temperature: 0 }, google: { model: "gemini-2.0-flash", temperature: 0 }, diff --git a/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts index 444a4c632..fdfe95fe2 100644 --- a/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts +++ b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts @@ -8,6 +8,7 @@ import { import { LlmCapabilityUnavailableError, + PROVIDERS, generateStructuredWithMetadata, type Provider, } from "~/lib/llm"; @@ -72,7 +73,11 @@ export class ProviderDocumentChangeMaterialityAnalyzer implements DocumentChange /** Production collection opts in explicitly; credentials alone never trigger analyzer calls. */ export function createConfiguredDocumentChangeMaterialityAnalyzer(): DocumentChangeMaterialityAnalyzer | undefined { - return process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED === "true" - ? new ProviderDocumentChangeMaterialityAnalyzer() - : undefined; + if (process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED !== "true") return undefined; + const configured = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER; + if (!configured) return new ProviderDocumentChangeMaterialityAnalyzer(); + if (!(PROVIDERS as readonly string[]).includes(configured)) { + throw new Error(`FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER must be one of: ${PROVIDERS.join(", ")}.`); + } + return new ProviderDocumentChangeMaterialityAnalyzer(configured as Provider); } From ff967ae698a77591eb7901373b867a5c666a75a5 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 8 Aug 2026 02:18:30 +0800 Subject: [PATCH 25/29] feat(founder-weekly-review): refine semantic materiality analysis --- ...hange-materiality-analyzer-adapter.test.ts | 30 ++- ...cument-change-materiality-analyzer.test.ts | 33 ++- .../document-change-materiality.test.ts | 36 +++ .../document-change-materiality-analyzer.ts | 22 +- .../document-change-materiality-analyzer.ts | 100 +++++--- .../document-change-materiality.ts | 227 ++++++++++++++++++ 6 files changed, 407 insertions(+), 41 deletions(-) diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts index ac90d8471..8bb7a5a8b 100644 --- a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts @@ -14,7 +14,9 @@ import { generateStructuredWithMetadata } from "~/lib/llm"; import { z } from "zod"; import { + DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT, ProviderDocumentChangeMaterialityAnalyzer, + buildDocumentChangeMaterialityAnalyzerPrompt, createConfiguredDocumentChangeMaterialityAnalyzer, } from "~/server/founder-weekly-review/document-change-materiality-analyzer"; @@ -25,9 +27,18 @@ const input: DocumentChangeMaterialityAnalysisInput = { documentTitle: "Synthetic plan", structurePath: "/ownership", structureTitle: "Ownership", - deterministicCategory: "uncertain", - deterministicConfidence: "uncertain", - deterministicSignals: ["no_strong_deterministic_signal"], + deterministicAssessment: { + category: "uncertain", + confidence: "uncertain", + detectedSignals: ["no_strong_deterministic_signal"], + confirmedFactualDeltas: [], + equivalentFactualValues: [{ + kind: "ownership", previousValue: "product", currentValue: "product", relation: "equivalent", confidence: "strong", + }], + possibleSignals: [], + groupShape: "modified", + semanticRisk: "paraphrase_possible", + }, changes: [{ changeType: "modified", previousExcerpt: "Product owns telemetry.", @@ -69,10 +80,21 @@ describe("document-change materiality provider adapter", () => { maxOutputTokens: 512, })); expect(mockGenerateStructuredWithMetadata.mock.calls[0]![0].prompt).toContain("Product owns telemetry."); - expect(result.metadata).toEqual({ provider: "openai", model: "fixture-model", promptVersion: "document-change-materiality/v1" }); + expect(result.metadata).toEqual({ provider: "openai", model: "fixture-model", promptVersion: "document-change-materiality/v2" }); expect(result.metadata).not.toHaveProperty("providerRequestId"); }); + it("defines the v2 task around underlying state, structural equivalence, mixed rewrites, and concise output", () => { + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("underlying business state")); + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("split or merged fragments")); + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("one real factual delta")); + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("at most 320 characters")); + const prompt = buildDocumentChangeMaterialityAnalyzerPrompt(input); + expect(prompt).toContain('"confirmedFactualDeltas":[]'); + expect(prompt).toContain('"equivalentFactualValues"'); + expect(prompt).toContain('"semanticRisk":"paraphrase_possible"'); + }); + it("routes only this analyzer to its explicitly configured provider", async () => { process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = "kimi"; diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts index 3e8ff3818..2ef4938ad 100644 --- a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS, + DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + DocumentChangeMaterialityAnalysisResultSchema, DocumentChangeMaterialityAnalyzerError, FounderWeeklyReviewEvidenceSnapshotSchema, buildDocumentChangeMaterialityAnalysisInput, @@ -71,7 +73,7 @@ class FakeAnalyzer implements DocumentChangeMaterialityAnalyzer { this.calls.push(value); return { result: await this.implementation(value), - metadata: { provider: "fixture", model: "fixture-v1", promptVersion: "document-change-materiality/v1" }, + metadata: { provider: "fixture", model: "fixture-v1", promptVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION }, }; } } @@ -106,9 +108,21 @@ describe("optional document-change materiality analyzer", () => { ["Product owns telemetry.", "Platform owns telemetry."], ["Launch is planned for Q3.", "Launch is planned for Q4."], ["Conversion is 10%.", "Conversion is 25%."], + ["Admins may enable SSO.", "Admins must enable SSO."], + ["The migration is planned.", "The migration is launched."], + ["The plan supports SSO.", "The plan does not support SSO."], + ["Migration priority: P2", "Migration priority: P0"], ]) { expect(shouldAnalyzeDocumentChangeGroup(analyzed(9n, [modified(1, 9n, before!, after!)]))).toMatchObject({ eligible: false }); } + for (const [before, after] of [ + ["The launch remains planned for Q3.", "We still expect the launch during the third quarter."], + ["Product owns telemetry.", "Telemetry is owned by Product."], + ["ARR target is $1M.", "ARR target is one million dollars."], + ["The migration is planned.", "The migration is still planned."], + ]) { + expect(shouldAnalyzeDocumentChangeGroup(analyzed(10n, [modified(1, 10n, before!, after!)]))).toMatchObject({ eligible: true }); + } }); it("selects at most four calls deterministically while retaining document diversity", () => { @@ -131,6 +145,12 @@ describe("optional document-change materiality analyzer", () => { expect(built.canonicalCharacterCount).toBeLessThanOrEqual(8_000); expect(built.truncated).toBe(true); expect(built.input.changes).toHaveLength(1); + expect(built.input.deterministicAssessment).toEqual(expect.objectContaining({ + groupShape: "modified", + semanticRisk: expect.any(String), + confirmedFactualDeltas: expect.any(Array), + possibleSignals: expect.any(Array), + })); expect(JSON.stringify(built.input)).not.toContain("customer_feedback"); expect(built.inputDigest).toMatch(/^[a-f0-9]{64}$/); }); @@ -211,6 +231,17 @@ describe("optional document-change materiality analyzer", () => { expect(result.warnings.map(warning => warning.code)).toEqual(expect.arrayContaining([warningCode, "materiality_analysis_partial"])); }); + it("keeps the strict concise-output boundary at the documented limits", () => { + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse(materialResult("x".repeat(320))).success).toBe(true); + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse(materialResult("x".repeat(321))).success).toBe(false); + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse({ + ...materialResult(), beforeKeyPoint: "b".repeat(240), afterKeyPoint: "a".repeat(240), + }).success).toBe(true); + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse({ + ...materialResult(), beforeKeyPoint: "b".repeat(241), + }).success).toBe(false); + }); + it("isolates partial failures and never exceeds concurrency or call budgets", async () => { let active = 0; let maximumActive = 0; diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts index fef33b4bf..a8c9673c0 100644 --- a/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts @@ -1,5 +1,6 @@ import { FounderWeeklyReviewEvidenceSnapshotSchema, + analyzeDocumentChangeFactualDeltas, analyzeDocumentChangeGroup, buildCondensedDocumentChangeEvidence, buildFounderWeeklyReviewEvidenceDigest, @@ -106,6 +107,41 @@ describe("deterministic document-change materiality", () => { }); }); + it.each([ + ["Q3", "Q4", "deadline", "changed"], + ["Q3", "third quarter", "deadline", "equivalent"], + ["Product owns telemetry.", "Platform owns telemetry.", "ownership", "changed"], + ["Product owns telemetry.", "Telemetry is owned by Product.", "ownership", "equivalent"], + ["10% conversion", "25% conversion", "metric", "changed"], + ["ARR target is $1M.", "ARR target is one million dollars.", "metric", "equivalent"], + ["The migration is planned.", "The migration is launched.", "status", "changed"], + ["The migration is planned.", "The migration is still planned.", "status", "equivalent"], + ] as const)("compares factual state in %s -> %s as %s %s", (before, after, kind, relation) => { + const group = groupFor(before, after); + const deterministic = analyzeDocumentChangeGroup(group); + const assessment = analyzeDocumentChangeFactualDeltas(group, deterministic.signals); + expect(assessment.factualComparisons).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind, relation }), + ])); + }); + + it("keeps one-sided business signals unconfirmed rather than manufacturing a delta", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [{ + changeType: "added", + currentChunk: chunk(2, 2n, "Admins must enable SSO."), + alignmentMethod: "unmatched", + }]).rawChanges; + const group = groupRawDocumentChanges(versionPair, raw).groups[0]!; + const deterministic = analyzeDocumentChangeGroup(group); + const assessment = analyzeDocumentChangeFactualDeltas(group, deterministic.signals); + expect(assessment.confirmedFactualDeltas).toHaveLength(0); + expect(assessment.possibleSignals).toContain("requirement"); + expect(assessment.factualComparisons).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind: "requirement", relation: "unknown" }), + ])); + }); + it("assigns editorial only to a narrow deterministic formatting rewrite", () => { expect(analyzeDocumentChangeGroup(groupFor("- First item", "* First item"))).toMatchObject({ category: "editorial_rewrite", diff --git a/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts index fdfe95fe2..bcfcf9d55 100644 --- a/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts +++ b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts @@ -14,19 +14,23 @@ import { } from "~/lib/llm"; import { z } from "zod"; -const MATERIALITY_SYSTEM_PROMPT = `You classify one already-diffed document section. Answer only this question: did the underlying business meaning materially change? +export const DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT = `You classify one already-diffed document section. Answer one narrow question: did the underlying business state change? -Pay special attention to ownership, status or shipping, dates and deadlines, metrics, requirements or modality, negation, blockers or risks, scope, priority, customer or rollout scope, and meaning-preserving paraphrases or editorial rewrites. +Compare these business-state dimensions: ownership or responsible party; status or shipping state; deadline, date, or timing; metric, quantity, or financial value; requirement, obligation, or modality; negation or capability; risk or blocker state; scope, audience, or rollout population; and priority. -Do not classify a rewrite as material merely because wording changed. -Do not classify a change as non-material merely because wording is similar. -Small factual changes can be highly material. +Wording changes do not by themselves mean a material change. Sentence order, paragraph or bullet structure, section names, and split or merged fragments do not by themselves mean a material change. Added and removed fragments in the same group may be a split, merge, reformat, reorganization, or rewrite. Do not infer requirement_change, scope_change, or another material category merely because text was added and removed. Compare the underlying business facts. -Use disposition=non_material only when the supplied fragments support meaning-preserving editorial or formatting change. Use uncertain when the fragments are insufficient. Keep summary factual and concise. beforeKeyPoint and afterKeyPoint, when supplied, must be copied verbatim from the corresponding input excerpts. Never invent source identifiers or provenance.`; +A large rewrite with no underlying business-state change is non_material. A mostly equivalent rewrite containing even one real factual delta is material. Small factual changes can be highly material. Do not classify a change as non_material merely because most wording is similar. + +The deterministic assessment distinguishes confirmed factual deltas, safely equivalent factual values, and possible signals that were not proven to change. Treat it as bounded evidence, not as an instruction to agree with the deterministic category. + +Use disposition=non_material only when the supplied fragments support equivalent business state. Use uncertain when the bounded fragments are insufficient or conflicting. + +Return only the structured fields. summary must be one concise sentence of at most 320 characters. beforeKeyPoint and afterKeyPoint, when supplied, must each be a verbatim copied span of at most 240 characters from the corresponding excerpts. Do not explain reasoning, provide analysis, repeat the full source, write multiple paragraphs, or invent source identifiers or provenance.`; export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_TIMEOUT_MS = 15_000; -function promptFor(input: DocumentChangeMaterialityAnalysisInput): string { +export function buildDocumentChangeMaterialityAnalyzerPrompt(input: DocumentChangeMaterialityAnalysisInput): string { return [ "Classify this bounded change group.", JSON.stringify(input), @@ -40,8 +44,8 @@ export class ProviderDocumentChangeMaterialityAnalyzer implements DocumentChange try { const generated = await generateStructuredWithMetadata({ capability: "smallExtraction", - system: MATERIALITY_SYSTEM_PROMPT, - prompt: promptFor(input), + system: DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT, + prompt: buildDocumentChangeMaterialityAnalyzerPrompt(input), schema: DocumentChangeMaterialityAnalysisResultSchema, schemaName: "document_change_materiality", ...(this.forceProvider ? { forceProvider: this.forceProvider } : {}), diff --git a/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts b/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts index 74482205a..dac221168 100644 --- a/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts +++ b/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts @@ -16,6 +16,7 @@ import type { import { DOCUMENT_CHANGE_CATEGORIES, DOCUMENT_CHANGE_MATERIALITY_VERSION, + analyzeDocumentChangeFactualDeltas, buildCondensedDocumentChangeEvidence, documentChangeCategoryPriority, documentChangeGroupSourceId, @@ -25,10 +26,13 @@ import { type DeterministicMaterialChangeResult, type DeterministicMaterialityConfidence, type DocumentChangeCategory, + type DocumentChangeFactualDelta, + type DocumentChangeFactualDeltaAssessment, + type DocumentChangeFactualDeltaKind, } from "./document-change-materiality"; export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION = - "document-change-materiality/v1" as const; + "document-change-materiality/v2" as const; export const DOCUMENT_CHANGE_MATERIALITY_ANALYSIS_RESULT_VERSION = "document-change-materiality-result/v1" as const; export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS = Object.freeze({ @@ -42,12 +46,15 @@ export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS = Object.freeze({ }); export const DocumentChangeMaterialityAnalysisResultSchema = z.object({ - disposition: z.enum(["material", "non_material", "uncertain"]), - category: z.enum(DOCUMENT_CHANGE_CATEGORIES), - summary: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.summaryCharacters), - beforeKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters).optional(), - afterKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters).optional(), - confidence: z.number().min(0).max(1), + disposition: z.enum(["material", "non_material", "uncertain"]).describe("Whether the underlying business state materially changed."), + category: z.enum(DOCUMENT_CHANGE_CATEGORIES).describe("The primary changed business-state dimension, editorial_rewrite, or uncertain."), + summary: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.summaryCharacters) + .describe("One concise sentence of at most 320 characters. Do not include reasoning or repeat the full source."), + beforeKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters) + .describe("Optional verbatim copied before span of at most 240 characters.").optional(), + afterKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters) + .describe("Optional verbatim copied after span of at most 240 characters.").optional(), + confidence: z.number().min(0).max(1).describe("Judgment confidence from 0 through 1."), }).strict().superRefine((result, context) => { if (result.disposition === "uncertain" && result.category !== "uncertain") { context.addIssue({ code: z.ZodIssueCode.custom, message: "Uncertain disposition requires uncertain category.", path: ["category"] }); @@ -61,14 +68,29 @@ export type DocumentChangeMaterialityAnalysisResult = z.infer< typeof DocumentChangeMaterialityAnalysisResultSchema >; +export type DocumentChangeGroupShape = "modified" | "added" | "removed" | "replacement" | "mixed"; +export type DocumentChangeSemanticRisk = + | "confirmed_factual_delta" + | "paraphrase_possible" + | "structural_rewrite" + | "split_merge_possible" + | "ambiguous_business_signal"; + export type DocumentChangeMaterialityAnalysisInput = { groupId: string; documentTitle?: string; structurePath?: string | null; structureTitle?: string | null; - deterministicCategory: DocumentChangeCategory; - deterministicConfidence: DeterministicMaterialityConfidence; - deterministicSignals: readonly string[]; + deterministicAssessment: { + category: DocumentChangeCategory; + confidence: DeterministicMaterialityConfidence; + detectedSignals: readonly string[]; + confirmedFactualDeltas: readonly DocumentChangeFactualDelta[]; + equivalentFactualValues: readonly DocumentChangeFactualDelta[]; + possibleSignals: readonly DocumentChangeFactualDeltaKind[]; + groupShape: DocumentChangeGroupShape; + semanticRisk: DocumentChangeSemanticRisk; + }; changes: readonly { changeType: RawDocumentChange["changeType"]; previousExcerpt?: string; @@ -107,6 +129,7 @@ export type DocumentChangeMaterialityEligibilityReason = | "complex_multi_change" | "large_rewrite" | "moderate_signal" + | "unconfirmed_business_signal" | "paraphrase_risk"; export type DocumentChangeMaterialityEligibility = { @@ -145,16 +168,6 @@ type SuccessfulAnalysis = { metadata: DocumentChangeMaterialityAnalyzerMetadata; }; -const SIMPLE_BYPASS_SIGNALS = new Set([ - "ownership_subject_changed", - "status_term_changed", - "date_or_deadline_changed", - "numeric_metric_changed", - "requirement_or_modality_changed", - "negation_changed", - "priority_marker_changed", -]); - function compareOrdinal(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } @@ -178,20 +191,42 @@ function lexicalSimilarity(change: RawDocumentChange): number { return union === 0 ? 1 : intersection / union; } -function isStrongDeterministicBypass(analyzed: AnalyzedDocumentChangeGroup): boolean { +function groupShape(group: DocumentChangeGroup): DocumentChangeGroupShape { + const types = new Set(group.rawChanges.map(change => change.changeType)); + if (types.has("added") && types.has("removed")) return "replacement"; + if (types.size > 1) return "mixed"; + return group.rawChanges[0]?.changeType ?? "mixed"; +} + +function semanticRisk( + analyzed: AnalyzedDocumentChangeGroup, + assessment: DocumentChangeFactualDeltaAssessment, +): DocumentChangeSemanticRisk { + const shape = groupShape(analyzed.group); + if (shape === "replacement") return "split_merge_possible"; + if (analyzed.group.rawChanges.length > 1 || shape === "mixed") return "structural_rewrite"; + if (assessment.confirmedFactualDeltas.length > 0 && assessment.possibleSignals.length === 0) return "confirmed_factual_delta"; + if (assessment.possibleSignals.length > 0) return "ambiguous_business_signal"; + return "paraphrase_possible"; +} + +export function hasStrongConfirmedFactualDeltaBypass(analyzed: AnalyzedDocumentChangeGroup): boolean { const { group, materiality } = analyzed; if (group.rawChanges.length !== 1 || group.rawChanges[0]!.changeType !== "modified") return false; if (changedCharacters(group) > 700 || materiality.category === "uncertain" || materiality.category === "editorial_rewrite") return false; - return materiality.signals.length > 0 - && materiality.signals.every(signal => SIMPLE_BYPASS_SIGNALS.has(signal)); + const assessment = analyzeDocumentChangeFactualDeltas(group, materiality.signals); + return assessment.confirmedFactualDeltas.length > 0 + && assessment.confirmedFactualDeltas.every(delta => delta.confidence === "strong") + && assessment.possibleSignals.length === 0; } /** Pure semantic-risk gate. It intentionally does not use embeddings or changelog text. */ export function shouldAnalyzeDocumentChangeGroup( analyzed: AnalyzedDocumentChangeGroup, ): DocumentChangeMaterialityEligibility { - if (isStrongDeterministicBypass(analyzed)) return { eligible: false, priority: Number.MAX_SAFE_INTEGER }; + if (hasStrongConfirmedFactualDeltaBypass(analyzed)) return { eligible: false, priority: Number.MAX_SAFE_INTEGER }; const { group, materiality } = analyzed; + const factualAssessment = analyzeDocumentChangeFactualDeltas(group, materiality.signals); if (materiality.category === "uncertain" && group.rawChanges.length > 1) { return { eligible: true, reason: "uncertain_multi_change", priority: 1 }; } @@ -205,6 +240,9 @@ export function shouldAnalyzeDocumentChangeGroup( if (group.rawChanges.length > 1) { return { eligible: true, reason: "complex_multi_change", priority: 4 }; } + if (factualAssessment.possibleSignals.length > 0 && factualAssessment.confirmedFactualDeltas.length === 0) { + return { eligible: true, reason: "unconfirmed_business_signal", priority: 4 }; + } if (changedCharacters(group) >= 1_200) { return { eligible: true, reason: "large_rewrite", priority: 4 }; } @@ -297,14 +335,22 @@ export function buildDocumentChangeMaterialityAnalysisInput(analyzed: AnalyzedDo ...(compactExcerpt(change.currentChunk?.content) ? { currentExcerpt: compactExcerpt(change.currentChunk?.content) } : {}), alignmentMethod: change.alignmentMethod, })); + const factualAssessment = analyzeDocumentChangeFactualDeltas(analyzed.group, analyzed.materiality.signals); const input: DocumentChangeMaterialityAnalysisInput = { groupId: analyzed.group.groupId, documentTitle: analyzed.pair.documentTitle.slice(0, 256), structurePath: analyzed.group.structurePath?.slice(0, 256) ?? null, structureTitle: analyzed.group.structureTitle?.slice(0, 256) ?? null, - deterministicCategory: analyzed.materiality.category, - deterministicConfidence: analyzed.materiality.confidence, - deterministicSignals: analyzed.materiality.signals.slice(0, 20), + deterministicAssessment: { + category: analyzed.materiality.category, + confidence: analyzed.materiality.confidence, + detectedSignals: analyzed.materiality.signals.slice(0, 20), + confirmedFactualDeltas: factualAssessment.confirmedFactualDeltas.slice(0, 16), + equivalentFactualValues: factualAssessment.equivalentFactualValues.slice(0, 16), + possibleSignals: factualAssessment.possibleSignals, + groupShape: groupShape(analyzed.group), + semanticRisk: semanticRisk(analyzed, factualAssessment), + }, changes, }; const canonical = JSON.stringify(canonicalize(input)); diff --git a/packages/features/src/founder-weekly-review/document-change-materiality.ts b/packages/features/src/founder-weekly-review/document-change-materiality.ts index 956fc80ee..a95ae42a3 100644 --- a/packages/features/src/founder-weekly-review/document-change-materiality.ts +++ b/packages/features/src/founder-weekly-review/document-change-materiality.ts @@ -45,6 +45,37 @@ export type DeterministicMaterialityResult = { signals: readonly string[]; }; +export const DOCUMENT_CHANGE_FACTUAL_DELTA_VERSION = "document-change-factual-delta/v2" as const; + +export const DOCUMENT_CHANGE_FACTUAL_DELTA_KINDS = [ + "ownership", + "status", + "deadline", + "metric", + "requirement", + "negation", + "risk_or_blocker", + "scope", + "priority", +] as const; + +export type DocumentChangeFactualDeltaKind = typeof DOCUMENT_CHANGE_FACTUAL_DELTA_KINDS[number]; +export type DocumentChangeFactualDelta = { + kind: DocumentChangeFactualDeltaKind; + previousValue?: string; + currentValue?: string; + relation: "changed" | "equivalent" | "unknown"; + confidence: "strong" | "moderate"; +}; + +export type DocumentChangeFactualDeltaAssessment = { + version: typeof DOCUMENT_CHANGE_FACTUAL_DELTA_VERSION; + factualComparisons: readonly DocumentChangeFactualDelta[]; + confirmedFactualDeltas: readonly DocumentChangeFactualDelta[]; + equivalentFactualValues: readonly DocumentChangeFactualDelta[]; + possibleSignals: readonly DocumentChangeFactualDeltaKind[]; +}; + export type AnalyzedDocumentChangeGroup = { pair: VersionPair; group: DocumentChangeGroup; @@ -172,6 +203,202 @@ function metricValues(value: string): string[] { ]); } +function uniqueSorted(values: readonly string[]): string[] { + return [...new Set(values)].sort(compareOrdinal); +} + +function canonicalDeadlineValues(value: string): string[] { + const quarterAliases: Record = { + "q1": "q1", "first": "q1", + "q2": "q2", "second": "q2", + "q3": "q3", "third": "q3", + "q4": "q4", "fourth": "q4", + }; + const quarters = [ + ...value.matchAll(/\b(q[1-4])\b/giu), + ...value.matchAll(/\b(first|second|third|fourth)[ -]quarter\b/giu), + ].map(match => quarterAliases[match[1]!.toLocaleLowerCase()]!); + return uniqueSorted([ + ...quarters, + ...valuesMatching(value, [ + /\b(20\d{2})\b/gu, + /\b((?:january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2}(?:,\s*20\d{2})?)\b/giu, + /\b(20\d{2}-\d{2}-\d{2})\b/gu, + ]), + ]); +} + +const NUMBER_WORD_VALUES: Record = { + zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, + ten: 10, eleven: 11, twelve: 12, thirteen: 13, fourteen: 14, fifteen: 15, sixteen: 16, + seventeen: 17, eighteen: 18, nineteen: 19, twenty: 20, thirty: 30, forty: 40, fifty: 50, + sixty: 60, seventy: 70, eighty: 80, ninety: 90, +}; + +function parseNumberWords(value: string): number | null { + const tokens = value.toLocaleLowerCase().split(/[\s-]+/).filter(Boolean); + if (tokens.length === 0 || tokens.some(token => !(token in NUMBER_WORD_VALUES) && !["hundred", "thousand", "million", "billion", "and"].includes(token))) { + return null; + } + let total = 0; + let current = 0; + for (const token of tokens) { + if (token === "and") continue; + if (token in NUMBER_WORD_VALUES) current += NUMBER_WORD_VALUES[token]!; + else if (token === "hundred") current = Math.max(1, current) * 100; + else { + const scale = token === "thousand" ? 1_000 : token === "million" ? 1_000_000 : 1_000_000_000; + total += Math.max(1, current) * scale; + current = 0; + } + } + return total + current; +} + +function canonicalMetricValues(value: string): string[] { + const result: string[] = []; + const currencyCodes: Record = { "$": "usd", "€": "eur", "£": "gbp" }; + for (const match of value.matchAll(/([$€£])\s*(\d+(?:\.\d+)?)\s*([kmb])?\b/giu)) { + const multiplier = match[3]?.toLocaleLowerCase() === "k" ? 1_000 + : match[3]?.toLocaleLowerCase() === "m" ? 1_000_000 + : match[3]?.toLocaleLowerCase() === "b" ? 1_000_000_000 : 1; + result.push(`${currencyCodes[match[1]!]!}:${Number(match[2]) * multiplier}`); + } + const numberWords = "zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|and"; + const wordCurrency = new RegExp(`\\b((?:(?:${numberWords})[\\s-]+){1,8})(dollars?|euros?|pounds?)\\b`, "giu"); + for (const match of value.matchAll(wordCurrency)) { + const amount = parseNumberWords(match[1]!); + const unit = match[2]!.toLocaleLowerCase().startsWith("dollar") ? "usd" + : match[2]!.toLocaleLowerCase().startsWith("euro") ? "eur" : "gbp"; + if (amount !== null) result.push(`${unit}:${amount}`); + } + for (const match of value.matchAll(/(\d+(?:\.\d+)?)\s*%/gu)) result.push(`percent:${Number(match[1])}`); + for (const match of value.matchAll(/\b(\d+(?:\.\d+)?)\s*(customers?|users?|accounts?|employees?|days?|weeks?|months?|revenue|arr|mrr)\b/giu)) { + result.push(`${match[2]!.toLocaleLowerCase().replace(/s$/, "")}:${Number(match[1])}`); + } + return uniqueSorted(result); +} + +function canonicalPhraseValues(value: string, phrases: readonly string[], aliases: Record = {}): string[] { + return uniqueSorted(phraseValues(value, phrases).map(phrase => aliases[phrase] ?? phrase)); +} + +type FactualValueExtractor = (value: string) => readonly string[]; + +const FACTUAL_EXTRACTORS: ReadonlyArray<{ + kind: Exclude; + extract: FactualValueExtractor; +}> = [ + { kind: "ownership", extract: ownershipValues }, + { kind: "status", extract: value => canonicalPhraseValues(value, STATUS_TERMS, { canceled: "cancelled" }) }, + { kind: "deadline", extract: canonicalDeadlineValues }, + { kind: "metric", extract: canonicalMetricValues }, + { kind: "requirement", extract: value => canonicalPhraseValues(value, REQUIREMENT_TERMS) }, + { kind: "risk_or_blocker", extract: value => canonicalPhraseValues(value, RISK_TERMS, { risks: "risk", blockers: "blocker" }) }, + { kind: "scope", extract: value => canonicalPhraseValues(value, SCOPE_TERMS, { "company wide": "company-wide" }) }, + { kind: "priority", extract: value => canonicalPhraseValues(value, PRIORITY_TERMS) }, +]; + +const SIGNAL_KIND: Record = { + ownership_subject_changed: "ownership", + status_term_changed: "status", + date_or_deadline_changed: "deadline", + numeric_metric_changed: "metric", + requirement_or_modality_changed: "requirement", + negation_changed: "negation", + risk_or_blocker_term_changed: "risk_or_blocker", + scope_marker_changed: "scope", + priority_marker_changed: "priority", +}; + +function comparison( + change: RawDocumentChange, + kind: DocumentChangeFactualDeltaKind, + previous: readonly string[], + current: readonly string[], +): DocumentChangeFactualDelta | null { + if (previous.length === 0 && current.length === 0) return null; + const previousValue = previous.join(" | ") || undefined; + const currentValue = current.join(" | ") || undefined; + if (change.changeType !== "modified" || !change.previousChunk || !change.currentChunk || !previousValue || !currentValue) { + return { kind, ...(previousValue ? { previousValue } : {}), ...(currentValue ? { currentValue } : {}), relation: "unknown", confidence: "moderate" }; + } + return { + kind, + previousValue, + currentValue, + relation: previousValue === currentValue ? "equivalent" : "changed", + confidence: "strong", + }; +} + +/** + * Separates business-token presence from a confirmed two-sided factual delta. + * It is intentionally narrow: unknown values remain analyzer candidates rather than being normalized away. + */ +export function analyzeDocumentChangeFactualDeltas( + group: DocumentChangeGroup, + deterministicSignals: readonly string[] = [], +): DocumentChangeFactualDeltaAssessment { + const factualComparisons: DocumentChangeFactualDelta[] = []; + for (const change of group.rawChanges) { + const before = change.previousNormalizedContent ?? ""; + const after = change.currentNormalizedContent ?? ""; + for (const extractor of FACTUAL_EXTRACTORS) { + const result = comparison(change, extractor.kind, extractor.extract(before), extractor.extract(after)); + if (result) factualComparisons.push(result); + } + if (change.changeType === "modified" && change.previousChunk && change.currentChunk) { + const negations = (value: string) => phraseValues(value, ["not", "no", "never", "unavailable"]); + const beforeNegated = negations(before).length > 0; + const afterNegated = negations(after).length > 0; + if (beforeNegated || afterNegated) { + factualComparisons.push({ + kind: "negation", + previousValue: beforeNegated ? "negated" : "affirmed", + currentValue: afterNegated ? "negated" : "affirmed", + relation: beforeNegated === afterNegated ? "equivalent" : "changed", + confidence: "strong", + }); + } + } else { + const negations = phraseValues(`${before} ${after}`, ["not", "no", "never", "unavailable"]); + if (negations.length > 0) { + factualComparisons.push({ + kind: "negation", + ...(before ? { previousValue: negations.join(" | ") } : {}), + ...(after ? { currentValue: negations.join(" | ") } : {}), + relation: "unknown", + confidence: "moderate", + }); + } + } + } + const deduplicated = [...new Map(factualComparisons.map(value => [ + [value.kind, value.previousValue ?? "", value.currentValue ?? "", value.relation].join(":"), value, + ])).values()].sort((a, b) => compareOrdinal(a.kind, b.kind) + || compareOrdinal(a.previousValue ?? "", b.previousValue ?? "") + || compareOrdinal(a.currentValue ?? "", b.currentValue ?? "") + || compareOrdinal(a.relation, b.relation)); + const confirmedFactualDeltas = deduplicated.filter(value => value.relation === "changed"); + const equivalentFactualValues = deduplicated.filter(value => value.relation === "equivalent"); + const confirmedKinds = new Set(confirmedFactualDeltas.map(value => value.kind)); + const possibleSignals = uniqueSorted([ + ...deduplicated.filter(value => value.relation === "unknown").map(value => value.kind), + ...deterministicSignals.flatMap(signal => { + const kind = SIGNAL_KIND[signal]; + return kind && !confirmedKinds.has(kind) ? [kind] : []; + }), + ]) as DocumentChangeFactualDeltaKind[]; + return { + version: DOCUMENT_CHANGE_FACTUAL_DELTA_VERSION, + factualComparisons: deduplicated, + confirmedFactualDeltas, + equivalentFactualValues, + possibleSignals, + }; +} + function punctuationOnlyEditorial(before: string, after: string): boolean { const stripBullet = (value: string) => value.replace(/^\s*[-*+]\s+/gm, "").replace(/\s+/g, " ").trim(); return before !== after && stripBullet(before) === stripBullet(after); From c23c5446a5a4a02ac46d3a7b23a38b823994b522 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sat, 8 Aug 2026 02:18:48 +0800 Subject: [PATCH 26/29] test(founder-weekly-review): add materiality analyzer v2 evaluation --- .../materiality-evaluation.test.ts | 19 ++++- ...review-materiality-evaluation-analyzers.ts | 5 +- ...-review-materiality-evaluation-baseline.ts | 38 +++++++++ ...er-weekly-review-materiality-evaluation.ts | 77 +++++++++++++++++-- 4 files changed, 129 insertions(+), 10 deletions(-) diff --git a/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts index 38dcc66d3..854636925 100644 --- a/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts @@ -7,7 +7,10 @@ import { runMaterialityAnalyzerEvaluation, runMaterialityEvaluation, } from "../../scripts/founder-weekly-review-materiality-evaluation"; -import { FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE } from "../../scripts/founder-weekly-review-materiality-evaluation-baseline"; +import { + FOUNDER_WEEKLY_REVIEW_MATERIALITY_ANALYZER_V1_BASELINES, + FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE, +} from "../../scripts/founder-weekly-review-materiality-evaluation-baseline"; import { MATERIALITY_EVALUATION_SCENARIOS, } from "../../scripts/founder-weekly-review-materiality-evaluation-fixtures"; @@ -30,6 +33,20 @@ describe("Founder Weekly Review realistic materiality evaluation harness", () => } }); + it("freezes prior offline, failed OpenAI, and canonical Kimi-v1 analyzer reference points", () => { + expect(FOUNDER_WEEKLY_REVIEW_MATERIALITY_ANALYZER_V1_BASELINES).toEqual(expect.objectContaining({ + offline: expect.objectContaining({ materialRecall: 0.9677, falseMaterialRate: 0.6429 }), + openAiLive: expect.objectContaining({ semanticQualityMeasured: false, timeouts: 18 }), + kimiLive: expect.objectContaining({ + promptVersion: "document-change-materiality/v1", + materialRecall: 1, + materialPrecision: 0.7949, + falseMaterialRate: 0.5714, + invalidAnalyzerResponses: 3, + }), + })); + }); + it("calculates explicit confusion, alignment, condensation, and budget metrics", () => { const result = runMaterialityEvaluation(); expect(result.summary.scenarioCount).toBe(46); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts index d1f68e6ae..22dcc9b9a 100644 --- a/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts @@ -41,10 +41,11 @@ export class OfflineFixtureDocumentChangeMaterialityAnalyzer implements Document ) / combined.length; const replacement = input.changes.some(change => change.changeType === "added") && input.changes.some(change => change.changeType === "removed"); - const result = input.deterministicCategory !== "uncertain" && input.deterministicCategory !== "editorial_rewrite" + const deterministicCategory = input.deterministicAssessment.category; + const result = deterministicCategory !== "uncertain" && deterministicCategory !== "editorial_rewrite" ? { disposition: "material" as const, - category: input.deterministicCategory, + category: deterministicCategory, summary: "The supplied fragments contain a deterministic factual change.", confidence: 0.86, } diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts index ada3ad7f3..94bb0ee31 100644 --- a/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts @@ -35,3 +35,41 @@ export const FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE = Object.f reductionRatio: "bounded raw audit excerpt characters divided by structurally selected condensed evidence characters", }, } as const); + +/** Historical analyzer reference points; these are not regenerated or tuned by v2 evaluation. */ +export const FOUNDER_WEEKLY_REVIEW_MATERIALITY_ANALYZER_V1_BASELINES = Object.freeze({ + offline: { + strategy: "materiality-analyzer-offline-v1", + materialRecall: 0.9677, + materialPrecision: 0.7692, + falseMaterialRate: 0.6429, + uncertainRate: 0.2381, + missedMaterialRate: 0.0323, + }, + openAiLive: { + strategy: "materiality-analyzer-live-v1", + provider: "openai", + model: "gpt-4o-mini", + semanticQualityMeasured: false, + calls: 18, + successfulCalls: 0, + timeouts: 18, + }, + kimiLive: { + strategy: "materiality-analyzer-kimi-live-v1", + provider: "kimi", + model: "kimi-k2.6", + promptVersion: "document-change-materiality/v1", + categoryAccuracy: 0.9355, + materialRecall: 1, + materialPrecision: 0.7949, + falseMaterialRate: 0.5714, + uncertainRate: 0.1667, + missedMaterialRate: 0, + uncertainMaterialRate: 0, + uncertainNonMaterialRate: 0.2143, + validAnalyzerCalls: 15, + invalidAnalyzerResponses: 3, + timeouts: 0, + }, +} as const); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts index 4f9a19963..e5619d94d 100644 --- a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts @@ -26,6 +26,7 @@ import { OfflineFixtureDocumentChangeMaterialityAnalyzer } from "./founder-weekl export const MATERIALITY_EVALUATION_RUN_ID = "deterministic-v1" as const; export const MATERIALITY_ANALYZER_EVALUATION_RUN_ID = "materiality-analyzer-v1" as const; +export const MATERIALITY_ANALYZER_KIMI_LIVE_V2_RUN_ID = "materiality-analyzer-kimi-live-v2" as const; export const MATERIALITY_EVALUATION_ARTIFACT_ROOT = ".artifacts/founder-weekly-review/materiality-evaluation" as const; @@ -679,7 +680,10 @@ function percentage(value: number): string { return `${(value * 100).toFixed(1)}%`; } -export function renderMaterialityEvaluationArtifacts(result: MaterialityEvaluationResult): { +export function renderMaterialityEvaluationArtifacts( + result: MaterialityEvaluationResult, + options: { providerInvoked?: boolean } = {}, +): { summaryJson: string; failuresJson: string; evaluationMarkdown: string; @@ -734,7 +738,9 @@ export function renderMaterialityEvaluationArtifacts(result: MaterialityEvaluati "- Alignment miss rate is unrecovered intended modified/unchanged/split/merge relations divided by all such intended relations.", "- Reduction ratio is bounded raw audit excerpt characters divided by structurally selected condensed evidence characters.", "", - "Generated artifacts contain synthetic fixture text only; no provider was invoked.", + options.providerInvoked + ? "Generated artifacts contain synthetic fixture text only; the explicitly enabled materiality analyzer provider was invoked." + : "Generated artifacts contain synthetic fixture text only; no provider was invoked.", "", ].join("\n"); return { @@ -746,10 +752,11 @@ export function renderMaterialityEvaluationArtifacts(result: MaterialityEvaluati export async function writeMaterialityEvaluationArtifacts( result: MaterialityEvaluationResult, - runId: string = MATERIALITY_EVALUATION_RUN_ID + runId: string = MATERIALITY_EVALUATION_RUN_ID, + options: { providerInvoked?: boolean } = {}, ): Promise<{ directory: string; summary: string; failures: string; evaluation: string }> { const directory = evaluationArtifactDirectory(runId); - const artifacts = renderMaterialityEvaluationArtifacts(result); + const artifacts = renderMaterialityEvaluationArtifacts(result, options); await mkdir(directory, { recursive: true }); const paths = { directory, @@ -769,6 +776,14 @@ async function main(): Promise { const mode = process.env.FWR_MATERIALITY_EVAL_MODE ?? "offline"; let result: MaterialityEvaluationResult; let defaultRunId: string; + let liveCallRecords: Array<{ + durationMs: number; + status: "success" | "failure"; + provider?: string; + model?: string; + promptVersion?: string; + errorCode?: string; + }> | undefined; if (mode === "deterministic") { result = runMaterialityEvaluation(); defaultRunId = MATERIALITY_EVALUATION_RUN_ID; @@ -776,27 +791,75 @@ async function main(): Promise { result = await runMaterialityAnalyzerEvaluation(); defaultRunId = MATERIALITY_ANALYZER_EVALUATION_RUN_ID; } else if (mode === "live") { + const { config: loadDotenv } = await import("dotenv"); + loadDotenv({ path: resolve(process.cwd(), "../../.env"), quiet: true }); const { createConfiguredDocumentChangeMaterialityAnalyzer } = await import("../src/server/founder-weekly-review/document-change-materiality-analyzer"); const maximumCalls = Math.max(1, Math.min(64, Number(process.env.FWR_MATERIALITY_EVAL_MAX_CALLS ?? 64) || 64)); const live = createConfiguredDocumentChangeMaterialityAnalyzer(); if (!live) throw new Error("Live materiality evaluation requires FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED=true."); let calls = 0; + liveCallRecords = []; const bounded: DocumentChangeMaterialityAnalyzer = { - analyze(input) { + async analyze(input) { if (calls >= maximumCalls) throw new Error("Live materiality evaluation reached its explicit global call limit."); calls++; - return live.analyze(input); + const startedAt = Date.now(); + try { + const response = await live.analyze(input); + liveCallRecords!.push({ + durationMs: Date.now() - startedAt, + status: "success", + ...(response.metadata?.provider ? { provider: response.metadata.provider } : {}), + ...(response.metadata?.model ? { model: response.metadata.model } : {}), + ...(response.metadata?.promptVersion ? { promptVersion: response.metadata.promptVersion } : {}), + }); + return response; + } catch (error) { + const errorCode = error && typeof error === "object" && "code" in error && typeof error.code === "string" + ? error.code : "unknown"; + liveCallRecords!.push({ durationMs: Date.now() - startedAt, status: "failure", errorCode }); + throw error; + } }, }; result = await runMaterialityAnalyzerEvaluation(bounded); - defaultRunId = "materiality-analyzer-live-v1"; + defaultRunId = MATERIALITY_ANALYZER_KIMI_LIVE_V2_RUN_ID; } else { throw new Error("FWR_MATERIALITY_EVAL_MODE must be deterministic, offline, or live."); } const artifacts = await writeMaterialityEvaluationArtifacts( result, process.env.FWR_MATERIALITY_EVALUATION_RUN_ID ?? defaultRunId, + { providerInvoked: mode === "live" }, ); + if (liveCallRecords) { + const durations = liveCallRecords.map(record => record.durationMs).sort((a, b) => a - b); + const percentile = (fraction: number) => durations.length === 0 ? 0 : durations[Math.min(durations.length - 1, Math.ceil(durations.length * fraction) - 1)]!; + const firstSuccess = liveCallRecords.find(record => record.status === "success"); + await writeFile(resolve(artifacts.directory, "call-summary.json"), `${JSON.stringify({ + strategy: defaultRunId, + provider: firstSuccess?.provider ?? null, + model: firstSuccess?.model ?? null, + promptVersion: firstSuccess?.promptVersion ?? null, + totalCalls: liveCallRecords.length, + successfulCalls: liveCallRecords.filter(record => record.status === "success").length, + failedCalls: liveCallRecords.filter(record => record.status === "failure").length, + failuresByCode: Object.fromEntries([...new Set(liveCallRecords.flatMap(record => record.errorCode ? [record.errorCode] : []))] + .sort().map(code => [code, liveCallRecords!.filter(record => record.errorCode === code).length])), + latencyMs: { + minimum: durations[0] ?? 0, + median: percentile(0.5), + p95: percentile(0.95), + maximum: durations.at(-1) ?? 0, + under5Seconds: durations.filter(value => value < 5_000).length, + from5To10Seconds: durations.filter(value => value >= 5_000 && value < 10_000).length, + from10To15Seconds: durations.filter(value => value >= 10_000 && value < 15_000).length, + atLeast15Seconds: durations.filter(value => value >= 15_000).length, + total: durations.reduce((total, value) => total + value, 0), + }, + providerTokenUsageAvailable: false, + }, null, 2)}\n`, "utf8"); + } console.log(JSON.stringify({ ...result.summary, artifactDirectory: artifacts.directory })); } From 2e038de201d032f3b8fb51e38336b9753eaae76c Mon Sep 17 00:00:00 2001 From: Peace Odetola Date: Sat, 8 Aug 2026 04:07:16 -0500 Subject: [PATCH 27/29] Integrate Founder Weekly Review evaluation and LLM grader --- .../founderWeeklyReview/generation.test.ts | 123 +- .../founderWeeklyReview/runner.test.ts | 139 ++ ...ounder-weekly-review-synthetic-fixtures.ts | 4 +- .../run-founder-weekly-review-benchmark.ts | 16 + ...founder-weekly-review-inngest-transport.ts | 61 +- ...run-founder-weekly-review-realistic-e2e.ts | 21 +- .../evaluation-markdown.ts | 67 + .../generation-adapter.ts | 1 + packages/features/package.json | 2 + .../benchmarks/baseline-output.json | 1396 ++++++++++++++++- .../founder-weekly-review/benchmarks/cases.ts | 345 +++- .../benchmarks/evaluate-generated-review.ts | 58 + .../founder-weekly-review/benchmarks/index.ts | 3 + .../founder-weekly-review/benchmarks/run.ts | 1 + .../benchmarks/runner.ts | 371 +++-- .../src/founder-weekly-review/contracts.ts | 2 +- .../founder-weekly-review/document-change.ts | 32 +- .../evaluation-prompt.ts | 50 + .../src/founder-weekly-review/evaluation.ts | 62 +- .../generation-validation.ts | 63 + .../src/founder-weekly-review/generator.ts | 2 +- .../src/founder-weekly-review/grader.ts | 55 + .../src/founder-weekly-review/index.ts | 1 + .../src/founder-weekly-review/llm-grader.ts | 32 + .../src/founder-weekly-review/prompts.ts | 68 +- 25 files changed, 2714 insertions(+), 261 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/runner.test.ts create mode 100644 apps/web/scripts/run-founder-weekly-review-benchmark.ts create mode 100644 apps/web/src/server/founder-weekly-review/evaluation-markdown.ts create mode 100644 packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts create mode 100644 packages/features/src/founder-weekly-review/benchmarks/index.ts create mode 100644 packages/features/src/founder-weekly-review/benchmarks/run.ts create mode 100644 packages/features/src/founder-weekly-review/evaluation-prompt.ts create mode 100644 packages/features/src/founder-weekly-review/grader.ts create mode 100644 packages/features/src/founder-weekly-review/llm-grader.ts diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts index f324053ce..50c133d1d 100644 --- a/apps/web/__tests__/founderWeeklyReview/generation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -28,12 +28,12 @@ function snapshot(items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderW }; } -const source = (sourceId: string, sourceType: FounderWeeklyReviewEvidenceSnapshot["items"][number]["sourceType"], excerpt = "Evidence excerpt") => ({ +const source = (sourceId: string, sourceType: FounderWeeklyReviewEvidenceSnapshot["items"][number]["sourceType"], excerpt = "Evidence excerpt", metadata: Record = {}) => ({ sourceId, sourceType, title: `${sourceType} title`, excerpt, - metadata: {}, + metadata, }); function noEvidence(message = "No evidence", cta = "Add evidence") { @@ -48,7 +48,7 @@ function validPayload(): FounderWeeklyReviewV2Payload { whatShipped: { state: "evidence", items: [{ kind: "observed_fact", text: "A release shipped.", sourceIds: ["doc-1"], confidence: 1 }] }, whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "A customer requested audit logs.", sourceIds: ["feedback-1"], confidence: 0.5 }] }, currentBlockers: { state: "evidence", items: [{ kind: "observed_fact", text: "SSO remains blocked.", sourceIds: ["context-1"], confidence: 0.8 }] }, - nextPriorities: { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Prioritize SSO.", sourceIds: ["context-1"], confidence: 0.8 }] }, + nextPriorities: { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Prioritize SSO.", sourceIds: ["context-1"], confidence: 0.8, rationale: null }] }, }, }; } @@ -175,7 +175,7 @@ describe("Founder Weekly Review generation", () => { it("allows workspace_document for blockers and priorities while customer-only enforcement remains", async () => { const payload = validPayload(); payload.sections.currentBlockers = { state: "evidence", items: [{ kind: "observed_fact", text: "Current context", sourceIds: ["workspace-1"], confidence: 0.5 }] }; - payload.sections.nextPriorities = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Act on current context", sourceIds: ["workspace-1"], confidence: 0.5 }] }; + payload.sections.nextPriorities = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Act on current context", sourceIds: ["workspace-1"], confidence: 0.5, rationale: null }] }; await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([...completeSnapshot().items, source("workspace-1", "workspace_document")]), generate: fake(payload) })).resolves.toBeDefined(); }); @@ -265,4 +265,119 @@ describe("Founder Weekly Review generation", () => { sections: Object.fromEntries(["whatChanged", "whatShipped", "whatCustomersSaid", "currentBlockers", "nextPriorities"].map((key) => [key, { heading: key, items: [{ kind: "no_evidence", code: "not_assessed" }] }])), }).schemaVersion).toBe("founder-weekly-review/v1"); }); + + it("allows planned evidence to describe future shipment", async () => { + const payload = validPayload(); + + payload.sections.whatChanged = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Feature will ship next week.", + sourceIds: ["doc-1"], + confidence: 0.8, + }], + }; + + payload.sections.whatShipped = { + state: "no_evidence", + noEvidence: { + code: "no_relevant_evidence", + message: "No shipped work yet.", + cta: "Add shipped evidence", + }, + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Feature roadmap item.", { + evidenceStatus: "planned", + }), + source("feedback-1", "customer_feedback"), + source("context-1", "founder_context"), + ]), + generate: fake(payload), + }) + ).resolves.toBeDefined(); + }); + + + it("rejects planned evidence described as already shipped", async () => { + const payload = validPayload(); + payload.sections.whatChanged = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Feature shipped yesterday.", + sourceIds: ["doc-1"], + confidence: 0.8, + }], + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Feature roadmap item.", { + evidenceStatus: "planned", + }), + ]), + generate: fake(payload), + }) + ).rejects.toThrow("Claim describes planned work as completed."); + }); + + + it("allows shipped evidence to describe shipped work", async () => { + const payload = validPayload(); + payload.sections.whatShipped = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Feature shipped yesterday.", + sourceIds: ["doc-1"], + confidence: 1, + }], + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Feature shipped yesterday.", { + evidenceStatus: "shipped", + }), + source("feedback-1", "customer_feedback"), + source("context-1", "founder_context"), + ]), + generate: fake(payload), + }) + ).resolves.toBeDefined(); + }); + + + it("allows changed evidence to describe updates without implying shipment", async () => { + const payload = validPayload(); + payload.sections.whatChanged = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Documentation was updated.", + sourceIds: ["doc-1"], + confidence: 0.8, + }], + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Documentation updated.", { + evidenceStatus: "changed", + }), + source("feedback-1", "customer_feedback"), + source("context-1", "founder_context"), + ]), + generate: fake(payload), + }) + ).resolves.toBeDefined(); + }); }); diff --git a/apps/web/__tests__/founderWeeklyReview/runner.test.ts b/apps/web/__tests__/founderWeeklyReview/runner.test.ts new file mode 100644 index 000000000..017cef211 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/runner.test.ts @@ -0,0 +1,139 @@ +import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; +import { FounderWeeklyReviewEvidenceSnapshotSchema } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewV2PayloadSchema } from "@launchstack/features/founder-weekly-review"; + +import type { FounderWeeklyReviewGenerateFn } from "@launchstack/features/founder-weekly-review"; + +describe("Founder Weekly Review evaluation pipeline", () => { + const fakeEvidenceSnapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-13T00:00:00.000Z", + reportingPeriod: { + start: "2026-07-06", + end: "2026-07-12", + }, + workspaceTimezone: "UTC", + items: [ + { + sourceType: "document_change", + sourceId: "doc:release", + title: "Release notes", + sourceTimestamp: "2026-07-10T00:00:00.000Z", + excerpt: "Export shipped.", + metadata: {}, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback:export", + title: "Customer feedback", + sourceTimestamp: "2026-07-11T00:00:00.000Z", + excerpt: "Customers requested export support.", + metadata: {}, + }, + ], + sourceWarnings: [], + }); + + const fakeReport = FounderWeeklyReviewV2PayloadSchema.parse({ + schemaVersion: "founder-weekly-review/v2", + sections: { + whatChanged: { + state: "no_evidence", + noEvidence: { + code: "no_changes", + message: "No changes were reported.", + cta: "Discuss recent updates.", + }, + }, + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Export shipped.", + sourceIds: ["doc:release"], + confidence: 0.8, + }, + ], + }, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customers requested export support.", + sourceIds: ["feedback:export"], + confidence: 0.8, + }, + ], + }, + currentBlockers: { + state: "no_evidence", + noEvidence: { + code: "no_blockers", + message: "No blockers were reported.", + cta: "Reach out if blockers appear.", + }, + }, + nextPriorities: { + state: "no_evidence", + noEvidence: { + code: "no_priorities", + message: "No next priorities were specified.", + cta: "Discuss upcoming priorities.", + }, + }, + }, + }); + it("runs deterministic evaluation and LLM grading with structured output", async () => { + const mockGrader: FounderWeeklyReviewGenerateFn = async (input) => { + expect(input.schemaName).toBe("founder_weekly_review_grader"); + expect(input.prompt).toContain("Evidence:"); + expect(input.prompt).toContain("Report:"); + expect(input.system).toContain("groundedness"); + + return { + object: { + overallScore: 0.8, + dimensions: { + groundedness: 0.9, + materiality: 0.8, + temporalAccuracy: 0.7, + synthesisQuality: 0.75, + actionability: 0.85, + }, + findings: [], + summary: "Looks good", + }, + metadata: { + provider: "mock", + model: "mock-model", + }, + }; + }; + + const result = await evaluateGeneratedFounderWeeklyReview( + fakeEvidenceSnapshot, + fakeReport, + mockGrader + ); + + expect(result.deterministic).toBeDefined(); + expect(result.llmGrader?.overallScore).toBe(0.8); + expect(result.llmGrader?.metadata?.provider).toBe("mock"); + }); + + it("rejects malformed LLM grader output", async () => { + const badGrader: FounderWeeklyReviewGenerateFn = async () => ({ + overallScore: 5, + } as any); + + await expect( + evaluateGeneratedFounderWeeklyReview( + fakeEvidenceSnapshot, + fakeReport, + badGrader + ) + ).rejects.toThrow(); + }); +}); \ No newline at end of file diff --git a/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts b/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts index ce458688f..50b396dc4 100644 --- a/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts +++ b/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts @@ -12,11 +12,11 @@ const make = (items: unknown[]) => FounderWeeklyReviewEvidenceSnapshotSchema.par export const syntheticFounderWeeklyReviewFixtures = { partial: make([ - { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1" } }, + { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1", evidenceStatus: "shipped" } }, { sourceType: "founder_context", sourceId: "synthetic:context:priority", title: "Founder context", excerpt: "Prioritize onboarding reliability.", metadata: { fixture: "synthetic-v1" } }, ]), full: make([ - { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1" } }, + { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1", evidenceStatus: "shipped" } }, { sourceType: "customer_feedback", sourceId: "synthetic:feedback:export", title: "Customer feedback", sourceTimestamp: "2026-02-21T12:00:00.000Z", excerpt: "A customer requested saved export filters.", metadata: { fixture: "synthetic-v1" } }, { sourceType: "founder_context", sourceId: "synthetic:context:priority", title: "Founder context", excerpt: "Prioritize onboarding reliability.", metadata: { fixture: "synthetic-v1" } }, ]), diff --git a/apps/web/scripts/run-founder-weekly-review-benchmark.ts b/apps/web/scripts/run-founder-weekly-review-benchmark.ts new file mode 100644 index 000000000..cacf7fcdd --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-benchmark.ts @@ -0,0 +1,16 @@ +import dotenv from "dotenv"; +import path from "path"; + +dotenv.config({ + path: path.resolve(import.meta.dirname, "../../../.env"), +}); + +import { generateFounderWeeklyReviewStructured } from "../src/server/founder-weekly-review/generation-adapter"; +import { main } from "@launchstack/features/founder-weekly-review/benchmarks/run"; + +console.log( + "OPENAI:", + process.env.OPENAI_API_KEY ? "loaded" : "missing" +); + +await main(generateFounderWeeklyReviewStructured); \ No newline at end of file diff --git a/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts b/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts index 390802f95..2e5b25ddb 100644 --- a/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts +++ b/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts @@ -19,6 +19,9 @@ import { FounderWeeklyReviewRepository } from "@launchstack/features/founder-wee import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { renderFounderWeeklyReviewEvaluationMarkdown } from "~/server/founder-weekly-review/evaluation-markdown"; +import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; const require = createRequire(import.meta.url); const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); @@ -373,8 +376,8 @@ async function main(): Promise { if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(localUrl)) { throw new Error("Refusing non-local database."); } - if (!STARTUP_SMOKE_ONLY && !process.env.MOONSHOT_API_KEY?.trim()) { - throw new Error("MOONSHOT_API_KEY is required for the real callback generation."); + if (!STARTUP_SMOKE_ONLY && !process.env.OPENAI_API_KEY?.trim()) { + throw new Error("OPENAI_API_KEY is required for the real callback generation."); } const testDb = await createFounderWeeklyReviewTestDatabase(); @@ -550,12 +553,50 @@ async function main(): Promise { await sleep(500); } if (!final?.reviewPayload || !final.evidenceSnapshot) throw new Error("Timed out before persisted draft read-back."); + const evaluation = await evaluateGeneratedFounderWeeklyReview( + final.evidenceSnapshot, + final.reviewPayload as any, + generateFounderWeeklyReviewStructured + ); const dispatch = (await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, created.dispatch.id)))[0]; const rendered = renderFounderWeeklyReviewMarkdown(final); - const directory = resolve(process.cwd(), ".artifacts/founder-weekly-review"); + const directory = resolve(process.cwd(), `.artifacts/founder-weekly-review/transport/${final.id}`); await mkdir(directory, { recursive: true }); - const markdownPath = resolve(directory, `${final.id}-transport.md`); + const reportPath = resolve(directory, "report.json"); + const markdownPath = resolve(directory, "report.md"); + const evaluationJsonPath = resolve(directory, "evaluation.json"); + const evaluationMarkdownPath = resolve(directory, "evaluation.md"); + + await writeFile( + reportPath, + JSON.stringify( + { + runId: final.id, + status: final.status, + provider: final.modelMetadata?.provider, + model: final.modelMetadata?.model, + reportingPeriod: final.reportingPeriod, + review: final.reviewPayload, + }, + null, + 2 + ), + "utf8" + ); + await writeFile(markdownPath, rendered, "utf8"); + + await writeFile( + evaluationJsonPath, + JSON.stringify(evaluation, null, 2), + "utf8" + ); + + await writeFile( + evaluationMarkdownPath, + renderFounderWeeklyReviewEvaluationMarkdown(evaluation), + "utf8" + ); if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); @@ -563,7 +604,17 @@ async function main(): Promise { } const generationEventKeys = ["runId", "companyId", "generationJobId", "generationClaimId"]; const forbiddenEventKeys = ["evidenceSnapshot", "founderContext", "documentContent", "customerFeedback", "prompt", "reviewPayload", "providerResponse", "databaseUrl", "credentials", "token"]; - console.log(JSON.stringify({ runId: final.id, lifecycle: seen, dispatch: { initialStatus: initialDispatches[0]!.status, finalStatus: dispatch?.status, attempts: dispatch?.attemptCount }, ingressEventIds: requested.ids, callbackUrl, functionId: "founder-weekly-review-generation", transportEvent: { name: "founder-weekly-review/generation.requested", keys: generationEventKeys, companyIdSerializedAsString: true, forbiddenKeysAbsent: forbiddenEventKeys.every((key) => !generationEventKeys.includes(key)) }, steps: ["claim-evidence", "collect-evidence", "persist-evidence", "claim", "generate", "persist"], evidenceCounts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, final!.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])), snapshotDigest: digest(final.evidenceSnapshot), retryCount: final.retryCount, generationAttempt: final.generationAttempt, provider: final.modelMetadata?.provider, model: final.modelMetadata?.model, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, markdownPath, terminalEqualsExport: (await readFile(markdownPath, "utf8")) === rendered, devLogsMentionFunction: inngestLogs.join("").includes("founder-weekly-review-generation") })); + console.log(JSON.stringify({ runId: final.id, lifecycle: seen, dispatch: { initialStatus: initialDispatches[0]!.status, finalStatus: dispatch?.status, attempts: dispatch?.attemptCount }, ingressEventIds: requested.ids, callbackUrl, functionId: "founder-weekly-review-generation", transportEvent: { name: "founder-weekly-review/generation.requested", keys: generationEventKeys, companyIdSerializedAsString: true, forbiddenKeysAbsent: forbiddenEventKeys.every((key) => !generationEventKeys.includes(key)) }, steps: ["claim-evidence", "collect-evidence", "persist-evidence", "claim", "generate", "persist"], evidenceCounts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, final!.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])), snapshotDigest: digest(final.evidenceSnapshot), retryCount: final.retryCount, generationAttempt: final.generationAttempt, provider: final.modelMetadata?.provider, model: final.modelMetadata?.model, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, evaluation: { + deterministicScore: evaluation.deterministic?.overallScore ?? null, + llmScore: evaluation.llmGrader?.overallScore ?? null, + failures: evaluation.failures, + }, + evaluationPaths: { + report: reportPath, + markdown: markdownPath, + evaluation: evaluationJsonPath, + evaluationMarkdown: evaluationMarkdownPath, + }, terminalEqualsExport: (await readFile(markdownPath, "utf8")) === rendered, devLogsMentionFunction: inngestLogs.join("").includes("founder-weekly-review-generation") })); shutdownDrain.finalDiagnosticsCaptured = true; } catch (error) { printRecentDiagnostics("next", nextLogs); diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts index 1b94b478c..4d59541e2 100644 --- a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -11,7 +11,9 @@ import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-rev import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { renderFounderWeeklyReviewEvaluationMarkdown } from "~/server/founder-weekly-review/evaluation-markdown"; import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; +import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; const require = createRequire(import.meta.url); const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); @@ -19,7 +21,7 @@ const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/ type Fixture = { reportingPeriod: { start: string; end: string }; workspaceTimezone: string; founderContext: string; documents: Array<{ title: string; category: string; changelog: string; timestamp: string; chunks?: string[] }> }; type EvidenceMode = ReturnType; type ComputedTexts = { before: string; after: string; v3: string; bHistorical: string; nullVersion: string; foreign: string; unrelated: string }; -type ArtifactPaths = { evidence: string; report: string; markdown: string; summary: string }; +type ArtifactPaths = { evidence: string; report: string; markdown: string; evaluation: string; evaluationMarkdown: string; summary: string }; function canonicalize(value: unknown): unknown { if (value === null || ["string", "boolean"].includes(typeof value)) return value; if (typeof value === "number" && Number.isFinite(value)) return value; if (Array.isArray(value)) return value.map(canonicalize); if (typeof value === "object") return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalize((value as Record)[key])])); throw new Error("Cannot canonicalize snapshot."); } function digest(value: unknown) { return createHash("sha256").update(JSON.stringify(canonicalize(value)), "utf8").digest("hex"); } @@ -126,11 +128,26 @@ try { const generated = await generateFounderWeeklyReview({ evidenceSnapshot: generating.evidenceSnapshot, generate: async (request) => { generationCalls++; return generateFounderWeeklyReviewStructured(request); } }); validateFounderWeeklyReviewV2Citations(generated.reviewPayload as never, generating.evidenceSnapshot); const saved = await worker.saveGeneratedDraft(generationContext, generated.reviewPayload, generated.modelMetadata); const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft" || !readBack.evidenceSnapshot || digest(readBack.evidenceSnapshot) !== beforeDigest) throw new Error("Validated draft read-back or snapshot immutability failed."); + + const evaluation = await evaluateGeneratedFounderWeeklyReview( + readBack.evidenceSnapshot, + readBack.reviewPayload as unknown as Parameters[1], + generateFounderWeeklyReviewStructured + ); + + console.log(JSON.stringify({ + evaluation: { + deterministicScore: evaluation.deterministic?.overallScore, + failures: evaluation.failures, + llmScore: evaluation.llmGrader?.overallScore, + } + })); + if (mode === "computed") assertComputedReport(readBack.reviewPayload, readBack.evidenceSnapshot); const rendered = renderFounderWeeklyReviewMarkdown(readBack); const dispatchRows = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, saved.id)); const runRows = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); let artifactPaths: ArtifactPaths | null = null; - if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const exportRoot = founderWeeklyReviewRealisticExportRoot(mode, process.env.SYNTHETIC_FWR_EXPORT_DIR); const directory = resolve(process.cwd(), exportRoot, saved.id); await mkdir(directory, { recursive: true }); artifactPaths = { evidence: resolve(directory, "evidence.json"), report: resolve(directory, "report.json"), markdown: resolve(directory, "report.md"), summary: resolve(directory, "run-summary.json") }; await writeAtomic(artifactPaths.evidence, JSON.stringify(readBack.evidenceSnapshot, null, 2)); await writeAtomic(artifactPaths.report, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, reportingPeriod: readBack.reportingPeriod, review: readBack.reviewPayload }, null, 2)); await writeAtomic(artifactPaths.markdown, rendered); await writeAtomic(artifactPaths.summary, JSON.stringify({ runId: saved.id, scenario: "realistic-company", mode, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), repairCount: generationCalls - 1, retryCount: saved.retryCount, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, snapshotDigestBefore: beforeDigest, snapshotDigestAfter: digest(readBack.evidenceSnapshot), artifactPaths }, null, 2)); } + if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const exportRoot = founderWeeklyReviewRealisticExportRoot(mode, process.env.SYNTHETIC_FWR_EXPORT_DIR); const directory = resolve(process.cwd(), exportRoot, saved.id); await mkdir(directory, { recursive: true }); artifactPaths = { evidence: resolve(directory, "evidence.json"), report: resolve(directory, "report.json"), markdown: resolve(directory, "report.md"), evaluation: resolve(directory, "evaluation.json"), evaluationMarkdown: resolve(directory, "evaluation.md"), summary: resolve(directory, "run-summary.json") }; await writeAtomic(artifactPaths.evidence, JSON.stringify(readBack.evidenceSnapshot, null, 2)); await writeAtomic(artifactPaths.report, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, reportingPeriod: readBack.reportingPeriod, review: readBack.reviewPayload }, null, 2)); await writeAtomic(artifactPaths.markdown, rendered); await writeAtomic(artifactPaths.evaluation, JSON.stringify(evaluation, null, 2)); await writeAtomic (artifactPaths.evaluationMarkdown, renderFounderWeeklyReviewEvaluationMarkdown(evaluation)); await writeAtomic(artifactPaths.summary, JSON.stringify({ runId: saved.id, scenario: "realistic-company", mode, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), repairCount: generationCalls - 1, retryCount: saved.retryCount, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, snapshotDigestBefore: beforeDigest, snapshotDigestAfter: digest(readBack.evidenceSnapshot), artifactPaths }, null, 2)); } if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); console.log("===== END FOUNDER WEEKLY REVIEW ====="); } console.log(JSON.stringify({ runId: saved.id, mode, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, repairCount: generationCalls - 1, dispatchCount: dispatchRows.length, runRowCount: runRows.length, artifactPaths })); } finally { await testDb.close(); } diff --git a/apps/web/src/server/founder-weekly-review/evaluation-markdown.ts b/apps/web/src/server/founder-weekly-review/evaluation-markdown.ts new file mode 100644 index 000000000..de2d9ba53 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/evaluation-markdown.ts @@ -0,0 +1,67 @@ +import type { GeneratedReviewEvaluation } from "@launchstack/features/founder-weekly-review/benchmarks"; + +export function renderFounderWeeklyReviewEvaluationMarkdown( + evaluation: GeneratedReviewEvaluation +) { + return ` +# Founder Weekly Review Evaluation + +## Deterministic Evaluation + +${ + evaluation.deterministic + ? ` +Score: ${evaluation.deterministic.overallScore} + +Failures: + +${ + evaluation.deterministic.failures.length + ? evaluation.deterministic.failures + .map( + (failure) => + `- ${failure.category}: ${failure.explanation}` + ) + .join("\n") + : "- None" +} +` + : "Malformed payload" +} + + +## LLM Grader + +${ + evaluation.llmGrader + ? ` +Overall Score: ${evaluation.llmGrader.overallScore} + +## Dimensions + +- Groundedness: ${evaluation.llmGrader.dimensions.groundedness} +- Materiality: ${evaluation.llmGrader.dimensions.materiality} +- Temporal Accuracy: ${evaluation.llmGrader.dimensions.temporalAccuracy} +- Synthesis Quality: ${evaluation.llmGrader.dimensions.synthesisQuality} +- Actionability: ${evaluation.llmGrader.dimensions.actionability} + + +## Findings + +${evaluation.llmGrader.findings + .map( + (finding) => + `### ${finding.section} +${finding.severity}: ${finding.explanation}` + ) + .join("\n\n")} + + +## Summary + +${evaluation.llmGrader.summary} +` + : "No LLM grader result" +} +`; +} \ No newline at end of file diff --git a/apps/web/src/server/founder-weekly-review/generation-adapter.ts b/apps/web/src/server/founder-weekly-review/generation-adapter.ts index 7cea36e43..f591bacc4 100644 --- a/apps/web/src/server/founder-weekly-review/generation-adapter.ts +++ b/apps/web/src/server/founder-weekly-review/generation-adapter.ts @@ -15,6 +15,7 @@ export function generateFounderWeeklyReviewStructured(i schema: TSchema; schemaName?: string; generationPhase?: "initial" | "semantic-repair"; + temperature?: number; }): Promise<{ object: ReturnType; metadata: StructuredGenerationMetadata; diff --git a/packages/features/package.json b/packages/features/package.json index bccc018c9..0b6a9667b 100644 --- a/packages/features/package.json +++ b/packages/features/package.json @@ -13,6 +13,8 @@ "types": "./src/index.ts", "exports": { ".": "./src/index.ts", + "./founder-weekly-review/benchmarks": "./src/founder-weekly-review/benchmarks/index.ts", + "./founder-weekly-review/benchmarks/run": "./src/founder-weekly-review/benchmarks/run.ts", "./adeu": { "types": "./src/adeu/index.ts", "default": "./src/adeu/index.ts" diff --git a/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json b/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json index 08dfcab9b..bb461dd06 100644 --- a/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json +++ b/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json @@ -1,21 +1,22 @@ { "summary": { - "totalCases": 19, - "passed": 19, + "totalCases": 26, + "passed": 26, "failed": 0, - "hardFailures": 9, + "hardFailures": 12, "passRate": 1, - "overallScore": 0.48157894736842105 + "deterministicScore": 0.5240384615384616, + "llmScore": 0.9291666666666667 }, "metrics": { - "citationValidity": 0.8823529411764706, + "citationValidity": 0.9166666666666666, "citationCoverage": 1, - "unsupportedClaimRate": 0.058823529411764705, - "unsupportedShippedClaimRate": 0.11764705882352941, - "sourceTypeViolationRate": 0.11764705882352941, - "evidenceCoverage": 0.7156862745098039, + "unsupportedClaimRate": 0.125, + "unsupportedShippedClaimRate": 0.08333333333333333, + "sourceTypeViolationRate": 0.16666666666666666, + "evidenceCoverage": 0.8541666666666666, "emptySectionCorrectness": 1, - "duplicateClaimRate": 0.058823529411764705 + "duplicateClaimRate": 0.041666666666666664 }, "weakestCases": [ { @@ -42,12 +43,12 @@ ], "commonFailures": { "invalid_citation": 2, - "invalid_source_type": 3, + "invalid_source_type": 5, "duplicate_claim": 1, "malformed_payload": 2, - "unsupported_shipped_claim": 2, - "unsupported_claim": 1, - "conflicting_evidence": 1 + "unsupported_claim": 3, + "conflicting_evidence": 1, + "unsupported_shipped_claim": 2 }, "cases": [ { @@ -63,10 +64,55 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook potential internal changes that could be relevant." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not reflect actual progress if there were minor updates." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The customer feedback is well-supported by the cited evidence, but it lacks broader context." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable next steps. It correctly identifies no changes or shipped work but misses potential internal developments and future priorities.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "invalid_citation_report", @@ -80,7 +126,8 @@ "unsupportedClaimRate": 0, "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, - "evidenceCoverage": 1, + "evidenceCoverage": 0, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -90,7 +137,57 @@ "section": "whatCustomersSaid", "explanation": "Unknown sourceId: fake_feedback_999" } - ] + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "fake_feedback_999" + ], + "explanation": "The claim about customer feedback is unsupported as the cited sourceId does not match any valid evidence." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report fails to identify any changes during the reporting period, which may overlook important updates." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states that nothing was shipped, but it does not provide context or evidence for this claim." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + }, + { + "section": "synthesisQuality", + "severity": "high", + "sourceIds": [], + "explanation": "The report lacks synthesis of evidence and merely states that no changes or priorities exist without connecting to broader themes." + } + ], + "summary": "The report has significant issues with groundedness due to unsupported claims about customer feedback. It also lacks material updates and actionable insights, making it difficult for the founder to make informed decisions. The synthesis quality is poor, as it fails to connect evidence or summarize key themes.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "empty_workspace", @@ -105,15 +202,64 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 0, + "dimensions": { + "groundedness": 0, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for changes during the reporting period, leading to unsupported claims." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of shipments, making it impossible to assess what was delivered." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "Lack of customer feedback evidence results in unsupported claims about customer insights." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of blockers, leaving the report without context on potential issues." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence to ground priorities, making it unclear what the next steps should be." + } + ], + "summary": "The report lacks any evidence across all sections, resulting in a complete failure to provide grounded, actionable insights for the reporting period.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "complete_workspace", "passed": true, - "score": 0.8999999999999999, + "score": 1, "hasHardFailure": false, "metrics": { "canonicalSchemaValid": true, @@ -122,11 +268,60 @@ "unsupportedClaimRate": 0, "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, - "evidenceCoverage": 0.6666666666666666, + "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 0.7, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.6, + "temporalAccuracy": 0.9, + "synthesisQuality": 0.5, + "actionability": 0.7 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states 'No changes were reported for this period,' which may overlook minor internal changes that could be relevant." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [ + "context_1" + ], + "explanation": "The recommendation to improve onboarding lacks a clear rationale or connection to evidence, making it less actionable." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The customer feedback is appropriately cited and relevant." + }, + { + "section": "whatShipped", + "severity": "low", + "sourceIds": [ + "github_1" + ], + "explanation": "The shipping of the CSV export feature is well-supported by evidence." + } + ], + "summary": "The report effectively documents the shipping of the CSV export feature and customer feedback but fails to capture any internal changes and lacks depth in the rationale for recommendations. Overall, it provides a solid foundation but needs improvement in materiality and synthesis quality.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "partial_workspace", @@ -141,10 +336,55 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook potential internal changes that could be relevant." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not reflect actual progress if there were minor updates." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The customer feedback is well-supported by the cited evidence, but it lacks broader context." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable priorities. It correctly identifies no changes or shipped work, but this may not fully represent the team's activities.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "invalid_missing_source_citation", @@ -158,7 +398,8 @@ "unsupportedClaimRate": 0, "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, - "evidenceCoverage": 1, + "evidenceCoverage": 0, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -168,7 +409,57 @@ "section": "whatCustomersSaid", "explanation": "Unknown sourceId: missing_source" } - ] + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "missing_source" + ], + "explanation": "The claim about customer feedback lacks a valid source ID, making it unsupported." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook important updates." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which could miss significant developments." + }, + { + "section": "currentBlockers", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of reported blockers may not reflect the actual situation." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "Not specifying next priorities may lead to a lack of direction." + } + ], + "summary": "The report has significant issues with groundedness due to unsupported claims, lacks material updates, and fails to provide actionable insights or synthesis of evidence. It needs improvement in identifying key changes and providing clear next steps for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "founder_context_as_customer_feedback", @@ -183,6 +474,7 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 1, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -192,7 +484,51 @@ "section": "whatCustomersSaid", "explanation": "Source type \"founder_context\" is not valid for section \"whatCustomersSaid\"" } - ] + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "context_1" + ], + "explanation": "The claim about customer requests for export is supported by the founder's notes, but the report lacks a broader context or additional evidence to strengthen the claim." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no changes were reported, which may overlook important internal updates that could be relevant." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which could be misleading if there were minor updates or deployments that were not documented." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report provides some grounded evidence regarding customer feedback but lacks material updates and actionable priorities. It effectively maintains temporal accuracy but fails to synthesize information or provide clear next steps for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "founder_context_only_valid", @@ -207,10 +543,61 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for changes during the reporting period." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for shipped items during the reporting period." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for customer feedback." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for current blockers." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [ + "context_1" + ], + "explanation": "Recommendation to improve onboarding is grounded in founder context but lacks detailed rationale." + } + ], + "summary": "The report lacks evidence across all sections except for a single recommendation. It fails to provide any information on changes, shipped items, customer feedback, or blockers, significantly impacting its overall quality.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "multi_source_customer_feedback", @@ -225,10 +612,56 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states 'No changes were reported for this period,' which may not accurately reflect the customer feedback indicating a request for a CSV export feature." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states 'No shipped work was reported this period,' which is accurate but does not acknowledge the customer feedback that could inform future shipping priorities." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1", + "feedback_2" + ], + "explanation": "The report correctly cites customer feedback regarding the CSV export request, but it lacks depth in synthesizing this information into actionable insights." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The report fails to identify any next priorities or actions based on customer feedback, which is critical for guiding future work." + } + ], + "summary": "The report effectively captures customer feedback but fails to translate this into actionable priorities or changes. It lacks depth in synthesizing the feedback into a strategic direction for the team.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "duplicate_customer_claims", @@ -243,6 +676,7 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0.5, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 1, "emptySectionCorrectness": 1 }, @@ -258,7 +692,53 @@ "section": "whatChanged", "explanation": "Source type \"customer_feedback\" is not valid for section \"whatChanged\"" } - ] + ], + "llmGrader": { + "overallScore": 0.6, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 0.9, + "synthesisQuality": 0.6, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim about customer feedback is supported by the cited evidence, but the report could benefit from more diverse feedback." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The report repeats the same customer feedback without additional context or changes, limiting its materiality." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no work was shipped, which is accurate but lacks any context or follow-up on future shipping plans." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of next priorities is a significant gap, as it leaves the report without actionable next steps." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable next steps. It needs to better synthesize information and clarify future priorities.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "duplicate_source_ids_schema_validation", @@ -282,9 +762,10 @@ "citationValidity": 1, "citationCoverage": 1, "unsupportedClaimRate": 0, - "unsupportedShippedClaimRate": 1, + "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0.5, "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -293,14 +774,54 @@ "category": "invalid_source_type", "section": "whatShipped", "explanation": "Source type \"customer_feedback\" is not valid for section \"whatShipped\"" + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0.5, + "temporalAccuracy": 0.5, + "synthesisQuality": 0.5, + "actionability": 0.5 }, - { - "category": "unsupported_shipped_claim", - "section": "whatShipped", - "claim": "CSV export was shipped.", - "explanation": "Evidence does not contain a shipping signal." + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim about the customer requesting a CSV export is supported by the cited evidence, but the report lacks a clear distinction between customer feedback and shipped features." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The report incorrectly states that the CSV export was shipped based solely on customer feedback, without evidence of actual shipping." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states no changes were reported, which is misleading as it does not acknowledge the customer request as a significant change." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The lack of specified next priorities limits actionable insights for the founder." + } + ], + "summary": "The report has significant issues with groundedness and temporal accuracy, particularly in misrepresenting customer feedback as shipped work. It fails to highlight important changes and lacks actionable recommendations for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" } - ] + } }, { "case": "malformed_payload", @@ -317,7 +838,7 @@ { "case": "evidence_omitted", "passed": true, - "score": 0.8500000000000001, + "score": 0.875, "hasHardFailure": false, "metrics": { "canonicalSchemaValid": true, @@ -327,15 +848,60 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 0.5, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim about customer feedback is supported by the cited evidence." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook important updates." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which could miss significant releases." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable next steps, limiting its overall effectiveness.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "exaggerated_claim", "passed": true, - "score": 0.7, + "score": 1, "hasHardFailure": false, "metrics": { "canonicalSchemaValid": true, @@ -344,7 +910,8 @@ "unsupportedClaimRate": 1, "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, - "evidenceCoverage": 0, + "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -355,7 +922,114 @@ "claim": "Customers cannot use the product without CSV export.", "explanation": "Claim is not directly supported by cited evidence." } - ] + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim that customers cannot use the product without CSV export is supported by customer feedback, but it lacks a broader context or additional evidence to strengthen the assertion." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no changes were reported, which may overlook important internal changes that could have been relevant." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not accurately reflect the team's activities or progress." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of specified next priorities limits the report's actionability and clarity on future directions." + } + ], + "summary": "The report demonstrates strong groundedness and temporal accuracy but falls short in materiality, synthesis quality, and actionability. It effectively cites customer feedback but lacks depth in reporting changes, shipped work, and future priorities, which diminishes its overall utility for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "weak_customer_signal_overstated", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "feedback_weak" + ], + "explanation": "The claim that 'CSV export is a major customer pain point affecting adoption' is not fully supported by the cited customer feedback, which only mentions a request for CSV export without indicating it as a major pain point." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report fails to identify any changes during the reporting period, which is a critical oversight." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no work was shipped, but it does not provide any context or evidence to support this claim." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of specified next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report has significant issues with groundedness and materiality, particularly in how customer feedback is interpreted and the lack of reported changes or priorities. While it accurately reflects temporal aspects, it fails to provide actionable insights for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "conflicting_evidence", @@ -369,7 +1043,8 @@ "unsupportedClaimRate": 0, "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, - "evidenceCoverage": 0, + "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -380,7 +1055,52 @@ "claim": "All customers want CSV export.", "explanation": "Claim ignores conflicting cited evidence." } - ] + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "feedback_1", + "feedback_2" + ], + "explanation": "The claim 'All customers want CSV export' is unsupported as it contradicts the second piece of feedback stating that a customer does not need the export." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook important updates that should have been included." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which is accurate but lacks context on any planned work or expectations." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report contains significant groundedness issues due to unsupported claims about customer feedback. It lacks material updates and actionable priorities, which diminishes its overall utility for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "document_change_as_shipped", @@ -394,7 +1114,8 @@ "unsupportedClaimRate": 0, "unsupportedShippedClaimRate": 1, "sourceTypeViolationRate": 0, - "evidenceCoverage": 0, + "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, @@ -405,12 +1126,212 @@ "claim": "CSV export was shipped.", "explanation": "Evidence does not contain a shipping signal." } - ] + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 0.5, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [ + "docs_1" + ], + "explanation": "The claim that 'CSV export was shipped' is not supported by the evidence provided, as the document change only mentions an update to the README with export instructions, not the actual shipping of the feature." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report fails to identify any changes during the reporting period, which is critical for understanding progress." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No priorities or actionable next steps are provided, leaving the founder without guidance on what to focus on next." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of customer feedback is a significant gap, as it is essential for understanding user needs and reactions." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not address any current blockers, which is important for transparency and problem-solving." + } + ], + "summary": "The report lacks critical evidence and insights, failing to provide a clear picture of progress, customer feedback, or actionable next steps. It misrepresents the shipping status of the CSV export feature, leading to confusion about what has actually been accomplished.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "document_change_as_customer_feedback", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 1, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 1, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatCustomersSaid", + "explanation": "Source type \"document_change\" is not valid for section \"whatCustomersSaid\"" + }, + { + "category": "unsupported_claim", + "section": "whatCustomersSaid", + "claim": "Customers requested export.", + "explanation": "Claim is not directly supported by cited evidence." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "docs_1" + ], + "explanation": "The claim about customer requests is supported by the cited document, but the confidence level suggests some uncertainty." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states no changes were reported, which is misleading given the updated README." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report claims nothing was shipped, ignoring the update to the README." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The lack of specified priorities limits actionable insights for the founder." + } + ], + "summary": "The report has strong groundedness and temporal accuracy but fails significantly in materiality and synthesis quality. It overlooks important changes, such as the README update, and lacks actionable next steps.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "workspace_document_as_customer_feedback", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 1, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 1, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatCustomersSaid", + "explanation": "Source type \"workspace_document\" is not valid for section \"whatCustomersSaid\"" + }, + { + "category": "unsupported_claim", + "section": "whatCustomersSaid", + "claim": "Customers requested better onboarding.", + "explanation": "Claim is not directly supported by cited evidence." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "workspace_doc_1" + ], + "explanation": "The claim about customer feedback is based on a workspace document, but the excerpt does not directly support the specific request for better onboarding." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no changes were reported, which may overlook important internal updates that could have been included." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not accurately reflect any minor updates or deployments that occurred." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The lack of specified next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report demonstrates strong temporal accuracy and groundedness in some areas, but it lacks materiality and synthesis quality. Key changes and priorities are missing, which diminishes its actionability for founders.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "unavailable_source_warning", "passed": true, - "score": 0.7, + "score": 0.75, "hasHardFailure": false, "metrics": { "canonicalSchemaValid": true, @@ -420,10 +1341,61 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 0, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for any changes, leading to unsupported claims." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for any shipped work, leading to unsupported claims." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "feedback_missing" + ], + "explanation": "Customer feedback is marked as unavailable, which undermines the credibility of any claims made." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for current blockers, leading to unsupported claims." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for next priorities, leading to unsupported claims." + } + ], + "summary": "The report lacks any evidence across all sections, resulting in a complete failure to provide grounded, actionable insights. All claims are unsupported, and there is no material information to guide decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } }, { "case": "valid_contradictory_evidence", @@ -438,10 +1410,340 @@ "unsupportedShippedClaimRate": 0, "sourceTypeViolationRate": 0, "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook minor updates that could be relevant." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1", + "feedback_2" + ], + "explanation": "While the report captures contradictory customer feedback, it does not synthesize this information into actionable insights." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of specified next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report effectively captures customer feedback but lacks material updates and actionable next steps, limiting its overall effectiveness for founders.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "planned_work_marked_as_shipped", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 1, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "unsupported_shipped_claim", + "section": "whatShipped", + "claim": "CSV export shipped.", + "explanation": "Evidence does not contain a shipping signal." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [ + "plan_doc_1" + ], + "explanation": "The report claims that 'CSV export shipped' based on the plan document, which states that implementation has not started yet. This is a groundedness failure as the claim is unsupported." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not identify any changes during the reporting period, which is a materiality failure as it ignores important updates." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report incorrectly presents planned work (CSV export) as shipped, which is a temporal accuracy failure." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "The report lacks any evidence or insights from customers, which is a materiality failure." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not mention any current blockers, which is a materiality failure." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not outline any next priorities, which is a materiality failure." + } + ], + "summary": "The report contains significant failures in groundedness, materiality, and temporal accuracy. It inaccurately claims that a planned feature has shipped, lacks evidence of customer feedback, changes, blockers, and next priorities, leading to a lack of actionable insights for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "valid_document_change_evidence", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, "duplicateClaimRate": 0, "emptySectionCorrectness": 1 }, - "failures": [] + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 0.9, + "synthesisQuality": 0.6, + "actionability": 0.4 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "document_change_1" + ], + "explanation": "The claim about ownership change is supported by the cited document change, but lacks additional context or evidence to fully substantiate its significance." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of customer feedback is provided, which is critical for understanding customer sentiment." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of shipped work is included, which is essential for tracking progress." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of current blockers is provided, which is necessary for identifying challenges." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of next priorities is included, which is important for future planning." + } + ], + "summary": "The report provides limited evidence regarding changes but lacks comprehensive insights into customer feedback, shipped work, blockers, and future priorities. This limits its overall effectiveness for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "multiple_adjacent_version_changes", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.6, + "actionability": 0.4 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "document_change_v1_v2", + "document_change_v2_v3" + ], + "explanation": "The report combines two changes into one statement without clearly distinguishing them, which could lead to confusion about the timeline of changes." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "The section lacks any evidence or customer feedback, which is critical for understanding customer sentiment." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of shipped work is provided, which is essential for tracking progress." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of any blockers reported limits the understanding of potential issues affecting progress." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No priorities are outlined, which is necessary for guiding future actions." + } + ], + "summary": "The report has significant gaps in customer feedback, shipped work, blockers, and next priorities, which undermines its overall effectiveness. While it does provide some evidence of changes, the synthesis of these changes lacks clarity and could confuse readers.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "low_materiality_summary", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "change_1" + ], + "explanation": "The report mentions a change in ownership but lacks detail on its significance or implications." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "No customer feedback was reported, which is critical for understanding user needs." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No next priorities were specified, leaving a gap in actionable insights." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No shipped work was reported, which is essential for tracking progress." + }, + { + "section": "currentBlockers", + "severity": "medium", + "sourceIds": [], + "explanation": "While no blockers were reported, this section could benefit from context on potential challenges." + } + ], + "summary": "The report provides some evidence of a change in ownership but lacks critical customer feedback, shipped work, and next priorities, which diminishes its overall effectiveness. It needs to focus on actionable insights and material changes.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } } ] } \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/cases.ts b/packages/features/src/founder-weekly-review/benchmarks/cases.ts index 1acfec6f9..ba76de19c 100644 --- a/packages/features/src/founder-weekly-review/benchmarks/cases.ts +++ b/packages/features/src/founder-weekly-review/benchmarks/cases.ts @@ -155,7 +155,20 @@ const weakEvidence = { ], } satisfies FounderWeeklyReviewEvidenceSnapshot; -const conflictingEvidence = { +const weakCustomerSignalEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_weak", + title: "Customer feedback", + excerpt: "Customer asked if CSV export could be added.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const conflictingCustomerFeedbackEvidence = { ...validEvidence, items:[ { @@ -208,6 +221,19 @@ const documentChangeEvidence = { ], } satisfies FounderWeeklyReviewEvidenceSnapshot; +const workspaceDocumentEvidence = { + ...validEvidence, + items: [ + { + sourceType: "workspace_document", + sourceId: "workspace_doc_1", + title: "Onboarding Plan", + excerpt: "Platform owns retry telemetry.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + const unavailableSourceEvidence = { ...validEvidence, items:[ @@ -228,6 +254,87 @@ const unavailableSourceEvidence = { ], } satisfies FounderWeeklyReviewEvidenceSnapshot; +const plannedWorkEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "plan_doc_1", + title: "Launch Plan", + excerpt: "CSV export is planned for next sprint. Implementation has not started yet.", + metadata: { + changeType: "modified", + }, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + + +const validDocumentChangeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "document_change_1", + title: "Onboarding Plan", + excerpt: "Ownership changed from Product to Platform.", + metadata: { + previousVersionId: "v1", + currentVersionId: "v2", + previousChunkId: "chunk_101", + currentChunkId: "chunk_202", + changeType: "modified", + }, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + + +const multipleVersionChangeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "document_change_v1_v2", + title: "Reliability Plan", + excerpt: "Retry ownership moved to Platform.", + metadata: { + previousVersionId: "v1", + currentVersionId: "v2", + previousChunkId: "chunk_101", + currentChunkId: "chunk_202", + changeType: "modified", + }, + }, + { + sourceType: "document_change", + sourceId: "document_change_v2_v3", + title: "Reliability Plan", + excerpt: "Monitoring requirements were added.", + metadata: { + previousVersionId: "v2", + currentVersionId: "v3", + previousChunkId: "chunk_202", + currentChunkId: "chunk_303", + changeType: "modified", + }, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const meaningfulChangeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "change_1", + title: "Retry ownership update", + excerpt: "Ownership moved from Product to Platform.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + const validReport = { schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, sections: { @@ -360,6 +467,7 @@ const founderContextOnlyReport = { sourceIds: ["context_1"], confidence: 0.9, label: "Recommendation", + rationale: null, }, ], }, @@ -425,6 +533,7 @@ const completeReport = { sourceIds: ["context_1"], confidence: 0.9, label: "Recommendation", + rationale: null, }, ], }, @@ -558,6 +667,24 @@ const exaggeratedClaimReport = { }, } satisfies FounderWeeklyReviewV2Payload; +const weakSynthesisReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export is a major customer pain point affecting adoption.", + sourceIds: ["feedback_weak"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + const conflictingEvidenceReport = { ...validReport, sections:{ @@ -618,10 +745,131 @@ const documentChangeShippedReport = { }, } satisfies FounderWeeklyReviewV2Payload; +const documentChangeCustomerReport = { + ...validReport, + sections:{ + ...validReport.sections, + whatCustomersSaid:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customers requested export.", + sourceIds:["docs_1"], + confidence:0.9, + } + ] + } + } +} satisfies FounderWeeklyReviewV2Payload; + +const workspaceDocumentAsCustomerReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customers requested better onboarding.", + sourceIds: ["workspace_doc_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + const unavailableSourceSafeReport = { ...emptyReport, } satisfies FounderWeeklyReviewV2Payload; +const plannedWorkMarkedShippedReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export shipped.", + sourceIds: ["plan_doc_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + + +const validDocumentChangeReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + whatChanged: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Ownership changed from Product to Platform.", + sourceIds: ["document_change_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + + +const multipleVersionChangeReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + whatChanged: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Retry ownership moved to Platform and monitoring requirements were added.", + sourceIds: [ + "document_change_v1_v2", + "document_change_v2_v3", + ], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const vagueSummaryReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "no_evidence", + noEvidence: { + code: "no_customer_feedback", + message: "No customer feedback was reported.", + cta: "Collect customer feedback.", + }, + }, + whatChanged: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "There was a change related to retry ownership.", + sourceIds: ["change_1"], + confidence: 0.8, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + export const benchmarkCases: BenchmarkCase[] = [ { id: "valid_customer_feedback_report", @@ -768,7 +1016,7 @@ export const benchmarkCases: BenchmarkCase[] = [ expectations: { shouldPass: false, expectedFailureCategories: [ - "unsupported_shipped_claim", + "invalid_source_type", ], }, }, @@ -812,11 +1060,23 @@ export const benchmarkCases: BenchmarkCase[] = [ }, }, + { + id: "weak_customer_signal_overstated", + runThroughGeneration: false, + description: + "Report overstates a small customer signal into a major business conclusion.", + evidenceSnapshot: weakCustomerSignalEvidence, + generatedReport: weakSynthesisReport, + expectations: { + shouldPass: true, + }, + }, + { id:"conflicting_evidence", runThroughGeneration: false, description:"Report ignores conflicting evidence.", - evidenceSnapshot:conflictingEvidence, + evidenceSnapshot:conflictingCustomerFeedbackEvidence, generatedReport:conflictingEvidenceReport, expectations:{ shouldPass:false, @@ -840,6 +1100,34 @@ export const benchmarkCases: BenchmarkCase[] = [ }, }, + { + id:"document_change_as_customer_feedback", + runThroughGeneration:false, + description:"Document changes cannot represent customer feedback.", + evidenceSnapshot:documentChangeEvidence, + generatedReport:documentChangeCustomerReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "invalid_source_type" + ] + } + }, + + { + id: "workspace_document_as_customer_feedback", + runThroughGeneration: false, + description: "Workspace documents cannot be used as customer feedback.", + evidenceSnapshot: workspaceDocumentEvidence, + generatedReport: workspaceDocumentAsCustomerReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_source_type", + ], + }, + }, + { id:"unavailable_source_warning", runThroughGeneration: true, @@ -861,4 +1149,55 @@ export const benchmarkCases: BenchmarkCase[] = [ shouldPass: true, }, }, + + { + id: "planned_work_marked_as_shipped", + runThroughGeneration: false, + description: "Planned document work cannot be represented as shipped work.", + evidenceSnapshot: plannedWorkEvidence, + generatedReport: plannedWorkMarkedShippedReport, + expectations: { + shouldPass: false, + forbiddenClaims: [ + "CSV export shipped", + ], + expectedFailureCategories: [ + "unsupported_shipped_claim", + ], + }, + }, + + { + id: "valid_document_change_evidence", + runThroughGeneration: false, + description: "Document changes can support what changed when properly cited.", + evidenceSnapshot: validDocumentChangeEvidence, + generatedReport: validDocumentChangeReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "multiple_adjacent_version_changes", + runThroughGeneration: false, + description: "Multiple document versions in a reporting period should preserve adjacent changes.", + evidenceSnapshot: multipleVersionChangeEvidence, + generatedReport: multipleVersionChangeReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "low_materiality_summary", + runThroughGeneration: false, + description: + "Report cites evidence but provides a vague low-value summary.", + evidenceSnapshot: meaningfulChangeEvidence, + generatedReport: vagueSummaryReport, + expectations: { + shouldPass: true, + }, + }, ]; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts b/packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts new file mode 100644 index 000000000..42c6ebfd2 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts @@ -0,0 +1,58 @@ +import { + evaluateFounderWeeklyReview, + type EvaluationFailure, +} from "../evaluation"; +import { + FounderWeeklyReviewV2PayloadSchema, + type FounderWeeklyReviewV2Payload, +} from "../contracts"; +import { + type FounderWeeklyReviewGenerateFn, + gradeFounderWeeklyReview, +} from "@launchstack/features/founder-weekly-review"; +import { LLMGraderResult } from "../llm-grader"; + +export type GeneratedReviewEvaluation = { + deterministic: ReturnType | null; + llmGrader?: LLMGraderResult; + failures: EvaluationFailure[]; +}; + +export async function evaluateGeneratedFounderWeeklyReview( + evidenceSnapshot: Parameters[0], + generatedReport: FounderWeeklyReviewV2Payload, + generate: FounderWeeklyReviewGenerateFn +): Promise { + const schemaResult = FounderWeeklyReviewV2PayloadSchema.safeParse( + generatedReport + ); + + if (!schemaResult.success) { + return { + deterministic: null, + failures: [ + { + category: "malformed_payload", + explanation: "Report failed schema validation.", + } satisfies EvaluationFailure + ], + }; + } + + const result = evaluateFounderWeeklyReview( + evidenceSnapshot, + generatedReport + ); + + const llmGrader = await gradeFounderWeeklyReview({ + evidenceSnapshot, + report: generatedReport, + generate, + }); + + return { + deterministic: result, + llmGrader, + failures: result.failures, + }; +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/index.ts b/packages/features/src/founder-weekly-review/benchmarks/index.ts new file mode 100644 index 000000000..42b597ec2 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/index.ts @@ -0,0 +1,3 @@ +export * from "./evaluate-generated-review"; +export * from "../grader"; +export * from "../llm-grader"; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/run.ts b/packages/features/src/founder-weekly-review/benchmarks/run.ts new file mode 100644 index 000000000..b634f610b --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/run.ts @@ -0,0 +1 @@ +export { main } from "./runner"; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/runner.ts b/packages/features/src/founder-weekly-review/benchmarks/runner.ts index d3a7ae2a4..0073944fe 100644 --- a/packages/features/src/founder-weekly-review/benchmarks/runner.ts +++ b/packages/features/src/founder-weekly-review/benchmarks/runner.ts @@ -4,16 +4,16 @@ import { } from "../evaluation"; import { benchmarkCases } from "./cases"; import { - FounderWeeklyReviewV2PayloadSchema, type FounderWeeklyReviewV2Payload, } from "../contracts"; -import { generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { + generateFounderWeeklyReview, + type FounderWeeklyReviewGenerateFn, +} from "@launchstack/features/founder-weekly-review"; import { writeFileSync } from "fs"; - -let passedCount = 0; -let failedCount = 0; - -let hardFailureCount = 0; +import type { LLMGraderResult } from "../llm-grader"; +import path from "path"; +import { evaluateGeneratedFounderWeeklyReview } from "./evaluate-generated-review"; type BenchmarkMetrics = { citationValidity: number; @@ -33,200 +33,225 @@ type BenchmarkResult = { hasHardFailure: boolean; metrics?: BenchmarkMetrics; failures: EvaluationFailure[]; + llmGrader?: LLMGraderResult; }; -const results: BenchmarkResult[] = []; - -for (const testCase of benchmarkCases) { - - if (!testCase.generatedReport) { - continue; - } +type BenchmarkState = { + passedCount: number; + failedCount: number; + hardFailureCount: number; +}; - let generatedReport: FounderWeeklyReviewV2Payload; - - if(testCase.runThroughGeneration) { - const generated = await generateFounderWeeklyReview({ - evidenceSnapshot: testCase.evidenceSnapshot, - generate: async ({schema}) => ({ - object: schema.parse(testCase.generatedReport), - metadata: { - provider: "benchmark", - model: "fixture", - capability: "founderWeeklyReview", - temperature: 0, - }, - }), - }); - - generatedReport = generated.reviewPayload; +export async function runBenchmarks( + generate: FounderWeeklyReviewGenerateFn, + results: BenchmarkResult[], + state: BenchmarkState +): Promise { + for (const testCase of benchmarkCases) { - } else { - generatedReport = testCase.generatedReport; - } + if (!testCase.generatedReport) { + continue; + } - const schemaResult = FounderWeeklyReviewV2PayloadSchema.safeParse( - generatedReport - ); + let generatedReport: FounderWeeklyReviewV2Payload; + + if(testCase.runThroughGeneration) { + const generated = await generateFounderWeeklyReview({ + evidenceSnapshot: testCase.evidenceSnapshot, + generate: async ({schema}) => ({ + object: schema.parse(testCase.generatedReport), + metadata: { + provider: "benchmark", + model: "fixture", + capability: "founderWeeklyReview", + temperature: 0, + }, + }), + }); + + generatedReport = generated.reviewPayload; - if (!schemaResult.success) { - - const passed = - testCase.expectations.expectedFailureCategories?.includes( - "malformed_payload" - ) ?? false; + } else { + generatedReport = testCase.generatedReport; + } - console.log( - `${passed ? "✅" : "❌"} ${testCase.id} (schema validation failure)` + const evaluation = await evaluateGeneratedFounderWeeklyReview( + testCase.evidenceSnapshot, + generatedReport, + generate ); - if (!passed) { - console.log(JSON.stringify({ + if (evaluation.deterministic === null) { + const passed = + testCase.expectations.expectedFailureCategories?.includes( + "malformed_payload" + ) ?? false; + + results.push({ case: testCase.id, - expected: testCase.expectations.expectedFailureCategories, - actual: "malformed_payload", - zodErrors: schemaResult.error.issues, - }, null, 2)); + passed, + score: 0, + hasHardFailure: true, + failures: evaluation.failures, + }); + + state.hardFailureCount++; + + if (passed) { + state.passedCount++; + } else { + state.failedCount++; + } + + console.log( + `${passed ? "✅" : "❌"} ${testCase.id} (schema validation failure)` + ); + + continue; } - results.push({ - case: testCase.id, - passed, - score: 0, - hasHardFailure: true, - failures: [ - { - category: "malformed_payload", - explanation: "Report failed schema validation.", - }, - ], - }); + const result = evaluation.deterministic; + const llmGrade = evaluation.llmGrader ?? undefined; - // Malformed payload cases are expected failures, but still count as hard failures. - hardFailureCount++; + const actualFailures = result.failures.map(f => f.category); + + const passed = + testCase.expectations.shouldPass + ? actualFailures.length === 0 + : testCase.expectations.expectedFailureCategories?.every( + category => actualFailures.includes(category) + ) ?? false; + + if (result.hasHardFailure) { + state.hardFailureCount++; + } if (passed) { - passedCount++; + state.passedCount++; } else { - failedCount++; + state.failedCount++; } - continue; - } - - const result = evaluateFounderWeeklyReview( - testCase.evidenceSnapshot, - generatedReport - ); - - const actualFailures = result.failures.map(f => f.category); + results.push({ + case: testCase.id, + passed, + score: result.overallScore, + hasHardFailure: result.hasHardFailure, + metrics: result.deterministic, + failures: result.failures, + llmGrader: llmGrade, + }); - const passed = - testCase.expectations.shouldPass - ? actualFailures.length === 0 - : testCase.expectations.expectedFailureCategories?.every( - category => actualFailures.includes(category) - ) ?? false; + console.log( + `${passed ? "✅" : "❌"} ${testCase.id}` + ); - if (result.hasHardFailure) { - hardFailureCount++; + if (!passed) { + console.log(JSON.stringify({ + case: testCase.id, + deterministicFailures: result.failures, + llmGrader: llmGrade, + }, null, 2)); + } } +} - if (passed) { - passedCount++; - } else { - failedCount++; - } +export async function main(generate: FounderWeeklyReviewGenerateFn) { + const results: BenchmarkResult[] = []; - results.push({ - case: testCase.id, - passed, - score: result.overallScore, - hasHardFailure: result.hasHardFailure, - metrics: result.deterministic, - failures: result.failures, - }); - - console.log( - `${passed ? "✅" : "❌"} ${testCase.id}` - ); + const state: BenchmarkState = { + passedCount: 0, + failedCount: 0, + hardFailureCount: 0 + }; - if (!passed) { - console.log(JSON.stringify({ - case:testCase.id, - passed: passed, - score:result.overallScore, - failures:result.failures - },null,2)); - } -} + await runBenchmarks(generate, results, state); -const averageScore = - results.length === 0 - ? 0 - : results.reduce( - (sum, result) => sum + result.score, - 0 - ) / results.length; - -const averageMetric = (key: keyof BenchmarkMetrics) => { - const values = results - .map(result => result.metrics?.[key]) - .filter((value): value is number => value !== undefined); - - return values.length === 0 - ? 0 - : values.reduce((sum, value) => sum + value, 0) / values.length; -}; + const averageScore = + results.length === 0 + ? 0 + : results.reduce( + (sum, result) => sum + result.score, + 0 + ) / results.length; -const weakestCases = [...results] - .sort((a,b) => a.score - b.score) - .slice(0,3) - .map(result => ({ - case: result.case, - score: result.score, - failures: result.failures.map(f => f.category), - })); - -const failureCounts = results - .flatMap(result => result.failures) - .reduce>((acc, failure) => { - acc[failure.category] = - (acc[failure.category] ?? 0) + 1; - - return acc; - }, {}); - -const benchmarkOutput = { - summary: { - totalCases: benchmarkCases.length, - passed: passedCount, - failed: failedCount, - hardFailures: hardFailureCount, - passRate: benchmarkCases.length === 0 + const llmScores = results + .map(r => r.llmGrader?.overallScore) + .filter((v): v is number => v !== undefined); + + const averageLLMScore = + llmScores.length === 0 ? 0 - : passedCount / benchmarkCases.length, - overallScore: averageScore, - }, - metrics: { - citationValidity: averageMetric("citationValidity"), - citationCoverage: averageMetric("citationCoverage"), - unsupportedClaimRate: averageMetric("unsupportedClaimRate"), - unsupportedShippedClaimRate: averageMetric("unsupportedShippedClaimRate"), - sourceTypeViolationRate: averageMetric("sourceTypeViolationRate"), - evidenceCoverage: averageMetric("evidenceCoverage"), - emptySectionCorrectness: averageMetric("emptySectionCorrectness"), - duplicateClaimRate: averageMetric("duplicateClaimRate"), - }, - weakestCases, - commonFailures: failureCounts, - cases: results, -}; + : llmScores.reduce((sum, v) => sum + v, 0) / llmScores.length; -console.log("\n===== Benchmark Summary ====="); + const averageMetric = (key: keyof BenchmarkMetrics) => { + const values = results + .map(result => result.metrics?.[key]) + .filter((value): value is number => value !== undefined); -console.log(JSON.stringify(benchmarkOutput, null, 2)); + return values.length === 0 + ? 0 + : values.reduce((sum, value) => sum + value, 0) / values.length; + }; + + const weakestCases = [...results] + .sort((a,b) => a.score - b.score) + .slice(0,3) + .map(result => ({ + case: result.case, + score: result.score, + failures: result.failures.map(f => f.category), + })); + + const failureCounts = results + .flatMap(result => result.failures) + .reduce>((acc, failure) => { + acc[failure.category] = + (acc[failure.category] ?? 0) + 1; + + return acc; + }, {}); + + const benchmarkOutput = { + summary: { + totalCases: benchmarkCases.length, + passed: state.passedCount, + failed: state.failedCount, + hardFailures: state.hardFailureCount, + passRate: benchmarkCases.length === 0 + ? 0 + : state.passedCount / benchmarkCases.length, + deterministicScore: averageScore, + llmScore: averageLLMScore, + }, + metrics: { + citationValidity: averageMetric("citationValidity"), + citationCoverage: averageMetric("citationCoverage"), + unsupportedClaimRate: averageMetric("unsupportedClaimRate"), + unsupportedShippedClaimRate: averageMetric("unsupportedShippedClaimRate"), + sourceTypeViolationRate: averageMetric("sourceTypeViolationRate"), + evidenceCoverage: averageMetric("evidenceCoverage"), + emptySectionCorrectness: averageMetric("emptySectionCorrectness"), + duplicateClaimRate: averageMetric("duplicateClaimRate"), + }, + weakestCases, + commonFailures: failureCounts, + cases: results, + }; + + console.log("\n===== Benchmark Summary ====="); + + console.log(JSON.stringify(benchmarkOutput, null, 2)); + + const outputPath = path.resolve( + import.meta.dirname, + "./baseline-output.json" + ); -writeFileSync("packages/features/src/founder-weekly-review/benchmarks/baseline-output.json", JSON.stringify(benchmarkOutput, null, 2)); + writeFileSync( + outputPath, + JSON.stringify(benchmarkOutput, null, 2) + ); -process.exitCode = failedCount > 0 ? 1 : 0; \ No newline at end of file + process.exitCode = state.failedCount > 0 ? 1 : 0; +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts index d32626b7b..a4c97d112 100644 --- a/packages/features/src/founder-weekly-review/contracts.ts +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -181,7 +181,7 @@ export const FounderWeeklyReviewV2RecommendationSchema = z.object({ kind: z.literal("recommendation"), label: z.literal("Recommendation"), text: z.string().min(1).max(2000), - rationale: z.string().min(1).max(2000).optional(), + rationale: z.string().min(1).max(2000).nullable(), sourceIds: v2SourceIdsSchema(1), confidence: V2ConfidenceSchema, }).strict(); diff --git a/packages/features/src/founder-weekly-review/document-change.ts b/packages/features/src/founder-weekly-review/document-change.ts index 17b244b45..c4e40d402 100644 --- a/packages/features/src/founder-weekly-review/document-change.ts +++ b/packages/features/src/founder-weekly-review/document-change.ts @@ -61,6 +61,11 @@ const compareChunks = (a: VersionChunk, b: VersionChunk) => (a.lineStart ?? -1) - (b.lineStart ?? -1) || (a.structureOrdering ?? -1) - (b.structureOrdering ?? -1) || a.chunkId - b.chunkId; +export type EvidenceStatus = + | "changed" + | "shipped" + | "planned"; + /** Select only adjacent pairs whose current version is in the reporting period. */ export function selectVersionPairsForReportingPeriod( versions: readonly DocumentVersionForComparison[], startInclusive: Date, endExclusive: Date @@ -146,6 +151,30 @@ export function alignVersionChunks(previousChunks: readonly VersionChunk[], curr function bound(value: string, max = MAX_EXCERPT) { return value.length <= max ? value : `${value.slice(0, max - 1)}…`; } function preview(value: string) { return bound(value.replace(/\s+/g, " ").trim(), 900); } +function classifyDocumentChangeStatus( + alignment: ChunkAlignment, + changelog: string | null +): EvidenceStatus { + const text = `${changelog ?? ""} ${ + alignment.currentChunk?.content ?? + alignment.previousChunk?.content ?? "" + }`.toLowerCase(); + + if ( + /\b(released|launched|shipped|rolled out|available now)\b/i.test(text) + ) { + return "shipped"; + } + + if ( + /\b(planned|upcoming|will|next|roadmap)\b/i.test(text) + ) { + return "planned"; + } + + return "changed"; +} + export function buildDocumentChangeEvidence(pair: VersionPair, alignments: readonly ChunkAlignment[]): FounderWeeklyReviewEvidenceItem[] { return alignments.filter((alignment) => alignment.changeType !== "unchanged").map((alignment) => { const previous = alignment.previousChunk; const current = alignment.currentChunk; @@ -154,9 +183,10 @@ export function buildDocumentChangeEvidence(pair: VersionPair, alignments: reado const excerpt = alignment.changeType === "modified" ? `Section modified. Before: ${preview(previous!.content)} After: ${preview(current!.content)}` : alignment.changeType === "added" ? `Section added: ${preview(current!.content)}` : `Section removed: ${preview(previous!.content)}`; + const status = classifyDocumentChangeStatus(alignment, pair.currentChangelog); return { sourceType: "document_change", sourceId, title: pair.documentTitle, sourceTimestamp: pair.currentCreatedAt.toISOString(), excerpt: bound(excerpt), workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, - metadata: { documentId: pair.documentId.toString(), previousVersionId: pair.previousVersionId, currentVersionId: pair.currentVersionId, previousVersionNumber: pair.previousVersionNumber, currentVersionNumber: pair.currentVersionNumber, + metadata: { evidenceStatus: status, documentId: pair.documentId.toString(), previousVersionId: pair.previousVersionId, currentVersionId: pair.currentVersionId, previousVersionNumber: pair.previousVersionNumber, currentVersionNumber: pair.currentVersionNumber, previousChunkId: previous?.chunkId ?? null, currentChunkId: current?.chunkId ?? null, changeType: alignment.changeType, alignmentMethod: alignment.alignmentMethod, previousContentHash: previous?.contentHash ?? null, currentContentHash: current?.contentHash ?? null, structurePath: current?.structurePath ?? previous?.structurePath ?? null, userChangelog: pair.currentChangelog ? bound(pair.currentChangelog.replace(/\s+/g, " ").trim(), MAX_METADATA_TEXT) : null } }; diff --git a/packages/features/src/founder-weekly-review/evaluation-prompt.ts b/packages/features/src/founder-weekly-review/evaluation-prompt.ts new file mode 100644 index 000000000..db305d9d5 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evaluation-prompt.ts @@ -0,0 +1,50 @@ +export const founderWeeklyReviewGraderRubric = ` +Evaluate the Founder Weekly Review based on: + +1. Groundedness +- Are every factual claims directly supported by cited evidence? +- Are cited sourceIds semantically appropriate for the claim? +- Does the report avoid inventing customer feedback, shipped work, blockers, or outcomes? +- Does the report avoid turning plans or documents into facts? + +2. Materiality +- Did the report identify the most important changes during the reporting period? +- Did it avoid spending attention on minor or irrelevant updates? +- Did it include meaningful evidence themes instead of ignoring important changes? + +3. Temporal Accuracy +- Does the report correctly distinguish between: + - changed this week + - shipped this week + - planned work + - recommendations + - historical context + +- Planned work must not be represented as shipped. +- Document changes should only appear as "what changed" unless there is separate shipping evidence. +- Multiple document versions should preserve the sequence of changes instead of collapsing unrelated updates incorrectly. + +- workspace_document evidence represents current workspace context. +- It must not be represented as an in-period change unless there is explicit change evidence. +- workspace_document evidence may support blockers, context, or recommendations. + +A technically valid citation may still be semantically unsupported if the cited evidence does not actually justify the claim being made. + +4. Synthesis Quality +- Does the report connect multiple evidence sources when appropriate? +- Does it summarize patterns across evidence rather than simply repeating individual events? +- Does it provide founder-level interpretation while remaining grounded in evidence? + +5. Actionability +- Are recommendations specific and useful? +- Do recommendations clearly separate evidence-backed observations from suggested actions? +- Would a founder know what decision, follow-up, or next step to take? +- Does the report help a founder make decisions? +- Does it highlight what deserves attention instead of merely summarizing activity? + +When evaluating failures: +- Mark unsupported claims as groundedness failures. +- Mark planned work presented as shipped as temporal accuracy failures. +- Mark missing important evidence as materiality failures. +- Mark weak summaries that only restate evidence as synthesis quality failures. +`; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/evaluation.ts b/packages/features/src/founder-weekly-review/evaluation.ts index bfbabfc37..8952acdce 100644 --- a/packages/features/src/founder-weekly-review/evaluation.ts +++ b/packages/features/src/founder-weekly-review/evaluation.ts @@ -17,6 +17,7 @@ export interface EvaluationResult { unsupportedShippedClaimRate: number; sourceTypeViolationRate: number; evidenceCoverage: number; + materialityScore: number; duplicateClaimRate: number; emptySectionCorrectness: number; }; @@ -39,18 +40,21 @@ const SECTION_SOURCE_RULES: Record = { "document_change", "workspace_document", "github_activity", + "founder_context", ], whatShipped: [ "github_activity", "document_change", + "workspace_document", ], currentBlockers: [ "founder_context", - "manual_note", + "workspace_document", + "github_activity", ], nextPriorities: [ "founder_context", - "manual_note", + "workspace_document", ], }; @@ -255,8 +259,9 @@ export function evaluateFounderWeeklyReview( citationCoverage: 0, unsupportedShippedClaimRate: 0, unsupportedClaimRate: 0, - sourceTypeViolationRate: 1, + sourceTypeViolationRate: 0, evidenceCoverage: 0, + materialityScore: 0, duplicateClaimRate: 0, emptySectionCorrectness, }, @@ -273,6 +278,8 @@ export function evaluateFounderWeeklyReview( const reportClaims = new Set(); + const citedEvidenceIds = new Set(); + for (const [sectionName, section] of Object.entries(report.sections)) { for (const item of getSectionItems(section)) { const text = typeof item === "object" && @@ -325,7 +332,7 @@ export function evaluateFounderWeeklyReview( ) { for (const sourceId of item.sourceIds) { - + citedEvidenceIds.add(sourceId); const evidence = evidenceById.get(sourceId); if (evidence) { @@ -350,7 +357,10 @@ export function evaluateFounderWeeklyReview( ) { shippedClaimChecks++; - if (!evidenceIndicatesShipped(evidence)) { + if ( + ["github_activity", "document_change", "workspace_document"].includes(evidence.sourceType) && + !evidenceIndicatesShipped(evidence) + ) { unsupportedShippedClaims++; failures.push({ @@ -461,25 +471,44 @@ export function evaluateFounderWeeklyReview( ); const coveredEvidence = - evidenceSnapshot.items.filter((evidence) => { - const evidenceText = normalizeClaim( - `${evidence.title} ${evidence.excerpt}`); - - return [...reportClaims].some((claim) => - claimSupportedByEvidence(claim, evidenceText) - ); - }).length; + evidenceSnapshot.items.filter((evidence) => + citedEvidenceIds.has(evidence.sourceId) + ).length; const evidenceCoverage = evidenceSnapshot.items.length === 0 ? 1 : coveredEvidence / evidenceSnapshot.items.length; + const materialEvidence = evidenceSnapshot.items.filter( + (evidence) => + [ + "github_activity", + "document_change", + "founder_context", + "workspace_document", + ].includes(evidence.sourceType) + ); + + + const materialEvidenceCovered = + materialEvidence.filter((evidence) => + citedEvidenceIds.has(evidence.sourceId) + ).length; + + + const materialityScore = + materialEvidence.length === 0 + ? 1 + : materialEvidenceCovered / materialEvidence.length; + + const overallScore = hasHardFailure ? 0 - : citationValidity * 0.30 + - citationCoverage * 0.20 + - evidenceCoverage * 0.30 + + : citationValidity * 0.25 + + citationCoverage * 0.15 + + evidenceCoverage * 0.25 + + materialityScore * 0.15 + (1 - sourceTypeViolationRate) * 0.20; return { @@ -502,6 +531,7 @@ export function evaluateFounderWeeklyReview( : unsupportedShippedClaims / shippedClaimChecks, sourceTypeViolationRate, evidenceCoverage, + materialityScore, duplicateClaimRate: claims.size === 0 ? 0 diff --git a/packages/features/src/founder-weekly-review/generation-validation.ts b/packages/features/src/founder-weekly-review/generation-validation.ts index b0f2a3c09..9533ce61c 100644 --- a/packages/features/src/founder-weekly-review/generation-validation.ts +++ b/packages/features/src/founder-weekly-review/generation-validation.ts @@ -25,6 +25,58 @@ const TEMPORAL_EVIDENCE_SOURCE_TYPES = new Set(["document_change"]); const isTemporalEvidenceSource = (source: { sourceType?: string } | undefined) => source !== undefined && TEMPORAL_EVIDENCE_SOURCE_TYPES.has(source.sourceType ?? ""); +type EvidenceStatus = + | "planned" + | "changed" + | "shipped"; + +function getEvidenceStatus( + source: { metadata?: Record } | undefined +): EvidenceStatus | null { + const status = source?.metadata?.evidenceStatus; + + if ( + status === "planned" || + status === "changed" || + status === "shipped" + ) { + return status; + } + + return null; +} + +function assertClaimDoesNotOverstateEvidence( + text: string, + sources: Array<{ metadata?: Record } | undefined>, + section: string, + itemIndex: number, + sourceIds: readonly string[] +): void { + const claim = text.toLowerCase(); + + for (const source of sources) { + const status = getEvidenceStatus(source); + + if (!status) continue; + + if ( + status === "planned" && + /\b(shipped|released|launched|completed|delivered)\b/i.test(claim) + ) { + throw new FounderWeeklyReviewGenerationValidationError( + "Claim describes planned work as completed.", + [{ + code: "claim_overstates_planned_evidence", + section, + itemIndex, + sourceId: sourceIds[0], + }] + ); + } + } +} + export function assertUniqueSnapshotSourceIds( evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot ): void { @@ -55,6 +107,17 @@ export function validateFounderWeeklyReviewV2Citations( if (section.state === "no_evidence") continue; for (const [itemIndex, item] of section.items.entries()) { assertCitations(item.sourceIds, evidenceBySourceId, item.kind); + + const citedSources = item.sourceIds.map((sourceId) => evidenceBySourceId.get(sourceId)); + + assertClaimDoesNotOverstateEvidence( + item.text, + citedSources, + sectionName, + itemIndex, + item.sourceIds + ); + if (item.kind === "contradictory_evidence" && item.sourceIds.length < 2) { throw new FounderWeeklyReviewGenerationValidationError( `${sectionName} contradictory_evidence must cite at least two sources.` diff --git a/packages/features/src/founder-weekly-review/generator.ts b/packages/features/src/founder-weekly-review/generator.ts index 9a963ae91..6f728c234 100644 --- a/packages/features/src/founder-weekly-review/generator.ts +++ b/packages/features/src/founder-weekly-review/generator.ts @@ -122,7 +122,7 @@ function buildSemanticRepairPrompt( const errors = error.details.length > 0 ? error.details : [{ code: "report_validation_failed" }]; - const sources = evidenceSnapshot.items.map(({ sourceId, sourceType }) => ({ sourceId, sourceType })); + const sources = evidenceSnapshot.items.map(({ sourceId, sourceType, metadata }) => ({ sourceId, sourceType, evidenceStatus: metadata.evidenceStatus ?? null })); return [ "Correct the complete canonical Founder Weekly Review JSON candidate below.", "Customer Signals may cite only customer_feedback sources.", diff --git a/packages/features/src/founder-weekly-review/grader.ts b/packages/features/src/founder-weekly-review/grader.ts new file mode 100644 index 000000000..0e6c4269e --- /dev/null +++ b/packages/features/src/founder-weekly-review/grader.ts @@ -0,0 +1,55 @@ +import { z, type ZodType } from "zod"; +import { founderWeeklyReviewGraderRubric } from "./evaluation-prompt"; +import { LLMGraderResultSchema, type LLMGraderResult } from "./llm-grader"; +import{ + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewV2Payload, +} from "./contracts"; + +export type FounderWeeklyReviewGenerateFn = ( + input: { + system?: string; + prompt: string; + schema: TSchema; + schemaName?: string; + temperature?: number; + }, +) => Promise<{ + object: z.infer; + metadata: { + provider: string; + model: string; + promptVersion?: string; + }; +}>; + +export async function gradeFounderWeeklyReview( + params: { + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + report: FounderWeeklyReviewV2Payload; + generate: FounderWeeklyReviewGenerateFn; + }, +): Promise { + const result = await params.generate({ + schema: LLMGraderResultSchema, + schemaName: "founder_weekly_review_grader", + temperature: 0, + system: founderWeeklyReviewGraderRubric, + prompt: ` +Evidence: +${JSON.stringify(params.evidenceSnapshot, null, 2)} + +Report: +${JSON.stringify(params.report, null, 2)} +`, + }); + + return LLMGraderResultSchema.parse({ + ...result.object, + metadata: { + provider: result.metadata.provider, + model: result.metadata.model, + promptVersion: "founder_weekly_review_grader_v1", + }, + }); +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 535060b42..827218c2d 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -10,3 +10,4 @@ export * from "./generation-validation"; export * from "./prompts"; export * from "./document-change"; export * from "./workspace-document"; +export * from "./grader"; diff --git a/packages/features/src/founder-weekly-review/llm-grader.ts b/packages/features/src/founder-weekly-review/llm-grader.ts new file mode 100644 index 000000000..4d5d128c4 --- /dev/null +++ b/packages/features/src/founder-weekly-review/llm-grader.ts @@ -0,0 +1,32 @@ +import { z } from "zod"; + +export const LLMGraderResultSchema = z.object({ + overallScore: z.number().min(0).max(1), + + dimensions: z.object({ + groundedness: z.number().min(0).max(1), + materiality: z.number().min(0).max(1), + temporalAccuracy: z.number().min(0).max(1), + synthesisQuality: z.number().min(0).max(1), + actionability: z.number().min(0).max(1), + }), + + findings: z.array( + z.object({ + section: z.string(), + severity: z.enum(["low", "medium", "high"]), + sourceIds: z.array(z.string()), + explanation: z.string(), + }) + ), + + summary: z.string(), + + metadata: z.object({ + provider: z.string(), + model: z.string(), + promptVersion: z.string(), + }), +}); + +export type LLMGraderResult = z.infer; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts index 01aca49a7..b6c98ba60 100644 --- a/packages/features/src/founder-weekly-review/prompts.ts +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -5,21 +5,76 @@ export const FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION = export const FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT = `You generate a structured Founder Weekly Review from supplied evidence only. -Never invent, assume, infer, or embellish customers, dates, metrics, people, decisions, shipped work, blockers, outcomes, or source IDs. Every factual item must cite one or more supplied source IDs exactly as given. Do not create or modify source IDs. Confidence is how strongly the generated claim is supported by its cited supplied evidence; it is not a score for source reliability or truthfulness. Omit unsupported claims or use no_evidence rather than assigning them a low confidence. +Never invent, assume, infer, or embellish customers, dates, metrics, people, decisions, shipped work, blockers, outcomes, or source IDs. Every factual item must cite one or more supplied source IDs exactly as given. Do not create or modify source IDs. Confidence is how strongly the generated claim is supported by its cited supplied evidence; it is not a score for source reliability or truthfulness. -Write a concise, natural, professional review for a founder. Prefer a few substantive items over many one-sentence paraphrases. When related evidence supports it, synthesize the relationship into one focused item: describe the development or signal, why it matters, what the evidence does and does not establish, and an evidence-backed next action where the section permits it. Aim for roughly 2–4 sentences per substantive item when the supplied evidence supports that depth, and for approximately 600–1,000 words overall when the evidence supports it. Do not add filler to reach a length target. +Omit unsupported claims or use no_evidence rather than assigning them a low confidence. Do not lower confidence simply because the underlying source describes uncertainty; confidence measures support for the wording of the claim, not certainty about the business. -Keep the distinctions below explicit. A document change can establish that work was released or that preparation was documented; it does not by itself prove adoption, a measured outcome, or that an underlying issue is resolved. Treat retry telemetry, ownership, plans, and similar records as operational preparation unless evidence proves execution. Customer feedback is customer-only evidence: whatCustomersSaid may cite only customer_feedback, and it must not represent founder_context as customer testimony. founder_context is founder-provided context, not shipped work or external validation. Describe qualitative or limited feedback as limited; do not present one signal as broad proof. +Write a concise, natural, professional review for a founder. Prefer a few substantive items over many one-sentence paraphrases. When multiple evidence items describe the same customer problem, request, or reaction, synthesize them into a shared theme rather than summarizing each source independently. Explicitly identify the recurring pattern, indicate how many or which sources support it when useful, explain why the pattern matters, and preserve the limits of the evidence. Do not generalize beyond the supplied customer evidence. -Use whatShipped only for work that the evidence establishes as released during this reporting period. Use whatChanged only for separate, non-shipped developments such as operational preparation; do not restate a shipped item there. Do not turn the administrative processing of a feedback document into a meaningful change when the customer signals themselves are already covered in whatCustomersSaid. +Aim for roughly 2–4 sentences per substantive item when the supplied evidence supports that depth, and for approximately 600–1,000 words overall when the evidence supports it. Do not add filler to reach a length target. Avoid repeating the same insight across multiple sections unless each section serves a distinct purpose. + +Keep the distinctions below explicit. A document change can establish that work was released or that preparation was documented; it does not by itself prove adoption, a measured outcome, or that an underlying issue is resolved. Treat retry telemetry, ownership, plans, and similar records as operational preparation unless evidence proves execution. Customer feedback is customer-only evidence: whatCustomersSaid may cite only customer_feedback, and it must not represent founder_context as customer testimony. + +Customer feedback describes customer requests, reactions, complaints, or opinions; it does not establish that implementation occurred. founder_context is founder-provided context, not shipped work or external validation. Describe qualitative or limited feedback as limited; do not present one signal as broad proof. + +Use whatShipped only for work that the evidence explicitly establishes as released during this reporting period. Do not infer that nothing shipped merely because no shipped evidence is present. When no shipped work is supported, use qualified language such as "No shipped work was identified in the available evidence" rather than asserting that nothing shipped. + +Use whatChanged for meaningful developments supported by the evidence, including separate non-shipped operational developments. Do not infer that nothing changed merely because there are no document_change items. If the available evidence does not establish a meaningful change, say that no documented change was identified rather than claiming that nothing changed. Do not restate a shipped item in whatChanged. Use currentBlockers for evidence-backed execution blockers and for material product, customer, or operational risks. When there is no explicit execution blocker but evidence shows a risk, say that distinction plainly. State evidence gaps and open questions rather than filling them with assumptions. Do not use generic language such as "continue monitoring" unless paired with a concrete action grounded in cited evidence. -For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must be evidence-backed and explain its rationale in the optional rationale field when useful. Avoid repetitive wording across sections. +For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must identify a concrete action, explain why that action is warranted by the cited evidence, and be specific enough that a founder could act on it without guessing what "improve" or "address" means. When appropriate, include a concrete validation step, decision, owner-facing follow-up, or outcome to pursue. + +Prioritize recommendations by likely impact or urgency when the evidence supports that distinction. Avoid generic recommendations such as "monitor," "improve," or "continue working on" unless they name the specific thing to monitor, improve, or continue and the action is grounded in evidence. Avoid repetitive wording across sections. When evidence conflicts, return contradictory_evidence with the conflicting source IDs. Do not choose a winner or reconcile it unless supplied evidence explicitly resolves the conflict. -nextPriorities contains recommendations only. Every recommendation must have label "Recommendation" and be grounded in supplied evidence. If a section lacks relevant evidence, return its typed no_evidence state with a concrete CTA. sourceWarnings may inform the CTA but are not factual evidence and cannot be cited.`; +nextPriorities contains recommendations only. Every recommendation must have label "Recommendation" and be grounded in supplied evidence. If a section lacks relevant evidence, return its typed no_evidence state with a concrete CTA. sourceWarnings may inform the CTA but are not factual evidence and cannot be cited. + +For nextPriorities, sourceIds have a strict citation allowlist. + +ONLY these source types may appear in nextPriorities.sourceIds: +- founder_context +- workspace_document + +NEVER put any other source type in nextPriorities.sourceIds, including: +- customer_feedback +- document_change +- github_activity + +Customer feedback and document changes may influence the recommendation's wording or rationale, but they are not valid citation sources for nextPriorities. Do not include their sourceIds under any circumstances. + +Before producing each nextPriorities recommendation, check every sourceId: +1. Look up the source's sourceType. +2. If sourceType is founder_context or workspace_document, it may be cited. +3. If sourceType is anything else, do not include that sourceId. +4. If no founder_context or workspace_document evidence independently supports the priority, do not create the recommendation. Return no_evidence instead. + +Important: Do not "fix" an invalid recommendation by returning an empty sourceIds array. Omit the recommendation and use no_evidence when the citation allowlist cannot be satisfied. + +Example: +If founder_context supports "follow up on saved filter adoption" and customer_feedback says customers want saved filters, the recommendation MAY mention the customer signal in its rationale, but sourceIds MUST contain only the founder_context sourceId. +Evidence status rules: + +- Only write shipped claims when evidenceStatus is "shipped". + +- "planned" evidence cannot be described as completed. + +- Customer feedback, requests, suggestions, complaints, or discussions never prove that a feature shipped, released, launched, or was completed. + +- Never place customer_feedback evidence in whatShipped unless separate cited evidence explicitly establishes shipped work. + +- A feature request or customer desire may be described in whatCustomersSaid, but it cannot be used to infer implementation status. + +- Claim strength must not exceed evidence strength. + +Temporal interpretation rules: + +- Prefer the evidence's sourceTimestamp and evidenceStatus when determining whether something happened during the reporting period. +- Do not describe ongoing work, preparation, plans, or current context as completed work. +- Do not treat the absence of evidence as evidence that an event did not occur. +- When timing or completion status is not established, use qualified wording such as "was documented," "was identified," "was in progress," or "was not established in the available evidence." +- Preserve the distinction between a development being documented this week and the underlying work actually being completed this week.` /** Canonical, stable prompt serialization: preserve snapshot item order and avoid wall-clock data. */ export function buildFounderWeeklyReviewPrompt( @@ -32,6 +87,7 @@ export function buildFounderWeeklyReviewPrompt( evidence: evidenceSnapshot.items.map((item) => ({ sourceId: item.sourceId, sourceType: item.sourceType, + evidenceStatus: item.metadata.evidenceStatus ?? null, title: item.title, sourceTimestamp: item.sourceTimestamp ?? null, excerpt: item.excerpt, From 72a9e62e1a3e0a379037d8e6f29607948a67c5c9 Mon Sep 17 00:00:00 2001 From: EricLiu2795 Date: Sun, 9 Aug 2026 02:16:40 +0800 Subject: [PATCH 28/29] Complete grounded Founder Weekly Review evidence pipeline --- .../founderWeeklyReview/demo.test.ts | 27 ++++ .../document-change-condensation.test.ts | 7 + .../founderWeeklyReview/evaluation.test.ts | 81 +++++++++++ .../founderWeeklyReview/generation.test.ts | 14 ++ .../kimi-chat-completions.test.ts | 48 ++++++- .../scenario-collection.integration.test.ts | 83 +++++++++++ .../scenario-contract.test.ts | 33 +++++ .../scenario-loader.test.ts | 19 +++ apps/web/package.json | 3 +- .../web/scripts/founder-weekly-review-demo.ts | 136 ++++++++++++++++++ .../founder-weekly-review-scenario-loader.ts | 11 ++ .../founder-weekly-review-scenario-seeder.ts | 48 +++++++ ...run-founder-weekly-review-realistic-e2e.ts | 136 ++++++++++++++++-- apps/web/src/lib/llm/generate.ts | 2 +- .../scenarios/01-empty-evidence/scenario.json | 18 +++ .../02-founder-context-only/scenario.json | 12 ++ .../scenario.json | 28 ++++ .../scenario.json | 25 ++++ .../scenario.json | 26 ++++ .../scenarios/contracts.ts | 118 +++++++++++++++ .../evaluation-prompt.ts | 18 +++ .../src/founder-weekly-review/evaluation.ts | 94 ++++++++++++ .../founder-weekly-review/evidence-service.ts | 3 + .../src/founder-weekly-review/index.ts | 2 + .../src/founder-weekly-review/prompts.ts | 8 +- 25 files changed, 985 insertions(+), 15 deletions(-) create mode 100644 apps/web/__tests__/founderWeeklyReview/demo.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/evaluation.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts create mode 100644 apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts create mode 100644 apps/web/scripts/founder-weekly-review-demo.ts create mode 100644 apps/web/scripts/founder-weekly-review-scenario-loader.ts create mode 100644 apps/web/scripts/founder-weekly-review-scenario-seeder.ts create mode 100644 apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json create mode 100644 apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json create mode 100644 apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json create mode 100644 apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json create mode 100644 apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json create mode 100644 apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts create mode 100644 packages/features/src/founder-weekly-review/evaluation-prompt.ts create mode 100644 packages/features/src/founder-weekly-review/evaluation.ts diff --git a/apps/web/__tests__/founderWeeklyReview/demo.test.ts b/apps/web/__tests__/founderWeeklyReview/demo.test.ts new file mode 100644 index 000000000..56621f6c1 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/demo.test.ts @@ -0,0 +1,27 @@ +import { FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET } from "@launchstack/features/founder-weekly-review"; +import { formatFounderWeeklyReviewDemo } from "../../scripts/founder-weekly-review-demo"; + +describe("Founder Weekly Review demo presentation", () => { + it("formats a safe staged report from pipeline outputs", () => { + const output = formatFounderWeeklyReviewDemo({ + rawChanges: [{ documentId: "1", previousVersionId: 1, currentVersionId: 2, alignmentMethod: "structure_path", previousStructureTitle: "Ownership", previousExcerpt: "Product owns telemetry.", currentExcerpt: "Platform owns telemetry.", changeType: "modified" }], + groups: [{ structureTitle: "Ownership", category: "ownership_change" }], + promptItems: [{ sourceType: "document_change", title: "Ownership", excerpt: "Ownership changed.", metadata: { category: "ownership_change" } }, { sourceType: "customer_feedback" }], + envelopeDiagnostics: { selectedItemCount: 2, serializedCharacterCount: 100, estimatedTokenCount: 25, truncated: false }, + analyzerCalls: [], eligibleGroups: 0, warnings: [], reviewMarkdown: "# Founder Weekly Review\n\nConcise result.", provider: "kimi", model: "kimi-k2.6", promptVersion: "founder-weekly-review-generation/v2", outputCeiling: 2400, repairCount: 0, snapshotVersion: "founder-weekly-review-evidence/v2", evidenceDigest: "a".repeat(64), persistencePassed: true, readBackPassed: true, noOpRemoved: 0, budget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + }); + expect(output).toContain("CHANGE PROCESSING"); + expect(output).toContain("IMMUTABLE EVIDENCE SNAPSHOT v2"); + expect(output).toContain("FINAL FOUNDER WEEKLY REVIEW"); + expect(output).toContain("Product owns telemetry."); + expect(output).not.toContain("api-key"); + expect(output).not.toContain("provider response"); + }); + + it("renders the configured budget rather than presentation literals", () => { + const output = formatFounderWeeklyReviewDemo({ + rawChanges: [], groups: [], promptItems: [], envelopeDiagnostics: { selectedItemCount: 0, serializedCharacterCount: 0, estimatedTokenCount: 0, truncated: false }, analyzerCalls: [], eligibleGroups: 0, warnings: [], reviewMarkdown: "", provider: "kimi", model: "kimi-k2.6", promptVersion: "v2", outputCeiling: 2400, repairCount: 0, snapshotVersion: "v2", evidenceDigest: "a".repeat(64), persistencePassed: true, readBackPassed: true, noOpRemoved: 0, budget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + }); + expect(output).toContain(`${FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.totalSerializedCharacters.toLocaleString()} chars`); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts index cfd91c620..e922c4b02 100644 --- a/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts @@ -284,6 +284,13 @@ describe("deterministic document-change condensation", () => { expect(FounderWeeklyReviewEvidenceItemSchema.safeParse(evidence[0]).success).toBe(true); }); + it("reports the actual deterministic no-op count used by the collector", () => { + const result = condenseDocumentChanges([{ pair: pair(), alignments: [modified(7, " same\r\nvalue\u00a0 ", "same\nvalue ")] }]); + expect(result.diagnostics.deterministicNoOpCount).toBe(1); + expect(result.rawChanges).toHaveLength(0); + expect(result.selectedGroups).toHaveLength(0); + }); + it("condenses a deterministic 20-page, 40-chunk enterprise fixture before the prompt envelope", () => { const versionPair = pair(44n); const alignments: ChunkAlignment[] = []; diff --git a/apps/web/__tests__/founderWeeklyReview/evaluation.test.ts b/apps/web/__tests__/founderWeeklyReview/evaluation.test.ts new file mode 100644 index 000000000..137fc07f9 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evaluation.test.ts @@ -0,0 +1,81 @@ +import { evaluateFounderWeeklyReview, buildFounderWeeklyReviewEvaluationPrompt } from "@launchstack/features/founder-weekly-review"; +import type { FounderWeeklyReviewEvidenceSnapshot, FounderWeeklyReviewV2Payload } from "@launchstack/features/founder-weekly-review"; + +const snapshot = (items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderWeeklyReviewEvidenceSnapshot => ({ schemaVersion: "founder-weekly-review-evidence/v2", capturedAt: "2026-01-01T00:00:00.000Z", reportingPeriod: { start: "2026-01-01", end: "2026-01-07" }, workspaceTimezone: "UTC", items, sourceWarnings: [], documentChangeAudit: { schemaVersion: "document-change-audit/v1", rawChanges: [], groups: [] } }); +const payload = (whatCustomersSaid: FounderWeeklyReviewV2Payload["sections"]["whatCustomersSaid"], whatChanged: FounderWeeklyReviewV2Payload["sections"]["whatChanged"]): FounderWeeklyReviewV2Payload => ({ schemaVersion: "founder-weekly-review/v2", sections: { whatChanged, whatShipped: { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, whatCustomersSaid, currentBlockers: { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, nextPriorities: { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } } } }); +const change = { sourceType: "document_change" as const, sourceId: "change-1", title: "Launch", excerpt: "The launch shipped.", metadata: {} }; +const workspace = { sourceType: "workspace_document" as const, sourceId: "workspace-1", title: "Current", excerpt: "Current blocker context.", metadata: {} }; +const feedback = { sourceType: "customer_feedback" as const, sourceId: "feedback-1", title: "Interview", excerpt: "Customers requested reliable recovery.", metadata: {} }; + +describe("Founder Weekly Review evaluation", () => { + it("enforces customer-only attribution and temporal workspace semantics", () => { + const evidence = snapshot([change, workspace, feedback]); + const invalid = payload({ state: "evidence", items: [{ kind: "observed_fact", text: "Customers requested reliable recovery.", sourceIds: ["workspace-1"], confidence: 0.8 }] }, { state: "evidence", items: [{ kind: "observed_fact", text: "A current document changed this week.", sourceIds: ["workspace-1"], confidence: 0.8 }] }); + const result = evaluateFounderWeeklyReview(evidence, invalid); + expect(result.hasHardFailure).toBe(true); + expect(result.failures.map(failure => failure.category)).toEqual(expect.arrayContaining(["invalid_source_type"])); + }); + + it("allows workspace context alongside temporal evidence", () => { + const evidence = snapshot([change, workspace, feedback]); + const valid = payload({ state: "evidence", items: [{ kind: "observed_fact", text: "Customers requested reliable recovery.", sourceIds: ["feedback-1"], confidence: 0.8 }] }, { state: "evidence", items: [{ kind: "observed_fact", text: "The launch shipped with current blocker context.", sourceIds: ["change-1", "workspace-1"], confidence: 0.8 }] }); + expect(evaluateFounderWeeklyReview(evidence, valid).hasHardFailure).toBe(false); + }); + + it("flags a future release as not shipped in the reporting period", () => { + const evidence = snapshot([{ ...change, excerpt: "Status changed. Before: planned for the April release. After: launched in the April release.", sourceTimestamp: "2026-02-20T10:00:00.000Z" }, workspace]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }); + review.sections.whatShipped = { state: "evidence", items: [{ kind: "observed_fact", text: "Recovery workflow launched in the April release.", sourceIds: ["change-1", "workspace-1"], confidence: 0.8 }] }; + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.failures).toEqual(expect.arrayContaining([expect.objectContaining({ category: "future_release_claim" })])); + expect(result.hasHardFailure).toBe(true); + }); + + it("rejects workspace-only temporal claims but permits workspace context with temporal evidence", () => { + const evidence = snapshot([{ ...change, excerpt: "The workflow shipped during the reporting period." }, workspace]); + const invalid = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "The current document changed this week.", sourceIds: ["workspace-1"], confidence: 0.8 }] }); + expect(evaluateFounderWeeklyReview(evidence, invalid).failures).toEqual(expect.arrayContaining([expect.objectContaining({ category: "invalid_source_type" })])); + const valid = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "The workflow shipped this period and remains current.", sourceIds: ["change-1", "workspace-1"], confidence: 0.8 }] }); + expect(evaluateFounderWeeklyReview(evidence, valid).failures).not.toEqual(expect.arrayContaining([expect.objectContaining({ category: "invalid_source_type" })])); + }); + + it("measures theme coverage rather than requiring one bullet per source", () => { + const evidence = snapshot([{ ...change, sourceId: "change-1" }, { ...change, sourceId: "change-2", title: "Scope", excerpt: "Rollout expanded globally." }, feedback]); + const review = payload({ state: "evidence", items: [{ kind: "observed_fact", text: "Customers requested reliable recovery.", sourceIds: ["feedback-1"], confidence: 0.8 }] }, { state: "evidence", items: [{ kind: "observed_fact", text: "The launch shipped and rollout expanded globally.", sourceIds: ["change-1", "change-2"], confidence: 0.8 }] }); + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.deterministic.evidenceCoverage).toBe(1); + expect(result.deterministic.duplicateClaimRate).toBe(0); + }); + + it("allows section-specific reuse of the same evidence", () => { + const evidence = snapshot([change]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "The launch shipped.", sourceIds: ["change-1"], confidence: 0.8 }] }); + review.sections.currentBlockers = { state: "evidence", items: [{ kind: "observed_fact", text: "The shipped launch still has a current blocker.", sourceIds: ["change-1"], confidence: 0.7 }] }; + expect(evaluateFounderWeeklyReview(evidence, review).deterministic.duplicateClaimRate).toBe(0); + }); + + it("accepts an evidence-grounded action without requiring invented owner or deadline", () => { + const evidence = snapshot([change]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }); + review.sections.nextPriorities = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Confirm the launch readiness gap.", rationale: "The release evidence indicates the gap remains relevant.", sourceIds: ["change-1"], confidence: 0.7 }] }; + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.failures.filter(failure => failure.category === "unsupported_claim")).toHaveLength(0); + }); + + it("keeps lexical support as a soft diagnostic rather than a hard gate", () => { + const evidence = snapshot([change]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "A release occurred.", sourceIds: ["change-1"], confidence: 0.8 }] }); + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.hasHardFailure).toBe(false); + expect(result.deterministic.unsupportedClaimRate).toBeGreaterThanOrEqual(0); + }); + + it("builds a rubric-aware semantic grading prompt", () => { + const prompt = buildFounderWeeklyReviewEvaluationPrompt(snapshot([change]), payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } })); + expect(prompt).toContain("document_change is reporting-period temporal evidence"); + expect(prompt).toContain("concise founder-level synthesis"); + expect(prompt).toContain("future-dated release is not shipped"); + expect(prompt).toContain("supports, indicates, and suggests"); + expect(prompt).toContain("invented owners"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts index daab5a053..18a2867a3 100644 --- a/apps/web/__tests__/founderWeeklyReview/generation.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -241,6 +241,20 @@ describe("Founder Weekly Review generation", () => { expect(first.modelMetadata.promptHash).toBe(second.modelMetadata.promptHash); }); + it("uses the concise founder-review generation contract", async () => { + const generate = fake(validPayload()); + await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate }); + const system = generate.mock.calls[0][0].system as string; + expect(system).toContain("decision-oriented founder review, not an evidence transcript"); + expect(system).toContain("at most 3 items in each section"); + expect(system).toContain("one concise sentence whenever possible"); + expect(system).toContain("Synthesize related evidence into one focused claim"); + expect(system).toContain("do not create one output item per evidence source"); + expect(system).toContain("optional rationale should be brief"); + expect(system).not.toContain("2–4 sentences per substantive item"); + expect(system).not.toContain("600–1,000 words overall"); + }); + it("canonicalizes metadata key order before building the prompt and hash", async () => { const firstSnapshot = completeSnapshot(); firstSnapshot.items[0]!.metadata = { changeType: "modified", alignmentMethod: "structure_path" }; diff --git a/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts index 40dd57d05..c580ce3fb 100644 --- a/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts @@ -38,7 +38,7 @@ describe("Founder Weekly Review Kimi transport", () => { expect(String(url)).not.toContain("/responses"); const body = JSON.parse(String(init?.body)) as Record; expect(body.model).toBe("kimi-k2.6"); - expect(body.max_tokens).toBe(1800); + expect(body.max_tokens).toBe(2400); expect(body.messages).toEqual(expect.any(Array)); expect(body).not.toHaveProperty("temperature"); expect(body.response_format).toEqual({ type: "json_object" }); @@ -81,6 +81,52 @@ describe("Founder Weekly Review Kimi transport", () => { expect(resolved).not.toHaveProperty("thinking"); }); + it("keeps an explicit Founder Weekly Review output override", async () => { + const fetchMock = jest.fn(async (_url: string | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + expect(body.max_tokens).toBe(777); + return new Response(JSON.stringify({ + id: "chatcmpl-override", + object: "chat.completion", + created: 0, + model: "kimi-k2.6", + choices: [{ index: 0, message: { role: "assistant", content: "{\"ok\":true}" }, finish_reason: "stop", logprobs: null }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + global.fetch = fetchMock as typeof fetch; + await expect(generateStructuredWithMetadata({ + capability: "founderWeeklyReview", + maxOutputTokens: 777, + system: "Return valid JSON.", + prompt: "Generate the object.", + schema: z.object({ ok: z.boolean() }), + })).resolves.toMatchObject({ object: { ok: true } }); + }); + + it("does not apply the Founder Weekly Review default to small extraction", async () => { + const fetchMock = jest.fn(async (_url: string | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + expect(body).not.toHaveProperty("max_tokens"); + return new Response(JSON.stringify({ + id: "chatcmpl-extraction", + object: "chat.completion", + created: 0, + model: "kimi-k2.6", + choices: [{ index: 0, message: { role: "assistant", content: "{\"ok\":true}" }, finish_reason: "stop", logprobs: null }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + global.fetch = fetchMock as typeof fetch; + await expect(generateStructuredWithMetadata({ + capability: "smallExtraction", + forceProvider: "kimi", + system: "Return valid JSON.", + prompt: "Extract the object.", + schema: z.object({ ok: z.boolean() }), + })).resolves.toMatchObject({ object: { ok: true } }); + }); + it("uses explicit OpenAI without requiring Moonshot configuration", () => { process.env.FWR_GENERATION_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "openai-test-key"; diff --git a/apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts new file mode 100644 index 000000000..d87f5a022 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts @@ -0,0 +1,83 @@ +import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewEvidenceSnapshotSchema } from "@launchstack/features/founder-weekly-review"; +import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; + +import { loadScenario } from "../../scripts/founder-weekly-review-scenario-loader"; +import { seedScenario } from "../../scripts/founder-weekly-review-scenario-seeder"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeIfDatabase = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ? describe : describe.skip; +const vector = () => Array.from({ length: 1536 }, (_, index) => index === 0 ? 1 : 0); +const fixturePath = (name: string) => `${__dirname}/../../test-fixtures/founder-weekly-review/scenarios/${name}/scenario.json`; +type V2Snapshot = Extract, { schemaVersion: "founder-weekly-review-evidence/v2" }>; + +async function collect(name: string, deterministicWorkspaceEmbeddings = false) { + const scenario = await loadScenario(fixturePath(name)); + const testDb = await createFounderWeeklyReviewTestDatabase(); + try { + const seeded = await seedScenario(testDb.db, scenario, { deterministicWorkspaceEmbeddings }); + const collector = new FounderWeeklyReviewEvidenceService( + testDb.db, + () => new Date("2026-07-27T00:00:00.000Z"), + { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(testDb.db) }, + deterministicWorkspaceEmbeddings ? new StrictCurrentWorkspaceDocumentStore(testDb.db, { embedQuery: async () => vector() }) : undefined, + ); + const snapshot = await collector.collectFounderWeeklyReviewEvidence({ + companyId: seeded.underReviewCompanyId, + reportingPeriod: scenario.reportingPeriod, + workspaceTimezone: scenario.workspaceTimezone, + founderContext: scenario.founderContext, + actor: scenario.founderContext ? { externalUserId: "scenario-test" } : undefined, + requestKey: `scenario-${scenario.name}`, + capturedAt: new Date("2026-07-27T00:00:00.000Z"), + }); + return { scenario, snapshot: FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot) as V2Snapshot }; + } finally { + await testDb.close(); + } +} + +function assertExpectations(scenario: Awaited>, snapshot: V2Snapshot) { + expect(snapshot.schemaVersion).toBe("founder-weekly-review-evidence/v2"); + const expected = scenario.expect; + for (const [sourceType, bounds] of Object.entries(expected?.evidence?.sourceTypeCounts ?? {})) { + const count = snapshot.items.filter((item) => item.sourceType === sourceType).length; + if (bounds?.exact !== undefined) expect(count).toBe(bounds.exact); + if (bounds?.min !== undefined) expect(count).toBeGreaterThanOrEqual(bounds.min); + if (bounds?.max !== undefined) expect(count).toBeLessThanOrEqual(bounds.max); + } + for (const warningCode of expected?.evidence?.warningCodes ?? []) expect(snapshot.sourceWarnings.map((warning) => warning.code)).toContain(warningCode); + if (expected?.documentChanges) { + const groups = snapshot.documentChangeAudit.groups; + if (expected.documentChanges.minGroups !== undefined) expect(groups.length).toBeGreaterThanOrEqual(expected.documentChanges.minGroups); + if (expected.documentChanges.maxGroups !== undefined) expect(groups.length).toBeLessThanOrEqual(expected.documentChanges.maxGroups); + if (expected.documentChanges.requiredCategories) for (const category of expected.documentChanges.requiredCategories) expect(groups.some((group) => group.category === category)).toBe(true); + if (expected.documentChanges.requireNoInventedBaseline) expect(groups).toHaveLength(0); + } +} + +describeIfDatabase("provider-free scenario collection against current LAU-9", () => { + jest.setTimeout(120_000); + + it.each([ + ["01-empty-evidence", false], + ["02-founder-context-only", false], + ["03-relevant-workspace-document", true], + ["04-first-ever-version-no-invented-diff", false], + ["05-multiple-versions-in-period", false], + ] as const)("collects %s with current snapshot v2 semantics", async (name, deterministicWorkspaceEmbeddings) => { + const { scenario, snapshot } = await collect(name, deterministicWorkspaceEmbeddings); + assertExpectations(scenario, snapshot); + if (name === "03-relevant-workspace-document") { + expect(snapshot.items.some((item) => item.sourceType === "workspace_document")).toBe(true); + expect(snapshot.items.some((item) => item.sourceType === "document_change")).toBe(false); + expect(snapshot.items.filter((item) => item.sourceType === "workspace_document").every((item) => item.metadata.documentVersionId !== undefined)).toBe(true); + } + if (name === "05-multiple-versions-in-period") { + const pairKeys = new Set(snapshot.documentChangeAudit.groups.map((group) => `${group.previousVersionId}->${group.currentVersionId}`)); + expect(pairKeys.size).toBeGreaterThanOrEqual(2); + expect(snapshot.documentChangeAudit.rawChanges.length).toBeGreaterThan(0); + expect(snapshot.documentChangeAudit.groups.every((group) => group.previousVersionId !== group.currentVersionId)).toBe(true); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts b/apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts new file mode 100644 index 000000000..c453b7717 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts @@ -0,0 +1,33 @@ +import { FounderWeeklyReviewScenarioSchema } from "../../test-fixtures/founder-weekly-review/scenarios/contracts"; + +const baseScenario = () => ({ + name: "contract-test", + reportingPeriod: { start: "2026-07-20", end: "2026-07-26" }, + workspaceTimezone: "UTC", + companies: [{ name: "Northstar", underReview: true, documents: [] }], +}); + +describe("FounderWeeklyReviewScenarioSchema", () => { + it("accepts bounded invariant expectations", () => { + const parsed = FounderWeeklyReviewScenarioSchema.parse({ + ...baseScenario(), + expect: { + evidence: { sourceTypeCounts: { document_change: { min: 1 }, founder_context: { exact: 0 } } }, + documentChanges: { minGroups: 1, maxGroups: 8, requireNoInventedBaseline: true }, + sourceSemantics: { customerFeedbackOnly: true, currentWorkspaceOnly: true }, + review: { sectionStates: { whatShipped: "no_evidence" }, requiredThemes: ["ownership"] }, + }, + }); + expect(parsed.expect?.documentChanges?.requireNoInventedBaseline).toBe(true); + }); + + it("rejects contradictory count bounds and duplicate versions", () => { + expect(() => FounderWeeklyReviewScenarioSchema.parse({ ...baseScenario(), expect: { evidence: { sourceTypeCounts: { document_change: { min: 3, max: 1 } } } } })).toThrow(/min must not exceed max/); + const scenario = baseScenario() as any; + scenario.companies[0]!.documents = [{ title: "Plan", category: "Planning", versions: [ + { versionNumber: 1, timestamp: "2026-07-20T10:00:00.000Z", chunks: [] }, + { versionNumber: 1, timestamp: "2026-07-21T10:00:00.000Z", chunks: [] }, + ] }]; + expect(() => FounderWeeklyReviewScenarioSchema.parse(scenario)).toThrow(/versionNumber must be unique/); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts b/apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts new file mode 100644 index 000000000..7e965e387 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts @@ -0,0 +1,19 @@ +import { resolve } from "node:path"; + +import { loadScenario, parseScenario } from "../../scripts/founder-weekly-review-scenario-loader"; + +const scenarioPath = (name: string) => resolve(__dirname, `../../test-fixtures/founder-weekly-review/scenarios/${name}/scenario.json`); + +describe("Founder Weekly Review scenario loader", () => { + it("loads the first-ever-version regression fixture", async () => { + const scenario = await loadScenario(scenarioPath("04-first-ever-version-no-invented-diff")); + expect(scenario.name).toBe("first-ever-version-no-invented-diff"); + expect(scenario.expect?.documentChanges?.requireNoInventedBaseline).toBe(true); + }); + + it("applies bounded defaults while rejecting unknown fields", () => { + const parsed = parseScenario({ ...JSON.parse('{"name":"empty","reportingPeriod":{"start":"2026-07-20","end":"2026-07-26"},"workspaceTimezone":"UTC","companies":[{"name":"Northstar","underReview":true,"documents":[]}]}') }); + expect(parsed.schemaVersion).toBe("founder-weekly-review-scenario/v1"); + expect(() => parseScenario({ ...parsed, unexpected: true })).toThrow(); + }); +}); diff --git a/apps/web/package.json b/apps/web/package.json index 0faebc999..36bd459f7 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -22,7 +22,8 @@ "start": "next start", "test": "jest --passWithNoTests", "typecheck": "tsc --noEmit", - "inngest:dev": "pnpm dlx inngest-cli@latest dev -u http://localhost:3000/api/inngest" + "inngest:dev": "pnpm dlx inngest-cli@latest dev -u http://localhost:3000/api/inngest", + "founder-weekly-review:demo": "tsx scripts/run-founder-weekly-review-realistic-e2e.ts --demo" }, "dependencies": { "@ai-sdk/anthropic": "^3.0.69", diff --git a/apps/web/scripts/founder-weekly-review-demo.ts b/apps/web/scripts/founder-weekly-review-demo.ts new file mode 100644 index 000000000..28faab480 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-demo.ts @@ -0,0 +1,136 @@ +import type { FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET } from "@launchstack/features/founder-weekly-review"; + +type Budget = typeof FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET; + +export type FounderWeeklyReviewDemoInput = { + rawChanges: readonly Record[]; + groups: readonly Record[]; + promptItems: readonly Record[]; + envelopeDiagnostics: Record; + analyzerCalls: readonly Record[]; + eligibleGroups: number; + warnings: readonly string[]; + reviewMarkdown: string; + provider: string; + model: string; + promptVersion: string; + outputCeiling: number; + repairCount: number; + snapshotVersion: string; + evidenceDigest: string; + persistencePassed: boolean; + readBackPassed: boolean; + noOpRemoved: number; + budget: Budget; + scaleResults?: readonly Record[]; +}; + +const text = (value: unknown) => typeof value === "string" ? value : ""; +const number = (value: unknown) => typeof value === "number" ? value : 0; +const label = (value: unknown) => text(value) || "unknown"; + +function line(key: string, value: string | number) { + return `${key.padEnd(34)} ${value}`; +} + +function sourceCounts(items: readonly Record[]) { + const counts = new Map(); + for (const item of items) counts.set(label(item.sourceType), (counts.get(label(item.sourceType)) ?? 0) + 1); + return [...counts.entries()].sort(([a], [b]) => a.localeCompare(b)); +} + +function sampleChanges(rawChanges: readonly Record[]) { + const preferred = ["Launch timing", "Retry ownership", "Requirement"]; + const selected = preferred.map(title => rawChanges.find(change => text(change.previousStructureTitle) === title)).filter((change): change is Record => Boolean(change)); + return selected.length ? selected : rawChanges.slice(0, 3); +} + +export function formatFounderWeeklyReviewDemo(input: FounderWeeklyReviewDemoInput): string { + const raw = input.rawChanges; + const auditGroups = input.groups; + const analyzed = input.analyzerCalls; + const nonMaterial = auditGroups.filter(group => { + const analysis = group.analysis as Record | undefined; + return analysis?.disposition === "non_material"; + }); + const alignmentCounts = new Map(); + for (const change of raw) { + const method = label(change.alignmentMethod); + alignmentCounts.set(method, (alignmentCounts.get(method) ?? 0) + 1); + } + const unmatched = raw.filter(change => ["added", "removed"].includes(text(change.changeType))).length; + const selectedChanges = input.promptItems.filter(item => item.sourceType === "document_change"); + const diagnostics = input.envelopeDiagnostics; + const scale = input.scaleResults ?? []; + const out: string[] = []; + out.push("FOUNDER WEEKLY REVIEW — LIVE E2E", "=".repeat(34), ""); + out.push("INPUT SCENARIO", "-------------"); + for (const change of sampleChanges(raw)) { + out.push(`${text(change.currentStructureTitle) || "Document change"}`); + out.push(` Before: ${text(change.previousExcerpt)}`); + out.push(` After: ${text(change.currentExcerpt)}`); + } + out.push("", "VERSION / ALIGNMENT", "-------------------"); + out.push(line("Documents compared", new Set(raw.map(change => text(change.documentId))).size)); + out.push(line("Version pairs", new Set(raw.map(change => `${change.documentId}:${change.previousVersionId}:${change.currentVersionId}`)).size)); + out.push(line("Historical chunks represented", raw.length * 2)); + for (const [method, count] of alignmentCounts) out.push(line(`Matched by ${method}`, count)); + out.push(line("Unmatched", unmatched)); + out.push("", "CHANGE PROCESSING", "-----------------"); + out.push(line("Raw changes", raw.length)); + out.push(line("No-op changes removed", input.noOpRemoved)); + out.push(line("DocumentChangeGroups", auditGroups.length)); + out.push("", "MATERIALITY"); + out.push(line("Analyzer eligible", input.eligibleGroups)); + out.push(line("Kimi analyzed", `${analyzed.length} / 4 cap`)); + out.push(line("Kimi non-material", analyzed.filter(call => call.disposition === "non_material").length)); + out.push(line("Analyzer failures", analyzed.filter(call => call.errorCode).length)); + out.push(line("Budget-excluded", Math.max(0, input.eligibleGroups - analyzed.length))); + out.push(line("Deterministic fallback", Math.max(0, input.eligibleGroups - analyzed.length))); + out.push(line("Prompt-facing document changes", selectedChanges.length)); + out.push("", "SELECTED MATERIAL CHANGES", "-------------------------"); + for (const item of selectedChanges) out.push(`[${label((item.metadata as Record | undefined)?.category)}] ${text(item.title)} — ${text(item.excerpt).split("\n")[0]}`); + out.push("", "FILTERED SEMANTIC REWRITES", "--------------------------"); + for (const group of nonMaterial) { + const analysis = group.analysis as Record; + out.push(`- ${text(group.structureTitle)}: ${text(analysis.beforeKeyPoint)} → ${text(analysis.afterKeyPoint)}`); + } + out.push("", "IMMUTABLE EVIDENCE SNAPSHOT v2", "-----------------------------"); + for (const [type, count] of sourceCounts(input.promptItems)) out.push(line(type, count)); + out.push(line("Prompt-facing items", input.promptItems.length)); + out.push(line("Evidence digest", `${input.evidenceDigest.slice(0, 16)}...`)); + out.push(line("Immutable", "✓")); + out.push("", "GENERATION ENVELOPE", "-------------------"); + out.push("Snapshot = complete frozen provenance"); + out.push("Envelope = bounded projection for generation", ""); + out.push(line("Selected evidence items", number(diagnostics.selectedItemCount))); + out.push(line("Serialized characters", number(diagnostics.serializedCharacterCount))); + out.push(line("Estimated input tokens", number(diagnostics.estimatedTokenCount))); + out.push(line("Envelope truncation", diagnostics.truncated ? "yes" : "no")); + out.push("", "Evidence budgets"); + out.push(line("Global", `${input.budget.totalSerializedCharacters.toLocaleString()} chars`)); + out.push(line("Document changes", `${input.budget.documentChangeSerializedCharacters.toLocaleString()} chars / max ${input.budget.documentChangeItems}`)); + out.push(line("Per document", `max ${input.budget.documentChangeItemsPerDocument} document changes`)); + out.push(line("Workspace reserve", `${input.budget.workspaceDocumentReservedCharacters.toLocaleString()} chars`)); + out.push(line("Customer feedback", `${input.budget.customerFeedbackReservedCharacters.toLocaleString()} chars`)); + out.push(line("Founder Context", `${input.budget.founderContextReservedCharacters.toLocaleString()} chars / max 1`)); + out.push("", "GENERATION", "----------"); + out.push(line("Provider / model", `${input.provider} / ${input.model}`)); + out.push(line("Prompt version", input.promptVersion)); + out.push(line("Output ceiling", `${input.outputCeiling} tokens`)); + out.push(line("Schema validation", "✓")); + out.push(line("Citation validation", "✓")); + out.push(line("Source semantics", "✓")); + out.push(line("Semantic repair", input.repairCount)); + out.push(line("Persistence", input.persistencePassed ? "✓" : "✗")); + out.push(line("Repository read-back", input.readBackPassed ? "✓" : "✗")); + out.push("", "FINAL FOUNDER WEEKLY REVIEW", "---------------------------", input.reviewMarkdown.trim()); + if (scale.length) { + out.push("", "SCALE BEHAVIOR", "-------------"); + out.push("Raw changes Prompt envelope Audit snapshot"); + for (const row of scale) out.push(`${number(row.rawChanged).toString().padEnd(16)}${number(row.envelopeCharacters).toString().padEnd(20)}${number(row.auditCharacters)}`); + out.push("", "Generation input remains bounded.", "Immutable audit grows with source volume.", "Kimi materiality calls remain capped at 4."); + } + out.push("", `Warnings: ${input.warnings.join(", ") || "none"}`); + return out.join("\n"); +} diff --git a/apps/web/scripts/founder-weekly-review-scenario-loader.ts b/apps/web/scripts/founder-weekly-review-scenario-loader.ts new file mode 100644 index 000000000..fa04ee07d --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-scenario-loader.ts @@ -0,0 +1,11 @@ +import { readFile } from "node:fs/promises"; + +import { FounderWeeklyReviewScenarioSchema, type FounderWeeklyReviewScenario } from "../test-fixtures/founder-weekly-review/scenarios/contracts"; + +export function parseScenario(input: unknown): FounderWeeklyReviewScenario { + return FounderWeeklyReviewScenarioSchema.parse(input); +} + +export async function loadScenario(path: string): Promise { + return parseScenario(JSON.parse(await readFile(path, "utf8"))); +} diff --git a/apps/web/scripts/founder-weekly-review-scenario-seeder.ts b/apps/web/scripts/founder-weekly-review-scenario-seeder.ts new file mode 100644 index 000000000..cb35b2ed0 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-scenario-seeder.ts @@ -0,0 +1,48 @@ +import { createHash } from "node:crypto"; + +import type { DbClient } from "@launchstack/core/db"; +import { company, document, documentContextChunks, documentStructure, documentVersions } from "@launchstack/core/db/schema"; +import { eq, sql } from "drizzle-orm"; + +import type { FounderWeeklyReviewScenario } from "../test-fixtures/founder-weekly-review/scenarios/contracts"; + +export interface SeededScenario { + companyNamesToId: Map; + underReviewCompanyId: bigint; +} + +export type ScenarioSeederOptions = { deterministicWorkspaceEmbeddings?: boolean }; + +const workspaceVector = () => Array.from({ length: 1536 }, (_, index) => index === 0 ? 1 : 0); +const contentHash = (content: string) => createHash("sha256").update(content, "utf8").digest("hex"); +const structurePath = (section: string, index: number) => `/${section.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "section"}-${index + 1}`; + +/** Seeds business state only; the current collector computes evidence from these rows. */ +export async function seedScenario(db: DbClient, scenario: FounderWeeklyReviewScenario, options: ScenarioSeederOptions = {}): Promise { + const companyNamesToId = new Map(); + let underReviewCompanyId: bigint | undefined; + for (const companyEntry of scenario.companies) { + const [companyRow] = await db.insert(company).values({ name: companyEntry.name, numberOfEmployees: "10" }).returning(); + const companyId = BigInt(companyRow!.id); + companyNamesToId.set(companyEntry.name, companyId); + if (companyEntry.underReview) underReviewCompanyId = companyId; + for (const doc of companyEntry.documents) { + const [documentRow] = await db.insert(document).values({ companyId, url: `local://${doc.title}`, category: doc.category, title: doc.title }).returning(); + const documentId = BigInt(documentRow!.id); + let latestVersion: { id: bigint; versionNumber: number } | undefined; + for (const version of doc.versions) { + const [versionRow] = await db.insert(documentVersions).values({ documentId, versionNumber: version.versionNumber, url: `local://${doc.title}/v${version.versionNumber}`, mimeType: "text/plain", uploadedBy: version.uploadedBy ?? "scenario-seed", changelog: version.changelog ?? null, createdAt: new Date(version.timestamp) }).returning(); + const versionId = BigInt(versionRow!.id); + if (!latestVersion || version.versionNumber > latestVersion.versionNumber) latestVersion = { id: versionId, versionNumber: version.versionNumber }; + for (const [index, chunk] of version.chunks.entries()) { + const [structure] = await db.insert(documentStructure).values({ documentId, versionId, ordering: index + 1, title: chunk.section, path: structurePath(chunk.section, index), startPage: chunk.pageNumber ?? index + 1, endPage: chunk.pageNumber ?? index + 1 }).returning(); + const values = { documentId, versionId, structureId: BigInt(structure!.id), content: chunk.content, contentHash: contentHash(chunk.content), tokenCount: chunk.content.split(/\s+/).length, charCount: chunk.content.length, pageNumber: chunk.pageNumber ?? index + 1, lineStart: chunk.lineStart ?? 1, lineEnd: chunk.lineEnd ?? (chunk.lineStart ?? 1) }; + await db.insert(documentContextChunks).values(options.deterministicWorkspaceEmbeddings ? { ...values, embedding: sql`${JSON.stringify(workspaceVector())}::vector(1536)` } : values); + } + } + if (latestVersion) await db.update(document).set({ currentVersionId: latestVersion.id }).where(eq(document.id, documentRow!.id)); + } + } + if (!underReviewCompanyId) throw new Error("scenario has no under-review company (expected exactly one)."); + return { companyNamesToId, underReviewCompanyId }; +} diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts index d697ae843..53b00375c 100644 --- a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -4,14 +4,18 @@ import { createRequire } from "node:module"; import { readFile, mkdir, rename, writeFile, access } from "node:fs/promises"; import { resolve } from "node:path"; import { eq, sql } from "drizzle-orm"; +import { z } from "zod"; import { company, document, documentContextChunks, documentStructure, documentVersions, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; -import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, generateFounderWeeklyReview, validateFounderWeeklyReviewV2Citations } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, buildFounderWeeklyReviewEvaluationPrompt, buildGenerationEvidenceEnvelope, evaluateFounderWeeklyReview, FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION, generateFounderWeeklyReview, validateFounderWeeklyReviewV2Citations, type DeterministicMaterialChangeDiagnostics, type DocumentChangeMaterialityAnalyzer } from "@launchstack/features/founder-weekly-review"; import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { createConfiguredDocumentChangeMaterialityAnalyzer } from "~/server/founder-weekly-review/document-change-materiality-analyzer"; +import { generateStructuredWithMetadata } from "~/lib/llm"; import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; +import { formatFounderWeeklyReviewDemo } from "./founder-weekly-review-demo"; const require = createRequire(import.meta.url); const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); @@ -19,7 +23,14 @@ const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/ type Fixture = { reportingPeriod: { start: string; end: string }; workspaceTimezone: string; founderContext: string; documents: Array<{ title: string; category: string; changelog: string; timestamp: string; chunks?: string[] }> }; type EvidenceMode = ReturnType; type ComputedTexts = { before: string; after: string; v3: string; bHistorical: string; nullVersion: string; foreign: string; unrelated: string }; -type ArtifactPaths = { evidence: string; report: string; markdown: string; summary: string }; +type ArtifactPaths = { evidence: string; report: string; markdown: string; summary: string; envelope: string; validation: string; e2eReport: string; analyzerCalls: string; evaluation: string; evaluationMarkdown: string }; + +const SemanticEvaluationSchema = z.object({ + overallScore: z.number().min(0).max(1), + dimensions: z.object({ groundedness: z.number().min(0).max(1), materiality: z.number().min(0).max(1), temporalAccuracy: z.number().min(0).max(1), synthesisQuality: z.number().min(0).max(1), actionability: z.number().min(0).max(1) }), + findings: z.array(z.object({ category: z.string().min(1).max(80), severity: z.enum(["info", "minor", "major"]), explanation: z.string().min(1).max(2000) })).max(8), + summary: z.string().min(1).max(2000), +}); function canonicalize(value: unknown): unknown { if (value === null || ["string", "boolean"].includes(typeof value)) return value; if (typeof value === "number" && Number.isFinite(value)) return value; if (Array.isArray(value)) return value.map(canonicalize); if (typeof value === "object") return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalize((value as Record)[key])])); throw new Error("Cannot canonicalize snapshot."); } function digest(value: unknown) { return createHash("sha256").update(JSON.stringify(canonicalize(value)), "utf8").digest("hex"); } @@ -64,12 +75,34 @@ if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.te const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as Fixture; const mode = parseFounderWeeklyReviewRealisticEvidenceMode(process.env.FWR_EVIDENCE_MODE); +const demoMode = process.argv.includes("--demo") || process.env.FWR_DEMO_MODE === "true"; +const originalConsoleLog = console.log; +const originalConsoleInfo = console.info; +if (demoMode) { console.log = () => undefined; console.info = () => undefined; } const testDb = await createFounderWeeklyReviewTestDatabase(); try { const [target] = await testDb.db.insert(company).values({ name: "Northstar Analytics", numberOfEmployees: "24" }).returning(); const [other] = await testDb.db.insert(company).values({ name: "Other Company", numberOfEmployees: "4" }).returning(); const actor = { externalUserId: "realistic-owner", internalUserId: 1n, companyId: BigInt(target!.id), role: "owner" }; let collector: FounderWeeklyReviewEvidenceService; + const configuredAnalyzer = mode === "computed" ? createConfiguredDocumentChangeMaterialityAnalyzer() : undefined; + if (mode === "computed" && process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED === "true" && !configuredAnalyzer) throw new Error("Analyzer was explicitly enabled but not configured."); + const analyzerCalls: Array> = []; + let changeDiagnostics: DeterministicMaterialChangeDiagnostics | undefined; + const analyzer: DocumentChangeMaterialityAnalyzer | undefined = configuredAnalyzer ? { + async analyze(input) { + const started = Date.now(); + try { + const response = await configuredAnalyzer.analyze(input); + const result = response.result as Record; + analyzerCalls.push({ groupId: input.groupId, structurePath: input.structurePath, structureTitle: input.structureTitle, deterministicCategory: input.deterministicAssessment.category, semanticRisk: input.deterministicAssessment.semanticRisk, durationMs: Date.now() - started, disposition: result.disposition, category: result.category, confidence: result.confidence, provider: response.metadata?.provider, model: response.metadata?.model, promptVersion: response.metadata?.promptVersion }); + return response; + } catch (error) { + analyzerCalls.push({ groupId: input.groupId, structurePath: input.structurePath, structureTitle: input.structureTitle, deterministicCategory: input.deterministicAssessment.category, semanticRisk: input.deterministicAssessment.semanticRisk, durationMs: Date.now() - started, errorCode: error instanceof Error ? error.name : "unknown" }); + throw error; + } + }, + } : undefined; let computedIds: Record | undefined; const texts: ComputedTexts = { before: "Product owns retry telemetry.", after: "Platform owns retry telemetry and recovery monitoring.", v3: "v3 future-only operational note", bHistorical: "historical workspace reliability text", nullVersion: "null-version workspace reliability text", foreign: "foreign workspace reliability text", unrelated: "unrelated current cafeteria menu" }; @@ -109,8 +142,27 @@ try { { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: texts.foreign, contentHash: "3".repeat(64), tokenCount: 4, charCount: texts.foreign.length, embedding: vectorSql(0) }, ]).returning(); const bCurrentChunk = inserted.find((chunk) => chunk.documentId === BigInt(b!.id) && chunk.versionId === BigInt(b2!.id))!; + const enterpriseScenarios: Array<{ title: string; category: string; sections: Array<[string, string, string]> }> = [ + { title: "Roadmap and launch plan", category: "Planning", sections: [["Launch timing", "The analytics launch remains planned for Q3.", "We still expect the analytics launch during the third quarter."], ["Launch narrative", "The roadmap tracks launch readiness, dependencies, and adoption preparation.", "Launch readiness, dependencies, and adoption preparation remain tracked in the roadmap."], ["Customer rollout", "The pilot serves selected enterprise customers in the United States.", "The pilot serves selected enterprise customers in the United States."], ["Decision notes", "The team will review launch readiness with the founder next week.", "The founder will review launch readiness with the team next week."], ["Owner", "Product owns roadmap coordination.", "Platform owns roadmap coordination."]], }, + { title: "Enterprise rollout plan", category: "Operations", sections: [["Release status", "The recovery workflow is planned for the April release.", "The recovery workflow launched in the April release."], ["Requirement", "SAML support is optional for the pilot.", "SAML support is required for the pilot."], ["Scope", "The rollout is limited to US enterprise customers.", "The rollout is global for enterprise customers."], ["Rollout narrative", "The rollout plan covers enablement, support readiness, and customer communications.", "Enablement, support readiness, and customer communications are covered by the rollout plan."], ["Risk", "A rollout risk remains documented for support capacity.", "Support capacity remains a documented rollout risk."]], }, + { title: "Operating metrics", category: "Operating Plan", sections: [["Activation target", "The activation target is 10% for the quarter.", "The activation target is 25% for the quarter."], ["Deadline", "The onboarding milestone is due June 1.", "The onboarding milestone is due July 15."], ["Metric narrative", "The operating plan tracks activation, retention, and onboarding throughput.", "Activation, retention, and onboarding throughput are tracked in the operating plan."], ["Customer count", "The target is 20 enterprise customers by year end.", "The target is twenty enterprise customers by year end."], ["Review cadence", "The team reviews operating metrics every Friday.", "Operating metrics are reviewed by the team every Friday."]], }, + { title: "Risk and ownership register", category: "Risk", sections: [["Blocker status", "The data migration is blocked by missing export access.", "The data migration is resolved after export access was granted."], ["Risk rewrite", "The main risk is delayed onboarding for enterprise accounts.", "Enterprise onboarding delay remains the primary operating risk."], ["Priority", "Migration follow-up is P2 and deferred.", "Migration follow-up is P0 and immediate."], ["Ownership narrative", "Marketing owns the customer readiness checklist.", "The customer readiness checklist is owned by Marketing."], ["Decision context", "The register summarizes active risks and their next review dates.", "Active risks and next review dates are summarized in the register."]], }, + ]; + for (const [scenarioIndex, scenario] of enterpriseScenarios.slice(0, 2).entries()) { + const sections = scenario.sections.slice(0, 4); + const [scenarioDocument] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: `local://scenario-${scenarioIndex}`, category: scenario.category, title: scenario.title }).returning(); + const [previousVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(scenarioDocument!.id), versionNumber: 1, url: `local://scenario-${scenarioIndex}/v1`, mimeType: "text/plain", createdAt: new Date("2026-02-10T10:00:00.000Z") }).returning(); + const [currentVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(scenarioDocument!.id), versionNumber: 2, url: `local://scenario-${scenarioIndex}/v2`, mimeType: "text/plain", changelog: "Operating plan sections updated for this reporting period.", createdAt: new Date("2026-02-20T10:00:00.000Z") }).returning(); + const previousStructures = await testDb.db.insert(documentStructure).values(sections.map(([title], index) => ({ documentId: BigInt(scenarioDocument!.id), versionId: BigInt(previousVersion!.id), ordering: index + 1, title, path: `${index + 1}` }))).returning(); + const currentStructures = await testDb.db.insert(documentStructure).values(sections.map(([title], index) => ({ documentId: BigInt(scenarioDocument!.id), versionId: BigInt(currentVersion!.id), ordering: index + 1, title, path: `${index + 1}` }))).returning(); + await testDb.db.insert(documentContextChunks).values(sections.flatMap(([, before, after], index) => [ + { documentId: BigInt(scenarioDocument!.id), versionId: BigInt(previousVersion!.id), structureId: BigInt(previousStructures[index]!.id), content: before, contentHash: `${scenarioIndex}${index}a`.padEnd(64, "a"), tokenCount: before.split(/\s+/).length, charCount: before.length, pageNumber: index + 1, lineStart: 1, lineEnd: 1, ...(index < 3 ? { embedding: vectorSql(0) } : {}) }, + { documentId: BigInt(scenarioDocument!.id), versionId: BigInt(currentVersion!.id), structureId: BigInt(currentStructures[index]!.id), content: after, contentHash: `${scenarioIndex}${index}b`.padEnd(64, "b"), tokenCount: after.split(/\s+/).length, charCount: after.length, pageNumber: index + 1, lineStart: 1, lineEnd: 1, ...(index < 3 ? { embedding: vectorSql(0) } : {}) }, + ])); + await testDb.db.update(document).set({ currentVersionId: BigInt(currentVersion!.id) }).where(eq(document.id, scenarioDocument!.id)); + } computedIds = { a: BigInt(a!.id), a1: a1!.id, a2: a2!.id, a3: a3!.id, b: BigInt(b!.id), b2: BigInt(b2!.id), bCurrentChunk: bCurrentChunk.id }; - collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-03T00:00:00.000Z"), { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(testDb.db) }, new StrictCurrentWorkspaceDocumentStore(testDb.db, { embedQuery: async () => vector(0) })); + collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-03T00:00:00.000Z"), { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(testDb.db) }, new StrictCurrentWorkspaceDocumentStore(testDb.db, { embedQuery: async () => vector(0) }), analyzer, (diagnostics) => { changeDiagnostics = diagnostics; }); } const dispatchService = createFounderWeeklyReviewDispatchService(testDb.db); @@ -119,8 +171,8 @@ try { const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); const collectionContext = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: created.dispatch.generationClaimId }; const collecting = await worker.claimEvidenceCollection(collectionContext); const input = { companyId: actor.companyId, reportingPeriod: fixture.reportingPeriod, workspaceTimezone: fixture.workspaceTimezone, founderContext: mode === "computed" ? "Assess whether onboarding reliability and retry monitoring are blocking enterprise expansion." : fixture.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: created.run.requestKey }; - const snapshot = await collector.collectFounderWeeklyReviewEvidence(input); const repeated = await collector.collectFounderWeeklyReviewEvidence(input); - if (JSON.stringify(snapshot.items) !== JSON.stringify(repeated.items) || JSON.stringify(snapshot.sourceWarnings) !== JSON.stringify(repeated.sourceWarnings)) throw new Error("Evidence collection was not deterministic."); + const snapshot = await collector.collectFounderWeeklyReviewEvidence(input); const repeated = analyzer ? snapshot : await collector.collectFounderWeeklyReviewEvidence(input); + if (!analyzer && (JSON.stringify(snapshot.items) !== JSON.stringify(repeated.items) || JSON.stringify(snapshot.sourceWarnings) !== JSON.stringify(repeated.sourceWarnings))) throw new Error("Evidence collection was not deterministic."); const beforeDigest = digest(snapshot); const attached = await worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot); const afterDigest = digest(attached.evidenceSnapshot); const checked = mode === "computed" ? assertComputedSnapshot(attached.evidenceSnapshot, computedIds!, texts) : { parsed: attached.evidenceSnapshot!, counts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, attached.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])) }; if (attached.status !== "queued" || beforeDigest !== afterDigest || !checked.counts.document_change || !checked.counts.customer_feedback || checked.counts.founder_context !== 1) throw new Error("Realistic collector assertions failed."); @@ -130,10 +182,76 @@ try { validateFounderWeeklyReviewV2Citations(generated.reviewPayload as never, generating.evidenceSnapshot); const saved = await worker.saveGeneratedDraft(generationContext, generated.reviewPayload, generated.modelMetadata); const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft" || !readBack.evidenceSnapshot || digest(readBack.evidenceSnapshot) !== beforeDigest) throw new Error("Validated draft read-back or snapshot immutability failed."); if (mode === "computed") assertComputedReport(readBack.reviewPayload, readBack.evidenceSnapshot); + const deterministicEvaluation = evaluateFounderWeeklyReview(readBack.evidenceSnapshot, readBack.reviewPayload as any); + let semanticEvaluation: { result: z.infer; metadata: Record } | null = null; + let evaluationCalls = 0; + if (process.env.FWR_FOUNDER_REVIEW_EVALUATION_ENABLED === "true") { + evaluationCalls++; + try { + const graded = await generateStructuredWithMetadata({ + capability: "smallExtraction", + forceProvider: "kimi", + maxOutputTokens: 2048, + system: "You are a strict Founder Weekly Review quality grader. Return one JSON object only. Do not repeat the review or evidence. Score each dimension from 0 to 1. Return at most 6 concise findings and a summary under 900 characters.", + prompt: buildFounderWeeklyReviewEvaluationPrompt(readBack.evidenceSnapshot, readBack.reviewPayload as any), + schema: SemanticEvaluationSchema, + schemaName: "founder_weekly_review_evaluation", + }); + semanticEvaluation = { result: SemanticEvaluationSchema.parse(graded.object), metadata: { provider: graded.metadata.provider, model: graded.metadata.model, promptVersion: FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION } }; + } catch (error) { + semanticEvaluation = null; + console.error(`[fwr-evaluation] semantic grader unavailable: ${error instanceof Error ? error.name : "unknown"}`); + } + } const rendered = renderFounderWeeklyReviewMarkdown(readBack); const dispatchRows = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, saved.id)); const runRows = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); let artifactPaths: ArtifactPaths | null = null; - if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const exportRoot = founderWeeklyReviewRealisticExportRoot(mode, process.env.SYNTHETIC_FWR_EXPORT_DIR); const directory = resolve(process.cwd(), exportRoot, saved.id); await mkdir(directory, { recursive: true }); artifactPaths = { evidence: resolve(directory, "evidence.json"), report: resolve(directory, "report.json"), markdown: resolve(directory, "report.md"), summary: resolve(directory, "run-summary.json") }; await writeAtomic(artifactPaths.evidence, JSON.stringify(readBack.evidenceSnapshot, null, 2)); await writeAtomic(artifactPaths.report, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, reportingPeriod: readBack.reportingPeriod, review: readBack.reviewPayload }, null, 2)); await writeAtomic(artifactPaths.markdown, rendered); await writeAtomic(artifactPaths.summary, JSON.stringify({ runId: saved.id, scenario: "realistic-company", mode, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), repairCount: generationCalls - 1, retryCount: saved.retryCount, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, snapshotDigestBefore: beforeDigest, snapshotDigestAfter: digest(readBack.evidenceSnapshot), artifactPaths }, null, 2)); } - if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); console.log("===== END FOUNDER WEEKLY REVIEW ====="); } - console.log(JSON.stringify({ runId: saved.id, mode, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, repairCount: generationCalls - 1, dispatchCount: dispatchRows.length, runRowCount: runRows.length, artifactPaths })); -} finally { await testDb.close(); } + if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const exportRoot = founderWeeklyReviewRealisticExportRoot(mode, process.env.SYNTHETIC_FWR_EXPORT_DIR); const directory = resolve(process.cwd(), exportRoot, saved.id); await mkdir(directory, { recursive: true }); artifactPaths = { evidence: resolve(directory, "evidence-snapshot.json"), report: resolve(directory, "review.json"), markdown: resolve(directory, "review.md"), summary: resolve(directory, "summary.json"), envelope: resolve(directory, "generation-envelope.json"), validation: resolve(directory, "validation.json"), e2eReport: resolve(directory, "e2e-report.md"), analyzerCalls: resolve(directory, "analyzer-calls.json"), evaluation: resolve(directory, "evaluation.json"), evaluationMarkdown: resolve(directory, "evaluation.md") }; const envelope = buildGenerationEvidenceEnvelope(readBack.evidenceSnapshot); const validation = { schema: true, citations: true, sourceSemantics: true }; const evaluation = { deterministic: deterministicEvaluation, semantic: semanticEvaluation }; await writeAtomic(artifactPaths.evidence, JSON.stringify(readBack.evidenceSnapshot, null, 2)); await writeAtomic(artifactPaths.report, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, reportingPeriod: readBack.reportingPeriod, review: readBack.reviewPayload }, null, 2)); await writeAtomic(artifactPaths.markdown, rendered); await writeAtomic(artifactPaths.envelope, JSON.stringify(envelope, null, 2)); await writeAtomic(artifactPaths.validation, JSON.stringify(validation, null, 2)); await writeAtomic(artifactPaths.analyzerCalls, JSON.stringify({ callCount: analyzerCalls.length, calls: analyzerCalls }, null, 2)); await writeAtomic(artifactPaths.evaluation, JSON.stringify(evaluation, null, 2)); await writeAtomic(artifactPaths.evaluationMarkdown, `# Founder Weekly Review Evaluation\n\nDeterministic: ${deterministicEvaluation.passed ? "PASS" : "FAIL"}\n\n${semanticEvaluation ? `Overall: ${semanticEvaluation.result.overallScore.toFixed(2)}\n\n${semanticEvaluation.result.summary}` : "Semantic grading was not enabled."}\n`); await writeAtomic(artifactPaths.e2eReport, `# Analyzer-enabled realistic E2E\n\n- Materiality calls: ${analyzerCalls.length}\n- Evaluation calls: ${evaluationCalls}\n- Materiality provider: ${analyzerCalls.find(call => call.provider)?.provider ?? "none"}\n- Materiality model: ${analyzerCalls.find(call => call.model)?.model ?? "none"}\n- Generation provider: ${generated.modelMetadata.provider}\n- Generation model: ${generated.modelMetadata.model}\n- Snapshot schema: ${readBack.evidenceSnapshot.schemaVersion}\n- Evidence digest: ${digest(readBack.evidenceSnapshot)}\n- Prompt hash: ${generated.modelMetadata.promptHash}\n`); await writeAtomic(artifactPaths.summary, JSON.stringify({ runId: saved.id, scenario: "realistic-company", mode, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), repairCount: generationCalls - 1, retryCount: saved.retryCount, validation, snapshotDigestBefore: beforeDigest, snapshotDigestAfter: digest(readBack.evidenceSnapshot), analyzerCallCount: analyzerCalls.length, reviewGenerationCallCount: generationCalls, evaluationCallCount: evaluationCalls, analyzerCalls, evaluation, artifactPaths }, null, 2)); } + if (demoMode) { + console.log = originalConsoleLog; + console.info = originalConsoleInfo; + const demoSnapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse(readBack.evidenceSnapshot) as any; + const envelope = buildGenerationEvidenceEnvelope(demoSnapshot); + const rawChanges = demoSnapshot.documentChangeAudit.rawChanges as unknown as Record[]; + const groups = demoSnapshot.documentChangeAudit.groups as unknown as Record[]; + const eligibleGroups = analyzerCalls.length + (readBack.evidenceSnapshot.sourceWarnings.some((warning) => warning.code === "materiality_analysis_budget_truncated") ? 1 : 0); + let scaleResults: Record[] = []; + try { + const scale = JSON.parse(await readFile(resolve(process.cwd(), ".artifacts/founder-weekly-review/scale-analysis/scale-analysis.json"), "utf8")) as { results?: Record[] }; + scaleResults = scale.results ?? []; + } catch { /* Scale artifacts are optional for the presentation. */ } + console.log(formatFounderWeeklyReviewDemo({ + rawChanges, + groups, + promptItems: demoSnapshot.items as unknown as Record[], + envelopeDiagnostics: envelope.diagnostics as unknown as Record, + analyzerCalls, + warnings: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), + reviewMarkdown: rendered, + provider: String(generated.modelMetadata.provider), + model: String(generated.modelMetadata.model), + promptVersion: String(generated.modelMetadata.promptVersion ?? "founder-weekly-review-generation/v2"), + outputCeiling: 2400, + repairCount: generationCalls - 1, + snapshotVersion: demoSnapshot.schemaVersion, + evidenceDigest: digest(demoSnapshot), + persistencePassed: saved.status === "draft", + readBackPassed: Boolean(readBack.reviewPayload), + noOpRemoved: changeDiagnostics?.deterministicNoOpCount ?? 0, + eligibleGroups, + budget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + scaleResults, + })); + if (semanticEvaluation) { + console.log("\nEVALUATION\n----------"); + console.log(`Deterministic checks ${deterministicEvaluation.passed ? "PASS" : "FAIL"}`); + for (const [dimension, score] of Object.entries(semanticEvaluation.result.dimensions)) console.log(`${dimension.padEnd(28)} ${score.toFixed(2)}`); + console.log(`${"Overall".padEnd(28)} ${semanticEvaluation.result.overallScore.toFixed(2)}`); + for (const finding of semanticEvaluation.result.findings) console.log(`- ${finding.explanation}`); + } else { + console.log("\nEVALUATION\n----------\nSemantic grading disabled; set FWR_FOUNDER_REVIEW_EVALUATION_ENABLED=true."); + } + console.log(`\nArtifacts:\n${artifactPaths ? Object.values(artifactPaths).join("\n") : "export disabled"}`); + } else if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); console.log("===== END FOUNDER WEEKLY REVIEW ====="); } + if (!demoMode) console.log(JSON.stringify({ runId: saved.id, mode, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, repairCount: generationCalls - 1, dispatchCount: dispatchRows.length, runRowCount: runRows.length, artifactPaths })); +} finally { console.log = originalConsoleLog; console.info = originalConsoleInfo; await testDb.close(); } diff --git a/apps/web/src/lib/llm/generate.ts b/apps/web/src/lib/llm/generate.ts index e4bd6eb11..0c07c529a 100644 --- a/apps/web/src/lib/llm/generate.ts +++ b/apps/web/src/lib/llm/generate.ts @@ -79,7 +79,7 @@ export async function generateStructuredWithMetadata( ...(resolved.temperature === undefined ? {} : { temperature: resolved.temperature }), ...(input.maxOutputTokens !== undefined ? { maxOutputTokens: input.maxOutputTokens } - : input.capability === "founderWeeklyReview" ? { maxOutputTokens: 1800 } : {}), + : input.capability === "founderWeeklyReview" ? { maxOutputTokens: 2400 } : {}), ...((input.timeoutMs !== undefined || resolved.structuredOutputMode === "json_object") ? { abortSignal: AbortSignal.timeout(input.timeoutMs ?? 90_000) } : {}), diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json new file mode 100644 index 000000000..1ea6b28e6 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json @@ -0,0 +1,18 @@ +{ + "name": "empty-evidence", + "description": "No in-window versions and no founder context produce an empty evidence snapshot.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Legacy Handbook", + "category": "Reference", + "versions": [{ "versionNumber": 1, "timestamp": "2026-05-01T10:00:00.000Z", "changelog": "Outside the reporting period.", "chunks": [] }] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "document_change": { "exact": 0 }, "workspace_document": { "exact": 0 }, "customer_feedback": { "exact": 0 }, "founder_context": { "exact": 0 } } } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json new file mode 100644 index 000000000..9577612ca --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json @@ -0,0 +1,12 @@ +{ + "name": "founder-context-only", + "description": "Founder Context is the only collected source.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Focus on onboarding reliability and the billing revamp decision.", + "companies": [{ "name": "Northstar Analytics", "underReview": true, "documents": [] }], + "expect": { + "evidence": { "sourceTypeCounts": { "founder_context": { "exact": 1 }, "document_change": { "exact": 0 }, "workspace_document": { "exact": 0 }, "customer_feedback": { "exact": 0 } } }, + "sourceSemantics": { "customerFeedbackOnly": true, "temporalEvidenceRequired": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json new file mode 100644 index 000000000..5178df203 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json @@ -0,0 +1,28 @@ +{ + "name": "relevant-workspace-document", + "description": "An unchanged current document is retrieved for Founder Context without becoming a temporal document change.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Focus on onboarding reliability and who owns retry telemetry.", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Onboarding Reliability Plan", + "category": "Planning", + "versions": [{ + "versionNumber": 1, + "timestamp": "2026-06-15T10:00:00.000Z", + "changelog": "Last updated before the reporting period.", + "chunks": [ + { "section": "Ownership", "content": "Platform owns retry telemetry and recovery monitoring for onboarding." }, + { "section": "Runbook", "content": "On failure, onboarding retries with backoff and pages the platform on-call." } + ] + }] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "workspace_document": { "min": 1 }, "document_change": { "exact": 0 } } }, + "sourceSemantics": { "currentWorkspaceOnly": true, "temporalEvidenceRequired": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json new file mode 100644 index 000000000..2ff8c83b3 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json @@ -0,0 +1,25 @@ +{ + "name": "first-ever-version-no-invented-diff", + "description": "A first-ever in-window version has no predecessor and must not create invented temporal change evidence.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Review onboarding ownership.", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Onboarding Plan", + "category": "Planning", + "versions": [{ + "versionNumber": 1, + "timestamp": "2026-07-24T10:00:00.000Z", + "changelog": "Documented retry telemetry ownership.", + "chunks": [{ "section": "Ownership", "content": "Platform owns retry telemetry and recovery monitoring." }] + }] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "document_change": { "exact": 0 } }, "warningCodes": ["document_change_baseline_missing"] }, + "documentChanges": { "requireNoInventedBaseline": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json new file mode 100644 index 000000000..6a4136a13 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json @@ -0,0 +1,26 @@ +{ + "name": "multiple-versions-in-period", + "description": "A pre-window baseline and three adjacent in-window versions exercise pair chaining, alignment, grouping, and audit provenance.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Track how ownership of retry telemetry changed this week.", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Onboarding Reliability Plan", + "category": "Planning", + "versions": [ + { "versionNumber": 1, "timestamp": "2026-07-10T10:00:00.000Z", "changelog": "Baseline before the reporting window.", "chunks": [{ "section": "Ownership", "content": "Product owns retry telemetry." }] }, + { "versionNumber": 2, "timestamp": "2026-07-21T10:00:00.000Z", "changelog": "Moved retry telemetry ownership to Platform.", "chunks": [{ "section": "Ownership", "content": "Platform owns retry telemetry." }] }, + { "versionNumber": 3, "timestamp": "2026-07-23T10:00:00.000Z", "changelog": "Added recovery monitoring to Platform ownership.", "chunks": [{ "section": "Ownership", "content": "Platform owns retry telemetry and recovery monitoring." }] }, + { "versionNumber": 4, "timestamp": "2026-07-25T10:00:00.000Z", "changelog": "Clarified on-call paging.", "chunks": [{ "section": "Runbook", "content": "On failure, page the Platform on-call." }] } + ] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "document_change": { "min": 1 } } }, + "documentChanges": { "minGroups": 1, "maxGroups": 8 }, + "sourceSemantics": { "temporalEvidenceRequired": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts b/apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts new file mode 100644 index 000000000..18ff1f249 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts @@ -0,0 +1,118 @@ +import { z } from "zod"; + +import { + DOCUMENT_CHANGE_CATEGORIES, + ReportingPeriodSchema, + type FounderWeeklyReviewEvidenceItem, +} from "@launchstack/features/founder-weekly-review"; + +export const FOUNDER_WEEKLY_REVIEW_SCENARIO_SCHEMA_VERSION = "founder-weekly-review-scenario/v1" as const; + +const ScenarioSourceTypeSchema = z.enum([ + "workspace_document", + "document_change", + "customer_feedback", + "github_activity", + "manual_note", + "founder_context", + "other", +]); +export type ScenarioSourceType = z.infer; + +const ScenarioSectionNameSchema = z.enum([ + "whatChanged", + "whatShipped", + "whatCustomersSaid", + "currentBlockers", + "nextPriorities", +]); +export type ScenarioSectionName = z.infer; + +const CountExpectationSchema = z.object({ + min: z.number().int().nonnegative().optional(), + max: z.number().int().nonnegative().optional(), + exact: z.number().int().nonnegative().optional(), +}).strict().superRefine((value, ctx) => { + if (value.min !== undefined && value.max !== undefined && value.min > value.max) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "min must not exceed max" }); + if (value.exact !== undefined && ((value.min !== undefined && value.exact < value.min) || (value.max !== undefined && value.exact > value.max))) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "exact must satisfy min/max" }); +}); + +export const ScenarioExpectSchema = z.object({ + evidence: z.object({ + sourceTypeCounts: z.record(ScenarioSourceTypeSchema, CountExpectationSchema).optional(), + warningCodes: z.array(z.string().min(1).max(128)).max(32).optional(), + }).strict().optional(), + documentChanges: z.object({ + minGroups: z.number().int().nonnegative().optional(), + maxGroups: z.number().int().nonnegative().optional(), + requiredCategories: z.array(z.enum(DOCUMENT_CHANGE_CATEGORIES)).max(16).optional(), + requireNoInventedBaseline: z.boolean().optional(), + }).strict().optional(), + sourceSemantics: z.object({ + customerFeedbackOnly: z.boolean().optional(), + temporalEvidenceRequired: z.boolean().optional(), + noCrossCompanyLeakage: z.boolean().optional(), + currentWorkspaceOnly: z.boolean().optional(), + }).strict().optional(), + review: z.object({ + sectionStates: z.record(ScenarioSectionNameSchema, z.enum(["evidence", "no_evidence"])).optional(), + requiredThemes: z.array(z.string().min(1).max(256)).max(16).optional(), + forbiddenThemes: z.array(z.string().min(1).max(256)).max(16).optional(), + }).strict().optional(), +}).strict(); +export type FounderWeeklyReviewScenarioExpect = z.infer; + +export const ScenarioChunkSchema = z.object({ + section: z.string().min(1).max(256), + content: z.string().min(1).max(2000), + pageNumber: z.number().int().positive().optional(), + lineStart: z.number().int().positive().optional(), + lineEnd: z.number().int().positive().optional(), +}).strict(); +export type FounderWeeklyReviewScenarioChunk = z.infer; + +export const ScenarioVersionSchema = z.object({ + versionNumber: z.number().int().positive(), + timestamp: z.string().datetime({ offset: true }), + changelog: z.string().max(1000).optional(), + uploadedBy: z.string().min(1).max(256).optional(), + chunks: z.array(ScenarioChunkSchema).max(200).default([]), +}).strict(); +export type FounderWeeklyReviewScenarioVersion = z.infer; + +export const ScenarioDocumentSchema = z.object({ + title: z.string().min(1).max(512), + category: z.string().min(1).max(256), + versions: z.array(ScenarioVersionSchema).max(50), +}).strict(); +export type FounderWeeklyReviewScenarioDocument = z.infer; + +export const ScenarioCompanySchema = z.object({ + name: z.string().min(1).max(256), + underReview: z.boolean().default(false), + documents: z.array(ScenarioDocumentSchema).max(200).default([]), +}).strict(); +export type FounderWeeklyReviewScenarioCompany = z.infer; + +export const FounderWeeklyReviewScenarioSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_SCENARIO_SCHEMA_VERSION).default(FOUNDER_WEEKLY_REVIEW_SCENARIO_SCHEMA_VERSION), + name: z.string().min(1).max(128), + description: z.string().max(1000).optional(), + reportingPeriod: ReportingPeriodSchema, + workspaceTimezone: z.string().min(1).max(128), + founderContext: z.string().min(1).max(1000).optional(), + companies: z.array(ScenarioCompanySchema).min(1).max(10), + expect: ScenarioExpectSchema.optional(), +}).strict().superRefine((scenario, ctx) => { + const underReview = scenario.companies.filter((company) => company.underReview); + if (underReview.length !== 1) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["companies"], message: "Exactly one company must have underReview: true" }); + for (const [companyIndex, company] of scenario.companies.entries()) { + for (const [documentIndex, doc] of company.documents.entries()) { + const versionNumbers = doc.versions.map((version) => version.versionNumber); + if (new Set(versionNumbers).size !== versionNumbers.length) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["companies", companyIndex, "documents", documentIndex, "versions"], message: "versionNumber must be unique within a document" }); + } + } +}); +export type FounderWeeklyReviewScenario = z.infer; + +export type ScenarioEvidenceSourceType = FounderWeeklyReviewEvidenceItem["sourceType"]; diff --git a/packages/features/src/founder-weekly-review/evaluation-prompt.ts b/packages/features/src/founder-weekly-review/evaluation-prompt.ts new file mode 100644 index 000000000..442026903 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evaluation-prompt.ts @@ -0,0 +1,18 @@ +import type { FounderWeeklyReviewEvidenceSnapshot, FounderWeeklyReviewV2Payload } from "./contracts"; + +export const FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION = "founder-weekly-review-evaluation/v2" as const; +export function buildFounderWeeklyReviewEvaluationPrompt(snapshot: FounderWeeklyReviewEvidenceSnapshot, review: FounderWeeklyReviewV2Payload): string { + return JSON.stringify({ + promptVersion: FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION, + task: "Grade the persisted Founder Weekly Review against its immutable evidence snapshot. Return judgments, not a restatement of the review.", + rubric: { + groundedness: "Claims must be supported by cited snapshot evidence. Treat supports, indicates, and suggests as appropriately bounded language; penalize confirms/proves when the evidence is only a limited signal.", + materiality: "Prioritize meaningful business-state changes over editorial rewrites. Do not reward one-bullet-per-source enumeration.", + temporalAccuracy: "document_change is reporting-period temporal evidence. workspace_document is current context and cannot independently prove whatChanged or whatShipped. document_change plus workspace_document is valid when the document change establishes the temporal proposition and workspace only adds context. Explicitly distinguish planned, scheduled, or future releases from actually shipped during the reporting period; a future-dated release is not shipped in the current period.", + synthesisQuality: "Judge coverage of major material business themes, not source-level citation coverage. Reward concise founder-level synthesis; several related sources may be synthesized into one founder-level claim. Reusing evidence across sections is acceptable when the section-specific meaning changes; penalize only repetition that adds no new meaning.", + actionability: "Reward a clear action plus an evidence-grounded reason or dependency. Do not require or reward invented owners, dates, metrics, or sequencing that are absent from the evidence.", + }, + snapshot, + review, + }); +} diff --git a/packages/features/src/founder-weekly-review/evaluation.ts b/packages/features/src/founder-weekly-review/evaluation.ts new file mode 100644 index 000000000..9a5fa68c0 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evaluation.ts @@ -0,0 +1,94 @@ +import { FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewV2PayloadSchema, type FounderWeeklyReviewEvidenceSnapshot, type FounderWeeklyReviewV2Payload } from "./contracts"; + +export type EvaluationFailure = { category: string; section?: string; claim?: string; explanation: string }; +export type DeterministicEvaluation = { + canonicalSchemaValid: boolean; + citationValidity: number; + citationCoverage: number; + sourceSemanticViolationRate: number; + unsupportedClaimRate: number; + unsupportedShippedClaimRate: number; + duplicateClaimRate: number; + evidenceCoverage: number; + emptySectionCorrectness: number; +}; +export type EvaluationResult = { + passed: boolean; + hasHardFailure: boolean; + deterministic: DeterministicEvaluation; + failures: EvaluationFailure[]; +}; + +const normalize = (value: string) => value.toLowerCase().replace(/[^a-z0-9%$]+/g, " ").replace(/\s+/g, " ").trim(); +const words = (value: string) => new Set(normalize(value).split(" ").filter(word => word.length > 3)); +const overlap = (claim: string, evidence: string) => { + const claimWords = words(claim); const evidenceWords = words(evidence); + return [...claimWords].filter(word => evidenceWords.has(word)).length >= 2; +}; +const items = (section: unknown): Array> => { + if (!section || typeof section !== "object" || !("items" in section) || !Array.isArray(section.items)) return []; + return section.items.filter((item): item is Record => Boolean(item && typeof item === "object")); +}; +const isTemporal = (sourceTypes: string[]) => sourceTypes.includes("document_change"); + +const monthNumbers: Record = { january: 1, february: 2, march: 3, april: 4, may: 5, june: 6, july: 7, august: 8, september: 9, october: 10, november: 11, december: 12 }; +function futureReleaseMentioned(text: string, periodEnd: string): boolean { + const lower = text.toLowerCase(); + if (!/(planned|scheduled|expected|launch|ship|release|deploy)/i.test(lower)) return false; + const end = new Date(`${periodEnd}T23:59:59.999Z`); + const year = end.getUTCFullYear(); + for (const [month, number] of Object.entries(monthNumbers)) { + if (!new RegExp(`\\b${month}\\b`, "i").test(lower)) continue; + const candidate = new Date(Date.UTC(year, number - 1, 1)); + if (candidate > end) return true; + } + const yearMention = lower.match(/\b(20\d{2})\b/); + return yearMention ? Number(yearMention[1]) > year : false; +} + +export function evaluateFounderWeeklyReview(evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, report: FounderWeeklyReviewV2Payload): EvaluationResult { + const failures: EvaluationFailure[] = []; + const evidence = new Map(evidenceSnapshot.items.map(item => [item.sourceId, item])); + let canonicalSchemaValid = true; + try { FounderWeeklyReviewEvidenceSnapshotSchema.parse(evidenceSnapshot); FounderWeeklyReviewV2PayloadSchema.parse(report); } catch { canonicalSchemaValid = false; failures.push({ category: "malformed_payload", explanation: "Snapshot or review does not satisfy its canonical schema." }); } + if (!canonicalSchemaValid) return { passed: false, hasHardFailure: true, deterministic: { canonicalSchemaValid, citationValidity: 0, citationCoverage: 0, sourceSemanticViolationRate: 1, unsupportedClaimRate: 0, unsupportedShippedClaimRate: 0, duplicateClaimRate: 0, evidenceCoverage: 0, emptySectionCorrectness: 0 }, failures }; + let citations = 0, validCitations = 0, claimCount = 0, citedClaims = 0, semanticChecks = 0, semanticViolations = 0, unsupported = 0, supportChecks = 0, shippedChecks = 0, unsupportedShipped = 0; + const claims = new Set(); let duplicateClaims = 0; + const covered = new Set(); + for (const [sectionName, section] of Object.entries(report.sections)) { + const sectionItems = items(section); + for (const item of sectionItems) { + const claim = typeof item.text === "string" ? item.text : ""; + const normalizedClaim = normalize(claim); + if (normalizedClaim) { claimCount++; const sectionClaim = `${sectionName}:${normalizedClaim}`; if (claims.has(sectionClaim)) { duplicateClaims++; failures.push({ category: "duplicate_claim", section: sectionName, claim, explanation: "Repeated claim within the same section." }); } claims.add(sectionClaim); } + const ids = Array.isArray(item.sourceIds) ? item.sourceIds.filter((id): id is string => typeof id === "string") : []; + if (["observed_fact", "contradictory_evidence", "recommendation"].includes(String(item.kind))) { if (ids.length) citedClaims++; } + if (claim && ids.length) { + const cited = ids.map(id => evidence.get(id)).filter(Boolean); + const combined = cited.map(source => `${source!.title} ${source!.excerpt}`).join(" "); + supportChecks++; if (overlap(claim, combined)) cited.forEach(source => covered.add(source!.sourceId)); else if (item.kind !== "recommendation" && item.kind !== "contradictory_evidence") { unsupported++; failures.push({ category: "unsupported_claim", section: sectionName, claim, explanation: "Soft lexical diagnostic: claim has limited overlap with cited evidence." }); } + } + for (const id of ids) { + citations++; const source = evidence.get(id); + if (!source) { failures.push({ category: "invalid_citation", section: sectionName, claim, explanation: `Unknown sourceId: ${id}` }); continue; } + validCitations++; + const sourceType = source.sourceType; + semanticChecks++; + const allowed = sectionName === "whatCustomersSaid" ? sourceType === "customer_feedback" : sectionName === "whatChanged" || sectionName === "whatShipped" ? (sourceType === "document_change" || (sourceType === "workspace_document" && ids.some(other => evidence.get(other)?.sourceType === "document_change"))) : true; + if (!allowed) { semanticViolations++; failures.push({ category: "invalid_source_type", section: sectionName, claim, explanation: `${sourceType} cannot independently support ${sectionName}.` }); } + if (sectionName === "whatShipped") { + shippedChecks++; + const sourceText = `${source.title} ${source.excerpt}`; + if (!/launch|ship|release|deploy|complete|implement/i.test(sourceText)) { unsupportedShipped++; failures.push({ category: "unsupported_shipped_claim", section: sectionName, claim, explanation: "Cited source has no explicit shipping signal." }); } + if (sourceType === "document_change" && futureReleaseMentioned(`${claim} ${sourceText}`, evidenceSnapshot.reportingPeriod.end)) { + failures.push({ category: "future_release_claim", section: sectionName, claim, explanation: "A planned or launched release is dated after the reporting period and cannot establish shipped-this-period." }); + } + } + } + } + if (section && typeof section === "object" && "state" in section && section.state === "no_evidence" && "items" in section) failures.push({ category: "invalid_empty_section", section: sectionName, explanation: "No-evidence sections must not include items." }); + } + const hard = failures.some(failure => ["malformed_payload", "invalid_citation", "invalid_source_type", "unsupported_shipped_claim", "future_release_claim", "invalid_empty_section"].includes(failure.category)); + const qualityEvidence = evidenceSnapshot.items.filter(item => item.sourceType === "document_change" || item.sourceType === "customer_feedback"); + return { passed: failures.length === 0, hasHardFailure: hard, deterministic: { canonicalSchemaValid, citationValidity: citations ? validCitations / citations : 1, citationCoverage: claimCount ? citedClaims / claimCount : 1, sourceSemanticViolationRate: semanticChecks ? semanticViolations / semanticChecks : 0, unsupportedClaimRate: supportChecks ? unsupported / supportChecks : 0, unsupportedShippedClaimRate: shippedChecks ? unsupportedShipped / shippedChecks : 0, duplicateClaimRate: claimCount ? duplicateClaims / claimCount : 0, evidenceCoverage: qualityEvidence.length ? qualityEvidence.filter(item => covered.has(item.sourceId)).length / qualityEvidence.length : 1, emptySectionCorrectness: failures.some(failure => failure.category === "invalid_empty_section") ? 0 : 1 }, failures }; +} diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts index a6f3cd0c7..bc4b7f55d 100644 --- a/packages/features/src/founder-weekly-review/evidence-service.ts +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -26,6 +26,7 @@ import { materializeDocumentChangesWithAnalyzer, type DocumentChangeMaterialityAnalyzer, } from "./document-change-materiality-analyzer"; +import type { DeterministicMaterialChangeDiagnostics } from "./document-change-materiality"; import { buildWorkspaceDocumentEvidence, normalizeFounderContextRetrievalQuery, @@ -173,6 +174,7 @@ export class FounderWeeklyReviewEvidenceService { private readonly documentChangeSource: FounderWeeklyReviewDocumentChangeSource = { kind: "unconfigured" }, private readonly workspaceDocumentStore?: FounderWeeklyReviewWorkspaceDocumentStore, private readonly documentChangeMaterialityAnalyzer?: DocumentChangeMaterialityAnalyzer, + private readonly onDocumentChangeDiagnostics?: (diagnostics: DeterministicMaterialChangeDiagnostics) => void, ) {} async collectDocumentChangeEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { @@ -204,6 +206,7 @@ export class FounderWeeklyReviewEvidenceService { pairInputs, this.documentChangeMaterialityAnalyzer, ); + this.onDocumentChangeDiagnostics?.(materialized.diagnostics); const items = [...materialized.items]; warnings.push(...materialized.warnings.map((item) => warning(item.code, item.message, "document_change"))); const pairedCurrentVersionIds = new Set(pairs.map((pair) => pair.currentVersionId)); diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts index 210d3a497..9d6b06c03 100644 --- a/packages/features/src/founder-weekly-review/index.ts +++ b/packages/features/src/founder-weekly-review/index.ts @@ -8,6 +8,8 @@ export * from "./worker-service"; export * from "./generator"; export * from "./generation-validation"; export * from "./generation-evidence-envelope"; +export * from "./evaluation"; +export * from "./evaluation-prompt"; export * from "./prompts"; export * from "./document-change"; export * from "./document-change-materiality"; diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts index 4ddb7112d..f53624fb4 100644 --- a/packages/features/src/founder-weekly-review/prompts.ts +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -7,13 +7,15 @@ import { } from "./generation-evidence-envelope"; export const FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION = - "founder-weekly-review-generation/v1" as const; + "founder-weekly-review-generation/v2" as const; export const FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT = `You generate a structured Founder Weekly Review from supplied evidence only. Never invent, assume, infer, or embellish customers, dates, metrics, people, decisions, shipped work, blockers, outcomes, or source IDs. Every factual item must cite one or more supplied source IDs exactly as given. Do not create or modify source IDs. Confidence is how strongly the generated claim is supported by its cited supplied evidence; it is not a score for source reliability or truthfulness. Omit unsupported claims or use no_evidence rather than assigning them a low confidence. -Write a concise, natural, professional review for a founder. Prefer a few substantive items over many one-sentence paraphrases. When related evidence supports it, synthesize the relationship into one focused item: describe the development or signal, why it matters, what the evidence does and does not establish, and an evidence-backed next action where the section permits it. Aim for roughly 2–4 sentences per substantive item when the supplied evidence supports that depth, and for approximately 600–1,000 words overall when the evidence supports it. Do not add filler to reach a length target. +Write a concise, decision-oriented founder review, not an evidence transcript. Prioritize the few most material founder-level conclusions. Synthesize related evidence into one focused claim where possible; do not create one output item per evidence source, and do not repeat the same fact across sections unless the section semantics genuinely require it. Prefer concise factual statements over explanatory prose. + +Use at most 3 items in each section: whatChanged, whatShipped, whatCustomersSaid, currentBlockers, and nextPriorities. Use fewer items when fewer material conclusions are justified; never add filler to reach a limit. For observed facts, use one concise sentence whenever possible and do not restate the entire evidence excerpt. Keep the distinctions below explicit. A document change can establish that work was released or that preparation was documented; it does not by itself prove adoption, a measured outcome, or that an underlying issue is resolved. Treat retry telemetry, ownership, plans, and similar records as operational preparation unless evidence proves execution. Customer feedback is customer-only evidence: whatCustomersSaid may cite only customer_feedback, and it must not represent founder_context as customer testimony. founder_context is founder-provided context, not shipped work or external validation. Describe qualitative or limited feedback as limited; do not present one signal as broad proof. @@ -21,7 +23,7 @@ Use whatShipped only for work that the evidence establishes as released during t Use currentBlockers for evidence-backed execution blockers and for material product, customer, or operational risks. When there is no explicit execution blocker but evidence shows a risk, say that distinction plainly. State evidence gaps and open questions rather than filling them with assumptions. Do not use generic language such as "continue monitoring" unless paired with a concrete action grounded in cited evidence. -For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must be evidence-backed and explain its rationale in the optional rationale field when useful. Avoid repetitive wording across sections. +For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must be evidence-backed. The text should be one concise recommended action. The optional rationale should be brief (preferably one short sentence), should add decision context without restating the recommendation, and should not reproduce source evidence. Omit rationale when it adds no useful context. Avoid repetitive wording across sections. When evidence conflicts, return contradictory_evidence with the conflicting source IDs. Do not choose a winner or reconcile it unless supplied evidence explicitly resolves the conflict. From 4c012df72806c73107d5094460da84fab178205b Mon Sep 17 00:00:00 2001 From: Peace Odetola Date: Sat, 8 Aug 2026 18:58:27 -0500 Subject: [PATCH 29/29] Add realistic Founder Weekly Review E2E test --- .../lifecycle.integration.test.ts | 1 + ...run-founder-weekly-review-realistic-e2e.ts | 14 ++++---- package.json | 3 +- .../founder-weekly-review/benchmarks/cases.ts | 8 +++++ .../src/founder-weekly-review/evaluation.ts | 34 ++++++++++++++++++- .../src/founder-weekly-review/prompts.ts | 4 +-- .../founder-weekly-review/reporting-period.ts | 18 ++++++---- 7 files changed, 63 insertions(+), 19 deletions(-) diff --git a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts index 43dcf3c03..7adbeca56 100644 --- a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts +++ b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts @@ -146,6 +146,7 @@ function createV2Payload(): FounderWeeklyReviewV2Payload { text: "Prioritize SSO setup.", sourceIds: ["founder-context-1"], confidence: 0.8, + rationale: null, }], }, }, diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts index 0402e0ee1..e951e449d 100644 --- a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -14,7 +14,6 @@ import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-r import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; import { createConfiguredDocumentChangeMaterialityAnalyzer } from "~/server/founder-weekly-review/document-change-materiality-analyzer"; import { generateStructuredWithMetadata } from "~/lib/llm"; -import { renderFounderWeeklyReviewEvaluationMarkdown } from "~/server/founder-weekly-review/evaluation-markdown"; import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; import { formatFounderWeeklyReviewDemo } from "./founder-weekly-review-demo"; import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; @@ -23,9 +22,8 @@ const require = createRequire(import.meta.url); const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/realistic-company/seed.json"); type Fixture = { reportingPeriod: { start: string; end: string }; workspaceTimezone: string; founderContext: string; documents: Array<{ title: string; category: string; changelog: string; timestamp: string; chunks?: string[] }> }; -type EvidenceMode = ReturnType; type ComputedTexts = { before: string; after: string; v3: string; bHistorical: string; nullVersion: string; foreign: string; unrelated: string }; -type ArtifactPaths = { evidence: string; report: string; markdown: string; evaluation: string; evaluationMarkdown: string; summary: string; envelope: string; validation: string; e2eReport: string; analyzerCalls: string; evaluation: string; evaluationMarkdown: string }; +type ArtifactPaths = { evidence: string; report: string; markdown: string; evaluation: string; evaluationMarkdown: string; summary: string; envelope: string; validation: string; e2eReport: string; analyzerCalls: string }; const SemanticEvaluationSchema = z.object({ overallScore: z.number().min(0).max(1), @@ -72,7 +70,7 @@ function assertComputedReport(payload: any, snapshot: ReturnType[1], generateFounderWeeklyReviewStructured @@ -192,9 +190,9 @@ try { console.log(JSON.stringify({ evaluation: { - deterministicScore: evaluation.deterministic?.overallScore, - failures: evaluation.failures, - llmScore: evaluation.llmGrader?.overallScore, + deterministicScore: benchmarkEvaluation.deterministic?.overallScore, + failures: benchmarkEvaluation.failures, + llmScore: benchmarkEvaluation.llmGrader?.overallScore, } })); diff --git a/package.json b/package.json index 52c46903e..903e37424 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "changeset": "changeset", "version": "changeset version", "release": "pnpm --filter @launchstack/core build && changeset publish", - "eval:founder-weekly-review": "tsx packages/features/src/founder-weekly-review/benchmarks/runner.ts" + "test:fwr:realistic": "cross-env SYNTHETIC_FWR_LOCAL=1 pnpm --filter @launchstack/web exec tsx scripts/run-founder-weekly-review-realistic-e2e.ts" }, "devDependencies": { "@changesets/cli": "^2.31.0", @@ -38,6 +38,7 @@ "@types/node": "^24.3.1", "@typescript-eslint/eslint-plugin": "^8.42.0", "@typescript-eslint/parser": "^8.42.0", + "cross-env": "^10.1.0", "eslint": "^9.34.0", "prettier": "^3.6.2", "tsx": "^4.21.0", diff --git a/packages/features/src/founder-weekly-review/benchmarks/cases.ts b/packages/features/src/founder-weekly-review/benchmarks/cases.ts index ba76de19c..17f1beec0 100644 --- a/packages/features/src/founder-weekly-review/benchmarks/cases.ts +++ b/packages/features/src/founder-weekly-review/benchmarks/cases.ts @@ -3,6 +3,7 @@ import { FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, FounderWeeklyReviewV2Payload, type FounderWeeklyReviewEvidenceSnapshot, + type DocumentChangeAuditSnapshot, } from "../contracts"; export type BenchmarkCase = { @@ -21,6 +22,12 @@ export type BenchmarkCase = { }; }; +const emptyDocumentChangeAudit: DocumentChangeAuditSnapshot = { + schemaVersion: "document-change-audit/v1", + rawChanges: [], + groups: [], +}; + const noEvidenceSection = { state: "no_evidence" as const, noEvidence: { @@ -48,6 +55,7 @@ const validEvidence = { }, ], sourceWarnings: [], + documentChangeAudit: emptyDocumentChangeAudit, } satisfies FounderWeeklyReviewEvidenceSnapshot; const emptyEvidence = { diff --git a/packages/features/src/founder-weekly-review/evaluation.ts b/packages/features/src/founder-weekly-review/evaluation.ts index 940bcc356..1f4d62611 100644 --- a/packages/features/src/founder-weekly-review/evaluation.ts +++ b/packages/features/src/founder-weekly-review/evaluation.ts @@ -232,6 +232,20 @@ function evidenceConflicts( return supporting && opposing; } +function isTemporalChangeClaim(text: string): boolean { + const lower = text.toLowerCase(); + + const changeLanguage = + /\b(changed|updated|modified|added|removed|introduced)\b/.test(lower); + + const temporalLanguage = + /\b(this week|this period|during the reporting period|during this period|recently)\b/.test( + lower + ); + + return changeLanguage && temporalLanguage; +} + export function evaluateFounderWeeklyReview( evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, report: FounderWeeklyReviewPayload, @@ -367,7 +381,25 @@ export function evaluateFounderWeeklyReview( if (evidence) { totalSourceTypeChecks++; - if(!isValidSourceForSection(sectionName, evidence.sourceType)) { + if ( + sectionName === "whatChanged" && + typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" && + isTemporalChangeClaim(item.text) && + evidence.sourceType === "workspace_document" + ) { + sourceTypeViolations++; + + failures.push({ + category: "invalid_source_type", + section: sectionName, + claim: item.text, + explanation: + "A workspace document can establish current document context, but cannot by itself establish that a change occurred during the reporting period.", + }); + } else if (!isValidSourceForSection(sectionName, evidence.sourceType)) { sourceTypeViolations++; failures.push({ diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts index 52c4cf98c..390a28f04 100644 --- a/packages/features/src/founder-weekly-review/prompts.ts +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -21,8 +21,6 @@ Use at most 3 items in each section: whatChanged, whatShipped, whatCustomersSaid Prefer a few substantive items over many one-sentence paraphrases. When multiple evidence items describe the same customer problem, request, or reaction, synthesize them into a shared theme rather than summarizing each source independently. Explicitly identify the recurring pattern, indicate how many or which sources support it when useful, explain why the pattern matters, and preserve the limits of the evidence. Do not generalize beyond the supplied customer evidence. -Aim for roughly 2–4 sentences per substantive item when the supplied evidence supports that depth. Do not add filler to reach a length target. Avoid repeating the same insight across multiple sections unless each section serves a distinct purpose. - Keep the distinctions below explicit. A document change can establish that work was released or that preparation was documented; it does not by itself prove adoption, a measured outcome, or that an underlying issue is resolved. Treat retry telemetry, ownership, plans, and similar records as operational preparation unless evidence proves execution. Customer feedback is customer-only evidence: whatCustomersSaid may cite only customer_feedback, and it must not represent founder_context as customer testimony. Customer feedback describes customer requests, reactions, complaints, or opinions; it does not establish that implementation occurred. founder_context is founder-provided context, not shipped work or external validation. Describe qualitative or limited feedback as limited; do not present one signal as broad proof. @@ -33,7 +31,7 @@ Use whatChanged for meaningful developments supported by the evidence, including Use currentBlockers for evidence-backed execution blockers and for material product, customer, or operational risks. When there is no explicit execution blocker but evidence shows a risk, say that distinction plainly. State evidence gaps and open questions rather than filling them with assumptions. Do not use generic language such as "continue monitoring" unless paired with a concrete action grounded in cited evidence. -For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must identify a concrete action, explain why that action is warranted by the cited evidence, and be specific enough that a founder could act on it without guessing what "improve" or "address" means. When appropriate, include a concrete validation step, decision, owner-facing follow-up, or outcome to pursue. +For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must be evidence-backed. The text should be one concise recommended action. The optional rationale should be brief (preferably one short sentence), should add decision context without restating the recommendation, and should not reproduce source evidence. Omit rationale when it adds no useful context. Avoid repetitive wording across sections. Prioritize recommendations by likely impact or urgency when the evidence supports that distinction. Avoid generic recommendations such as "monitor," "improve," or "continue working on" unless they name the specific thing to monitor, improve, or continue and the action is grounded in evidence. Avoid repetitive wording across sections. diff --git a/packages/features/src/founder-weekly-review/reporting-period.ts b/packages/features/src/founder-weekly-review/reporting-period.ts index 61ae9ab11..e26d113a9 100644 --- a/packages/features/src/founder-weekly-review/reporting-period.ts +++ b/packages/features/src/founder-weekly-review/reporting-period.ts @@ -20,13 +20,19 @@ export function resolveReportingPeriodBounds( ): ReportingPeriodBounds { assertValidTimeZone(workspaceTimezone); + const startInclusive = dayjs.tz( + `${period.start}T00:00:00`, + workspaceTimezone + ); + + const endExclusive = dayjs.tz( + `${period.end}T00:00:00`, + workspaceTimezone + ).add(1, "day"); + return { - startInclusive: dayjs.tz(period.start, workspaceTimezone).startOf("day").toDate(), - endExclusive: dayjs - .tz(period.end, workspaceTimezone) - .add(1, "day") - .startOf("day") - .toDate(), + startInclusive: startInclusive.toDate(), + endExclusive: endExclusive.toDate(), }; }