diff --git a/.changeset/README.md b/.changeset/README.md new file mode 100644 index 000000000..654c6d475 --- /dev/null +++ b/.changeset/README.md @@ -0,0 +1,8 @@ +# Changesets + +Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works +with multi-package repos, or single-package repos to help you version and publish your code. You can +find the full documentation for it [in our repository](https://github.com/changesets/changesets). + +We have a quick list of common questions to get you started engaging with this project in +[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md). diff --git a/.changeset/config.json b/.changeset/config.json new file mode 100644 index 000000000..bd36e8be2 --- /dev/null +++ b/.changeset/config.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://unpkg.com/@changesets/config@3.1.4/schema.json", + "changelog": "@changesets/cli/changelog", + "commit": false, + "fixed": [], + "linked": [], + "access": "public", + "baseBranch": "main", + "updateInternalDependencies": "patch", + "ignore": ["@launchstack/features", "@launchstack/web"] +} diff --git a/.changeset/fix-company-embedding-schema-drift.md b/.changeset/fix-company-embedding-schema-drift.md new file mode 100644 index 000000000..1068ac71b --- /dev/null +++ b/.changeset/fix-company-embedding-schema-drift.md @@ -0,0 +1,5 @@ +--- +"@launchstack/core": patch +--- + +Align the company Drizzle schema with migration 0011 by removing dropped plaintext embedding credential columns and legacy credential read fallbacks. diff --git a/.gitignore b/.gitignore index d6a7f3a9a..e03ea0c04 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ packages/*/dist/ .kiro backups/ /models/ +/apps/web/.artifacts/ .claude diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 000000000..5e1066bea --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,260 @@ +# Founder Weekly Review (LAU-9) Handoff + +## Current state + +- Branch: `lau-9-founder-review-generation-flow` +- Do not commit or push without explicit approval. +- Preserve all existing uncommitted work and the four untracked local Ollama probe files. +- Do not use shared Neon for this work. Use the local PostgreSQL URL only through the isolated-schema helper. +- Do not change embeddings, retrieval, frontend/LAU-10, PDF ingestion, Peace's evaluator, or provider defaults unless a later task explicitly authorizes it. + +## What has been completed locally + +### Async Inngest-owned workflow + +The prior synchronous POST collection flow has been refactored locally toward: + +```text +POST -> queued run without snapshot + transactional outbox +Inngest workflow -> collecting -> attach immutable snapshot -> queued + -> generating -> draft | failed +``` + +Key implementation files: + +- `apps/web/src/app/api/founder-weekly-reviews/route.ts` + - POST authorizes and validates, then persists durable collection input instead of collecting evidence synchronously. +- `packages/features/src/founder-weekly-review/contracts.ts` + - Adds `collecting` status and bounded `FounderWeeklyReviewCollectionInputSchema`: + - `workspaceTimezone` + - optional bounded `founderContext` + - `actorExternalUserId` +- `packages/core/src/db/schema/founder-weekly-review.ts` + - Allows a null `evidenceSnapshot` before collection; adds collection claim/timestamps/input fields. +- `apps/web/drizzle/0018_founder_weekly_review_async_collection.sql` + - Forward migration for nullable snapshot and durable collection fields. + - Backfills existing rows with bounded collection input. It must not be applied to shared Neon without separate approval. +- `packages/features/src/founder-weekly-review/repository.ts` + - Adds conditional collection claim, insert-once snapshot attachment, and collection failure methods. +- `packages/features/src/founder-weekly-review/worker-service.ts` + - Adds collection claim/attach/failure service methods. +- `apps/web/src/server/inngest/functions/founderWeeklyReview.ts` + - Uses the canonical evidence collector when no snapshot exists, then runs existing generation/validation. +- `apps/web/src/server/founder-weekly-review/dispatch-service.ts` + - Initial and retry outbox contracts accept optional snapshot plus collection input; event remains identifier-only. + +Important invariants implemented: + +- A generation claim requires an existing immutable snapshot. +- A collection claim only succeeds for `queued` runs whose snapshot is absent. +- Snapshot attachment requires the current collection claim and only writes when the snapshot remains absent. +- Failed retry remains the same run row and can recollect only if it still has no snapshot; otherwise it reuses the snapshot. +- Events do not contain evidence, founder context, prompts, credentials, or report text. + +### Local synthetic safety baselines + +Existing runner: + +- `apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts` + +It supports local-only synthetic fixtures, deterministic negative citation/source-semantic cases, retry snapshot immutability checks, bounded Kimi/Ollama diagnostics, optional export, and `FWR_PRINT_REPORT=1` terminal printing of a persisted/re-read report. + +Known validated local outcomes from earlier work: + +- Kimi partial and full synthetic cases reached persisted `draft`. +- Unknown citation IDs become sanitized `generation_failed` with no draft. +- Founder context cannot support customer testimony. +- Retry reused one row; retry count behaved `0 -> 1 -> 1`; snapshot digest was unchanged. + +### Realistic isolated-database E2E baseline + +New local-only assets: + +- `apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json` +- `apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts` +- `apps/web/__tests__/founderWeeklyReview/async-collection.test.ts` + +The fixture is fictional and seeds real collector source tables: + +- product release document/version; +- internal planning document/version; +- exact-category `Customer Feedback` document/version with processed context chunks; +- request-time founder context through durable collection input; +- an out-of-period control and another-company control. + +The driver uses the isolated schema helper, creates a queued run via the dispatch service, claims collection, invokes `FounderWeeklyReviewEvidenceService`, attaches the actual collector output, claims generation, invokes Kimi through the local runner adapter, validates via `generateFounderWeeklyReview`, persists/re-reads the draft, optionally prints and exports it, and then drops the isolated schema. + +Last successful realistic run: + +- Run ID: `fwr_0029d66b-99cd-4ebf-9f7a-3da904381be6` +- Lifecycle: `queued -> collecting -> queued -> generating -> draft` +- Provider/model: `kimi` / `kimi-k2.6` +- Evidence counts: `document_change=3`, `customer_feedback=3`, `founder_context=1` +- Warning codes: none +- Snapshot digest unchanged after attachment and repository read-back. +- Canonical schema, citation validation, and source-semantic validation passed. +- One run row and one initial outbox row were present. + +Export artifacts from that run are local and Git-ignored: + +- `apps/web/.artifacts/founder-weekly-review/fwr_0029d66b-99cd-4ebf-9f7a-3da904381be6.md` +- `apps/web/.artifacts/founder-weekly-review/fwr_0029d66b-99cd-4ebf-9f7a-3da904381be6.json` + +The Markdown did not contain the run ID or `evidenceSnapshot`. The JSON envelope did not contain the snapshot, prompts, credentials, raw provider output, or database URL. + +## Current quality-improvement task (interrupted before edits) + +The active next task is **Phase 1 report-quality improvement** within the existing Founder Weekly Review v2 payload schema. Do not add a V3 schema or change persisted payload shape. + +Requested improvements: + +1. Improve the generation prompt in `packages/features/src/founder-weekly-review/prompts.ts` so output synthesizes related evidence, explains why it matters, states limitations/evidence gaps, separates shipped work from preparation, treats customer feedback as customer-only evidence, and creates distinct grounded priorities. +2. Inspect and increase the founder-review output token budget only if an actual budget exists at the provider boundary. At the last inspection, `apps/web/src/lib/llm/generate.ts` delegates to AI SDK `generateObject` without an explicit founder-review max-output setting; the Kimi local adapters also do not currently send a max-token parameter. Do not invent a production-wide provider refactor. +3. Improve Markdown rendering in both runners (or extract a safe shared renderer) so it: + - has no empty `Key Outcomes` heading; + - omits or uses useful text for no-evidence sections; + - keeps citations next to claims; + - uses distinct evidence-reference labels for chunks from the same feedback document; + - uses title plus bounded existing metadata (page/section) rather than category-only labels; + - clearly labels founder context as founder-provided; + - does not show internal IDs, snapshots, prompts, provider responses, credentials, or database URLs. +4. Update only the fictional feedback **title** in the realistic fixture to something descriptive such as `Customer Interviews - February 2026`; retain its exact category `Customer Feedback` and retain the actual fixture facts. +5. Add deterministic renderer tests; do not call a provider in those tests. +6. Rerun the local realistic Kimi E2E with `FWR_PRINT_REPORT=1` and export enabled. + +Current prompt location and behavior: + +- `packages/features/src/founder-weekly-review/prompts.ts` + - system prompt currently emphasizes non-invention, citations, customer-source semantics, and no-evidence behavior but does not sufficiently ask for multi-evidence synthesis or natural founder-facing prose. +- `packages/features/src/founder-weekly-review/generator.ts` + - parses the current V2 schema, validates citations and semantics, and builds metadata. Do not weaken this path. +- `packages/features/src/founder-weekly-review/generation-validation.ts` + - contains the citation and source-semantic enforcement; preserve it unchanged. + +Current Markdown renderers are duplicated: + +- `markdownFor()` in `apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts` +- `markdown()` in `apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts` + +Both currently use simple title-only references and render a `## Key Outcomes` container even when only child headings carry content. A shared local renderer helper is preferable if extraction stays narrow and preserves the persisted-draft-only boundary. + +## Local database and test setup + +Use only: + +```powershell +$env:DATABASE_URL='postgresql://postgres:password@127.0.0.1:5433/pdr_ai_v2' +$env:LAUNCHSTACK_TEST_DATABASE_URL=$env:DATABASE_URL +``` + +The isolated helper is: + +- `apps/web/__tests__/founderWeeklyReview/testDb.ts` + +It creates a temporary schema, applies migration files, sets a schema-first search path, and drops the schema during cleanup. It was expanded to include local `document_versions` and `document_context_chunks` columns required by current Drizzle inserts; this prevents accidental fallback reads/writes to `public` during realistic seeding. + +Focused verification most recently passed: + +```powershell +pnpm.cmd --filter @launchstack/core typecheck +pnpm.cmd --filter @launchstack/features typecheck +pnpm.cmd --filter @launchstack/web typecheck + +Set-Location apps/web +pnpm.cmd exec jest --runInBand --detectOpenHandles founderWeeklyReview +``` + +Result: 13 suites passed, 74 tests passed, 0 skipped, no open-handle warning. + +`git diff --check` also passed. + +## Run commands + +### Deterministic async collection test + +```powershell +Set-Location apps/web +pnpm.cmd exec jest --runInBand --detectOpenHandles founderWeeklyReview/async-collection.test.ts +``` + +### Realistic isolated Kimi E2E + +`MOONSHOT_API_KEY` is loaded by the project `.env` mechanism. Never print it or `.env` content. + +```powershell +Set-Location apps/web +$env:SYNTHETIC_FWR_LOCAL='1' +$env:FWR_GENERATION_PROVIDER='kimi' +$env:KIMI_MODEL_ID='kimi-k2.6' +$env:MOONSHOT_BASE_URL='https://api.moonshot.ai/v1' +$env:FWR_PRINT_REPORT='1' +$env:SYNTHETIC_FWR_EXPORT_REPORT='1' +$env:SYNTHETIC_FWR_EXPORT_DIR='.artifacts/founder-weekly-review' +$env:DATABASE_URL='postgresql://postgres:password@127.0.0.1:5433/pdr_ai_v2' +$env:LAUNCHSTACK_TEST_DATABASE_URL=$env:DATABASE_URL +pnpm.cmd exec tsx ./scripts/run-founder-weekly-review-realistic-e2e.ts +``` + +The command performs one external Kimi generation request, but all database writes are inside a temporary local schema that the driver removes. + +### Founder Weekly Review generation provider + +Founder Weekly Review now selects its generation provider explicitly: + +```powershell +# Normal/default production behavior (also used when unset) +$env:FWR_GENERATION_PROVIDER='openai' +# Requires OPENAI_API_KEY; OPENAI_MODEL_ID is optional. + +# Local Moonshot/Kimi alternative +$env:FWR_GENERATION_PROVIDER='kimi' +$env:KIMI_MODEL_ID='kimi-k2.6' # optional; this is the default +# Requires MOONSHOT_API_KEY; MOONSHOT_BASE_URL is optional. +``` + +`FWR_GENERATION_PROVIDER` accepts only `openai` or `kimi`; an unset value is +`openai`. The Kimi path uses Moonshot Chat Completions with JSON-object output +and local canonical validation. It does not change the normal OpenAI protocol. +Kimi thinking is explicitly disabled for this path. Keep its current 1,800 +output-token budget for the next baseline; revisit approximately 2,500–3,000 +only if non-thinking real reports demonstrably approach the current limit. + +## Working-tree precautions + +Expected tracked modifications are the async workflow and test/E2E work listed by `git status -sb`. Expected untracked files include: + +- `apps/web/__tests__/founderWeeklyReview/async-collection.test.ts` +- `apps/web/drizzle/0018_founder_weekly_review_async_collection.sql` +- `apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts` +- `apps/web/test-fixtures/` +- four `ollama-*.json` local probe artifacts at repository root. + +The Ollama probe artifacts and `.artifacts/` exports must not be committed. No commit or push has been made for the current work. + +## Known limitations / next work + +### Latest local transport baseline + +The real isolated local transport reached a persisted draft for +`fwr_79958e72-d05c-4bc7-ad21-f6e76d490e2e` using `kimi` / `kimi-k2.6`: + +```text +queued without snapshot -> collecting -> queued with immutable snapshot +-> generating -> draft +``` + +Canonical schema, citation, and source-semantic validation passed, with +`retryCount=0`. Founder-context-as-customer-feedback remains forbidden. A +repairable post-generation validation failure now receives exactly one explicit +same-snapshot semantic-repair call; a second failure is non-retriable for +Inngest and fails normally. The local harness now has a bounded terminal +callback drain before scoped process-tree cleanup; rerun the local-only smoke +or transport baseline to confirm it with an explicitly local database URL. + +Next work is workflow comprehension and Peace evaluator integration. A broader +provider registry for Llama, Hugging Face, and Ollama remains a post-PR follow-up. + +- The realistic driver invokes production-equivalent collection/worker service boundaries directly, not a live Inngest dev-server callback. +- The flat V2 schema has only sections, items, text, source IDs, confidence, and typed no-evidence states. It cannot add an executive summary, overall status, owners, metrics, or decision records without a future schema version. +- The existing collector emits the feedback document version as `document_change` in addition to its processed feedback chunks. Customer-facing claims are still constrained to `customer_feedback` evidence. +- Peace evaluator integration, real PDF ingestion, shared-dev deployment, and frontend work remain out of scope. diff --git a/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts b/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts new file mode 100644 index 000000000..3bfd0b666 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/async-collection.test.ts @@ -0,0 +1,73 @@ +jest.mock("~/server/db", () => ({ db: { transaction: jest.fn() } })); +import { company } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; +const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }); +const snapshotV2 = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ ...snapshot, schemaVersion: "founder-weekly-review-evidence/v2", documentChangeAudit: { schemaVersion: "document-change-audit/v1", rawChanges: [], groups: [] } }); + +describeDb("Founder Weekly Review async evidence workflow", () => { + it("creates without a snapshot, attaches it once, and then permits generation claim", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [companyRow] = await test.db.insert(company).values({ name: "Workflow", numberOfEmployees: "1" }).returning(); + const actor = { externalUserId: "u", internalUserId: 1n, companyId: BigInt(companyRow!.id), role: "owner" }; + const created = await createFounderWeeklyReviewDispatchService(test.db).createRunWithDispatch({ actor, requestKey: "workflow", reportingPeriod: snapshot.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", founderContext: "bounded", actorExternalUserId: "u" } }); + expect(created.run.status).toBe("queued"); + expect(created.run.evidenceSnapshot).toBeNull(); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(test.db)); + const context = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: "collection-1" }; + const collecting = await worker.claimEvidenceCollection(context); + expect(collecting.status).toBe("collecting"); + const attached = await worker.attachEvidenceSnapshotIfAbsent(context, snapshot); + expect(attached.status).toBe("queued"); + expect(attached.evidenceSnapshot).toEqual(snapshot); + await expect(worker.attachEvidenceSnapshotIfAbsent(context, snapshot)).resolves.toMatchObject({ id: created.run.id }); + const generating = await worker.claimQueuedRun({ companyId: actor.companyId, runId: created.run.id, generationClaimId: "generation-1", generationJobId: "job-1" }); + expect(generating.status).toBe("generating"); + } finally { await test.close(); } + }); + + it("retries collection failures without creating a snapshot or another run", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [companyRow] = await test.db.insert(company).values({ name: "Recovery", numberOfEmployees: "1" }).returning(); + const actor = { externalUserId: "u", internalUserId: 1n, companyId: BigInt(companyRow!.id), role: "owner" }; + const service = createFounderWeeklyReviewDispatchService(test.db); + const created = await service.createRunWithDispatch({ actor, requestKey: "collection-failure", reportingPeriod: snapshot.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", actorExternalUserId: "u" } }); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(test.db)); + const collecting = await worker.claimEvidenceCollection({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: "collection-failure-claim" }); + expect(collecting.status).toBe("collecting"); + const failed = await worker.markCollectionFailed({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: "collection-failure-claim" }, { errorCode: "evidence_collection_failed" }); + expect(failed.status).toBe("failed"); expect(failed.evidenceSnapshot).toBeNull(); + const retried = await service.retryRunWithDispatch({ actor, runId: created.run.id, requestKey: "collection-retry" }); + expect(retried.run.id).toBe(created.run.id); expect(retried.run.retryCount).toBe(1); expect(retried.run.evidenceSnapshot).toBeNull(); + await expect(worker.claimEvidenceCollection({ companyId: actor.companyId, runId: created.run.id, collectionClaimId: retried.dispatch.generationClaimId })).resolves.toMatchObject({ status: "collecting" }); + } finally { await test.close(); } + }); + + it("reuses an attached v2 audit snapshot unchanged on generation retry", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [companyRow] = await test.db.insert(company).values({ name: "V2 Retry", numberOfEmployees: "1" }).returning(); + const actor = { externalUserId: "u", internalUserId: 1n, companyId: BigInt(companyRow!.id), role: "owner" }; + const service = createFounderWeeklyReviewDispatchService(test.db); + const created = await service.createRunWithDispatch({ actor, requestKey: "v2-retry", reportingPeriod: snapshotV2.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", actorExternalUserId: "u" } }); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(test.db)); + const collection = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: "v2-collection" }; + await worker.claimEvidenceCollection(collection); + const attached = await worker.attachEvidenceSnapshotIfAbsent(collection, snapshotV2); + const generation = { companyId: actor.companyId, runId: created.run.id, generationClaimId: "v2-generation", generationJobId: "v2-job" }; + await worker.claimQueuedRun(generation); + await worker.markGenerationFailed(generation, { errorCode: "provider_unavailable" }); + const retried = await service.retryRunWithDispatch({ actor, runId: created.run.id, requestKey: "v2-generation-retry" }); + + expect(attached.evidenceSnapshot).toEqual(snapshotV2); + expect(retried.run.evidenceSnapshot).toEqual(snapshotV2); + expect(retried.run.evidenceSchemaVersion).toBe("founder-weekly-review-evidence/v2"); + expect(retried.run.status).toBe("queued"); + } finally { await test.close(); } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts b/apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts new file mode 100644 index 000000000..51f227ae9 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/customer-feedback.test.ts @@ -0,0 +1,44 @@ +import { + mapCustomerFeedbackChunkToEvidenceItem, + type CustomerFeedbackChunkRow, +} from "@launchstack/features/founder-weekly-review"; + +function row(content: string | null): CustomerFeedbackChunkRow { + return { + documentId: 42n, + versionId: 100, + versionNumber: 3, + documentTitle: "Customer interview", + documentCategory: "Customer Feedback", + createdAt: new Date("2026-07-10T18:15:00.000Z"), + chunkId: 9, + chunkContent: content, + pageNumber: 4, + }; +} + +describe("Customer Feedback chunk mapping", () => { + it.each([null, "", " "])("does not create cited evidence for missing content", (content) => { + expect(mapCustomerFeedbackChunkToEvidenceItem(row(content))).toBeNull(); + }); + + it("maps valid content with only document-context-chunk provenance", () => { + expect(mapCustomerFeedbackChunkToEvidenceItem(row(" Export is too slow. "))).toEqual({ + sourceType: "customer_feedback", + sourceId: "customer_feedback:doc:42:version:100:section:9", + title: "Customer interview", + sourceTimestamp: "2026-07-10T18:15:00.000Z", + excerpt: "Export is too slow.", + workspaceDeepLink: "/employer/documents/viewer?docId=42", + metadata: { + documentId: "42", + documentVersionId: 100, + sectionId: 9, + versionNumber: 3, + documentCategory: "Customer Feedback", + pageNumber: 4, + excerptTruncated: false, + }, + }); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/demo.test.ts b/apps/web/__tests__/founderWeeklyReview/demo.test.ts new file mode 100644 index 000000000..56621f6c1 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/demo.test.ts @@ -0,0 +1,27 @@ +import { FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET } from "@launchstack/features/founder-weekly-review"; +import { formatFounderWeeklyReviewDemo } from "../../scripts/founder-weekly-review-demo"; + +describe("Founder Weekly Review demo presentation", () => { + it("formats a safe staged report from pipeline outputs", () => { + const output = formatFounderWeeklyReviewDemo({ + rawChanges: [{ documentId: "1", previousVersionId: 1, currentVersionId: 2, alignmentMethod: "structure_path", previousStructureTitle: "Ownership", previousExcerpt: "Product owns telemetry.", currentExcerpt: "Platform owns telemetry.", changeType: "modified" }], + groups: [{ structureTitle: "Ownership", category: "ownership_change" }], + promptItems: [{ sourceType: "document_change", title: "Ownership", excerpt: "Ownership changed.", metadata: { category: "ownership_change" } }, { sourceType: "customer_feedback" }], + envelopeDiagnostics: { selectedItemCount: 2, serializedCharacterCount: 100, estimatedTokenCount: 25, truncated: false }, + analyzerCalls: [], eligibleGroups: 0, warnings: [], reviewMarkdown: "# Founder Weekly Review\n\nConcise result.", provider: "kimi", model: "kimi-k2.6", promptVersion: "founder-weekly-review-generation/v2", outputCeiling: 2400, repairCount: 0, snapshotVersion: "founder-weekly-review-evidence/v2", evidenceDigest: "a".repeat(64), persistencePassed: true, readBackPassed: true, noOpRemoved: 0, budget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + }); + expect(output).toContain("CHANGE PROCESSING"); + expect(output).toContain("IMMUTABLE EVIDENCE SNAPSHOT v2"); + expect(output).toContain("FINAL FOUNDER WEEKLY REVIEW"); + expect(output).toContain("Product owns telemetry."); + expect(output).not.toContain("api-key"); + expect(output).not.toContain("provider response"); + }); + + it("renders the configured budget rather than presentation literals", () => { + const output = formatFounderWeeklyReviewDemo({ + rawChanges: [], groups: [], promptItems: [], envelopeDiagnostics: { selectedItemCount: 0, serializedCharacterCount: 0, estimatedTokenCount: 0, truncated: false }, analyzerCalls: [], eligibleGroups: 0, warnings: [], reviewMarkdown: "", provider: "kimi", model: "kimi-k2.6", promptVersion: "v2", outputCeiling: 2400, repairCount: 0, snapshotVersion: "v2", evidenceDigest: "a".repeat(64), persistencePassed: true, readBackPassed: true, noOpRemoved: 0, budget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + }); + expect(output).toContain(`${FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.totalSerializedCharacters.toLocaleString()} chars`); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts b/apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts new file mode 100644 index 000000000..18725f94e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/dispatch-atomicity.test.ts @@ -0,0 +1,18 @@ +jest.mock("~/server/db", () => ({ db: { transaction: jest.fn() } })); +import { eq } from "drizzle-orm"; +import { company, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; +const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }); +const actor = (companyId: bigint) => ({ externalUserId: "u", internalUserId: 1n, companyId, role: "owner" as const }); +async function companyId(db: Awaited>["db"]): Promise { const [row] = await db.insert(company).values({ name: "Atomic", numberOfEmployees: "1" }).returning(); return BigInt(row!.id); } + +describeDb("Founder Weekly Review dispatch atomicity", () => { + it("concurrent create persists one run and one dispatch", async () => { const test = await createFounderWeeklyReviewTestDatabase(); const a = await test.createSession(), b = await test.createSession(); try { const id = await companyId(test.db); const input = { actor: actor(id), requestKey: "create-race", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot }; const results = await Promise.all([createFounderWeeklyReviewDispatchService(a.db).createRunWithDispatch(input), createFounderWeeklyReviewDispatchService(b.db).createRunWithDispatch(input)]); const rows = await test.db.select().from(founderWeeklyReviewRuns); const dispatches = await test.db.select().from(founderWeeklyReviewDispatches); expect(rows).toHaveLength(1); expect(dispatches).toHaveLength(1); expect(results.map(x => x.created).sort()).toEqual([false, true]); expect(new Set(results.map(x => x.run.id)).size).toBe(1); expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(rows[0]!.evidenceSnapshot)).toEqual(snapshot); } finally { await a.close(); await b.close(); await test.close(); } }); + it("concurrent retry transitions once and preserves snapshot", async () => { const test = await createFounderWeeklyReviewTestDatabase(); const a = await test.createSession(), b = await test.createSession(); try { const id = await companyId(test.db); const repo = new FounderWeeklyReviewRepository(test.db); const created = await repo.createOrGetByRequestKey({ id: "fwr_failed", companyId: id, requestKey: "base", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot, createdByActorId: "user:u" }); await new FounderWeeklyReviewWorkerService(repo).markQueuedRunFailed(id, created.id, { errorCode: "x" }); const input = { actor: actor(id), runId: created.id, requestKey: "retry-race" }; const results = await Promise.all([createFounderWeeklyReviewDispatchService(a.db).retryRunWithDispatch(input), createFounderWeeklyReviewDispatchService(b.db).retryRunWithDispatch(input)]); const [row] = await test.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, created.id)); const dispatches = await test.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.operationType, "retry")); expect(results.map(x => x.transitionApplied).sort()).toEqual([false, true]); expect(row!.status).toBe("queued"); expect(row!.retryCount).toBe(1); expect(row!.evidenceSnapshot).toEqual(snapshot); expect(dispatches).toHaveLength(1); } finally { await a.close(); await b.close(); await test.close(); } }); + it("rolls back create when dispatch persistence fails", async () => { const test = await createFounderWeeklyReviewTestDatabase(); try { const id = await companyId(test.db); const service = createFounderWeeklyReviewDispatchService(test.db, { createDispatch: async () => { throw new Error("forced_dispatch_insert_failure"); } }); await expect(service.createRunWithDispatch({ actor: actor(id), requestKey: "rollback", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot })).rejects.toThrow("forced_dispatch_insert_failure"); expect(await test.db.select().from(founderWeeklyReviewRuns)).toHaveLength(0); expect(await test.db.select().from(founderWeeklyReviewDispatches)).toHaveLength(0); } finally { await test.close(); } }); + it("rolls back retry when dispatch persistence fails", async () => { const test = await createFounderWeeklyReviewTestDatabase(); try { const id = await companyId(test.db); const repo = new FounderWeeklyReviewRepository(test.db); const created = await repo.createOrGetByRequestKey({ id: "fwr_retry_rollback", companyId: id, requestKey: "base", reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot, createdByActorId: "user:u" }); await new FounderWeeklyReviewWorkerService(repo).markQueuedRunFailed(id, created.id, { errorCode: "x" }); const service = createFounderWeeklyReviewDispatchService(test.db, { createDispatch: async () => { throw new Error("forced_dispatch_insert_failure"); } }); await expect(service.retryRunWithDispatch({ actor: actor(id), runId: created.id, requestKey: "retry" })).rejects.toThrow("forced_dispatch_insert_failure"); const [row] = await test.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, created.id)); expect(row!.status).toBe("failed"); expect(row!.retryCount).toBe(0); expect(row!.evidenceSnapshot).toEqual(snapshot); expect(await test.db.select().from(founderWeeklyReviewDispatches)).toHaveLength(0); } finally { await test.close(); } }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts new file mode 100644 index 000000000..e922c4b02 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-condensation.test.ts @@ -0,0 +1,340 @@ +import { + DOCUMENT_CHANGE_GROUP_BUDGET, + FounderWeeklyReviewEvidenceItemSchema, + alignVersionChunks, + buildDocumentChangeEvidence, + buildDocumentChangeGroupEvidence, + buildGenerationEvidenceEnvelope, + buildRawDocumentChanges, + condenseDocumentChanges, + groupRawDocumentChanges, + normalizeDocumentChangeContent, + selectDocumentChangeGroups, + type ChunkAlignment, + type DocumentChangeGroup, + type VersionChunk, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +const pair = (documentId = 1n, previousVersionId = 1, currentVersionId = 2): VersionPair => ({ + documentId, + documentTitle: `Document ${documentId}`, + documentCategory: "Product", + previousVersionId, + previousVersionNumber: previousVersionId, + previousCreatedAt: new Date(`2026-01-${String(Math.min(previousVersionId, 28)).padStart(2, "0")}T00:00:00.000Z`), + currentVersionId, + currentVersionNumber: currentVersionId, + currentCreatedAt: new Date(`2026-02-${String(Math.min(currentVersionId, 28)).padStart(2, "0")}T00:00:00.000Z`), + currentChangelog: null, +}); + +const chunk = ( + chunkId: number, + versionId: bigint, + content: string, + overrides: Partial = {} +): VersionChunk => ({ + chunkId, + versionId, + documentId: 1n, + content, + contentHash: null, + structureId: BigInt(chunkId), + structurePath: "/overview", + structureTitle: "Overview", + structureOrdering: chunkId, + pageNumber: Math.ceil(chunkId / 2), + lineStart: chunkId * 10, + lineEnd: chunkId * 10 + 5, + ...overrides, +}); + +const modified = ( + id: number, + before: string, + after: string, + overrides: Partial = {} +): ChunkAlignment => ({ + changeType: "modified", + previousChunk: chunk(id, 1n, before, overrides), + currentChunk: chunk(10_000 + id, 2n, after, overrides), + alignmentMethod: "structure_path", +}); + +describe("deterministic document-change condensation", () => { + it("constructs stable, distinct raw IDs and preserves provenance independent of alignment order", () => { + const versionPair = pair(); + const alignments: ChunkAlignment[] = [ + modified(1, "before", "after"), + { changeType: "removed", previousChunk: chunk(2, 1n, "removed", { structurePath: "/removed" }), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(3, 2n, "added", { structurePath: "/added" }), alignmentMethod: "unmatched" }, + ]; + const first = buildRawDocumentChanges(versionPair, alignments); + const shuffled = buildRawDocumentChanges(versionPair, [alignments[2]!, alignments[0]!, alignments[1]!]); + + expect(shuffled).toEqual(first); + expect(new Set(first.rawChanges.map(change => change.rawChangeId)).size).toBe(3); + expect(first.rawChanges.every(change => /^raw_document_change:[a-f0-9]{64}$/.test(change.rawChangeId))).toBe(true); + expect(first.rawChanges.find(change => change.changeType === "modified")).toEqual(expect.objectContaining({ + previousChunk: expect.objectContaining({ chunkId: 1 }), + currentChunk: expect.objectContaining({ chunkId: 10_001 }), + })); + const changedContent = buildRawDocumentChanges(versionPair, [modified(1, "before", "different after")]); + expect(changedContent.rawChanges[0]!.rawChangeId).not.toBe(first.rawChanges.find(change => change.changeType === "modified")!.rawChangeId); + }); + + it.each([ + ["ordinary whitespace", "alpha beta", " alpha beta "], + ["CRLF", "alpha\nbeta", "alpha\r\nbeta"], + ["CR", "alpha\nbeta", "alpha\rbeta"], + ["non-breaking space", "alpha beta", "alpha\u00a0beta"], + ["leading and trailing whitespace", "alpha", "\t alpha \n"], + ["Unicode NFC", "Café", "Cafe\u0301"], + ])("filters %s-only modifications", (_name, before, after) => { + expect(normalizeDocumentChangeContent(before)).toBe(normalizeDocumentChangeContent(after)); + expect(buildRawDocumentChanges(pair(), [modified(1, before, after)])).toEqual({ + rawChanges: [], + deterministicNoOpCount: 1, + }); + }); + + it.each([ + ["punctuation", "Launch.", "Launch!"], + ["capitalization", "Product", "product"], + ["number", "10 users", "100 users"], + ["percentage", "10%", "20%"], + ["date", "August 7", "August 8"], + ["quarter", "Q3", "Q4"], + ["owner", "Product owns this", "Platform owns this"], + ["negation", "supports export", "does not support export"], + ["modal", "may ship", "must ship"], + ["requirement", "optional", "required"], + ["status", "planned", "launched"], + ])("retains a %s change", (_name, before, after) => { + const result = buildRawDocumentChanges(pair(), [modified(1, before, after)]); + expect(result.deterministicNoOpCount).toBe(0); + expect(result.rawChanges).toHaveLength(1); + }); + + it("groups adjacent mixed changes within a section and never crosses known section boundaries", () => { + const versionPair = pair(); + const alignments: ChunkAlignment[] = [ + modified(1, "old one", "new one", { structurePath: "/alpha", structureTitle: "Alpha" }), + { changeType: "removed", previousChunk: chunk(2, 1n, "old two", { structurePath: "/alpha", structureTitle: "Alpha" }), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(3, 2n, "new two", { structurePath: "/alpha", structureTitle: "Alpha" }), alignmentMethod: "unmatched" }, + modified(4, "old beta", "new beta", { structurePath: "/beta", structureTitle: "Beta", pageNumber: 2 }), + ]; + const raw = buildRawDocumentChanges(versionPair, alignments).rawChanges; + const grouped = groupRawDocumentChanges(versionPair, raw); + + expect(grouped.groups).toHaveLength(2); + expect(grouped.groups.map(group => group.rawChanges.length).sort()).toEqual([1, 3]); + expect(grouped.groups.find(group => group.structurePath === "/alpha")?.rawChanges.map(change => change.changeType).sort()).toEqual(["added", "modified", "removed"]); + }); + + it("keeps distinct known titles separate when paths are unavailable", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [ + modified(1, "old alpha", "new alpha", { structurePath: null, structureTitle: "Alpha", pageNumber: 1 }), + modified(2, "old beta", "new beta", { structurePath: null, structureTitle: "Beta", pageNumber: 1 }), + ]).rawChanges; + expect(groupRawDocumentChanges(versionPair, raw).groups).toHaveLength(2); + }); + + it("does not merge conflicting known paths solely because their titles match", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [ + modified(1, "old one", "new one", { structurePath: "/one", structureTitle: "Overview", pageNumber: 1 }), + modified(2, "old two", "new two", { structurePath: "/two", structureTitle: "Overview", pageNumber: 1 }), + ]).rawChanges; + expect(groupRawDocumentChanges(versionPair, raw).groups).toHaveLength(2); + }); + + it("groups renamed paths through connected aligned chunks and supports one-to-many and many-to-one boundaries", () => { + const versionPair = pair(); + const renamed: ChunkAlignment[] = [1, 2].map(id => ({ + changeType: "modified" as const, + previousChunk: chunk(id, 1n, `old ${id}`, { structurePath: "/old", structureTitle: "Plan" }), + currentChunk: chunk(100 + id, 2n, `new ${id}`, { structurePath: "/new", structureTitle: "Plan" }), + alignmentMethod: "section_title" as const, + })); + const oneToMany: ChunkAlignment[] = [ + modified(3, "one", "part one", { structurePath: "/split", structureTitle: "Split" }), + { changeType: "added", currentChunk: chunk(104, 2n, "part two", { structurePath: "/split", structureTitle: "Split" }), alignmentMethod: "unmatched" }, + ]; + const manyToOne: ChunkAlignment[] = [ + modified(5, "part one", "merged", { structurePath: "/merge", structureTitle: "Merge" }), + { changeType: "removed", previousChunk: chunk(6, 1n, "part two", { structurePath: "/merge", structureTitle: "Merge" }), alignmentMethod: "unmatched" }, + ]; + const groups = groupRawDocumentChanges(versionPair, buildRawDocumentChanges(versionPair, [...renamed, ...oneToMany, ...manyToOne]).rawChanges).groups; + + expect(groups).toHaveLength(3); + expect(groups.map(group => group.rawChanges.length).sort()).toEqual([2, 2, 2]); + }); + + it("keeps exact reordered content unchanged before grouping", () => { + const hashA = "a".repeat(64); const hashB = "b".repeat(64); + const previous = [ + chunk(1, 1n, "Alpha exact", { contentHash: hashA, structurePath: "/a" }), + chunk(2, 1n, "Beta exact", { contentHash: hashB, structurePath: "/b" }), + ]; + const current = [ + chunk(3, 2n, "Beta exact", { contentHash: hashB, structurePath: "/moved-b" }), + chunk(4, 2n, "Alpha exact", { contentHash: hashA, structurePath: "/moved-a" }), + ]; + const alignments = alignVersionChunks(previous, current); + const condensed = condenseDocumentChanges([{ pair: pair(), alignments }]); + expect(alignments.every(item => item.changeType === "unchanged")).toBe(true); + expect(condensed.rawChanges).toHaveLength(0); + expect(condensed.groups).toHaveLength(0); + }); + + it("produces identical groups and IDs for shuffled raw input", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [ + modified(1, "a", "aa", { structurePath: "/a", structureTitle: "A" }), + modified(2, "b", "bb", { structurePath: "/a", structureTitle: "A" }), + modified(3, "c", "cc", { structurePath: "/b", structureTitle: "B" }), + ]).rawChanges; + const first = groupRawDocumentChanges(versionPair, raw); + const shuffled = groupRawDocumentChanges(versionPair, [raw[2]!, raw[0]!, raw[1]!]); + expect(shuffled).toEqual(first); + expect(first.groups.every(group => /^document_change_group:[a-f0-9]{64}$/.test(group.groupId))).toBe(true); + const changedSupport = buildRawDocumentChanges(versionPair, [ + modified(1, "a", "different", { structurePath: "/a", structureTitle: "A" }), + modified(2, "b", "bb", { structurePath: "/a", structureTitle: "A" }), + ]).rawChanges; + expect(groupRawDocumentChanges(versionPair, changedSupport).groups[0]!.groupId).not.toBe(first.groups[0]!.groupId); + }); + + it("keeps 16 records together and splits 17 into stable bounded windows with one warning", () => { + const versionPair = pair(); + const sixteen = buildRawDocumentChanges(versionPair, Array.from({ length: 16 }, (_, index) => + modified(index + 1, `before ${index}`, `after ${index}`, { structurePath: "/large", structureTitle: "Large" }))).rawChanges; + expect(groupRawDocumentChanges(versionPair, sixteen).groups).toHaveLength(1); + + const seventeen = buildRawDocumentChanges(versionPair, [ + ...Array.from({ length: 16 }, (_, index) => modified(index + 1, `before ${index}`, `after ${index}`, { structurePath: "/large", structureTitle: "Large" })), + modified(17, "before 17", "after 17", { structurePath: "/large", structureTitle: "Large" }), + ]).rawChanges; + const first = groupRawDocumentChanges(versionPair, seventeen); + const shuffled = groupRawDocumentChanges(versionPair, [...seventeen].reverse()); + expect(first.groups.map(group => group.rawChanges.length)).toEqual([16, 1]); + expect(first.groups.map(group => group.splitOrdinal)).toEqual([0, 1]); + expect(first.groups.every(group => group.rawChanges.length <= DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup)).toBe(true); + expect(first.warnings).toEqual([expect.objectContaining({ code: "materiality_group_too_large" })]); + expect(shuffled).toEqual(first); + }); + + it("enforces pair, document, and review limits with deterministic round-robin diversity", () => { + const inputs = [1n, 2n, 3n, 4n].map(documentId => { + const versionPair = pair(documentId); + return { + pair: versionPair, + alignments: Array.from({ length: 12 }, (_, index) => modified( + Number(documentId) * 100 + index, + `before ${documentId}-${index}`, + `after ${documentId}-${index}`, + { documentId, structurePath: `/section-${index}`, structureTitle: `Section ${index}` } + )), + }; + }); + const first = condenseDocumentChanges(inputs); + const shuffled = condenseDocumentChanges([...inputs].reverse().map(input => ({ ...input, alignments: [...input.alignments].reverse() }))); + const byDocument = new Map(); + const byPair = new Map(); + for (const group of first.selectedGroups) { + const documentId = group.documentId.toString(); + const versionPair = `${documentId}:${group.previousVersionId}:${group.currentVersionId}`; + byDocument.set(documentId, (byDocument.get(documentId) ?? 0) + 1); + byPair.set(versionPair, (byPair.get(versionPair) ?? 0) + 1); + } + expect(first.selectedGroups).toHaveLength(24); + expect([...byDocument.values()].every(count => count <= 8)).toBe(true); + expect([...byPair.values()].every(count => count <= 8)).toBe(true); + expect([...byDocument.keys()]).toEqual(["1", "2", "3", "4"]); + expect(first.warnings).toContainEqual(expect.objectContaining({ code: "document_change_budget_truncated" })); + expect(shuffled.selectedGroups.map(group => group.groupId)).toEqual(first.selectedGroups.map(group => group.groupId)); + }); + + it("round-robins version pairs within a document", () => { + const inputs = [pair(1n, 1, 2), pair(1n, 2, 3)].map((versionPair, pairIndex) => ({ + pair: versionPair, + alignments: Array.from({ length: 6 }, (_, index) => modified( + pairIndex * 100 + index + 1, + `before ${pairIndex}-${index}`, + `after ${pairIndex}-${index}`, + { structurePath: `/pair-${pairIndex}-section-${index}`, structureTitle: `Pair ${pairIndex} Section ${index}` } + )), + })); + const result = condenseDocumentChanges(inputs); + expect(result.selectedGroups).toHaveLength(8); + expect(result.selectedGroups.filter(group => group.currentVersionId === 2)).toHaveLength(4); + expect(result.selectedGroups.filter(group => group.currentVersionId === 3)).toHaveLength(4); + }); + + it("keeps the existing evidence contract and source ID for a one-change group", () => { + const evidence = buildDocumentChangeEvidence(pair(), [modified(7, "Before", "After")]); + expect(evidence).toEqual([expect.objectContaining({ + sourceType: "document_change", + sourceId: "document_change:doc:1:v1:v2:chunk:7:10007", + excerpt: "Section modified. Before: Before After: After", + })]); + expect(FounderWeeklyReviewEvidenceItemSchema.safeParse(evidence[0]).success).toBe(true); + }); + + it("reports the actual deterministic no-op count used by the collector", () => { + const result = condenseDocumentChanges([{ pair: pair(), alignments: [modified(7, " same\r\nvalue\u00a0 ", "same\nvalue ")] }]); + expect(result.diagnostics.deterministicNoOpCount).toBe(1); + expect(result.rawChanges).toHaveLength(0); + expect(result.selectedGroups).toHaveLength(0); + }); + + it("condenses a deterministic 20-page, 40-chunk enterprise fixture before the prompt envelope", () => { + const versionPair = pair(44n); + const alignments: ChunkAlignment[] = []; + for (let index = 0; index < 20; index++) { + const exact = `unchanged section ${index}`; + alignments.push({ + changeType: "unchanged", + previousChunk: chunk(index + 1, 1n, exact, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}`, structureTitle: `Unchanged ${index}` }), + currentChunk: chunk(1_001 + index, 2n, exact, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}`, structureTitle: `Unchanged ${index}` }), + alignmentMethod: "content_hash", + }); + } + for (let index = 0; index < 5; index++) alignments.push(modified(100 + index, `wrapped line ${index}`, ` wrapped\r\n line\u00a0${index} `, { documentId: 44n, pageNumber: index + 1, structurePath: `/noop-${index}`, structureTitle: `No-op ${index}` })); + for (let index = 0; index < 4; index++) alignments.push(modified(200 + index, `planned metric ${index}`, `launched metric ${index + 1}`, { documentId: 44n, pageNumber: 6 + index, structurePath: `/material-${index}`, structureTitle: `Material ${index}` })); + for (let index = 0; index < 6; index++) alignments.push(modified(300 + index, `old rewritten ${index}`, `new rewritten ${index}`, { documentId: 44n, pageNumber: 10 + Math.floor(index / 2), structurePath: "/rewritten", structureTitle: "Rewritten section" })); + for (let index = 0; index < 5; index++) alignments.push(modified(400 + index, `editorial wording ${index}`, `rephrased wording ${index}`, { documentId: 44n, pageNumber: 15 + index, structurePath: "/editorial", structureTitle: "Editorial section" })); + + const first = condenseDocumentChanges([{ pair: versionPair, alignments }]); + const shuffled = condenseDocumentChanges([{ pair: versionPair, alignments: [...alignments].reverse() }]); + const evidence = first.selectedGroups.map(group => buildDocumentChangeGroupEvidence(versionPair, group)); + const snapshot = { + schemaVersion: "founder-weekly-review-evidence/v1" as const, + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: evidence, + sourceWarnings: [], + }; + const envelope = buildGenerationEvidenceEnvelope(snapshot); + + expect(alignments).toHaveLength(40); + expect(first.diagnostics).toEqual(expect.objectContaining({ + alignedChunkCount: 40, + rawModifiedCount: 15, + deterministicNoOpCount: 5, + groupCount: 6, + selectedGroupCount: 6, + truncatedGroupCount: 0, + })); + expect(first.selectedGroups.every(group => group.rawChanges.length <= 16)).toBe(true); + expect(first.selectedGroups.map(group => group.groupId)).toEqual(shuffled.selectedGroups.map(group => group.groupId)); + expect(evidence).toHaveLength(6); + expect(envelope.diagnostics.selectedBySourceType.document_change).toBe(6); + expect(envelope.diagnostics.serializedCharacterCount).toBeLessThanOrEqual(14_000); + expect(first.warnings).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts new file mode 100644 index 000000000..8bb7a5a8b --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer-adapter.test.ts @@ -0,0 +1,133 @@ +jest.mock("~/lib/llm", () => { + class LlmCapabilityUnavailableError extends Error {} + return { + generateStructuredWithMetadata: jest.fn(), + LlmCapabilityUnavailableError, + PROVIDERS: ["openai", "kimi", "anthropic", "google", "ollama"], + }; +}); + +import { + type DocumentChangeMaterialityAnalysisInput, +} from "@launchstack/features/founder-weekly-review"; +import { generateStructuredWithMetadata } from "~/lib/llm"; +import { z } from "zod"; + +import { + DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT, + ProviderDocumentChangeMaterialityAnalyzer, + buildDocumentChangeMaterialityAnalyzerPrompt, + createConfiguredDocumentChangeMaterialityAnalyzer, +} from "~/server/founder-weekly-review/document-change-materiality-analyzer"; + +const mockGenerateStructuredWithMetadata = jest.mocked(generateStructuredWithMetadata); + +const input: DocumentChangeMaterialityAnalysisInput = { + groupId: "document_change_group:synthetic", + documentTitle: "Synthetic plan", + structurePath: "/ownership", + structureTitle: "Ownership", + deterministicAssessment: { + category: "uncertain", + confidence: "uncertain", + detectedSignals: ["no_strong_deterministic_signal"], + confirmedFactualDeltas: [], + equivalentFactualValues: [{ + kind: "ownership", previousValue: "product", currentValue: "product", relation: "equivalent", confidence: "strong", + }], + possibleSignals: [], + groupShape: "modified", + semanticRisk: "paraphrase_possible", + }, + changes: [{ + changeType: "modified", + previousExcerpt: "Product owns telemetry.", + currentExcerpt: "Telemetry is owned by Product.", + alignmentMethod: "structure_path", + }], +}; + +describe("document-change materiality provider adapter", () => { + const originalEnabled = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + const originalProvider = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER; + + afterEach(() => { + mockGenerateStructuredWithMetadata.mockReset(); + if (originalEnabled === undefined) delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + else process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = originalEnabled; + if (originalProvider === undefined) delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER; + else process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = originalProvider; + }); + + it("requires explicit production opt-in instead of treating credentials as consent", () => { + delete process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED; + expect(createConfiguredDocumentChangeMaterialityAnalyzer()).toBeUndefined(); + expect(mockGenerateStructuredWithMetadata).not.toHaveBeenCalled(); + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; + expect(createConfiguredDocumentChangeMaterialityAnalyzer()).toBeInstanceOf(ProviderDocumentChangeMaterialityAnalyzer); + }); + + it("uses bounded smallExtraction structured generation and returns safe metadata", async () => { + mockGenerateStructuredWithMetadata.mockResolvedValue({ + object: { disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.94 }, + metadata: { provider: "openai", model: "fixture-model", capability: "smallExtraction", providerRequestId: "must-not-propagate" }, + }); + const result = await new ProviderDocumentChangeMaterialityAnalyzer().analyze(input); + expect(mockGenerateStructuredWithMetadata).toHaveBeenCalledWith(expect.objectContaining({ + capability: "smallExtraction", + schemaName: "document_change_materiality", + timeoutMs: 15_000, + maxOutputTokens: 512, + })); + expect(mockGenerateStructuredWithMetadata.mock.calls[0]![0].prompt).toContain("Product owns telemetry."); + expect(result.metadata).toEqual({ provider: "openai", model: "fixture-model", promptVersion: "document-change-materiality/v2" }); + expect(result.metadata).not.toHaveProperty("providerRequestId"); + }); + + it("defines the v2 task around underlying state, structural equivalence, mixed rewrites, and concise output", () => { + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("underlying business state")); + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("split or merged fragments")); + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("one real factual delta")); + expect(DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT).toEqual(expect.stringContaining("at most 320 characters")); + const prompt = buildDocumentChangeMaterialityAnalyzerPrompt(input); + expect(prompt).toContain('"confirmedFactualDeltas":[]'); + expect(prompt).toContain('"equivalentFactualValues"'); + expect(prompt).toContain('"semanticRisk":"paraphrase_possible"'); + }); + + it("routes only this analyzer to its explicitly configured provider", async () => { + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = "kimi"; + mockGenerateStructuredWithMetadata.mockResolvedValue({ + object: { disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.94 }, + metadata: { provider: "kimi", model: "kimi-k2.6", capability: "smallExtraction" }, + }); + const analyzer = createConfiguredDocumentChangeMaterialityAnalyzer(); + await analyzer!.analyze(input); + expect(mockGenerateStructuredWithMetadata).toHaveBeenCalledWith(expect.objectContaining({ + capability: "smallExtraction", + forceProvider: "kimi", + })); + }); + + it("rejects an invalid analyzer-specific provider before any request", () => { + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED = "true"; + process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER = "other"; + expect(() => createConfiguredDocumentChangeMaterialityAnalyzer()).toThrow("FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER"); + expect(mockGenerateStructuredWithMetadata).not.toHaveBeenCalled(); + }); + + it("maps provider failures to optional-unavailable fallback errors", async () => { + mockGenerateStructuredWithMetadata.mockRejectedValue(new Error("rate limited")); + await expect(new ProviderDocumentChangeMaterialityAnalyzer().analyze(input)).rejects.toEqual( + expect.objectContaining({ code: "unavailable" }), + ); + }); + + it("distinguishes invalid structured output from provider unavailability", async () => { + mockGenerateStructuredWithMetadata.mockRejectedValue(new z.ZodError([])); + await expect(new ProviderDocumentChangeMaterialityAnalyzer().analyze(input)).rejects.toEqual( + expect.objectContaining({ code: "invalid" }), + ); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts new file mode 100644 index 000000000..2ef4938ad --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality-analyzer.test.ts @@ -0,0 +1,307 @@ +import { createHash } from "node:crypto"; + +import { + DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS, + DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + DocumentChangeMaterialityAnalysisResultSchema, + DocumentChangeMaterialityAnalyzerError, + FounderWeeklyReviewEvidenceSnapshotSchema, + buildDocumentChangeMaterialityAnalysisInput, + buildFounderWeeklyReviewEvidenceDigest, + buildFounderWeeklyReviewPrompt, + buildRawDocumentChanges, + groupRawDocumentChanges, + materializeDocumentChanges, + materializeDocumentChangesWithAnalyzer, + selectDocumentChangeGroupsForMaterialityAnalysis, + shouldAnalyzeDocumentChangeGroup, + type AnalyzedDocumentChangeGroup, + type ChunkAlignment, + type DocumentChangeMaterialityAnalysisInput, + type DocumentChangeMaterialityAnalyzer, + type VersionChunk, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +const pair = (documentId = 1n, day = 2): VersionPair => ({ + documentId, + documentTitle: `Synthetic operating plan ${documentId}`, + documentCategory: "Operating plan", + previousVersionId: 1, + previousVersionNumber: 1, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), + currentVersionId: 2, + currentVersionNumber: 2, + currentCreatedAt: new Date(`2026-02-${String(day).padStart(2, "0")}T00:00:00.000Z`), + currentChangelog: null, +}); + +const chunk = (id: number, versionId: bigint, documentId: bigint, content: string, path = "/plan"): VersionChunk => ({ + chunkId: id, + versionId, + documentId, + content, + contentHash: null, + structureId: BigInt(id), + structurePath: path, + structureTitle: path.slice(1), + structureOrdering: id, + pageNumber: 1, + lineStart: id * 10, + lineEnd: id * 10 + 5, +}); + +const modified = (id: number, documentId: bigint, before: string, after: string, path = "/plan"): ChunkAlignment => ({ + changeType: "modified", + previousChunk: chunk(id, 1n, documentId, before, path), + currentChunk: chunk(10_000 + id, 2n, documentId, after, path), + alignmentMethod: "structure_path", +}); + +function input(documentId: bigint, alignments: ChunkAlignment[], day = 2) { + return { pair: pair(documentId, day), alignments }; +} + +function analyzed(documentId: bigint, alignments: ChunkAlignment[], day = 2): AnalyzedDocumentChangeGroup { + return materializeDocumentChanges([input(documentId, alignments, day)]).analyzedGroups[0]!; +} + +class FakeAnalyzer implements DocumentChangeMaterialityAnalyzer { + readonly calls: DocumentChangeMaterialityAnalysisInput[] = []; + constructor(private readonly implementation: (input: DocumentChangeMaterialityAnalysisInput) => unknown | Promise) {} + async analyze(value: DocumentChangeMaterialityAnalysisInput) { + this.calls.push(value); + return { + result: await this.implementation(value), + metadata: { provider: "fixture", model: "fixture-v1", promptVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION }, + }; + } +} + +const materialResult = (summary = "The operating meaning changed.") => ({ + disposition: "material" as const, + category: "scope_change" as const, + summary, + confidence: 0.91, +}); + +describe("optional document-change materiality analyzer", () => { + it("uses a conservative semantic-risk gate and bypasses simple factual deltas", () => { + const uncertainSingle = analyzed(1n, [modified(1, 1n, "Telemetry is reliable.", "Reliability is a property of telemetry.")]); + const uncertainMulti = analyzed(2n, [ + modified(1, 2n, "Telemetry is reliable.", "Reliability is a property of telemetry."), + modified(2, 2n, "Retries are automatic.", "Automatic retries remain enabled."), + ]); + const replacement = analyzed(3n, [ + { changeType: "removed", previousChunk: chunk(1, 1n, 3n, "The pilot serves invited accounts."), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(2, 2n, 3n, "The rollout continues with invited accounts."), alignmentMethod: "unmatched" }, + ]); + const complexStrong = analyzed(4n, [ + modified(1, 4n, "Product owns telemetry.", "Platform owns telemetry."), + modified(2, 4n, "Retries follow the established operating process.", "The operating process for retries remains established."), + ]); + expect(shouldAnalyzeDocumentChangeGroup(uncertainSingle)).toMatchObject({ eligible: true, reason: "uncertain_single_change" }); + expect(shouldAnalyzeDocumentChangeGroup(uncertainMulti)).toMatchObject({ eligible: true, reason: "uncertain_multi_change" }); + expect(shouldAnalyzeDocumentChangeGroup(replacement)).toMatchObject({ eligible: true }); + expect(shouldAnalyzeDocumentChangeGroup(complexStrong)).toMatchObject({ eligible: true, reason: "complex_multi_change" }); + for (const [before, after] of [ + ["Product owns telemetry.", "Platform owns telemetry."], + ["Launch is planned for Q3.", "Launch is planned for Q4."], + ["Conversion is 10%.", "Conversion is 25%."], + ["Admins may enable SSO.", "Admins must enable SSO."], + ["The migration is planned.", "The migration is launched."], + ["The plan supports SSO.", "The plan does not support SSO."], + ["Migration priority: P2", "Migration priority: P0"], + ]) { + expect(shouldAnalyzeDocumentChangeGroup(analyzed(9n, [modified(1, 9n, before!, after!)]))).toMatchObject({ eligible: false }); + } + for (const [before, after] of [ + ["The launch remains planned for Q3.", "We still expect the launch during the third quarter."], + ["Product owns telemetry.", "Telemetry is owned by Product."], + ["ARR target is $1M.", "ARR target is one million dollars."], + ["The migration is planned.", "The migration is still planned."], + ]) { + expect(shouldAnalyzeDocumentChangeGroup(analyzed(10n, [modified(1, 10n, before!, after!)]))).toMatchObject({ eligible: true }); + } + }); + + it("selects at most four calls deterministically while retaining document diversity", () => { + const groups = [1n, 2n, 3n, 4n, 5n].flatMap((documentId, index) => [ + analyzed(documentId, [modified(1, documentId, "The workflow remains reliable.", "Reliability remains part of the workflow.")], index + 2), + analyzed(documentId, [modified(2, documentId, "The plan remains durable.", "Durability remains in the plan.", "/other")], index + 2), + ]); + const forward = selectDocumentChangeGroupsForMaterialityAnalysis(groups); + const reverse = selectDocumentChangeGroupsForMaterialityAnalysis([...groups].reverse()); + expect(forward.selected).toHaveLength(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview); + expect(new Set(forward.selected.map(group => group.group.documentId.toString())).size).toBe(4); + expect(reverse.selected.map(group => group.group.groupId)).toEqual(forward.selected.map(group => group.group.groupId)); + expect(forward.skipped).toHaveLength(6); + }); + + it("bounds canonical inputs and keeps only changed fragment data", () => { + const value = "A realistic synthetic operating paragraph. ".repeat(200); + const group = analyzed(1n, [modified(1, 1n, value, `${value} revised`)]); + const built = buildDocumentChangeMaterialityAnalysisInput(group); + expect(built.canonicalCharacterCount).toBeLessThanOrEqual(8_000); + expect(built.truncated).toBe(true); + expect(built.input.changes).toHaveLength(1); + expect(built.input.deterministicAssessment).toEqual(expect.objectContaining({ + groupShape: "modified", + semanticRisk: expect.any(String), + confirmedFactualDeltas: expect.any(Array), + possibleSignals: expect.any(Array), + })); + expect(JSON.stringify(built.input)).not.toContain("customer_feedback"); + expect(built.inputDigest).toMatch(/^[a-f0-9]{64}$/); + }); + + it("freezes material analysis, validates copied key points, and preserves provider-owned provenance boundaries", async () => { + const analyzer = new FakeAnalyzer(() => ({ + ...materialResult("Rollout scope expanded."), + beforeKeyPoint: "The pilot is limited to invited accounts.", + afterKeyPoint: "Invited accounts remain the pilot audience.", + })); + const result = await materializeDocumentChangesWithAnalyzer([ + input(1n, [modified(1, 1n, "The pilot is limited to invited accounts.", "Invited accounts remain the pilot audience.")]), + ], analyzer); + expect(analyzer.calls).toHaveLength(1); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ + sourceType: "document_change", + excerpt: expect.stringContaining("Rollout scope expanded."), + metadata: expect.objectContaining({ materialityMethod: "llm", category: "scope_change" }), + }); + expect(result.items[0]!.excerpt).toContain("The pilot is limited to invited accounts."); + expect(result.audit.groups[0]!.analysis).toEqual(expect.objectContaining({ + disposition: "material", + analysisProvider: "fixture", + analysisModel: "fixture-v1", + analysisInputDigest: expect.stringMatching(/^[a-f0-9]{64}$/), + })); + expect(JSON.stringify(result.audit)).not.toContain("rawProviderPayload"); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: result.items, + sourceWarnings: [], + documentChangeAudit: result.audit, + }); + const firstPrompt = buildFounderWeeklyReviewPrompt(snapshot); + const retryPrompt = buildFounderWeeklyReviewPrompt(snapshot); + expect(retryPrompt).toBe(firstPrompt); + expect(firstPrompt).not.toContain("analysisInputDigest"); + expect(analyzer.calls).toHaveLength(1); + }); + + it("keeps non-material groups audit-only and retains uncertain groups conservatively", async () => { + const nonMaterial = new FakeAnalyzer(() => ({ + disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.95, + })); + const pairInput = input(1n, [modified(1, 1n, "Product owns telemetry.", "Telemetry is owned by Product.")]); + const filtered = await materializeDocumentChangesWithAnalyzer([pairInput], nonMaterial); + expect(filtered.items).toHaveLength(0); + expect(filtered.audit.rawChanges).toHaveLength(1); + expect(filtered.audit.groups[0]).toMatchObject({ evidenceSourceId: null, analysis: { disposition: "non_material" } }); + + const uncertain = new FakeAnalyzer(() => ({ + disposition: "uncertain", category: "uncertain", summary: "Meaning may have changed.", confidence: 0.4, + })); + const retained = await materializeDocumentChangesWithAnalyzer([pairInput], uncertain); + expect(retained.items).toHaveLength(1); + expect(retained.items[0]!.metadata).toMatchObject({ materialityMethod: "deterministic", category: "uncertain" }); + expect(retained.audit.groups[0]!.analysis?.disposition).toBe("uncertain"); + }); + + it.each([ + ["unavailable", new DocumentChangeMaterialityAnalyzerError("unavailable", "offline"), "materiality_analyzer_unavailable"], + ["timeout", new DocumentChangeMaterialityAnalyzerError("timeout", "slow"), "materiality_analysis_timeout"], + ["invalid confidence", { disposition: "material", category: "scope_change", summary: "x", confidence: 2 }, "materiality_result_invalid"], + ["oversized summary", materialResult("x".repeat(321)), "materiality_result_invalid"], + ])("falls back deterministically for %s", async (_name, response, warningCode) => { + const pairInput = input(1n, [modified(1, 1n, "Telemetry is reliable.", "Reliability is a property of telemetry.")]); + const deterministic = materializeDocumentChanges([pairInput]); + const analyzer = new FakeAnalyzer(() => { + if (response instanceof Error) throw response; + return response; + }); + const result = await materializeDocumentChangesWithAnalyzer([pairInput], analyzer); + expect(result.items).toEqual(deterministic.items); + expect(result.warnings.map(warning => warning.code)).toEqual(expect.arrayContaining([warningCode, "materiality_analysis_partial"])); + }); + + it("keeps the strict concise-output boundary at the documented limits", () => { + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse(materialResult("x".repeat(320))).success).toBe(true); + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse(materialResult("x".repeat(321))).success).toBe(false); + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse({ + ...materialResult(), beforeKeyPoint: "b".repeat(240), afterKeyPoint: "a".repeat(240), + }).success).toBe(true); + expect(DocumentChangeMaterialityAnalysisResultSchema.safeParse({ + ...materialResult(), beforeKeyPoint: "b".repeat(241), + }).success).toBe(false); + }); + + it("isolates partial failures and never exceeds concurrency or call budgets", async () => { + let active = 0; + let maximumActive = 0; + const analyzer = new FakeAnalyzer(async value => { + active++; + maximumActive = Math.max(maximumActive, active); + await Promise.resolve(); + active--; + if (value.documentTitle?.endsWith("2")) throw new Error("synthetic failure"); + return materialResult(); + }); + const inputs = [1n, 2n, 3n, 4n, 5n].map((documentId, index) => input(documentId, [ + modified(1, documentId, "The workflow remains reliable.", "Reliability remains part of the workflow."), + ], index + 2)); + const result = await materializeDocumentChangesWithAnalyzer(inputs, analyzer); + expect(analyzer.calls).toHaveLength(4); + expect(maximumActive).toBeLessThanOrEqual(2); + expect(result.analyzerDiagnostics).toMatchObject({ selectedAnalysisCount: 4, skippedByCallBudgetCount: 1, failedAnalysisCount: 1 }); + expect(result.warnings.map(warning => warning.code)).toEqual(expect.arrayContaining([ + "materiality_analysis_budget_truncated", "materiality_analysis_partial", "materiality_analyzer_unavailable", + ])); + }); + + it("keeps evidence digest distinct from the exact prompt consequence", async () => { + const pairInput = input(1n, [modified(1, 1n, "Product owns telemetry.", "Telemetry is owned by Product.")]); + const deterministic = materializeDocumentChanges([pairInput]); + const analyzedResult = await materializeDocumentChangesWithAnalyzer([pairInput], new FakeAnalyzer(() => ({ + disposition: "non_material", category: "editorial_rewrite", summary: "Meaning is unchanged.", confidence: 0.94, + }))); + const snapshot = (items: typeof deterministic.items, audit: typeof deterministic.audit) => FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], + documentChangeAudit: audit, + }); + const before = snapshot(deterministic.items, deterministic.audit); + const after = snapshot(analyzedResult.items, analyzedResult.audit); + expect(buildFounderWeeklyReviewEvidenceDigest(after)).not.toBe(buildFounderWeeklyReviewEvidenceDigest(before)); + const promptHash = (value: typeof before) => createHash("sha256").update(buildFounderWeeklyReviewPrompt(value)).digest("hex"); + expect(promptHash(after)).not.toBe(promptHash(before)); + expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(after)).toEqual(after); + }); + + it("does not invoke an analyzer when the optional dependency is absent", async () => { + const pairInput = input(1n, [modified(1, 1n, "Telemetry is reliable.", "Reliability is a property of telemetry.")]); + const deterministic = materializeDocumentChanges([pairInput]); + const optional = await materializeDocumentChangesWithAnalyzer([pairInput]); + expect(optional.items).toEqual(deterministic.items); + expect(optional.audit).toEqual(deterministic.audit); + expect(optional.warnings).toEqual(deterministic.warnings); + }); + + it("keeps raw/group construction stable before semantic analysis", () => { + const versionPair = pair(); + const alignment = modified(1, 1n, "Before", "After"); + const raw = buildRawDocumentChanges(versionPair, [alignment]); + const grouped = groupRawDocumentChanges(versionPair, raw.rawChanges); + expect(grouped.groups[0]!.rawChanges.map(change => change.rawChangeId)).toEqual(raw.rawChanges.map(change => change.rawChangeId)); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts new file mode 100644 index 000000000..a8c9673c0 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change-materiality.test.ts @@ -0,0 +1,370 @@ +import { + FounderWeeklyReviewEvidenceSnapshotSchema, + analyzeDocumentChangeFactualDeltas, + analyzeDocumentChangeGroup, + buildCondensedDocumentChangeEvidence, + buildFounderWeeklyReviewEvidenceDigest, + buildFounderWeeklyReviewPrompt, + buildGenerationEvidenceEnvelope, + buildRawDocumentChanges, + generateFounderWeeklyReview, + groupRawDocumentChanges, + materializeDocumentChanges, + resolveDocumentChangeEvidenceAudit, + selectMaterialDocumentChangeGroups, + type AnalyzedDocumentChangeGroup, + type ChunkAlignment, + type DocumentChangeCategory, + type VersionChunk, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +const pair = (documentId = 1n, previousVersionId = 1, currentVersionId = 2, currentDate = "2026-02-02T00:00:00.000Z"): VersionPair => ({ + documentId, + documentTitle: `Document ${documentId}`, + documentCategory: "Product", + previousVersionId, + previousVersionNumber: previousVersionId, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), + currentVersionId, + currentVersionNumber: currentVersionId, + currentCreatedAt: new Date(currentDate), + currentChangelog: null, +}); + +const chunk = (id: number, versionId: bigint, content: string, overrides: Partial = {}): VersionChunk => ({ + chunkId: id, + versionId, + documentId: 1n, + content, + contentHash: null, + structureId: BigInt(id), + structurePath: `/section-${id}`, + structureTitle: `Section ${id}`, + structureOrdering: id, + pageNumber: id, + lineStart: id * 10, + lineEnd: id * 10 + 5, + ...overrides, +}); + +const modified = (id: number, before: string, after: string, overrides: Partial = {}): ChunkAlignment => ({ + changeType: "modified", + previousChunk: chunk(id, 1n, before, overrides), + currentChunk: chunk(10_000 + id, 2n, after, overrides), + alignmentMethod: "structure_path", +}); + +function groupFor(before: string, after: string, id = 1, versionPair = pair()) { + const raw = buildRawDocumentChanges(versionPair, [modified(id, before, after, { documentId: versionPair.documentId })]).rawChanges; + return groupRawDocumentChanges(versionPair, raw).groups[0]!; +} + +function analyzed(category: DocumentChangeCategory, documentId: bigint, index: number, currentDate = "2026-02-02T00:00:00.000Z"): AnalyzedDocumentChangeGroup { + const versionPair = pair(documentId, 1, 2, currentDate); + const group = groupFor(`before ${documentId}-${index}`, `after ${documentId}-${index}`, Number(documentId) * 100 + index, versionPair); + const priority = ["ownership_change", "status_change", "deadline_change", "metric_change", "requirement_change", "risk_or_blocker_change", "scope_change", "priority_change", "uncertain", "editorial_rewrite"].indexOf(category) + 1; + return { pair: versionPair, group, materiality: { category, priority, confidence: category === "uncertain" ? "uncertain" : "strong", signals: [`${category}_test`] } }; +} + +describe("deterministic document-change materiality", () => { + it.each([ + ["Product", "Platform", "ownership_change"], + ["Product owns retry telemetry", "Platform owns retry telemetry", "ownership_change"], + ["planned", "launched", "status_change"], + ["Q3", "Q4", "deadline_change"], + ["June 1", "July 15", "deadline_change"], + ["10% conversion", "25% conversion", "metric_change"], + ["$1M ARR", "$750k ARR", "metric_change"], + ["optional", "required", "requirement_change"], + ["may launch", "must launch", "requirement_change"], + ["blocked", "resolved", "risk_or_blocker_change"], + ["P2", "P0", "priority_change"], + ["US only", "global", "scope_change"], + ] as const)("classifies %s → %s as %s", (before, after, category) => { + expect(analyzeDocumentChangeGroup(groupFor(before, after))).toMatchObject({ category }); + }); + + it("uses explicit precedence when multiple signals occur", () => { + const result = analyzeDocumentChangeGroup(groupFor( + "Product owns a planned Q3 launch for 10% of customers", + "Platform owns a launched Q4 release for 25% of customers" + )); + expect(result).toMatchObject({ + category: "ownership_change", + priority: 1, + signals: expect.arrayContaining(["ownership_subject_changed", "status_term_changed", "date_or_deadline_changed", "numeric_metric_changed"]), + }); + }); + + it("keeps punctuation, capitalization, and paraphrase changes uncertain while retaining negation", () => { + expect(analyzeDocumentChangeGroup(groupFor("Launch.", "Launch!"))).toMatchObject({ category: "uncertain" }); + expect(analyzeDocumentChangeGroup(groupFor("Product roadmap", "product roadmap"))).toMatchObject({ category: "uncertain" }); + expect(analyzeDocumentChangeGroup(groupFor("Reliable export retries", "Export retries are reliable"))).toMatchObject({ category: "uncertain" }); + expect(analyzeDocumentChangeGroup(groupFor("supports exports", "does not support exports"))).toMatchObject({ + category: "requirement_change", + signals: expect.arrayContaining(["negation_changed"]), + }); + }); + + it.each([ + ["Q3", "Q4", "deadline", "changed"], + ["Q3", "third quarter", "deadline", "equivalent"], + ["Product owns telemetry.", "Platform owns telemetry.", "ownership", "changed"], + ["Product owns telemetry.", "Telemetry is owned by Product.", "ownership", "equivalent"], + ["10% conversion", "25% conversion", "metric", "changed"], + ["ARR target is $1M.", "ARR target is one million dollars.", "metric", "equivalent"], + ["The migration is planned.", "The migration is launched.", "status", "changed"], + ["The migration is planned.", "The migration is still planned.", "status", "equivalent"], + ] as const)("compares factual state in %s -> %s as %s %s", (before, after, kind, relation) => { + const group = groupFor(before, after); + const deterministic = analyzeDocumentChangeGroup(group); + const assessment = analyzeDocumentChangeFactualDeltas(group, deterministic.signals); + expect(assessment.factualComparisons).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind, relation }), + ])); + }); + + it("keeps one-sided business signals unconfirmed rather than manufacturing a delta", () => { + const versionPair = pair(); + const raw = buildRawDocumentChanges(versionPair, [{ + changeType: "added", + currentChunk: chunk(2, 2n, "Admins must enable SSO."), + alignmentMethod: "unmatched", + }]).rawChanges; + const group = groupRawDocumentChanges(versionPair, raw).groups[0]!; + const deterministic = analyzeDocumentChangeGroup(group); + const assessment = analyzeDocumentChangeFactualDeltas(group, deterministic.signals); + expect(assessment.confirmedFactualDeltas).toHaveLength(0); + expect(assessment.possibleSignals).toContain("requirement"); + expect(assessment.factualComparisons).toEqual(expect.arrayContaining([ + expect.objectContaining({ kind: "requirement", relation: "unknown" }), + ])); + }); + + it("assigns editorial only to a narrow deterministic formatting rewrite", () => { + expect(analyzeDocumentChangeGroup(groupFor("- First item", "* First item"))).toMatchObject({ + category: "editorial_rewrite", + confidence: "moderate", + }); + }); + + it("builds one stable bounded group item with copied spans and complete audit membership", () => { + const versionPair = pair(); + const alignments: ChunkAlignment[] = [ + modified(1, "Product owns retry telemetry.", "Platform owns retry telemetry.", { structurePath: "/ownership", structureTitle: "Ownership" }), + modified(2, "Product owns alerting.", "Platform owns alerting.", { structurePath: "/ownership", structureTitle: "Ownership" }), + { changeType: "removed", previousChunk: chunk(3, 1n, "Product owns paging.", { structurePath: "/ownership", structureTitle: "Ownership" }), alignmentMethod: "unmatched" }, + { changeType: "added", currentChunk: chunk(4, 2n, "Platform owns paging.", { structurePath: "/ownership", structureTitle: "Ownership" }), alignmentMethod: "unmatched" }, + ]; + const result = materializeDocumentChanges([{ pair: versionPair, alignments }]); + const item = result.items[0]!; + const auditGroup = result.audit.groups[0]!; + + expect(result.items).toHaveLength(1); + expect(item).toEqual(buildCondensedDocumentChangeEvidence(versionPair, result.selectedGroups[0]!.group, result.selectedGroups[0]!.materiality)); + expect(item.sourceId).toMatch(/^document_change:group:[a-f0-9]{64}$/); + expect(item.metadata).toMatchObject({ category: "ownership_change", materialityMethod: "deterministic", rawChangeCount: 4 }); + expect(item.excerpt).toContain("Product owns retry telemetry."); + expect(item.excerpt).toContain("Platform owns retry telemetry."); + expect(item.excerpt).toContain("Section changed across 4 source fragments."); + expect(item.excerpt.length).toBeLessThanOrEqual(1800); + expect(auditGroup.rawChangeIds).toHaveLength(4); + expect(auditGroup.evidenceSourceId).toBe(item.sourceId); + expect(result.audit.rawChanges).toHaveLength(4); + expect(new Set(auditGroup.rawChangeIds)).toEqual(new Set(result.audit.rawChanges.map(change => change.rawChangeId))); + expect(result.audit.rawChanges.find(change => change.changeType === "modified")).toEqual(expect.objectContaining({ + documentId: "1", previousVersionId: 1, currentVersionId: 2, + alignmentMethod: expect.any(String), processingVersion: "raw-document-change/v1", + previousHash: expect.stringMatching(/^[a-f0-9]{64}$/), + })); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, workspaceTimezone: "UTC", + items: result.items, sourceWarnings: [], documentChangeAudit: result.audit, + }); + const promptItem = buildGenerationEvidenceEnvelope(snapshot).items[0]!; + expect(promptItem.metadata).toMatchObject({ category: "ownership_change", materialityMethod: "deterministic", materialityConfidence: "strong", structureTitle: "Ownership", rawChangeCount: 4 }); + expect(promptItem.metadata).not.toHaveProperty("groupId"); + }); + + it("preserves concise single-change before/after semantics", () => { + const versionPair = pair(); + const result = materializeDocumentChanges([{ pair: versionPair, alignments: [modified(1, "planned", "shipped")] }]); + expect(result.items).toHaveLength(1); + expect(result.items[0]).toMatchObject({ sourceType: "document_change", metadata: expect.objectContaining({ category: "status_change", rawChangeCount: 1 }) }); + expect(result.items[0]!.excerpt).toContain("Before:\n- planned"); + expect(result.items[0]!.excerpt).toContain("After:\n- shipped"); + }); + + it("selects by materiality within documents while preserving document diversity and final chronology", () => { + const candidates: AnalyzedDocumentChangeGroup[] = []; + for (const documentId of [1n, 2n, 3n, 4n]) { + candidates.push(analyzed("ownership_change", documentId, 1, "2026-02-09T00:00:00.000Z")); + candidates.push(analyzed("status_change", documentId, 2, "2026-02-08T00:00:00.000Z")); + candidates.push(analyzed("deadline_change", documentId, 3, "2026-02-07T00:00:00.000Z")); + candidates.push(analyzed("metric_change", documentId, 4, "2026-02-06T00:00:00.000Z")); + candidates.push(analyzed("requirement_change", documentId, 5, "2026-02-05T00:00:00.000Z")); + candidates.push(analyzed("uncertain", documentId, 6, "2026-02-04T00:00:00.000Z")); + candidates.push(analyzed("editorial_rewrite", documentId, 7, "2026-02-03T00:00:00.000Z")); + } + const selected = selectMaterialDocumentChangeGroups([...candidates].reverse()); + const counts = new Map(); + for (const entry of selected.selectedGroups) counts.set(entry.group.documentId.toString(), (counts.get(entry.group.documentId.toString()) ?? 0) + 1); + + expect(selected.selectedGroups).toHaveLength(24); + expect([...counts.values()]).toEqual([6, 6, 6, 6]); + expect(selected.selectedGroups.some(entry => entry.materiality.category === "editorial_rewrite")).toBe(false); + expect(selected.selectedGroups.filter(entry => entry.materiality.category === "ownership_change")).toHaveLength(4); + expect(selected.selectedGroups.map(entry => entry.pair.currentCreatedAt.getTime())).toEqual( + [...selected.selectedGroups].map(entry => entry.pair.currentCreatedAt.getTime()).sort((a, b) => a - b) + ); + }); + + it("enforces eight per document and pair and prefers newer equal-category events", () => { + const candidates = Array.from({ length: 9 }, (_, index) => analyzed( + "uncertain", + 1n, + index + 1, + `2026-02-${String(index + 1).padStart(2, "0")}T00:00:00.000Z` + )); + const selected = selectMaterialDocumentChangeGroups(candidates); + expect(selected.selectedGroups).toHaveLength(8); + expect(selected.truncatedGroups).toHaveLength(1); + expect(selected.truncatedGroups[0]!.pair.currentCreatedAt.toISOString()).toBe("2026-02-01T00:00:00.000Z"); + expect(selected.warnings).toContainEqual(expect.objectContaining({ code: "document_change_budget_truncated" })); + }); + + it("condenses the 40-chunk enterprise scenario while retaining every raw audit record", () => { + const versionPair = pair(44n); + const alignments: ChunkAlignment[] = []; + for (let index = 0; index < 20; index++) { + const content = `unchanged source fragment ${index}`; + alignments.push({ + changeType: "unchanged", + previousChunk: chunk(index + 1, 1n, content, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}` }), + currentChunk: chunk(1001 + index, 2n, content, { documentId: 44n, pageNumber: index + 1, structurePath: `/unchanged-${index}` }), + alignmentMethod: "content_hash", + }); + } + for (let index = 0; index < 5; index++) alignments.push(modified(100 + index, `wrapped line ${index}`, ` wrapped\r\n line\u00a0${index} `, { documentId: 44n, structurePath: `/noop-${index}` })); + alignments.push(modified(201, "Product owns retry telemetry.", "Platform owns retry telemetry.", { documentId: 44n, structurePath: "/ownership", structureTitle: "Ownership" })); + alignments.push(modified(202, "Launch deadline is Q3.", "Launch deadline is Q4.", { documentId: 44n, structurePath: "/deadline", structureTitle: "Deadline" })); + alignments.push(modified(203, "The release is planned.", "The release is launched.", { documentId: 44n, structurePath: "/status", structureTitle: "Status" })); + alignments.push(modified(204, "Retries are optional.", "Retries are required.", { documentId: 44n, structurePath: "/requirements", structureTitle: "Requirements" })); + const long = " Detailed source wording about the operating model and customer rollout remains copied for audit verification.".repeat(4); + for (let index = 0; index < 6; index++) alignments.push(modified(300 + index, `Old rewrite fragment ${index}.${long}`, `Rephrased rewrite fragment ${index}.${long}`, { documentId: 44n, structurePath: "/rewrite", structureTitle: "Large rewrite" })); + for (let index = 0; index < 5; index++) alignments.push(modified(400 + index, `- Editorial source fragment ${index}.${long}`, `* Editorial source fragment ${index}.${long}`, { documentId: 44n, structurePath: "/editorial", structureTitle: "Editorial" })); + + const result = materializeDocumentChanges([{ pair: versionPair, alignments }]); + const shuffled = materializeDocumentChanges([{ pair: versionPair, alignments: [...alignments].reverse() }]); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: result.items, + sourceWarnings: [], + documentChangeAudit: result.audit, + }); + const envelope = buildGenerationEvidenceEnvelope(snapshot); + const prompt = buildFounderWeeklyReviewPrompt(snapshot, envelope); + + expect(result.diagnostics).toMatchObject({ alignedChunkCount: 40, rawModifiedCount: 15, deterministicNoOpCount: 5, groupCount: 6, selectedGroupCount: 6, rawAuditCount: 15, condensedEvidenceCount: 6 }); + expect(result.analyzedGroups.map(group => group.materiality.category)).toEqual(expect.arrayContaining(["ownership_change", "deadline_change", "status_change", "requirement_change", "uncertain", "editorial_rewrite"])); + expect(result.audit.rawChanges).toHaveLength(15); + expect(result.items).toHaveLength(6); + expect(result.selectedGroups.map(entry => entry.group.groupId)).toEqual(shuffled.selectedGroups.map(entry => entry.group.groupId)); + expect(result.diagnostics.rawExcerptCharacters).toBeGreaterThan(result.diagnostics.condensedPromptFacingCharacters * 1.5); + expect(result.diagnostics.estimatedReductionRatio).toBeGreaterThan(1.5); + expect(result.diagnostics.rawExcerptCharacters).toBe(10_484); + expect(result.diagnostics.condensedPromptFacingCharacters).toBe(2_311); + expect(envelope.diagnostics.serializedCharacterCount).toBe(5_466); + expect(envelope.diagnostics.serializedCharacterCount).toBeLessThanOrEqual(14_000); + expect(prompt).not.toContain("rawChanges"); + expect(prompt).not.toContain("documentChangeAudit"); + }); +}); + +describe("Founder Weekly Review evidence snapshot v2", () => { + const v1 = { + schemaVersion: "founder-weekly-review-evidence/v1" as const, + capturedAt: "2026-02-28T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-02-28" }, + workspaceTimezone: "UTC", + items: [], + sourceWarnings: [], + }; + + it("parses v1 and v2 without mutating v1", () => { + const result = materializeDocumentChanges([{ pair: pair(), alignments: [modified(1, "planned", "launched")] }]); + const v2 = { ...v1, schemaVersion: "founder-weekly-review-evidence/v2" as const, items: result.items, documentChangeAudit: result.audit }; + expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(v1)).toEqual(v1); + expect(FounderWeeklyReviewEvidenceSnapshotSchema.parse(v2)).toEqual(v2); + expect(v1).not.toHaveProperty("documentChangeAudit"); + const parsedV2 = FounderWeeklyReviewEvidenceSnapshotSchema.parse(v2); + expect(resolveDocumentChangeEvidenceAudit(parsedV2, result.items[0]!.sourceId)?.rawChanges).toHaveLength(1); + if (parsedV2.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Expected v2 snapshot"); + const baseDigest = buildFounderWeeklyReviewEvidenceDigest(parsedV2); + const provenanceChanged = structuredClone(parsedV2); + provenanceChanged.documentChangeAudit.rawChanges[0]!.previousExcerpt = "different copied provenance"; + const materialityChanged = structuredClone(parsedV2); + materialityChanged.documentChangeAudit.groups[0]!.category = "uncertain"; + materialityChanged.documentChangeAudit.groups[0]!.priority = 9; + const evidenceChanged = structuredClone(parsedV2); + evidenceChanged.items[0]!.excerpt = "different condensed evidence"; + expect(buildFounderWeeklyReviewEvidenceDigest(provenanceChanged)).not.toBe(baseDigest); + expect(buildFounderWeeklyReviewEvidenceDigest(materialityChanged)).not.toBe(baseDigest); + expect(buildFounderWeeklyReviewEvidenceDigest(evidenceChanged)).not.toBe(baseDigest); + expect(buildFounderWeeklyReviewPrompt(provenanceChanged)).toBe(buildFounderWeeklyReviewPrompt(parsedV2)); + }); + + it("keeps audit text out of the prompt and digests the complete snapshot", () => { + const auditSecret = "AUDIT_ONLY_RAW_SOURCE_TEXT"; + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + ...v1, + schemaVersion: "founder-weekly-review-evidence/v2", + documentChangeAudit: { + schemaVersion: "document-change-audit/v1", + rawChanges: [{ + rawChangeId: `raw_document_change:${"a".repeat(64)}`, changeType: "removed", alignmentMethod: "unmatched", + documentId: "1", previousVersionId: 1, currentVersionId: 2, previousChunkId: 1, currentChunkId: null, + previousExcerpt: auditSecret, currentExcerpt: null, previousHash: "b".repeat(64), currentHash: null, + previousStructurePath: "/secret", currentStructurePath: null, previousStructureTitle: "Secret", currentStructureTitle: null, + previousPageNumber: 1, currentPageNumber: null, previousLineStart: 1, previousLineEnd: 2, currentLineStart: null, currentLineEnd: null, + processingVersion: "raw-document-change/v1", + }], + groups: [{ + groupId: `document_change_group:${"c".repeat(64)}`, evidenceSourceId: null, documentId: "1", previousVersionId: 1, currentVersionId: 2, + structurePath: "/secret", structureTitle: "Secret", splitOrdinal: 0, rawChangeIds: [`raw_document_change:${"a".repeat(64)}`], + category: "uncertain", priority: 9, confidence: "uncertain", signals: ["no_strong_deterministic_signal"], + materialityMethod: "deterministic", materialityVersion: "document-change-materiality/v1", + }], + }, + }); + const prompt = buildFounderWeeklyReviewPrompt(snapshot); + const firstDigest = buildFounderWeeklyReviewEvidenceDigest(snapshot); + const identicalDigest = buildFounderWeeklyReviewEvidenceDigest(FounderWeeklyReviewEvidenceSnapshotSchema.parse(JSON.parse(JSON.stringify(snapshot)))); + const changed = structuredClone(snapshot); + if (changed.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Expected v2 snapshot"); + changed.documentChangeAudit.groups[0]!.rawChangeIds = [`raw_document_change:${"d".repeat(64)}`]; + + expect(prompt).not.toContain(auditSecret); + expect(prompt).not.toContain("documentChangeAudit"); + expect(identicalDigest).toBe(firstDigest); + expect(buildFounderWeeklyReviewEvidenceDigest(changed)).not.toBe(firstDigest); + }); + + it("generates from v2 without a provider when prompt evidence is empty and records distinct digest metadata", async () => { + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + ...v1, + schemaVersion: "founder-weekly-review-evidence/v2", + documentChangeAudit: { schemaVersion: "document-change-audit/v1", rawChanges: [], groups: [] }, + }); + const generate = jest.fn(); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: snapshot, generate }); + expect(generate).not.toHaveBeenCalled(); + expect(result.modelMetadata.evidenceSchemaVersion).toBe("founder-weekly-review-evidence/v2"); + expect(result.modelMetadata.attributes.evidenceDigest).toBe(buildFounderWeeklyReviewEvidenceDigest(snapshot)); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-change.test.ts b/apps/web/__tests__/founderWeeklyReview/document-change.test.ts new file mode 100644 index 000000000..e7e85ac9f --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-change.test.ts @@ -0,0 +1,89 @@ +import { + alignVersionChunks, + buildDocumentChangeEvidence, + selectVersionPairsForReportingPeriod, + FounderWeeklyReviewEvidenceService, + FounderWeeklyReviewEvidenceItemSchema, + type DocumentVersionForComparison, + type VersionChunk, +} from "@launchstack/features/founder-weekly-review"; + +const version = (documentId: bigint, versionId: number, versionNumber: number, createdAt: string): DocumentVersionForComparison => ({ + documentId, documentTitle: `Document ${documentId}`, documentCategory: "Product", versionId, versionNumber, + createdAt: new Date(createdAt), changelog: versionId === 2 ? "Founder supplied note" : null, +}); +const chunk = (chunkId: number, versionId: bigint, content: string, overrides: Partial = {}): VersionChunk => ({ + chunkId, versionId, documentId: 1n, content, contentHash: null, structureId: 1n, structurePath: "/1", structureTitle: "Overview", + structureOrdering: 1, pageNumber: 1, lineStart: 1, lineEnd: 2, ...overrides, +}); + +describe("Week 3 document change domain logic", () => { + it("forms an adjacent chain including the predecessor before the period", () => { + const pairs = selectVersionPairsForReportingPeriod([ + version(1n, 1, 1, "2026-01-01T00:00:00.000Z"), version(1n, 3, 3, "2026-02-03T00:00:00.000Z"), + version(2n, 4, 1, "2026-02-02T00:00:00.000Z"), version(1n, 2, 2, "2026-02-02T00:00:00.000Z"), + ], new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-10T00:00:00.000Z")); + expect(pairs.map((pair) => [pair.documentId, pair.previousVersionId, pair.currentVersionId])).toEqual([[1n, 1, 2], [1n, 2, 3]]); + }); + + it("does not invent a predecessor for a first-ever in-period version and resolves timestamp ties deterministically", () => { + const pairs = selectVersionPairsForReportingPeriod([ + version(1n, 2, 2, "2026-02-02T00:00:00.000Z"), version(1n, 1, 1, "2026-02-02T00:00:00.000Z"), version(2n, 3, 1, "2026-02-02T00:00:00.000Z"), + ], new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z")); + expect(pairs.map((pair) => [pair.previousVersionId, pair.currentVersionId])).toEqual([[1, 2]]); + }); + + it("aligns once, classifies changes, and leaves exact hash matches unchanged", () => { + const hash = "a".repeat(64); + const prior = [chunk(1, 1n, "same", { contentHash: hash, structurePath: "/x" }), chunk(2, 1n, "old", { structurePath: "/y" }), chunk(3, 1n, "removed", { structurePath: "/z", structureTitle: "Old only" })]; + const current = [chunk(4, 2n, "same", { contentHash: hash, structurePath: "/other" }), chunk(5, 2n, "new", { structurePath: "/y" }), chunk(6, 2n, "added", { structurePath: "/new", structureTitle: "New only" })]; + const alignments = alignVersionChunks(prior, current); + expect(alignments.map((item) => [item.changeType, item.alignmentMethod])).toEqual([["added", "unmatched"], ["unchanged", "content_hash"], ["modified", "structure_path"], ["removed", "unmatched"]]); + expect(new Set(alignments.flatMap((item) => [item.previousChunk?.chunkId, item.currentChunk?.chunkId].filter(Boolean))).size).toBe(6); + }); + + it("does not trust malformed or contradictory matching hashes", () => { + const alignments = alignVersionChunks([chunk(1, 1n, "before", { contentHash: "bad", structurePath: "/same" })], [chunk(2, 2n, "after", { contentHash: "bad", structurePath: "/same" })]); + expect(alignments).toEqual([expect.objectContaining({ changeType: "modified", alignmentMethod: "structure_path" })]); + }); + + it("keeps maximum-size identifiers JSON-safe and within the evidence contract", () => { + const pair = { documentId: 99999999999999999999n, documentTitle: "D", documentCategory: null, previousVersionId: 2147483646, previousVersionNumber: 2147483646, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), currentVersionId: 2147483647, currentVersionNumber: 2147483647, currentCreatedAt: new Date("2026-02-02T00:00:00.000Z"), currentChangelog: "x" }; + const item = buildDocumentChangeEvidence(pair, [{ changeType: "added", currentChunk: chunk(2147483647, 2147483647n, "content"), alignmentMethod: "unmatched" }])[0]!; + expect(item.sourceId.length).toBeLessThanOrEqual(256); + expect(() => JSON.stringify(item)).not.toThrow(); + expect(FounderWeeklyReviewEvidenceItemSchema.safeParse(item).success).toBe(true); + }); + + it("creates stable computed evidence and keeps changelog separate", () => { + const pair = selectVersionPairsForReportingPeriod([version(1n, 1, 1, "2026-01-01T00:00:00.000Z"), version(1n, 2, 2, "2026-02-02T00:00:00.000Z")], new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z"))[0]!; + const evidence = buildDocumentChangeEvidence(pair, alignVersionChunks([chunk(10, 1n, "Before", { structurePath: "/plan" })], [chunk(20, 2n, "After", { structurePath: "/plan" })])); + expect(evidence).toEqual(buildDocumentChangeEvidence(pair, alignVersionChunks([chunk(10, 1n, "Before", { structurePath: "/plan" })], [chunk(20, 2n, "After", { structurePath: "/plan" })]))); + expect(evidence).toEqual([expect.objectContaining({ sourceId: "document_change:doc:1:v1:v2:chunk:10:20", excerpt: expect.stringContaining("Before: Before After: After"), metadata: expect.objectContaining({ changeType: "modified", userChangelog: "Founder supplied note", previousVersionId: 1, currentVersionId: 2 }) })]); + }); + + it("uses bounded deterministic text similarity only after structural strategies fail", () => { + const alignments = alignVersionChunks([chunk(1, 1n, "reliable export retry job", { structurePath: "/old", structureTitle: "Old" })], [chunk(2, 2n, "export retry job is now reliable", { structurePath: "/new", structureTitle: "New" })]); + expect(alignments).toEqual([expect.objectContaining({ changeType: "modified", alignmentMethod: "text_similarity" })]); + }); + + it("does not use text similarity to pair short generic fragments", () => { + const alignments = alignVersionChunks( + [chunk(1, 1n, "status update", { structurePath: "/old", structureTitle: "Old" })], + [chunk(2, 2n, "status update", { structurePath: "/new", structureTitle: "New" })] + ); + expect(alignments.map((item) => item.changeType)).toEqual(["added", "removed"]); + }); + + it("collects one computed change for the controlled v1-to-v2 case", async () => { + const versions = [version(1n, 1, 1, "2026-01-01T00:00:00.000Z"), version(1n, 2, 2, "2026-02-02T00:00:00.000Z")]; + const store = { listVersionsBeforePeriodEnd: jest.fn().mockResolvedValue(versions), getDocumentChunksForVersion: jest.fn() + .mockResolvedValueOnce({ state: "complete", chunks: [chunk(10, 1n, "Before", { structurePath: "/plan" })], warnings: [] }) + .mockResolvedValueOnce({ state: "complete", chunks: [chunk(20, 2n, "After", { structurePath: "/plan" })], warnings: [] }) }; + const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "computed", store }); + await expect(service.collectDocumentChangeEvidence(1n, new Date("2026-02-01T00:00:00.000Z"), new Date("2026-02-03T00:00:00.000Z"))).resolves.toEqual([ + expect.objectContaining({ sourceType: "document_change", sourceTimestamp: "2026-02-02T00:00:00.000Z", sourceId: expect.stringMatching(/^document_change:group:/), metadata: expect.objectContaining({ previousVersionId: 1, currentVersionId: 2, category: "uncertain", rawChangeCount: 1 }) }), + ]); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts b/apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts new file mode 100644 index 000000000..b23eafd6d --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/document-version-chunks.test.ts @@ -0,0 +1,34 @@ +import { company, document, documentContextChunks, documentVersions } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; + +describeDb("explicit document version chunks", () => { + it("loads only owned explicit-version chunks in deterministic order", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [firstCompany] = await test.db.insert(company).values({ name: "First", numberOfEmployees: "1" }).returning(); + const [secondCompany] = await test.db.insert(company).values({ name: "Second", numberOfEmployees: "1" }).returning(); + const [doc] = await test.db.insert(document).values({ companyId: BigInt(firstCompany!.id), url: "local://doc", category: "Product", title: "Plan" }).returning(); + const [otherDoc] = await test.db.insert(document).values({ companyId: BigInt(secondCompany!.id), url: "local://other", category: "Product", title: "Other" }).returning(); + const [v1] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: "local://v1", mimeType: "text/plain" }).returning(); + const [v2] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 2, url: "local://v2", mimeType: "text/plain" }).returning(); + const [v3] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 3, url: "local://v3", mimeType: "text/plain" }).returning(); + const [otherVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(otherDoc!.id), versionNumber: 1, url: "local://other-v1", mimeType: "text/plain" }).returning(); + await test.db.insert(documentContextChunks).values([ + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "page two", contentHash: "b", tokenCount: 1, charCount: 8, pageNumber: 2 }, + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "page one", contentHash: "a", tokenCount: 1, charCount: 8, pageNumber: 1 }, + { documentId: BigInt(doc!.id), versionId: BigInt(v1!.id), content: "old", contentHash: "old", tokenCount: 1, charCount: 3, pageNumber: 1 }, + { documentId: BigInt(doc!.id), versionId: null, content: "legacy", contentHash: "legacy", tokenCount: 1, charCount: 6, pageNumber: 0 }, + ]); + const store = new FounderWeeklyReviewDocumentVersionStore(test.db); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: v2!.id })).resolves.toMatchObject({ state: "partial", chunks: [{ content: "page one" }, { content: "page two" }] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: v1!.id })).resolves.toMatchObject({ state: "partial", chunks: [{ content: "old" }] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: v3!.id })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["version_chunks_missing"] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(secondCompany!.id), documentId: BigInt(doc!.id), versionId: v2!.id })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: otherVersion!.id })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }); + await expect(store.getDocumentChunksForVersion({ companyId: BigInt(firstCompany!.id), documentId: BigInt(doc!.id), versionId: 999_999 })).resolves.toEqual({ state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }); + } finally { await test.close(); } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts b/apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts new file mode 100644 index 000000000..c6a7c2579 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evaluation-fixtures.test.ts @@ -0,0 +1,65 @@ +import { + FounderWeeklyReviewEvidenceSnapshotSchema, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewV2Payload, + generateFounderWeeklyReview, +} from "@launchstack/features/founder-weekly-review"; + +const period = { start: "2026-07-06", end: "2026-07-12" }; +function fixture(items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderWeeklyReviewEvidenceSnapshot { + return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", + reportingPeriod: period, workspaceTimezone: "UTC", items, sourceWarnings: [], + }); +} +export const completeWorkspaceFixture = fixture([ + { sourceType: "document_change", sourceId: "doc:release", title: "Release notes", sourceTimestamp: "2026-07-10T00:00:00.000Z", excerpt: "Export shipped.", metadata: {} }, + { sourceType: "customer_feedback", sourceId: "feedback:1", title: "Customer call", excerpt: "Audit logs requested.", metadata: {} }, + { sourceType: "founder_context", sourceId: "context:1", title: "Founder context", excerpt: "SSO remains blocked.", metadata: {} }, +]); +export const partialWorkspaceFixture = fixture([ + { sourceType: "document_change", sourceId: "doc:release", title: "Release notes", sourceTimestamp: "2026-07-10T00:00:00.000Z", excerpt: "Export shipped.", metadata: {} }, + { sourceType: "founder_context", sourceId: "context:1", title: "Founder context", excerpt: "A founder heard customers ask about audit logs.", metadata: {} }, +]); +export const emptyWorkspaceFixture = fixture([]); + +function payload(customerEvidence: boolean): FounderWeeklyReviewV2Payload { + const noEvidence = { state: "no_evidence" as const, noEvidence: { code: "none", message: "No evidence", cta: "Add evidence" } }; + return { schemaVersion: "founder-weekly-review/v2", sections: { + whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "Exports shipped.", sourceIds: ["doc:release"], confidence: 0.9 }] }, + whatShipped: noEvidence, + whatCustomersSaid: customerEvidence ? { state: "evidence", items: [{ kind: "observed_fact", text: "Audit logs were requested.", sourceIds: ["feedback:1"], confidence: 0.8 }] } : noEvidence, + currentBlockers: noEvidence, + nextPriorities: noEvidence, + }}; +} +function assertGrounded(review: FounderWeeklyReviewV2Payload, snapshot: FounderWeeklyReviewEvidenceSnapshot) { + const ids = new Set(snapshot.items.map((item) => item.sourceId)); + for (const section of [review.sections.whatChanged, review.sections.whatShipped, review.sections.whatCustomersSaid, review.sections.currentBlockers]) { + if (section.state === "evidence") for (const item of section.items) { + expect(item.sourceIds.length).toBeGreaterThan(0); + for (const id of item.sourceIds) expect(ids.has(id)).toBe(true); + } + } +} + +describe("LAU-9 grounding evaluation fixtures", () => { + it("grounds the complete fixture, including customer attribution", async () => { + const generate = jest.fn().mockResolvedValue({ object: payload(true), metadata: { provider: "test", model: "fixed", capability: "founderWeeklyReview", temperature: 0 } }); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeWorkspaceFixture, generate }); + assertGrounded(result.reviewPayload, completeWorkspaceFixture); + expect(result.reviewPayload.sections.whatCustomersSaid).toMatchObject({ state: "evidence" }); + }); + it("keeps the partial fixture customer section at explicit no-evidence", async () => { + const generate = jest.fn().mockResolvedValue({ object: payload(false), metadata: { provider: "test", model: "fixed", capability: "founderWeeklyReview", temperature: 0 } }); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: partialWorkspaceFixture, generate }); + assertGrounded(result.reviewPayload, partialWorkspaceFixture); + expect(result.reviewPayload.sections.whatCustomersSaid.state).toBe("no_evidence"); + }); + it("completes the empty fixture without calling a provider or inventing facts", async () => { + const generate = jest.fn(); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: emptyWorkspaceFixture, generate }); + expect(generate).not.toHaveBeenCalled(); + for (const section of Object.values(result.reviewPayload.sections)) expect(section.state).toBe("no_evidence"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/evaluation.test.ts b/apps/web/__tests__/founderWeeklyReview/evaluation.test.ts new file mode 100644 index 000000000..137fc07f9 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evaluation.test.ts @@ -0,0 +1,81 @@ +import { evaluateFounderWeeklyReview, buildFounderWeeklyReviewEvaluationPrompt } from "@launchstack/features/founder-weekly-review"; +import type { FounderWeeklyReviewEvidenceSnapshot, FounderWeeklyReviewV2Payload } from "@launchstack/features/founder-weekly-review"; + +const snapshot = (items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderWeeklyReviewEvidenceSnapshot => ({ schemaVersion: "founder-weekly-review-evidence/v2", capturedAt: "2026-01-01T00:00:00.000Z", reportingPeriod: { start: "2026-01-01", end: "2026-01-07" }, workspaceTimezone: "UTC", items, sourceWarnings: [], documentChangeAudit: { schemaVersion: "document-change-audit/v1", rawChanges: [], groups: [] } }); +const payload = (whatCustomersSaid: FounderWeeklyReviewV2Payload["sections"]["whatCustomersSaid"], whatChanged: FounderWeeklyReviewV2Payload["sections"]["whatChanged"]): FounderWeeklyReviewV2Payload => ({ schemaVersion: "founder-weekly-review/v2", sections: { whatChanged, whatShipped: { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, whatCustomersSaid, currentBlockers: { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, nextPriorities: { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } } } }); +const change = { sourceType: "document_change" as const, sourceId: "change-1", title: "Launch", excerpt: "The launch shipped.", metadata: {} }; +const workspace = { sourceType: "workspace_document" as const, sourceId: "workspace-1", title: "Current", excerpt: "Current blocker context.", metadata: {} }; +const feedback = { sourceType: "customer_feedback" as const, sourceId: "feedback-1", title: "Interview", excerpt: "Customers requested reliable recovery.", metadata: {} }; + +describe("Founder Weekly Review evaluation", () => { + it("enforces customer-only attribution and temporal workspace semantics", () => { + const evidence = snapshot([change, workspace, feedback]); + const invalid = payload({ state: "evidence", items: [{ kind: "observed_fact", text: "Customers requested reliable recovery.", sourceIds: ["workspace-1"], confidence: 0.8 }] }, { state: "evidence", items: [{ kind: "observed_fact", text: "A current document changed this week.", sourceIds: ["workspace-1"], confidence: 0.8 }] }); + const result = evaluateFounderWeeklyReview(evidence, invalid); + expect(result.hasHardFailure).toBe(true); + expect(result.failures.map(failure => failure.category)).toEqual(expect.arrayContaining(["invalid_source_type"])); + }); + + it("allows workspace context alongside temporal evidence", () => { + const evidence = snapshot([change, workspace, feedback]); + const valid = payload({ state: "evidence", items: [{ kind: "observed_fact", text: "Customers requested reliable recovery.", sourceIds: ["feedback-1"], confidence: 0.8 }] }, { state: "evidence", items: [{ kind: "observed_fact", text: "The launch shipped with current blocker context.", sourceIds: ["change-1", "workspace-1"], confidence: 0.8 }] }); + expect(evaluateFounderWeeklyReview(evidence, valid).hasHardFailure).toBe(false); + }); + + it("flags a future release as not shipped in the reporting period", () => { + const evidence = snapshot([{ ...change, excerpt: "Status changed. Before: planned for the April release. After: launched in the April release.", sourceTimestamp: "2026-02-20T10:00:00.000Z" }, workspace]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }); + review.sections.whatShipped = { state: "evidence", items: [{ kind: "observed_fact", text: "Recovery workflow launched in the April release.", sourceIds: ["change-1", "workspace-1"], confidence: 0.8 }] }; + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.failures).toEqual(expect.arrayContaining([expect.objectContaining({ category: "future_release_claim" })])); + expect(result.hasHardFailure).toBe(true); + }); + + it("rejects workspace-only temporal claims but permits workspace context with temporal evidence", () => { + const evidence = snapshot([{ ...change, excerpt: "The workflow shipped during the reporting period." }, workspace]); + const invalid = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "The current document changed this week.", sourceIds: ["workspace-1"], confidence: 0.8 }] }); + expect(evaluateFounderWeeklyReview(evidence, invalid).failures).toEqual(expect.arrayContaining([expect.objectContaining({ category: "invalid_source_type" })])); + const valid = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "The workflow shipped this period and remains current.", sourceIds: ["change-1", "workspace-1"], confidence: 0.8 }] }); + expect(evaluateFounderWeeklyReview(evidence, valid).failures).not.toEqual(expect.arrayContaining([expect.objectContaining({ category: "invalid_source_type" })])); + }); + + it("measures theme coverage rather than requiring one bullet per source", () => { + const evidence = snapshot([{ ...change, sourceId: "change-1" }, { ...change, sourceId: "change-2", title: "Scope", excerpt: "Rollout expanded globally." }, feedback]); + const review = payload({ state: "evidence", items: [{ kind: "observed_fact", text: "Customers requested reliable recovery.", sourceIds: ["feedback-1"], confidence: 0.8 }] }, { state: "evidence", items: [{ kind: "observed_fact", text: "The launch shipped and rollout expanded globally.", sourceIds: ["change-1", "change-2"], confidence: 0.8 }] }); + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.deterministic.evidenceCoverage).toBe(1); + expect(result.deterministic.duplicateClaimRate).toBe(0); + }); + + it("allows section-specific reuse of the same evidence", () => { + const evidence = snapshot([change]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "The launch shipped.", sourceIds: ["change-1"], confidence: 0.8 }] }); + review.sections.currentBlockers = { state: "evidence", items: [{ kind: "observed_fact", text: "The shipped launch still has a current blocker.", sourceIds: ["change-1"], confidence: 0.7 }] }; + expect(evaluateFounderWeeklyReview(evidence, review).deterministic.duplicateClaimRate).toBe(0); + }); + + it("accepts an evidence-grounded action without requiring invented owner or deadline", () => { + const evidence = snapshot([change]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }); + review.sections.nextPriorities = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Confirm the launch readiness gap.", rationale: "The release evidence indicates the gap remains relevant.", sourceIds: ["change-1"], confidence: 0.7 }] }; + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.failures.filter(failure => failure.category === "unsupported_claim")).toHaveLength(0); + }); + + it("keeps lexical support as a soft diagnostic rather than a hard gate", () => { + const evidence = snapshot([change]); + const review = payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "evidence", items: [{ kind: "observed_fact", text: "A release occurred.", sourceIds: ["change-1"], confidence: 0.8 }] }); + const result = evaluateFounderWeeklyReview(evidence, review); + expect(result.hasHardFailure).toBe(false); + expect(result.deterministic.unsupportedClaimRate).toBeGreaterThanOrEqual(0); + }); + + it("builds a rubric-aware semantic grading prompt", () => { + const prompt = buildFounderWeeklyReviewEvaluationPrompt(snapshot([change]), payload({ state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } }, { state: "no_evidence", noEvidence: { code: "none", message: "None", cta: "None" } })); + expect(prompt).toContain("document_change is reporting-period temporal evidence"); + expect(prompt).toContain("concise founder-level synthesis"); + expect(prompt).toContain("future-dated release is not shipped"); + expect(prompt).toContain("supports, indicates, and suggests"); + expect(prompt).toContain("invented owners"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts new file mode 100644 index 000000000..8e4f5798e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evidence-assembly.test.ts @@ -0,0 +1,103 @@ +import { + dedupeEvidenceItems, + FounderWeeklyReviewEvidenceConflictError, + orderEvidenceItems, + type FounderWeeklyReviewEvidenceItem, +} from "@launchstack/features/founder-weekly-review"; + +function makeItem( + sourceId: string, + sourceTimestamp?: string, + sourceType: FounderWeeklyReviewEvidenceItem["sourceType"] = "document_change" +): FounderWeeklyReviewEvidenceItem { + return { + sourceType, + sourceId, + title: `Title ${sourceId}`, + sourceTimestamp, + excerpt: `Excerpt ${sourceId}`, + metadata: {}, + }; +} + +describe("dedupeEvidenceItems", () => { + it("collapses canonically identical items with the same sourceId", () => { + const result = dedupeEvidenceItems([ + makeItem("a", "2026-01-01T00:00:00.000Z"), + makeItem("a", "2026-01-01T00:00:00.000Z"), + makeItem("b"), + ]); + + expect(result.map((i) => i.sourceId)).toEqual(["a", "b"]); + expect(result[0]?.sourceTimestamp).toBe("2026-01-01T00:00:00.000Z"); + }); + + it("rejects sourceId conflicts even across source types", () => { + expect(() => dedupeEvidenceItems([ + makeItem("x", undefined, "document_change"), + makeItem("x", undefined, "customer_feedback"), + ])).toThrow(FounderWeeklyReviewEvidenceConflictError); + }); + + it("returns an empty array unchanged", () => { + expect(dedupeEvidenceItems([])).toEqual([]); + }); +}); + +describe("orderEvidenceItems", () => { + it("sorts by sourceTimestamp ascending", () => { + const result = orderEvidenceItems([ + makeItem("b", "2026-03-01T00:00:00.000Z"), + makeItem("a", "2026-01-01T00:00:00.000Z"), + makeItem("c", "2026-02-01T00:00:00.000Z"), + ]); + + expect(result.map((i) => i.sourceId)).toEqual(["a", "c", "b"]); + }); + + it("breaks ties on identity when timestamps are equal", () => { + const ts = "2026-01-01T00:00:00.000Z"; + const result = orderEvidenceItems([makeItem("z", ts), makeItem("a", ts)]); + + expect(result.map((i) => i.sourceId)).toEqual(["a", "z"]); + }); + + it("returns selected document changes to numeric structural order", () => { + const ts = "2026-01-01T00:00:00.000Z"; + const tenth = { ...makeItem("group-10", ts), metadata: { documentId: "1", previousVersionId: 1, currentVersionId: 2, structureOrdering: 10 } }; + const second = { ...makeItem("group-2", ts), metadata: { documentId: "1", previousVersionId: 1, currentVersionId: 2, structureOrdering: 2 } }; + expect(orderEvidenceItems([tenth, second]).map(item => item.sourceId)).toEqual(["group-2", "group-10"]); + }); + + it("orders source-type ties with ordinal comparison, not locale collation", () => { + const ts = "2026-01-01T00:00:00.000Z"; + const result = orderEvidenceItems([ + makeItem("same", ts, "github_activity"), + makeItem("same", ts, "customer_feedback"), + ]); + expect(result.map((item) => item.sourceType)).toEqual([ + "customer_feedback", "github_activity", + ]); + }); + + it("sorts items without a timestamp first", () => { + const result = orderEvidenceItems([ + makeItem("withTs", "2026-01-01T00:00:00.000Z"), + makeItem("noTs"), + ]); + + expect(result.map((i) => i.sourceId)).toEqual(["noTs", "withTs"]); + }); + + it("does not mutate the input array", () => { + const input = [ + makeItem("b", "2026-02-01T00:00:00.000Z"), + makeItem("a", "2026-01-01T00:00:00.000Z"), + ]; + const originalOrder = input.map((i) => i.sourceId); + + orderEvidenceItems(input); + + expect(input.map((i) => i.sourceId)).toEqual(originalOrder); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts b/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts new file mode 100644 index 000000000..328a90563 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/evidence-service.test.ts @@ -0,0 +1,78 @@ +import { + mapDocumentVersionToEvidenceItem, + FounderWeeklyReviewEvidenceItemSchema, + type DocumentVersionRow, +} from "@launchstack/features/founder-weekly-review"; + +function completeRow(): DocumentVersionRow { + return { + documentId: 42n, + documentTitle: "Weekly Product Notes", + documentCategory: "Product", + versionId: 100, + versionNumber: 3, + uploadedBy: "user_abc", + changelog: "Fixed onboarding activation delay.", + createdAt: new Date("2026-07-10T18:15:00.000Z"), + }; +} + +describe("mapDocumentVersionToEvidenceItem", () => { + it("maps a complete row to the exact expected evidence item", () => { + const item = mapDocumentVersionToEvidenceItem(completeRow()); + + expect(item).toEqual({ + sourceType: "document_change", + sourceId: "document_change:doc:42:version:100", + title: "Weekly Product Notes", + sourceTimestamp: "2026-07-10T18:15:00.000Z", + excerpt: "Fixed onboarding activation delay.", + workspaceDeepLink: "/employer/documents/viewer?docId=42", + metadata: { + documentId: "42", + documentVersionId: 100, + versionNumber: 3, + documentCategory: "Product", + uploadedBy: "user_abc", + hasChangelog: true, + changelogTruncated: false, + }, + }); + }); + + it("produces an item that satisfies the evidence-item contract", () => { + const item = mapDocumentVersionToEvidenceItem(completeRow()); + expect(() => + FounderWeeklyReviewEvidenceItemSchema.parse(item) + ).not.toThrow(); + }); + + it("falls back to a factual excerpt when changelog is absent", () => { + const item = mapDocumentVersionToEvidenceItem({ + ...completeRow(), + changelog: null, + }); + + expect(item.excerpt).toBe("Version 3 uploaded"); + expect(item.metadata.hasChangelog).toBe(false); + }); + + it("treats a whitespace-only changelog as absent", () => { + const item = mapDocumentVersionToEvidenceItem({ + ...completeRow(), + changelog: " ", + }); + + expect(item.excerpt).toBe("Version 3 uploaded"); + expect(item.metadata.hasChangelog).toBe(false); + }); + + it("carries a null uploader through without inventing a value", () => { + const item = mapDocumentVersionToEvidenceItem({ + ...completeRow(), + uploadedBy: null, + }); + + expect(item.metadata.uploadedBy).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/founder-context.test.ts b/apps/web/__tests__/founderWeeklyReview/founder-context.test.ts new file mode 100644 index 000000000..87f933f11 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/founder-context.test.ts @@ -0,0 +1,25 @@ +import { + FounderWeeklyReviewEvidenceService, +} from "@launchstack/features/founder-weekly-review"; + +describe("founder context evidence", () => { + const service = new FounderWeeklyReviewEvidenceService({} as never); + + it("normalizes request-time founder context with a stable non-content id", () => { + const result = service.collectFounderContextEvidence({ + founderContext: " Customers need faster exports. ", + contextEntryId: "request-123", + actor: { externalUserId: "user_123" }, + }); + expect(result.items).toEqual([expect.objectContaining({ + sourceType: "founder_context", + sourceId: "founder_context:entry:request-123", + excerpt: "Customers need faster exports.", + metadata: expect.objectContaining({ enteredBy: "user_123", provenance: "request_time_founder_input" }), + })]); + }); + + it("does not create evidence for blank context", () => { + expect(service.collectFounderContextEvidence({ founderContext: " " }).items).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts b/apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts new file mode 100644 index 000000000..2854857c4 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/generation-evidence-envelope.test.ts @@ -0,0 +1,268 @@ +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + FounderWeeklyReviewGenerationEvidenceBudgetError, + assertGenerationEvidenceEnvelopeWithinBudget, + buildFounderWeeklyReviewPrompt, + buildFounderWeeklyReviewPromptEvidenceItem, + buildGenerationEvidenceEnvelope, + type FounderWeeklyReviewEvidenceItem, + type FounderWeeklyReviewEvidenceSnapshot, + type GenerationEvidenceEnvelope, +} from "@launchstack/features/founder-weekly-review"; + +function snapshot(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceSnapshot { + return { + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-18T10:00:00.000Z", + reportingPeriod: { start: "2026-07-07", end: "2026-07-13" }, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], + }; +} + +function item( + sourceType: FounderWeeklyReviewEvidenceItem["sourceType"], + sourceId: string, + excerpt = "x".repeat(200), + metadata: FounderWeeklyReviewEvidenceItem["metadata"] = {}, + sourceTimestamp = "2026-07-10T10:00:00.000Z" +): FounderWeeklyReviewEvidenceItem { + return { + sourceType, + sourceId, + title: `${sourceType} title`, + sourceTimestamp, + excerpt, + metadata, + }; +} + +function documentChanges( + count: number, + documentCount: number, + excerpt = "x".repeat(200) +): FounderWeeklyReviewEvidenceItem[] { + return Array.from({ length: count }, (_, index) => { + const documentId = String((index % documentCount) + 1); + const sequence = Math.floor(index / documentCount) + 1; + return item( + "document_change", + `document_change:doc:${documentId}:v1:v2:chunk:${sequence}:${sequence}`, + excerpt, + { + documentId, + previousVersionId: 1, + currentVersionId: 2, + previousVersionNumber: 1, + currentVersionNumber: 2, + changeType: "modified", + alignmentMethod: "structure_path", + structurePath: `/section/${sequence}`, + previousChunkId: sequence, + currentChunkId: sequence, + previousContentHash: "a".repeat(64), + currentContentHash: "b".repeat(64), + }, + `2026-07-${String((sequence % 7) + 7).padStart(2, "0")}T10:00:00.000Z` + ); + }); +} + +describe("Founder Weekly Review generation evidence envelope", () => { + it("bounds 250 document changes with round-robin document diversity without mutating the snapshot", () => { + const evidenceSnapshot = snapshot(documentChanges(250, 3)); + const original = JSON.stringify(evidenceSnapshot); + const first = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const second = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const changes = first.items.filter(entry => entry.sourceType === "document_change"); + const perDocument = new Map(); + for (const change of changes) { + const documentId = /^document_change:doc:([^:]+)/.exec(change.sourceId)![1]!; + perDocument.set(documentId, (perDocument.get(documentId) ?? 0) + 1); + } + + expect(changes).toHaveLength(24); + expect(Math.max(...perDocument.values())).toBe(8); + expect(JSON.stringify(changes).length).toBeLessThanOrEqual(14_000); + expect(first.items.map(entry => entry.sourceId)).toEqual( + second.items.map(entry => entry.sourceId) + ); + expect(first.diagnostics).toEqual(second.diagnostics); + expect(evidenceSnapshot.items).toHaveLength(250); + expect(JSON.stringify(evidenceSnapshot)).toBe(original); + }); + + it("reserves Founder Context, customer feedback, and workspace evidence before bounded document changes", () => { + const items = [ + ...documentChanges(250, 3, "d".repeat(300)), + ...Array.from({ length: 250 }, (_, index) => + item("customer_feedback", `customer_feedback:${index}`, "c".repeat(500), { + pageNumber: index + 1, + }) + ), + ...Array.from({ length: 8 }, (_, index) => + item( + "workspace_document", + `workspace_document:${index}`, + "w".repeat(500), + { + similarityScore: 1 - index / 100, + retrievalReason: "founder_context_relevance", + }, + "" + ) + ), + item( + "founder_context", + "founder_context:entry:test", + "Founder direction", + { provenance: "request_time_founder_input" }, + "" + ), + ]; + // This intentionally models the pre-snapshot aggregate (509 items) + // that the 500-item snapshot cap subsequently trims. + const evidenceSnapshot = snapshot(items) as FounderWeeklyReviewEvidenceSnapshot; + const envelope = buildGenerationEvidenceEnvelope(evidenceSnapshot); + + expect(envelope.diagnostics.selectedBySourceType.founder_context).toBe(1); + expect(envelope.diagnostics.selectedBySourceType.customer_feedback).toBeGreaterThan(0); + expect(envelope.diagnostics.selectedBySourceType.workspace_document).toBe(8); + expect(envelope.diagnostics.selectedBySourceType.document_change).toBeLessThanOrEqual(24); + expect(envelope.diagnostics.selectedBySourceType.customer_feedback).toBeGreaterThan( + envelope.diagnostics.selectedBySourceType.document_change + ); + expect(envelope.diagnostics.serializedCharacterCount).toBeLessThanOrEqual(72_000); + expect(envelope.diagnostics.estimatedTokenCount).toBeLessThanOrEqual(18_000); + expect(envelope.diagnostics.truncated).toBe(true); + }); + + it("is input-order independent and produces byte-identical prompts", () => { + const items = [ + ...documentChanges(60, 4), + item("customer_feedback", "feedback-1", "Customer signal"), + item( + "workspace_document", + "workspace-1", + "Workspace context", + { similarityScore: 0.9 }, + "" + ), + item("founder_context", "context-1", "Founder context", {}, ""), + ]; + const shuffled = [...items].sort((a, b) => b.sourceId.localeCompare(a.sourceId)); + const first = buildGenerationEvidenceEnvelope(snapshot(items)); + const second = buildGenerationEvidenceEnvelope(snapshot(shuffled)); + + expect(first.items.map(entry => entry.sourceId)).toEqual( + second.items.map(entry => entry.sourceId) + ); + expect(buildFounderWeeklyReviewPrompt(snapshot(items))).toBe( + buildFounderWeeklyReviewPrompt(snapshot(shuffled)) + ); + }); + + it("allowlists prompt metadata while retaining complete immutable snapshot metadata and source IDs", () => { + const source = item( + "document_change", + "document_change:doc:7:v1:v2:chunk:1:2", + "Before and after", + { + documentId: "7", + previousChunkId: 1, + currentChunkId: 2, + previousContentHash: "a".repeat(64), + currentContentHash: "b".repeat(64), + previousVersionNumber: 1, + currentVersionNumber: 2, + changeType: "modified", + alignmentMethod: "structure_path", + structurePath: "/plan", + userChangelog: "Updated plan", + providerPayload: "must-not-leak", + credential: "must-not-leak", + } + ); + const evidenceSnapshot = snapshot([source]); + const prompt = JSON.parse(buildFounderWeeklyReviewPrompt(evidenceSnapshot)); + const promptItem = prompt.evidence[0]; + + expect(prompt.evidenceEnvelopeVersion).toBe( + FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION + ); + expect(prompt.evidenceEnvelopeBudget.totalSerializedCharacters).toBe(72_000); + expect(promptItem.sourceId).toBe(source.sourceId); + expect(promptItem.metadata).toEqual({ + alignmentMethod: "structure_path", + changeType: "modified", + currentVersionNumber: 2, + previousVersionNumber: 1, + structurePath: "/plan", + userChangelog: "Updated plan", + }); + expect(promptItem).not.toHaveProperty("canonicalUrl"); + expect(promptItem).not.toHaveProperty("workspaceDeepLink"); + expect(evidenceSnapshot.items[0]!.metadata).toHaveProperty("previousContentHash"); + expect(evidenceSnapshot.items[0]!.metadata).toHaveProperty("providerPayload"); + }); + + it("includes candidates atomically until the document-change character boundary", () => { + const evidenceSnapshot = snapshot(documentChanges(8, 1, "x".repeat(3_000))); + const envelope = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const selected = envelope.items.filter(entry => entry.sourceType === "document_change"); + const excluded = evidenceSnapshot.items.find( + entry => !selected.some(candidate => candidate.sourceId === entry.sourceId) + ); + + expect(selected.length).toBeGreaterThan(0); + expect(excluded).toBeDefined(); + expect(JSON.stringify(selected).length).toBeLessThanOrEqual( + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters + ); + expect( + JSON.stringify([...selected, buildFounderWeeklyReviewPromptEvidenceItem(excluded!)]) + .length + ).toBeGreaterThan( + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters + ); + expect(envelope.diagnostics.truncated).toBe(true); + }); + + it("rejects a corrupted over-budget envelope locally", () => { + const valid = buildGenerationEvidenceEnvelope(snapshot([item("manual_note", "note-1")])); + const corrupted: GenerationEvidenceEnvelope = { + ...valid, + items: Array.from({ length: 20 }, (_, index) => ({ + ...valid.items[0]!, + sourceId: `note-${index}`, + excerpt: "x".repeat(4_000), + })), + }; + + expect(() => assertGenerationEvidenceEnvelopeWithinBudget(corrupted)).toThrow( + FounderWeeklyReviewGenerationEvidenceBudgetError + ); + expect(() => buildFounderWeeklyReviewPrompt(snapshot([]), corrupted)).toThrow( + expect.objectContaining({ code: "generation_evidence_budget_exceeded" }) + ); + }); + + it("publishes a versioned envelope and safe aggregate diagnostics only", () => { + const envelope = buildGenerationEvidenceEnvelope(snapshot([item("manual_note", "note-1")])); + expect(envelope.version).toBe(FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION); + expect(envelope.diagnostics).toEqual( + expect.objectContaining({ + originalItemCount: 1, + selectedItemCount: 1, + excludedItemCount: 0, + serializedCharacterCount: JSON.stringify(envelope.items).length, + estimatedTokenCount: Math.ceil(JSON.stringify(envelope.items).length / 4), + truncated: false, + }) + ); + expect(JSON.stringify(envelope.diagnostics)).not.toContain("manual_note title"); + expect(JSON.stringify(envelope.diagnostics)).not.toContain("x".repeat(20)); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/generation.test.ts b/apps/web/__tests__/founderWeeklyReview/generation.test.ts new file mode 100644 index 000000000..ecd19aba8 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/generation.test.ts @@ -0,0 +1,454 @@ +jest.mock("~/lib/llm", () => ({ + generateStructuredWithMetadata: jest.fn(), +})); + +import { + FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + FounderWeeklyReviewV2PayloadSchema, + buildFounderWeeklyReviewPrompt, + buildGenerationEvidenceEnvelope, + generateFounderWeeklyReview, + parseFounderWeeklyReviewPayload, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewV2Payload, +} from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewGenerationValidationError } from "@launchstack/features/founder-weekly-review"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { generateStructuredWithMetadata } from "~/lib/llm"; + +const mockGenerateStructuredWithMetadata = generateStructuredWithMetadata as jest.Mock; + +function snapshot(items: FounderWeeklyReviewEvidenceSnapshot["items"]): FounderWeeklyReviewEvidenceSnapshot { + return { + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-18T10:00:00.000Z", + reportingPeriod: { start: "2026-07-07", end: "2026-07-13" }, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], + }; +} + +const source = (sourceId: string, sourceType: FounderWeeklyReviewEvidenceSnapshot["items"][number]["sourceType"], excerpt = "Evidence excerpt", metadata: Record = {}) => ({ + sourceId, + sourceType, + title: `${sourceType} title`, + excerpt, + metadata, +}); + +function noEvidence(message = "No evidence", cta = "Add evidence") { + return { state: "no_evidence" as const, noEvidence: { code: "no_relevant_evidence", message, cta } }; +} + +function validPayload(): FounderWeeklyReviewV2Payload { + return { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "The plan changed.", sourceIds: ["doc-1"], confidence: 0 }] }, + whatShipped: { state: "evidence", items: [{ kind: "observed_fact", text: "A release shipped.", sourceIds: ["doc-1"], confidence: 1 }] }, + whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "A customer requested audit logs.", sourceIds: ["feedback-1"], confidence: 0.5 }] }, + currentBlockers: { state: "evidence", items: [{ kind: "observed_fact", text: "SSO remains blocked.", sourceIds: ["context-1"], confidence: 0.8 }] }, + nextPriorities: { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Prioritize SSO.", sourceIds: ["context-1"], confidence: 0.8, rationale: null }] }, + }, + }; +} + +function fake(object: unknown, metadata = { provider: "openai", model: "test-model", capability: "founderWeeklyReview", temperature: 0 }) { + return jest.fn().mockResolvedValue({ object, metadata }); +} + +const completeSnapshot = () => snapshot([ + source("doc-1", "document_change", "A release shipped."), + source("feedback-1", "customer_feedback", "Please add audit logs."), + source("context-1", "founder_context", "SSO remains blocked."), +]); + +describe("Founder Weekly Review generation", () => { + it("generates complete evidence with numeric lower and upper confidence bounds", async () => { + const generate = fake(validPayload()); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate }); + expect(result.reviewPayload).toEqual(validPayload()); + expect(result.modelMetadata).toMatchObject({ provider: "openai", temperature: 0, capability: "founderWeeklyReview" }); + expect(result.modelMetadata.promptHash).toMatch(/^[a-f0-9]{64}$/); + expect(generate).toHaveBeenCalledTimes(1); + expect(generate.mock.calls[0][0]).toMatchObject({ generationPhase: "initial" }); + }); + + it("performs exactly one semantic repair against the same immutable snapshot", async () => { + const invalid = validPayload(); + invalid.sections.whatCustomersSaid = { + state: "evidence", + items: [{ kind: "observed_fact", text: "Founder direction was presented as customer feedback.", sourceIds: ["context-1"], confidence: 0.5 }], + }; + const repaired = validPayload(); + const generate = jest.fn() + .mockResolvedValueOnce({ object: invalid, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }) + .mockResolvedValueOnce({ object: repaired, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }); + const evidenceSnapshot = completeSnapshot(); + + await expect(generateFounderWeeklyReview({ evidenceSnapshot, generate })).resolves.toMatchObject({ reviewPayload: repaired }); + + expect(generate).toHaveBeenCalledTimes(2); + expect(generate.mock.calls[0][0]).toMatchObject({ generationPhase: "initial" }); + expect(generate.mock.calls[1][0]).toMatchObject({ generationPhase: "semantic-repair" }); + expect(generate.mock.calls[1][0].prompt).toContain("founder_context is founder-provided direction, not customer testimony."); + expect(generate.mock.calls[1][0].prompt).toContain("context-1"); + expect(evidenceSnapshot).toEqual(completeSnapshot()); + }); + + it("fails normally after one invalid semantic repair and never makes a third call", async () => { + const invalid = validPayload(); + invalid.sections.whatCustomersSaid = { + state: "evidence", + items: [{ kind: "observed_fact", text: "Invalid customer claim.", sourceIds: ["context-1"], confidence: 0.5 }], + }; + const generate = jest.fn() + .mockResolvedValueOnce({ object: invalid, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }) + .mockResolvedValueOnce({ object: invalid, metadata: { provider: "kimi", model: "kimi-k2.6", capability: "founderWeeklyReview" } }); + + await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate })).rejects.toMatchObject({ + name: "FounderWeeklyReviewGenerationValidationError", + }); + expect(generate).toHaveBeenCalledTimes(2); + }); + + it("does not semantic-repair provider failures", async () => { + const generate = jest.fn().mockRejectedValue(new Error("provider unavailable")); + await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate })).rejects.toThrow("provider unavailable"); + expect(generate).toHaveBeenCalledTimes(1); + }); + + it("allows partial reviews with typed no-evidence sections", async () => { + const payload = validPayload(); + payload.sections.whatShipped = noEvidence(); + payload.sections.whatCustomersSaid = noEvidence(); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) }); + expect(result.reviewPayload.sections.whatShipped.state).toBe("no_evidence"); + }); + + it("preserves contradictory evidence rather than resolving it", async () => { + const payload = validPayload(); + payload.sections.currentBlockers = { state: "evidence", items: [{ kind: "contradictory_evidence", text: "The blocker status conflicts.", sourceIds: ["doc-1", "context-1"], confidence: 0.6 }] }; + const result = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) }); + expect(result.reviewPayload.sections.currentBlockers).toMatchObject({ state: "evidence", items: [{ kind: "contradictory_evidence" }] }); + }); + + it("returns deterministic empty states and performs zero LLM calls for an empty snapshot", async () => { + const generate = fake(validPayload()); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: snapshot([]), generate }); + expect(generate).not.toHaveBeenCalled(); + expect(result.reviewPayload.sections.nextPriorities).toMatchObject({ state: "no_evidence" }); + expect(result.modelMetadata).toMatchObject({ provider: "skipped", model: "none", attributes: { generationSkipped: true } }); + for (const section of Object.values(result.reviewPayload.sections)) { + expect(section.state).toBe("no_evidence"); + if (section.state === "no_evidence") { + expect(section.noEvidence.cta.trim()).not.toBe(""); + } + } + }); + + it.each([ + ["unknown generated source ID", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["missing"], confidence: 0.5 }] }; }], + ["duplicate generated citations", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1", "doc-1"], confidence: 0.5 }] }; }], + ["customer section citing document_change", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }] }; }], + ["customer section citing founder_context", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["context-1"], confidence: 0.5 }] }; }], + ["contradiction with fewer than two citations", (p: FounderWeeklyReviewV2Payload) => { p.sections.currentBlockers = { state: "evidence", items: [{ kind: "contradictory_evidence", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }] }; }], + ["recommendation outside nextPriorities", (p: FounderWeeklyReviewV2Payload) => { p.sections.whatChanged = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }] } as never; }], + ])("rejects %s", async (_name, mutate) => { + const payload = validPayload(); + mutate(payload); + await expect(generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: fake(payload) })).rejects.toBeInstanceOf(Error); + }); + + it.each(["whatChanged", "whatShipped"] as const)("rejects workspace_document-only %s claims", async (sectionName) => { + const payload = validPayload(); + payload.sections[sectionName] = { state: "evidence", items: [{ kind: "observed_fact", text: "Current document implies a weekly event.", sourceIds: ["workspace-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([source("workspace-1", "workspace_document")]), generate: fake(payload) })).rejects.toBeInstanceOf(FounderWeeklyReviewGenerationValidationError); + }); + + it.each(["whatChanged", "whatShipped"] as const)("allows document_change plus workspace_document in %s", async (sectionName) => { + const payload = validPayload(); + payload.sections[sectionName] = { state: "evidence", items: [{ kind: "observed_fact", text: "A dated change has current context.", sourceIds: ["doc-1", "workspace-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([...completeSnapshot().items, source("workspace-1", "workspace_document")]), generate: fake(payload) })).resolves.toBeDefined(); + }); + + it("allows workspace_document for blockers and priorities while customer-only enforcement remains", async () => { + const payload = validPayload(); + payload.sections.currentBlockers = { state: "evidence", items: [{ kind: "observed_fact", text: "Current context", sourceIds: ["workspace-1"], confidence: 0.5 }] }; + payload.sections.nextPriorities = { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Act on current context", sourceIds: ["workspace-1"], confidence: 0.5, rationale: null }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([...completeSnapshot().items, source("workspace-1", "workspace_document")]), generate: fake(payload) })).resolves.toBeDefined(); + }); + + it("rejects duplicate input source IDs before calling the model", async () => { + const generate = fake(validPayload()); + await expect(generateFounderWeeklyReview({ evidenceSnapshot: snapshot([source("same", "manual_note"), source("same", "document_change")]), generate })).rejects.toBeInstanceOf(FounderWeeklyReviewGenerationValidationError); + expect(generate).not.toHaveBeenCalled(); + }); + + it("enforces numeric confidence bounds", () => { + const payload = validPayload(); + payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: -0.01 }] }; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + (payload.sections.whatChanged as { state: "evidence"; items: Array<{ confidence: number }> }).items[0]!.confidence = 1.01; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + }); + + it("rejects mixed v2 section states instead of normalizing them", () => { + const payload = validPayload(); + payload.sections.whatChanged = { + state: "evidence", + items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }], + noEvidence: { code: "no_relevant_evidence", message: "x", cta: "x" }, + } as never; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + + payload.sections.whatChanged = { + state: "no_evidence", + noEvidence: { code: "no_relevant_evidence", message: "x", cta: "x" }, + items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }], + } as never; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + }); + + it("rejects observed facts in nextPriorities", () => { + const payload = validPayload(); + payload.sections.nextPriorities = { + state: "evidence", + items: [{ kind: "observed_fact", text: "x", sourceIds: ["doc-1"], confidence: 0.5 }], + } as never; + expect(FounderWeeklyReviewV2PayloadSchema.safeParse(payload).success).toBe(false); + }); + + it("does not allow source warnings to be cited", async () => { + const evidenceSnapshot = completeSnapshot(); + evidenceSnapshot.sourceWarnings = [{ code: "warning-1", message: "Missing source", sourceType: "github_activity" }]; + const payload = validPayload(); + payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "x", sourceIds: ["warning-1"], confidence: 0.5 }] }; + await expect(generateFounderWeeklyReview({ evidenceSnapshot, generate: fake(payload) })).rejects.toBeInstanceOf(Error); + }); + + it("puts anti-invention requirements in the prompt and hashes a fixed fixture stably", async () => { + const generateA = fake(validPayload()); + const generateB = fake(validPayload()); + const first = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: generateA }); + const second = await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate: generateB }); + expect(generateA.mock.calls[0][0].system).toContain("Never invent"); + expect(generateA.mock.calls[0][0].system).toContain("whatCustomersSaid may cite only customer_feedback"); + for (const prohibitedFact of ["customers", "people", "dates", "metrics", "shipped work", "blockers", "outcomes", "source IDs"]) { + expect(generateA.mock.calls[0][0].system).toContain(prohibitedFact); + } + expect(first.modelMetadata.promptHash).toBe(second.modelMetadata.promptHash); + }); + + it("uses the concise founder-review generation contract", async () => { + const generate = fake(validPayload()); + await generateFounderWeeklyReview({ evidenceSnapshot: completeSnapshot(), generate }); + const system = generate.mock.calls[0][0].system as string; + expect(system).toContain("decision-oriented founder review, not an evidence transcript"); + expect(system).toContain("at most 3 items in each section"); + expect(system).toContain("one concise sentence whenever possible"); + expect(system).toContain("Synthesize related evidence into one focused claim"); + expect(system).toContain("do not create one output item per evidence source"); + expect(system).toContain("optional rationale should be brief"); + expect(system).not.toContain("2–4 sentences per substantive item"); + expect(system).not.toContain("600–1,000 words overall"); + }); + + it("canonicalizes metadata key order before building the prompt and hash", async () => { + const firstSnapshot = completeSnapshot(); + firstSnapshot.items[0]!.metadata = { changeType: "modified", alignmentMethod: "structure_path" }; + const secondSnapshot = completeSnapshot(); + secondSnapshot.items[0]!.metadata = { alignmentMethod: "structure_path", changeType: "modified" }; + expect(buildFounderWeeklyReviewPrompt(firstSnapshot)).toBe(buildFounderWeeklyReviewPrompt(secondSnapshot)); + + const first = await generateFounderWeeklyReview({ evidenceSnapshot: firstSnapshot, generate: fake(validPayload()) }); + const second = await generateFounderWeeklyReview({ evidenceSnapshot: secondSnapshot, generate: fake(validPayload()) }); + expect(first.modelMetadata.promptHash).toBe(second.modelMetadata.promptHash); + }); + + it("changes the prompt hash when selected evidence changes", async () => { + const firstSnapshot = completeSnapshot(); + const secondSnapshot = completeSnapshot(); + secondSnapshot.items[0]!.excerpt = "A different release shipped."; + const first = await generateFounderWeeklyReview({ evidenceSnapshot: firstSnapshot, generate: fake(validPayload()) }); + const second = await generateFounderWeeklyReview({ evidenceSnapshot: secondSnapshot, generate: fake(validPayload()) }); + expect(first.modelMetadata.promptHash).not.toBe(second.modelMetadata.promptHash); + }); + + it("normally truncates large evidence before generation and persists aggregate diagnostics", async () => { + const items = Array.from({ length: 250 }, (_, index) => ({ + ...source(`document_change:doc:${(index % 3) + 1}:v1:v2:chunk:${index}:${index}`, "document_change", "x".repeat(200)), + sourceTimestamp: `2026-07-${String((index % 7) + 7).padStart(2, "0")}T10:00:00.000Z`, + metadata: { documentId: String((index % 3) + 1), previousVersionId: 1, currentVersionId: 2, changeType: "modified" }, + })); + const payload: FounderWeeklyReviewV2Payload = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: noEvidence(), whatShipped: noEvidence(), whatCustomersSaid: noEvidence(), + currentBlockers: noEvidence(), nextPriorities: noEvidence(), + }, + }; + const generate = fake(payload); + const result = await generateFounderWeeklyReview({ evidenceSnapshot: snapshot(items), generate }); + + expect(generate).toHaveBeenCalledTimes(1); + const prompt = JSON.parse(generate.mock.calls[0][0].prompt); + expect(prompt.evidence).toHaveLength(24); + expect(result.modelMetadata.attributes).toEqual(expect.objectContaining({ + evidenceEnvelopeOriginalItems: 250, + evidenceEnvelopeSelectedItems: 24, + evidenceEnvelopeExcludedItems: 226, + evidenceEnvelopeTruncated: true, + })); + }); + + it("continues to validate citations against the complete immutable snapshot", async () => { + const items = Array.from({ length: 30 }, (_, index) => ({ + ...source(`document_change:doc:1:v1:v2:chunk:${index}:${index}`, "document_change", "x".repeat(200)), + metadata: { documentId: "1", previousVersionId: 1, currentVersionId: 2, changeType: "modified" }, + })); + const evidenceSnapshot = snapshot(items); + const selectedIds = new Set(buildGenerationEvidenceEnvelope(evidenceSnapshot).items.map((item) => item.sourceId)); + const excludedId = evidenceSnapshot.items.find((item) => !selectedIds.has(item.sourceId))!.sourceId; + const payload: FounderWeeklyReviewV2Payload = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "A source-backed change.", sourceIds: [excludedId], confidence: 0.5 }] }, + whatShipped: noEvidence(), whatCustomersSaid: noEvidence(), currentBlockers: noEvidence(), nextPriorities: noEvidence(), + }, + }; + + await expect(generateFounderWeeklyReview({ evidenceSnapshot, generate: fake(payload) })).resolves.toMatchObject({ reviewPayload: payload }); + expect(evidenceSnapshot.items).toHaveLength(30); + }); + + it("adapts the configured web abstraction and returns its metadata", async () => { + mockGenerateStructuredWithMetadata.mockResolvedValue({ object: { ok: true }, metadata: { provider: "openai", model: "adapter-model", capability: "founderWeeklyReview", temperature: 0 } }); + const schema = require("zod").z.object({ ok: require("zod").z.boolean() }); + await expect(generateFounderWeeklyReviewStructured({ prompt: "p", schema })).resolves.toMatchObject({ metadata: { model: "adapter-model" } }); + expect(mockGenerateStructuredWithMetadata).toHaveBeenCalledWith(expect.objectContaining({ capability: "founderWeeklyReview" })); + }); + + it("continues to parse legacy v1 payloads", () => { + expect(parseFounderWeeklyReviewPayload({ + schemaVersion: "founder-weekly-review/v1", + sections: Object.fromEntries(["whatChanged", "whatShipped", "whatCustomersSaid", "currentBlockers", "nextPriorities"].map((key) => [key, { heading: key, items: [{ kind: "no_evidence", code: "not_assessed" }] }])), + }).schemaVersion).toBe("founder-weekly-review/v1"); + }); + + it("allows planned evidence to describe future shipment", async () => { + const payload = validPayload(); + + payload.sections.whatChanged = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Feature will ship next week.", + sourceIds: ["doc-1"], + confidence: 0.8, + }], + }; + + payload.sections.whatShipped = { + state: "no_evidence", + noEvidence: { + code: "no_relevant_evidence", + message: "No shipped work yet.", + cta: "Add shipped evidence", + }, + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Feature roadmap item.", { + evidenceStatus: "planned", + }), + source("feedback-1", "customer_feedback"), + source("context-1", "founder_context"), + ]), + generate: fake(payload), + }) + ).resolves.toBeDefined(); + }); + + + it("rejects planned evidence described as already shipped", async () => { + const payload = validPayload(); + payload.sections.whatChanged = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Feature shipped yesterday.", + sourceIds: ["doc-1"], + confidence: 0.8, + }], + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Feature roadmap item.", { + evidenceStatus: "planned", + }), + ]), + generate: fake(payload), + }) + ).rejects.toThrow("Claim describes planned work as completed."); + }); + + + it("allows shipped evidence to describe shipped work", async () => { + const payload = validPayload(); + payload.sections.whatShipped = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Feature shipped yesterday.", + sourceIds: ["doc-1"], + confidence: 1, + }], + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Feature shipped yesterday.", { + evidenceStatus: "shipped", + }), + source("feedback-1", "customer_feedback"), + source("context-1", "founder_context"), + ]), + generate: fake(payload), + }) + ).resolves.toBeDefined(); + }); + + + it("allows changed evidence to describe updates without implying shipment", async () => { + const payload = validPayload(); + payload.sections.whatChanged = { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Documentation was updated.", + sourceIds: ["doc-1"], + confidence: 0.8, + }], + }; + + await expect( + generateFounderWeeklyReview({ + evidenceSnapshot: snapshot([ + source("doc-1", "document_change", "Documentation updated.", { + evidenceStatus: "changed", + }), + source("feedback-1", "customer_feedback"), + source("context-1", "founder_context"), + ]), + generate: fake(payload), + }) + ).resolves.toBeDefined(); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts new file mode 100644 index 000000000..c580ce3fb --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/kimi-chat-completions.test.ts @@ -0,0 +1,179 @@ +import { z } from "zod"; + +import { __resetLlmConfigForTests } from "~/lib/llm/config"; +import { generateStructuredWithMetadata } from "~/lib/llm/generate"; +import { __resetProviderCacheForTests } from "~/lib/llm/providers"; +import { resolveModel } from "~/lib/llm/providers"; + +describe("Founder Weekly Review Kimi transport", () => { + const originalEnv = { ...process.env }; + const originalFetch = global.fetch; + + beforeEach(() => { + process.env = { + ...originalEnv, + FWR_GENERATION_PROVIDER: "kimi", + MOONSHOT_API_KEY: "test-key-not-logged", + MOONSHOT_BASE_URL: "https://api.moonshot.ai/v1", + KIMI_MODEL_ID: "kimi-k2.6", + }; + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_MODEL_ID; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + }); + + afterEach(() => { + process.env = originalEnv; + global.fetch = originalFetch; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + }); + + it("uses Moonshot Chat Completions JSON mode and locally validates the result", async () => { + const logSpy = jest.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = jest.spyOn(console, "error").mockImplementation(() => undefined); + const fetchMock = jest.fn(async (url: string | URL, init?: RequestInit) => { + expect(String(url)).toMatch(/\/chat\/completions$/); + expect(String(url)).not.toContain("/responses"); + const body = JSON.parse(String(init?.body)) as Record; + expect(body.model).toBe("kimi-k2.6"); + expect(body.max_tokens).toBe(2400); + expect(body.messages).toEqual(expect.any(Array)); + expect(body).not.toHaveProperty("temperature"); + expect(body.response_format).toEqual({ type: "json_object" }); + expect(body.thinking).toEqual({ type: "disabled" }); + return new Response(JSON.stringify({ + id: "chatcmpl-test", + object: "chat.completion", + created: 0, + model: "kimi-k2.6", + choices: [{ index: 0, finish_reason: "stop", logprobs: null, message: { role: "assistant", content: "{\"ok\":true}" } }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + global.fetch = fetchMock as typeof fetch; + + await expect(generateStructuredWithMetadata({ + capability: "founderWeeklyReview", + system: "Return valid JSON.", + prompt: "Generate the object.", + schema: z.object({ ok: z.boolean() }), + schemaName: "founder_weekly_review", + })).resolves.toMatchObject({ object: { ok: true }, metadata: { provider: "kimi", model: "kimi-k2.6" } }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(`${logSpy.mock.calls.flat().join(" ")} ${errorSpy.mock.calls.flat().join(" ")}`).not.toContain("test-key-not-logged"); + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it("defaults to OpenAI and does not infer Kimi from Moonshot credentials", () => { + delete process.env.FWR_GENERATION_PROVIDER; + process.env.OPENAI_API_KEY = "openai-test-key"; + process.env.OPENAI_MODEL_ID = "gpt-test"; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + const resolved = resolveModel("founderWeeklyReview"); + expect(resolved).toMatchObject({ provider: "openai", modelId: "gpt-test" }); + expect(resolved.temperature).toBe(0); + expect(resolved.structuredOutputMode).toBeUndefined(); + expect(resolved).not.toHaveProperty("thinking"); + }); + + it("keeps an explicit Founder Weekly Review output override", async () => { + const fetchMock = jest.fn(async (_url: string | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + expect(body.max_tokens).toBe(777); + return new Response(JSON.stringify({ + id: "chatcmpl-override", + object: "chat.completion", + created: 0, + model: "kimi-k2.6", + choices: [{ index: 0, message: { role: "assistant", content: "{\"ok\":true}" }, finish_reason: "stop", logprobs: null }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + global.fetch = fetchMock as typeof fetch; + await expect(generateStructuredWithMetadata({ + capability: "founderWeeklyReview", + maxOutputTokens: 777, + system: "Return valid JSON.", + prompt: "Generate the object.", + schema: z.object({ ok: z.boolean() }), + })).resolves.toMatchObject({ object: { ok: true } }); + }); + + it("does not apply the Founder Weekly Review default to small extraction", async () => { + const fetchMock = jest.fn(async (_url: string | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + expect(body).not.toHaveProperty("max_tokens"); + return new Response(JSON.stringify({ + id: "chatcmpl-extraction", + object: "chat.completion", + created: 0, + model: "kimi-k2.6", + choices: [{ index: 0, message: { role: "assistant", content: "{\"ok\":true}" }, finish_reason: "stop", logprobs: null }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + global.fetch = fetchMock as typeof fetch; + await expect(generateStructuredWithMetadata({ + capability: "smallExtraction", + forceProvider: "kimi", + system: "Return valid JSON.", + prompt: "Extract the object.", + schema: z.object({ ok: z.boolean() }), + })).resolves.toMatchObject({ object: { ok: true } }); + }); + + it("uses explicit OpenAI without requiring Moonshot configuration", () => { + process.env.FWR_GENERATION_PROVIDER = "openai"; + process.env.OPENAI_API_KEY = "openai-test-key"; + process.env.OPENAI_MODEL_ID = "gpt-openai"; + delete process.env.MOONSHOT_API_KEY; + delete process.env.MOONSHOT_BASE_URL; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + + const resolved = resolveModel("founderWeeklyReview"); + expect(resolved).toMatchObject({ provider: "openai", modelId: "gpt-openai" }); + expect(resolved.structuredOutputMode).toBeUndefined(); + }); + + it("allows an explicit Kimi smallExtraction request without changing default priority", () => { + process.env.OPENAI_API_KEY = "openai-test-key"; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + const defaultExtraction = resolveModel("smallExtraction"); + const kimiExtraction = resolveModel("smallExtraction", "kimi"); + expect(defaultExtraction).toMatchObject({ provider: "openai", modelId: "gpt-4o-mini" }); + expect(kimiExtraction).toMatchObject({ provider: "kimi", modelId: "kimi-k2.6", structuredOutputMode: "json_object" }); + }); + + it("fails before any request for invalid or missing selected-provider configuration", () => { + process.env.FWR_GENERATION_PROVIDER = "other"; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + expect(() => resolveModel("founderWeeklyReview")).toThrow('FWR_GENERATION_PROVIDER must be "openai" or "kimi"'); + process.env.FWR_GENERATION_PROVIDER = "kimi"; + delete process.env.MOONSHOT_API_KEY; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + expect(() => resolveModel("founderWeeklyReview")).toThrow("FWR_GENERATION_PROVIDER=kimi requires MOONSHOT_API_KEY"); + + process.env.FWR_GENERATION_PROVIDER = "openai"; + process.env.MOONSHOT_API_KEY = "moonshot-secret-that-must-not-appear"; + delete process.env.OPENAI_API_KEY; + __resetLlmConfigForTests(); + __resetProviderCacheForTests(); + let thrown: Error | null = null; + try { + resolveModel("founderWeeklyReview"); + } catch (error) { + thrown = error as Error; + } + expect(thrown?.message).toContain("FWR_GENERATION_PROVIDER=openai requires OPENAI_API_KEY"); + expect(thrown?.message).not.toContain("moonshot-secret-that-must-not-appear"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts new file mode 100644 index 000000000..7adbeca56 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/lifecycle.integration.test.ts @@ -0,0 +1,568 @@ +import { randomUUID } from "node:crypto"; + +import { sql } from "drizzle-orm"; +import { company } from "@launchstack/core/db/schema"; +import { + FounderWeeklyReviewConflictError, + FounderWeeklyReviewEvidenceSnapshotSchema, + FounderWeeklyReviewInvalidPayloadError, + FounderWeeklyReviewInvalidTransitionError, + FounderWeeklyReviewNotFoundError, + FounderWeeklyReviewRepository, + FounderWeeklyReviewUserService, + FounderWeeklyReviewWorkerService, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewV2Payload, +} from "@launchstack/features/founder-weekly-review"; + +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeIfDatabase = + process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL + ? describe + : describe.skip; + +function createEvidenceSnapshot(): FounderWeeklyReviewEvidenceSnapshot { + return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-18T10:00:00.000Z", + reportingPeriod: { + start: "2026-07-07", + end: "2026-07-13", + }, + workspaceTimezone: "America/Los_Angeles", + items: [ + { + sourceType: "document_change", + sourceId: "doc-1", + title: "Weekly Product Notes", + sourceTimestamp: "2026-07-10T18:15:00.000Z", + excerpt: "Shipped billing exports and fixed onboarding activation delay.", + workspaceDeepLink: "launchstack://workspace/documents/doc-1", + metadata: { + documentType: "note", + tags: ["product", "ops"], + }, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback-1", + title: "Customer call summary", + excerpt: "Two enterprise prospects asked for stronger audit logging.", + canonicalUrl: "https://example.com/customer-feedback/1", + metadata: { + sentiment: "mixed", + }, + }, + { + sourceType: "founder_context", + sourceId: "founder-context-1", + title: "Founder weekly context", + excerpt: "Manual founder input: enterprise onboarding remains blocked on SSO setup.", + }, + ], + sourceWarnings: [ + { + code: "missing_github_activity", + message: "GitHub activity collection is not part of LAU-5.", + sourceType: "github_activity", + }, + ], + }); +} + +function createPayload(seed: string): FounderWeeklyReviewPayload { + const observed = { + kind: "observed_fact" as const, + text: `Observed fact ${seed}`, + sourceIds: ["doc-1"], + confidence: "high" as const, + }; + const recommended = { + kind: "recommended_item" as const, + text: `Recommended item ${seed}`, + sourceIds: ["feedback-1"], + confidence: "medium" as const, + }; + + return { + schemaVersion: "founder-weekly-review/v1", + sections: { + whatChanged: { heading: "What changed", items: [observed] }, + whatShipped: { heading: "What shipped", items: [observed] }, + whatCustomersSaid: { heading: "What customers said", items: [observed] }, + currentBlockers: { heading: "Current blockers", items: [recommended] }, + nextPriorities: { heading: "Next priorities", items: [recommended] }, + }, + }; +} + +function createV2Payload(): FounderWeeklyReviewV2Payload { + return { + schemaVersion: "founder-weekly-review/v2", + sections: { + whatChanged: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Billing exports changed.", + sourceIds: ["doc-1"], + confidence: 0.9, + }], + }, + whatShipped: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Billing exports shipped.", + sourceIds: ["doc-1"], + confidence: 0.9, + }], + }, + whatCustomersSaid: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "Prospects requested audit logging.", + sourceIds: ["feedback-1"], + confidence: 0.8, + }], + }, + currentBlockers: { + state: "evidence", + items: [{ + kind: "observed_fact", + text: "SSO setup remains blocked.", + sourceIds: ["founder-context-1"], + confidence: 0.8, + }], + }, + nextPriorities: { + state: "evidence", + items: [{ + kind: "recommendation", + label: "Recommendation", + text: "Prioritize SSO setup.", + sourceIds: ["founder-context-1"], + confidence: 0.8, + rationale: null, + }], + }, + }, + }; +} + +async function insertCompany( + db: Awaited>["db"], + name: string +): Promise { + const rows = await db.execute(sql` + INSERT INTO "pdr_ai_v2_company" ("name", "numberOfEmployees") + VALUES (${name}, '5') + RETURNING "id" + `); + + const [result] = rows; + + if (!result || typeof result !== "object" || !("id" in result)) { + throw new Error(`Failed to insert company fixture for ${name}`); + } + + const id = result.id; + + if ( + typeof id !== "number" && + typeof id !== "string" && + typeof id !== "bigint" + ) { + throw new Error(`Invalid company ID returned for ${name}`); + } + + return BigInt(id); +} + +describeIfDatabase("Founder Weekly Review lifecycle integration", () => { + it("covers create idempotency, isolation, lifecycle transitions, claims, retries, and immutability", async () => { + const testDb = await createFounderWeeklyReviewTestDatabase(); + const extraSession = await testDb.createSession(); + const thirdSession = await testDb.createSession(); + + try { + const companyAId = await insertCompany(testDb.db, "Alpha"); + const companyBId = await insertCompany(testDb.db, "Beta"); + const repositoryA = new FounderWeeklyReviewRepository(testDb.db); + const repositoryB = new FounderWeeklyReviewRepository(extraSession.db); + const userServiceA = new FounderWeeklyReviewUserService(repositoryA); + const userServiceB = new FounderWeeklyReviewUserService(repositoryB); + const workerA = new FounderWeeklyReviewWorkerService(repositoryA); + const workerB = new FounderWeeklyReviewWorkerService( + new FounderWeeklyReviewRepository(thirdSession.db) + ); + + const actorA = { + externalUserId: "clerk_alpha", + internalUserId: 101n, + companyId: companyAId, + role: "owner", + }; + const actorB = { + externalUserId: "clerk_beta", + internalUserId: 202n, + companyId: companyBId, + role: "owner", + }; + + const snapshot = createEvidenceSnapshot(); + expect(JSON.stringify(snapshot)).not.toMatch( + /token|api[_-]?key|authorization|signedUrl|presigned/i + ); + + const created = await userServiceA.createOrGetRun(actorA, { + requestKey: "create-1", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + expect(created.status).toBe("queued"); + expect(created.evidenceSnapshot).toEqual(snapshot); + + const createdAgain = await userServiceA.createOrGetRun(actorA, { + requestKey: "create-1", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: { + ...snapshot, + items: [], + }, + }); + expect(createdAgain.id).toBe(created.id); + expect(createdAgain.evidenceSnapshot).toEqual(snapshot); + + const companyBRun = await userServiceB.createOrGetRun(actorB, { + requestKey: "create-1", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + expect(companyBRun.id).not.toBe(created.id); + + await expect(userServiceB.getRun(actorB, created.id)).rejects.toBeInstanceOf( + FounderWeeklyReviewNotFoundError + ); + await expect( + userServiceB.updateDraft(actorB, created.id, createPayload("wrong-company")) + ).rejects.toBeInstanceOf(FounderWeeklyReviewNotFoundError); + await expect(userServiceB.publishDraft(actorB, created.id)).rejects.toBeInstanceOf( + FounderWeeklyReviewNotFoundError + ); + await expect( + userServiceB.retryFailedRun(actorB, created.id, "retry-wrong-company") + ).rejects.toBeInstanceOf(FounderWeeklyReviewNotFoundError); + await expect( + workerB.claimQueuedRun({ + companyId: companyBId, + runId: created.id, + generationClaimId: "claim-wrong-company", + }) + ).rejects.toBeInstanceOf(FounderWeeklyReviewNotFoundError); + + const claimOne = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-1", + generationJobId: "job-1", + }); + expect(claimOne.status).toBe("generating"); + expect(claimOne.generationAttempt).toBe(1); + + const claimOneAgain = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-1", + generationJobId: "job-1", + }); + expect(claimOneAgain.generationAttempt).toBe(1); + + await expect( + workerB.saveGeneratedDraft( + { + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-2", + }, + createPayload("claim-mismatch"), + null + ) + ).rejects.toMatchObject({ code: "claim_ownership_mismatch" }); + + const generatedDraft = await workerA.saveGeneratedDraft( + { + companyId: companyAId, + runId: created.id, + generationClaimId: "claim-1", + generationJobId: "job-1", + }, + createPayload("draft-1"), + { + provider: "openai", + model: "gpt-5-mini", + attributes: { latencyMs: 1300 }, + } + ); + expect(generatedDraft.status).toBe("draft"); + expect(generatedDraft.generatedAt).not.toBeNull(); + + const editedDraft = await userServiceA.updateDraft( + actorA, + created.id, + createPayload("draft-2") + ); + expect(editedDraft.status).toBe("draft"); + expect(editedDraft.reviewPayload?.schemaVersion).toBe( + "founder-weekly-review/v1" + ); + if ( + !editedDraft.reviewPayload || + editedDraft.reviewPayload.schemaVersion !== "founder-weekly-review/v1" + ) { + throw new Error("Expected v1 draft payload"); + } + expect(editedDraft.reviewPayload.sections.whatChanged.items[0]).toMatchObject({ + text: "Observed fact draft-2", + }); + + const published = await userServiceA.publishDraft(actorA, created.id); + expect(published.status).toBe("published"); + expect(published.publishedAt).not.toBeNull(); + expect(published.evidenceSnapshot).toEqual(snapshot); + + const republished = await userServiceA.publishDraft(actorA, created.id); + expect(republished.status).toBe("published"); + expect(republished.publishedAt?.toISOString()).toBe( + published.publishedAt?.toISOString() + ); + + await expect( + userServiceA.updateDraft(actorA, created.id, createPayload("after-publish")) + ).rejects.toBeInstanceOf(FounderWeeklyReviewInvalidTransitionError); + await expect( + userServiceA.retryFailedRun(actorA, created.id, "retry-published") + ).rejects.toBeInstanceOf(FounderWeeklyReviewInvalidTransitionError); + + const failedBeforeClaim = await userServiceA.createOrGetRun(actorA, { + requestKey: "queued-fail", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + const queuedFailure = await workerA.markQueuedRunFailed( + companyAId, + failedBeforeClaim.id, + { + errorCode: "EVIDENCE_INVALID", + errorMessage: "e".repeat(2000), + } + ); + expect(queuedFailure.status).toBe("failed"); + expect(queuedFailure.failureSequence).toBe(1); + expect(queuedFailure.errorMessage?.length).toBeLessThanOrEqual(1024); + + const retried = await userServiceA.retryFailedRun( + actorA, + failedBeforeClaim.id, + "retry-1" + ); + expect(retried.status).toBe("queued"); + expect(retried.retryCount).toBe(1); + expect(retried.errorCode).toBeNull(); + expect(retried.generationClaimId).toBeNull(); + expect(retried.evidenceSnapshot).toEqual(snapshot); + + const retriedAgain = await userServiceA.retryFailedRun( + actorA, + failedBeforeClaim.id, + "retry-1" + ); + expect(retriedAgain.retryCount).toBe(1); + + const claimRaceRun = await userServiceA.createOrGetRun(actorA, { + requestKey: "claim-race", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + const raceResults = await Promise.allSettled([ + workerA.claimQueuedRun({ + companyId: companyAId, + runId: claimRaceRun.id, + generationClaimId: "race-1", + }), + workerB.claimQueuedRun({ + companyId: companyAId, + runId: claimRaceRun.id, + generationClaimId: "race-2", + }), + ]); + const fulfilledClaims = raceResults.filter( + (result): result is PromiseFulfilledResult> => + result.status === "fulfilled" + ); + const rejectedClaims = raceResults.filter( + (result): result is PromiseRejectedResult => result.status === "rejected" + ); + expect(fulfilledClaims).toHaveLength(1); + expect(rejectedClaims).toHaveLength(1); + expect(rejectedClaims[0]?.reason).toBeInstanceOf(FounderWeeklyReviewConflictError); + + const generatingFailureRun = await userServiceA.createOrGetRun(actorA, { + requestKey: "generation-fail", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + const generatingFailureClaim = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-1", + }); + expect(generatingFailureClaim.status).toBe("generating"); + + const failed = await workerA.markGenerationFailed( + { + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-1", + }, + { + errorCode: "MODEL_TIMEOUT", + errorMessage: "provider timed out", + } + ); + expect(failed.status).toBe("failed"); + expect(failed.failureSequence).toBe(1); + + const failedAgain = await workerA.markGenerationFailed( + { + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-1", + }, + { + errorCode: "MODEL_TIMEOUT", + errorMessage: "provider timed out", + } + ); + expect(failedAgain.failureSequence).toBe(1); + + const retryRaceRun = await userServiceA.retryFailedRun( + actorA, + generatingFailureRun.id, + "retry-generation-1" + ); + expect(retryRaceRun.retryCount).toBe(1); + + const secondClaim = await workerA.claimQueuedRun({ + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-2", + }); + expect(secondClaim.generationAttempt).toBe(2); + + const secondFailure = await workerA.markGenerationFailed( + { + companyId: companyAId, + runId: generatingFailureRun.id, + generationClaimId: "gf-2", + }, + { + errorCode: "MODEL_TIMEOUT", + errorMessage: "provider timed out again", + } + ); + expect(secondFailure.failureSequence).toBe(2); + + await expect( + userServiceA.retryFailedRun(actorA, generatingFailureRun.id, "retry-generation-1") + ).rejects.toBeInstanceOf(FounderWeeklyReviewConflictError); + + const concurrentRetryRun = await userServiceA.createOrGetRun(actorA, { + requestKey: "retry-race", + reportingPeriod: snapshot.reportingPeriod, + evidenceSnapshot: snapshot, + }); + await workerA.markQueuedRunFailed(companyAId, concurrentRetryRun.id, { + errorCode: "DISPATCH_FAILED", + errorMessage: "queue dispatch failed", + }); + const concurrentRetryResults = await Promise.all([ + userServiceA.retryFailedRun(actorA, concurrentRetryRun.id, "same-retry-key"), + userServiceA.retryFailedRun(actorA, concurrentRetryRun.id, "same-retry-key"), + ]); + expect(concurrentRetryResults[0].retryCount).toBe(1); + expect(concurrentRetryResults[1].retryCount).toBe(1); + + const listed = await userServiceA.listRuns(actorA); + expect(listed.length).toBeGreaterThanOrEqual(5); + + await expect( + userServiceA.updateDraft( + actorA, + claimRaceRun.id, + { + schemaVersion: "founder-weekly-review/v1", + } as FounderWeeklyReviewPayload + ) + ).rejects.toBeInstanceOf(FounderWeeklyReviewInvalidPayloadError); + } finally { + await thirdSession.close(); + await extraSession.close(); + await testDb.close(); + } + }); + + it("round-trips v2 drafts and rejects mismatched persisted schema versions", async () => { + const testDb = await createFounderWeeklyReviewTestDatabase(); + try { + const companyId = await insertCompany(testDb.db, "V2 Review Co"); + const repository = new FounderWeeklyReviewRepository(testDb.db); + const userService = new FounderWeeklyReviewUserService(repository); + const worker = new FounderWeeklyReviewWorkerService(repository); + const actor = { + externalUserId: "clerk_v2", + companyId, + role: "owner", + }; + const evidenceSnapshot = createEvidenceSnapshot(); + const run = await userService.createOrGetRun(actor, { + requestKey: "v2-round-trip", + reportingPeriod: evidenceSnapshot.reportingPeriod, + evidenceSnapshot, + }); + const claim = { companyId, runId: run.id, generationClaimId: "v2-claim" }; + await worker.claimQueuedRun(claim); + + const payload = createV2Payload(); + const saved = await worker.saveGeneratedDraft(claim, payload, { + provider: "openai", + model: "gpt-4o-mini", + capability: "founderWeeklyReview", + temperature: 0, + attributes: {}, + }); + expect(saved.reviewPayload).toEqual(payload); + expect(saved.reviewPayload?.schemaVersion).toBe("founder-weekly-review/v2"); + expect(saved.reviewSchemaVersion).toBe("founder-weekly-review/v2"); + + const reread = await repository.getByCompanyAndRunId(companyId, run.id); + expect(reread?.reviewPayload).toEqual(payload); + expect(reread?.reviewPayload?.schemaVersion).toBe("founder-weekly-review/v2"); + expect(reread?.reviewSchemaVersion).toBe("founder-weekly-review/v2"); + + await testDb.db.execute(sql` + UPDATE "pdr_ai_v2_founder_weekly_review_runs" + SET "review_schema_version" = 'founder-weekly-review/v1' + WHERE "id" = ${run.id} + `); + await expect(repository.getByCompanyAndRunId(companyId, run.id)).rejects.toBeInstanceOf( + FounderWeeklyReviewInvalidPayloadError + ); + } finally { + await testDb.close(); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/markdown.test.ts b/apps/web/__tests__/founderWeeklyReview/markdown.test.ts new file mode 100644 index 000000000..ffbd29b76 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/markdown.test.ts @@ -0,0 +1,55 @@ +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; + +function reviewRun() { + return { + reportingPeriod: { start: "2026-02-16", end: "2026-02-28" }, + modelMetadata: { model: "kimi-k2.6" }, + evidenceSnapshot: { + items: [ + { sourceId: "customer_feedback:doc:7:version:3:section:15", sourceType: "customer_feedback", title: "Customer Interviews — February 2026", excerpt: "first raw excerpt", metadata: { pageNumber: 2, sectionId: 15, documentCategory: "Customer Feedback" } }, + { sourceId: "customer_feedback:doc:7:version:3:section:16", sourceType: "customer_feedback", title: "Customer Interviews — February 2026", excerpt: "second raw excerpt", metadata: { pageNumber: 3, sectionId: 16, documentCategory: "Customer Feedback" } }, + { sourceId: "document_change:doc:4:version:2", sourceType: "document_change", title: "Onboarding reliability plan", excerpt: "document changelog", metadata: { versionNumber: 2 } }, + { sourceId: "founder_context:request:secret", sourceType: "founder_context", title: "Founder context", excerpt: "private context", metadata: {} }, + ], + }, + reviewPayload: { + schemaVersion: "founder-weekly-review/v2", + sections: { + whatShipped: { state: "no_evidence", noEvidence: { code: "none", message: "No completed release evidence.", cta: "Add release evidence." } }, + whatChanged: { state: "no_evidence", noEvidence: { code: "none", message: "No other change evidence.", cta: "Add change evidence." } }, + whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "Two distinct interview excerpts identify reporting setup and recovery concerns.", sourceIds: ["customer_feedback:doc:7:version:3:section:15", "customer_feedback:doc:7:version:3:section:16"], confidence: 0.8 }] }, + currentBlockers: { state: "evidence", items: [{ kind: "observed_fact", text: "The documented plan is operational preparation, not proof that retries are fixed.", sourceIds: ["document_change:doc:4:version:2"], confidence: 0.8 }] }, + nextPriorities: { state: "evidence", items: [{ kind: "recommendation", label: "Recommendation", text: "Improve recovery feedback.", rationale: "Founder-provided context identifies it as a priority.", sourceIds: ["founder_context:request:secret"], confidence: 0.7 }] }, + }, + }, + } as any; +} + +describe("Founder Weekly Review Markdown", () => { + it("renders useful no-evidence sections without an empty Key Outcomes container", () => { + const rendered = renderFounderWeeklyReviewMarkdown(reviewRun()); + + expect(rendered).not.toContain("## Key Outcomes"); + expect(rendered).toContain("## Shipped This Period\n\nNo completed release evidence.\n\nNext: Add release evidence."); + }); + + it("uses deterministic, distinct human-readable evidence labels with metadata", () => { + const rendered = renderFounderWeeklyReviewMarkdown(reviewRun()); + + expect(rendered).toContain("[1] Customer Interviews — February 2026 — page 2 — section 15"); + expect(rendered).toContain("[2] Customer Interviews — February 2026 — page 3 — section 16"); + expect(rendered).toContain("[3] Document change — Onboarding reliability plan"); + expect(rendered).toContain("[4] Founder-provided context"); + expect(rendered).not.toContain("[1] Customer Feedback"); + }); + + it("uses the exact rendered Markdown for terminal and export paths without operational internals", () => { + const terminalMarkdown = renderFounderWeeklyReviewMarkdown(reviewRun()); + const exportedMarkdown = renderFounderWeeklyReviewMarkdown(reviewRun()); + + expect(exportedMarkdown).toBe(terminalMarkdown); + for (const forbidden of ["customer_feedback:doc:7", "founder_context:request:secret", "first raw excerpt", "evidenceSnapshot", "MOONSHOT_API_KEY", "DATABASE_URL", "provider raw response", "internal error"]) { + expect(terminalMarkdown).not.toContain(forbidden); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts new file mode 100644 index 000000000..854636925 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/materiality-evaluation.test.ts @@ -0,0 +1,178 @@ +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; + +import { + evaluationArtifactDirectory, + renderMaterialityEvaluationArtifacts, + runMaterialityAnalyzerEvaluation, + runMaterialityEvaluation, +} from "../../scripts/founder-weekly-review-materiality-evaluation"; +import { + FOUNDER_WEEKLY_REVIEW_MATERIALITY_ANALYZER_V1_BASELINES, + FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE, +} from "../../scripts/founder-weekly-review-materiality-evaluation-baseline"; +import { + MATERIALITY_EVALUATION_SCENARIOS, +} from "../../scripts/founder-weekly-review-materiality-evaluation-fixtures"; + +describe("Founder Weekly Review realistic materiality evaluation harness", () => { + it("loads a compact, stable, human-readable ground-truth matrix", () => { + expect(MATERIALITY_EVALUATION_SCENARIOS).toHaveLength(46); + expect(new Set(MATERIALITY_EVALUATION_SCENARIOS.map(scenario => scenario.id)).size).toBe(46); + expect(MATERIALITY_EVALUATION_SCENARIOS.filter(scenario => scenario.multiChunk).length).toBeGreaterThanOrEqual(10); + expect(MATERIALITY_EVALUATION_SCENARIOS.filter(scenario => scenario.largeDocument).length).toBeGreaterThanOrEqual(5); + for (const scenario of MATERIALITY_EVALUATION_SCENARIOS) { + expect(scenario.description.length).toBeGreaterThan(20); + expect(scenario.expected.expectedAlignmentRelations.length).toBeGreaterThan(0); + for (const expectation of [ + ...scenario.expected.meaningfulChanges, + ...scenario.expected.nonMaterialChanges, + ...scenario.expected.expectedNoOps, + ...scenario.expected.expectedAlignmentRelations, + ]) expect(expectation.id.startsWith(`${scenario.id}:`)).toBe(true); + } + }); + + it("freezes prior offline, failed OpenAI, and canonical Kimi-v1 analyzer reference points", () => { + expect(FOUNDER_WEEKLY_REVIEW_MATERIALITY_ANALYZER_V1_BASELINES).toEqual(expect.objectContaining({ + offline: expect.objectContaining({ materialRecall: 0.9677, falseMaterialRate: 0.6429 }), + openAiLive: expect.objectContaining({ semanticQualityMeasured: false, timeouts: 18 }), + kimiLive: expect.objectContaining({ + promptVersion: "document-change-materiality/v1", + materialRecall: 1, + materialPrecision: 0.7949, + falseMaterialRate: 0.5714, + invalidAnalyzerResponses: 3, + }), + })); + }); + + it("calculates explicit confusion, alignment, condensation, and budget metrics", () => { + const result = runMaterialityEvaluation(); + expect(result.summary.scenarioCount).toBe(46); + expect(result.summary.materiality.groundTruthMaterialChanges).toBeGreaterThan(20); + expect(result.summary.materiality.groundTruthNonMaterialChanges).toBeGreaterThan(8); + expect(result.summary.materiality.uncertainRate).toBeGreaterThanOrEqual(0); + expect(result.summary.materiality.falseMaterialRate).toBeGreaterThanOrEqual(0); + expect(result.summary.alignment.intendedSemanticRelations).toBeGreaterThan(40); + expect(result.summary.alignment.alignmentMissRate).toBeGreaterThanOrEqual(0); + expect(result.summary.condensation.rawChangedRecords).toBeGreaterThan(result.summary.condensation.condensedEvidenceItems); + expect(result.summary.condensation.reductionRatio).toBeCloseTo( + result.summary.condensation.rawCopiedCharacters / result.summary.condensation.condensedEvidenceCharacters, + 3 + ); + expect(result.summary.budget.groupBudgetTruncated).toBe(true); + expect(result.summary.budget.documentDiversityPreserved).toBe(true); + }); + + it("freezes the deterministic-v1 commit, fixture shape, formulas, and exact metrics", () => { + const result = runMaterialityEvaluation(); + const baseline = FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE; + expect(baseline.baselineCommit).toBe("630660fc861e77c464eaffc26ed6f04a7e38e7c7"); + expect(result.summary).toMatchObject({ + fixtureVersion: baseline.fixtureVersion, + scenarioCount: baseline.scenarioCount, + multiChunkScenarioCount: baseline.multiChunkScenarioCount, + largeDocumentScenarioCount: baseline.largeDocumentScenarioCount, + materiality: { + categoryAccuracy: baseline.metrics.categoryAccuracy, + materialRecall: baseline.metrics.materialRecall, + materialPrecision: baseline.metrics.materialPrecision, + uncertainRate: baseline.metrics.uncertainRate, + falseMaterialRate: baseline.metrics.falseMaterialRate, + missedMaterialRate: baseline.metrics.missedMaterialRate, + uncertainMaterialRate: baseline.metrics.uncertainMaterialRate, + uncertainNonMaterialRate: baseline.metrics.uncertainNonMaterialRate, + }, + alignment: { + alignmentMissRate: baseline.metrics.alignmentMissRate, + falseMatchRate: baseline.metrics.alignmentFalseMatchRate, + }, + condensation: { + rawChangedRecords: baseline.counts.rawChangedRecords, + groups: baseline.counts.groups, + condensedEvidenceItems: baseline.counts.structurallySelectedEvidenceItems, + rawCopiedCharacters: baseline.counts.rawCopiedCharacters, + condensedEvidenceCharacters: baseline.counts.condensedCharacters, + serializedPromptCharacters: baseline.counts.serializedPromptEvidenceCharacters, + reductionRatio: baseline.counts.reductionRatio, + }, + budget: { generationEnvelopeSelectedItems: baseline.counts.generationEnvelopeItems }, + }); + }); + + it("records all four critical materiality buckets separately", () => { + const result = runMaterialityEvaluation(); + expect(result.summary.failuresByKind).toEqual(expect.objectContaining({ + materiality_false_positive: expect.any(Number), + materiality_false_negative: expect.any(Number), + uncertain_material: expect.any(Number), + uncertain_non_material: expect.any(Number), + })); + expect(result.summary.materiality.falseMaterialCount).toBe(result.summary.failuresByKind.materiality_false_positive); + expect(result.summary.materiality.missedMaterialCount).toBe(result.summary.failuresByKind.materiality_false_negative); + }); + + it("calculates alignment misses and false matches from explicit intended relations", () => { + const result = runMaterialityEvaluation(); + expect(result.summary.alignment.alignmentMisses).toBe(result.summary.failuresByKind.alignment_miss); + expect(result.summary.alignment.falsePairings).toBe(result.summary.failuresByKind.alignment_false_match); + expect(result.summary.alignment.unmatchedOldChunks).toBeGreaterThanOrEqual(result.summary.alignment.intendedModifiedAsAddedRemoved); + expect(result.summary.alignment.unmatchedNewChunks).toBeGreaterThanOrEqual(result.summary.alignment.intendedModifiedAsAddedRemoved); + }); + + it("is byte-deterministic when scenario input order changes", () => { + const forward = runMaterialityEvaluation(MATERIALITY_EVALUATION_SCENARIOS); + const reversed = runMaterialityEvaluation([...MATERIALITY_EVALUATION_SCENARIOS].reverse()); + expect(renderMaterialityEvaluationArtifacts(reversed)).toEqual(renderMaterialityEvaluationArtifacts(forward)); + }); + + it("renders deterministic aggregate, synthetic-failure, and Markdown reports", () => { + const artifacts = renderMaterialityEvaluationArtifacts(runMaterialityEvaluation()); + expect(JSON.parse(artifacts.summaryJson)).toEqual(expect.objectContaining({ + summary: expect.objectContaining({ scenarioCount: 46 }), + scenarios: expect.any(Array), + })); + const failures = JSON.parse(artifacts.failuresJson) as { failures: Array<{ syntheticFixture: unknown }> }; + expect(failures.failures.length).toBeGreaterThan(0); + expect(failures.failures.every(failure => failure.syntheticFixture)).toBe(true); + expect(artifacts.evaluationMarkdown).toContain("## Recommendation"); + }); + + it("keeps generated report paths under the repository-gitignored artifact root", () => { + const directory = evaluationArtifactDirectory("test-run"); + expect(directory.replace(/\\/g, "/")).toContain("/apps/web/.artifacts/founder-weekly-review/materiality-evaluation/test-run"); + execFileSync("git", ["check-ignore", "-q", "apps/web/.artifacts/founder-weekly-review/materiality-evaluation/test-run/summary.json"], { + cwd: resolve(process.cwd(), "../.."), + }); + }); + + it("runs provider-free even when network access is made fatal", () => { + const fetchSpy = jest.spyOn(global, "fetch").mockImplementation(() => { + throw new Error("provider/network invocation is forbidden"); + }); + try { + expect(() => runMaterialityEvaluation()).not.toThrow(); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + fetchSpy.mockRestore(); + } + }); + + it("evaluates the analyzer strategy offline against the identical frozen fixtures", async () => { + const fetchSpy = jest.spyOn(global, "fetch").mockImplementation(() => { + throw new Error("provider/network invocation is forbidden"); + }); + try { + const forward = await runMaterialityAnalyzerEvaluation(undefined, MATERIALITY_EVALUATION_SCENARIOS); + const reversed = await runMaterialityAnalyzerEvaluation(undefined, [...MATERIALITY_EVALUATION_SCENARIOS].reverse()); + expect(forward.summary.scenarioCount).toBe(FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE.scenarioCount); + expect(forward.summary.groundTruthCategoryDistribution).toEqual(runMaterialityEvaluation().summary.groundTruthCategoryDistribution); + expect(forward.summary.alignment).toEqual(runMaterialityEvaluation().summary.alignment); + expect(renderMaterialityEvaluationArtifacts(reversed)).toEqual(renderMaterialityEvaluationArtifacts(forward)); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + fetchSpy.mockRestore(); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/migration.test.ts b/apps/web/__tests__/founderWeeklyReview/migration.test.ts new file mode 100644 index 000000000..fd11b5acf --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/migration.test.ts @@ -0,0 +1,87 @@ +import { sql } from "drizzle-orm"; + +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeIfDatabase = + process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL + ? describe + : describe.skip; + +describeIfDatabase("Founder Weekly Review migrations", () => { + it("replays all migrations into a clean schema and creates the new tables and constraints", async () => { + const testDb = await createFounderWeeklyReviewTestDatabase(); + try { + const runsTable = await testDb.db.execute(sql` + SELECT to_regclass('pdr_ai_v2_founder_weekly_review_runs') AS name + `); + const opsTable = await testDb.db.execute(sql` + SELECT to_regclass('pdr_ai_v2_founder_weekly_review_operations') AS name + `); + const indexes = await testDb.db.execute(sql` + SELECT indexname + FROM pg_indexes + WHERE tablename IN ( + 'pdr_ai_v2_founder_weekly_review_runs', + 'pdr_ai_v2_founder_weekly_review_operations' + ) + ORDER BY indexname + `); + + expect(runsTable[0]?.name).toBe("pdr_ai_v2_founder_weekly_review_runs"); + expect(opsTable[0]?.name).toBe("pdr_ai_v2_founder_weekly_review_operations"); + expect(indexes.map((row) => row.indexname)).toEqual( + expect.arrayContaining([ + "founder_weekly_review_runs_company_request_key_unique", + "founder_weekly_review_operations_run_type_request_key_unique", + "founder_weekly_review_runs_company_status_created_at_idx", + ]) + ); + + await testDb.db.execute(sql` + INSERT INTO "pdr_ai_v2_company" + ("name", "numberOfEmployees") + VALUES + ('Migration Test Co', '3') + `); + await testDb.db.execute(sql` + INSERT INTO "pdr_ai_v2_founder_weekly_review_runs" + ( + "id", + "company_id", + "request_key", + "reporting_period_start", + "reporting_period_end", + "status", + "review_schema_version", + "evidence_snapshot", + "evidence_schema_version", + "collection_input", + "created_by_actor_id" + ) + VALUES + ( + 'fwr_migration_test', + 1, + 'request-key', + DATE '2026-07-07', + DATE '2026-07-13', + 'queued', + 'founder-weekly-review/v1', + '{"schemaVersion":"founder-weekly-review-evidence/v1","capturedAt":"2026-07-18T00:00:00.000Z","reportingPeriod":{"start":"2026-07-07","end":"2026-07-13"},"workspaceTimezone":"UTC","items":[],"sourceWarnings":[]}'::jsonb, + 'founder-weekly-review-evidence/v1', + '{"workspaceTimezone":"UTC","actorExternalUserId":"test"}'::jsonb, + 'user:test' + ) + `); + + const inserted = await testDb.db.execute(sql` + SELECT "status" + FROM "pdr_ai_v2_founder_weekly_review_runs" + WHERE "id" = 'fwr_migration_test' + `); + expect(inserted[0]?.status).toBe("queued"); + } finally { + await testDb.close(); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts b/apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts new file mode 100644 index 000000000..bd4cb976f --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/read-retry-route.test.ts @@ -0,0 +1,20 @@ +jest.mock("@clerk/nextjs/server", () => ({ auth: jest.fn() })); +jest.mock("~/server/founder-weekly-review/actor-resolver", () => ({ productionFounderWeeklyReviewActorResolver: { resolve: jest.fn() } })); +jest.mock("~/server/founder-weekly-review/dispatch-service", () => ({ retryRunWithDispatch: jest.fn() })); +import { auth } from "@clerk/nextjs/server"; +import { createFounderWeeklyReviewGetHandler } from "~/app/api/founder-weekly-reviews/[runId]/route"; +import { createFounderWeeklyReviewRetryPostHandler } from "~/app/api/founder-weekly-reviews/[runId]/retry/route"; +import type { FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; +const mockAuth = auth as unknown as jest.Mock; +const actor = { externalUserId: "u", internalUserId: 1n, companyId: 1n, role: "owner" as const }; +function run(status: FounderWeeklyReviewRunRecord["status"] = "queued"): FounderWeeklyReviewRunRecord { return { id: "fwr_1", companyId: 1n, requestKey: "key", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, status, reviewPayload: status === "draft" ? { schemaVersion: "founder-weekly-review/v2", sections: {} } as never : null, reviewSchemaVersion: "founder-weekly-review/v1", evidenceSnapshot: { schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [{ sourceType: "founder_context", sourceId: "secret", title: "Founder context", excerpt: "DO NOT LEAK", metadata: {} }], sourceWarnings: [] }, evidenceSchemaVersion: "founder-weekly-review-evidence/v1", modelMetadata: null, createdByActorId: "user:u", retryCount: 0, failureSequence: 0, generationAttempt: 0, generationClaimId: null, generationJobId: null, queuedAt: new Date(), claimedAt: null, generationStartedAt: null, generatedAt: null, publishedAt: null, errorCode: "safe_code", errorMessage: "provider secret", createdAt: new Date(), updatedAt: null }; } +describe("Founder Weekly Review read and retry route handlers", () => { + beforeEach(() => mockAuth.mockResolvedValue({ userId: "u" })); + it("read authenticates before lookup and returns safe status payloads", async () => { const getRun = jest.fn().mockResolvedValue(run("failed")); const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, getRun }); const response = await handler(new Request("http://test"), { params: Promise.resolve({ runId: "fwr_1" }) }); const json = await response.json(); expect(response.status).toBe(200); expect(json.run.status).toBe("failed"); expect(JSON.stringify(json)).not.toContain("DO NOT LEAK"); expect(JSON.stringify(json)).not.toContain("provider secret"); }); + it("read returns 401 without lookup and maps company-scoped absence to 404", async () => { mockAuth.mockResolvedValue({ userId: null }); const getRun = jest.fn(); const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn() }, getRun }); expect((await handler(new Request("http://test"), { params: Promise.resolve({ runId: "x" }) })).status).toBe(401); expect(getRun).not.toHaveBeenCalled(); mockAuth.mockResolvedValue({ userId: "u" }); const missing = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, getRun: jest.fn().mockRejectedValue({ code: "not_found" }) }); expect((await missing(new Request("http://test"), { params: Promise.resolve({ runId: "x" }) })).status).toBe(404); }); + it.each(["queued", "generating", "draft", "failed", "published"] as const)("returns the %s status without snapshot internals", async (status) => { const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, getRun: jest.fn().mockResolvedValue(run(status)) }); const json = await (await handler(new Request("http://test"), { params: Promise.resolve({ runId: "fwr_1" }) })).json(); expect(json.run).toMatchObject({ id: "fwr_1", status }); expect(json.run).not.toHaveProperty("evidenceSnapshot"); expect(JSON.stringify(json)).not.toContain("DO NOT LEAK"); }); + it("does not look up a run when strict actor resolution fails", async () => { const getRun = jest.fn(); const handler = createFounderWeeklyReviewGetHandler({ actorResolver: { resolve: jest.fn().mockRejectedValue({ code: "forbidden" }) }, getRun }); expect((await handler(new Request("http://test"), { params: Promise.resolve({ runId: "fwr_1" }) })).status).toBe(403); expect(getRun).not.toHaveBeenCalled(); }); + it("retry increments only an applied transition and does not notify rejected callers", async () => { const incrementRetry = jest.fn(), sendDispatchRequested = jest.fn().mockResolvedValue(undefined); const retryRunWithDispatch = jest.fn().mockResolvedValue({ run: run("queued"), dispatch: {}, transitionApplied: true }); const handler = createFounderWeeklyReviewRetryPostHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, retryRunWithDispatch, incrementRetry, sendDispatchRequested }); expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r1" }) }), { params: Promise.resolve({ runId: "fwr_1" }) })).status).toBe(202); expect(incrementRetry).toHaveBeenCalledTimes(1); expect(sendDispatchRequested).toHaveBeenCalledTimes(1); retryRunWithDispatch.mockResolvedValue({ run: run("queued"), dispatch: {}, transitionApplied: false }); await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r1" }) }), { params: Promise.resolve({ runId: "fwr_1" }) }); expect(incrementRetry).toHaveBeenCalledTimes(1); }); + it("retry rejects auth before mutation or notification", async () => { mockAuth.mockResolvedValue({ userId: null }); const retryRunWithDispatch = jest.fn(), sendDispatchRequested = jest.fn(), incrementRetry = jest.fn(); const handler = createFounderWeeklyReviewRetryPostHandler({ actorResolver: { resolve: jest.fn() }, retryRunWithDispatch, sendDispatchRequested, incrementRetry }); expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r" }) }), { params: Promise.resolve({ runId: "x" }) })).status).toBe(401); expect(retryRunWithDispatch).not.toHaveBeenCalled(); expect(sendDispatchRequested).not.toHaveBeenCalled(); expect(incrementRetry).not.toHaveBeenCalled(); }); + it.each(["queued", "generating", "draft", "published"] as const)("maps %s retry conflict safely", async (status) => { const retryRunWithDispatch = jest.fn().mockRejectedValue({ code: "invalid_transition", status }); const sendDispatchRequested = jest.fn(), incrementRetry = jest.fn(); const handler = createFounderWeeklyReviewRetryPostHandler({ actorResolver: { resolve: jest.fn().mockResolvedValue(actor) }, retryRunWithDispatch, sendDispatchRequested, incrementRetry }); const response = await handler(new Request("http://test", { method: "POST", body: JSON.stringify({ requestKey: "r" }) }), { params: Promise.resolve({ runId: "fwr_1" }) }); expect(response.status).toBe(409); expect(incrementRetry).not.toHaveBeenCalled(); expect(sendDispatchRequested).not.toHaveBeenCalled(); }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts b/apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts new file mode 100644 index 000000000..67219135e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/realistic-e2e-mode.test.ts @@ -0,0 +1,18 @@ +import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; + +describe("realistic Founder Weekly Review evidence mode", () => { + it("preserves legacy mode by default and accepts computed mode explicitly", () => { + expect(parseFounderWeeklyReviewRealisticEvidenceMode(undefined)).toBe("legacy"); + expect(parseFounderWeeklyReviewRealisticEvidenceMode("legacy")).toBe("legacy"); + expect(parseFounderWeeklyReviewRealisticEvidenceMode("computed")).toBe("computed"); + }); + + it("rejects unknown evidence modes before any collection or provider call", () => { + expect(() => parseFounderWeeklyReviewRealisticEvidenceMode("other")).toThrow("Unsupported FWR_EVIDENCE_MODE"); + }); + + it("does not append computed twice when the configured export root already names it", () => { + expect(founderWeeklyReviewRealisticExportRoot("computed", ".artifacts/founder-weekly-review/computed")).toBe(".artifacts/founder-weekly-review/computed"); + expect(founderWeeklyReviewRealisticExportRoot("computed", undefined)).toBe(".artifacts/founder-weekly-review/computed"); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts b/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts new file mode 100644 index 000000000..899eb9b6e --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/reporting-period.test.ts @@ -0,0 +1,54 @@ +import { resolveReportingPeriodBounds } from "@launchstack/features/founder-weekly-review"; + +describe("resolveReportingPeriodBounds", () => { + it("resolves midnight-to-midnight bounds in the workspace timezone", () => { + // America/Los_Angeles in February is PST (UTC-8), so local midnight is 08:00 UTC. + const bounds = resolveReportingPeriodBounds( + { start: "2026-02-16", end: "2026-02-22" }, + "America/Los_Angeles" + ); + + expect(bounds.startInclusive.toISOString()).toBe("2026-02-16T08:00:00.000Z"); + // end is the last included day, so the exclusive bound is the next day's midnight. + expect(bounds.endExclusive.toISOString()).toBe("2026-02-23T08:00:00.000Z"); + }); + + it("accounts for daylight saving in the same timezone", () => { + // Same zone, but July is PDT (UTC-7), so local midnight is 07:00 UTC, not 08:00. + const bounds = resolveReportingPeriodBounds( + { start: "2026-07-06", end: "2026-07-12" }, + "America/Los_Angeles" + ); + + expect(bounds.startInclusive.toISOString()).toBe("2026-07-06T07:00:00.000Z"); + expect(bounds.endExclusive.toISOString()).toBe("2026-07-13T07:00:00.000Z"); + }); + + it("uses the following local midnight across the DST spring transition", () => { + const bounds = resolveReportingPeriodBounds( + { start: "2026-03-08", end: "2026-03-08" }, + "America/Los_Angeles" + ); + expect(bounds.startInclusive.toISOString()).toBe("2026-03-08T08:00:00.000Z"); + expect(bounds.endExclusive.toISOString()).toBe("2026-03-09T07:00:00.000Z"); + }); + + it("treats UTC boundaries as literal midnight", () => { + const bounds = resolveReportingPeriodBounds( + { start: "2026-02-16", end: "2026-02-22" }, + "UTC" + ); + + expect(bounds.startInclusive.toISOString()).toBe("2026-02-16T00:00:00.000Z"); + expect(bounds.endExclusive.toISOString()).toBe("2026-02-23T00:00:00.000Z"); + }); + + it("throws on an invalid timezone", () => { + expect(() => + resolveReportingPeriodBounds( + { start: "2026-02-16", end: "2026-02-22" }, + "Not/AZone" + ) + ).toThrow(/invalid workspace timezone/i); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/routes.test.ts b/apps/web/__tests__/founderWeeklyReview/routes.test.ts new file mode 100644 index 000000000..2c57d8c41 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/routes.test.ts @@ -0,0 +1,53 @@ +jest.mock("@clerk/nextjs/server", () => ({ auth: jest.fn() })); +jest.mock("~/server/founder-weekly-review/actor-resolver", () => ({ + productionFounderWeeklyReviewActorResolver: { resolve: jest.fn() }, +})); +jest.mock("~/server/founder-weekly-review/dispatch-service", () => ({ + createRunWithDispatch: jest.fn(), +})); + +import { createFounderWeeklyReviewPostHandler } from "~/app/api/founder-weekly-reviews/route"; +import type { FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; +import { auth } from "@clerk/nextjs/server"; +const mockAuth = auth as unknown as jest.Mock; + +const run = (id = "fwr_1"): FounderWeeklyReviewRunRecord => ({ + id, companyId: 1n, requestKey: "key", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, status: "queued", reviewPayload: null, + reviewSchemaVersion: "founder-weekly-review/v1", evidenceSnapshot: { schemaVersion: "founder-weekly-review-evidence/v1", capturedAt: "2026-07-13T00:00:00.000Z", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", items: [], sourceWarnings: [] }, evidenceSchemaVersion: "founder-weekly-review-evidence/v1", modelMetadata: null, createdByActorId: "user:u", retryCount: 0, failureSequence: 0, generationAttempt: 0, generationClaimId: null, generationJobId: null, queuedAt: new Date(), claimedAt: null, generationStartedAt: null, generatedAt: null, publishedAt: null, errorCode: null, errorMessage: null, createdAt: new Date(), updatedAt: null, +}); +const body = { requestKey: "key", reportingPeriod: { start: "2026-07-06", end: "2026-07-12" }, workspaceTimezone: "UTC", founderContext: "Context" }; +function setup(overrides: Partial[0]> = {}) { + const collector = { collectFounderWeeklyReviewEvidence: jest.fn().mockResolvedValue(run().evidenceSnapshot) }; + const createRunWithDispatch = jest.fn().mockResolvedValue({ run: run(), dispatch: {}, created: true }); + const deps = { actorResolver: { resolve: jest.fn().mockResolvedValue({ externalUserId: "u", internalUserId: 1n, companyId: 1n, role: "owner" }) }, evidenceCollector: collector, repository: { getByCompanyAndRequestKey: jest.fn().mockResolvedValue(null) }, createRunWithDispatch, sendDispatchRequested: jest.fn().mockResolvedValue(undefined), recordRunCreated: jest.fn(), ...overrides } as Parameters[0]; + return { deps, collector, createRunWithDispatch, handler: createFounderWeeklyReviewPostHandler(deps) }; +} +describe("Founder Weekly Review create route", () => { + beforeEach(() => mockAuth.mockResolvedValue({ userId: "u" })); + it("returns 401 without auth and never resolves or collects", async () => { + mockAuth.mockResolvedValue({ userId: null }); const { handler, collector, deps } = setup(); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(401); + expect(deps.actorResolver.resolve).not.toHaveBeenCalled(); expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); + }); + it("authorizes before validation and never collects rejected requests", async () => { + const { handler, collector, createRunWithDispatch } = setup({ actorResolver: { resolve: jest.fn().mockRejectedValue({ code: "forbidden" }) } }); + expect((await handler(new Request("http://test", { method: "POST", body: "{" }))).status).toBe(403); + expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); expect(createRunWithDispatch).not.toHaveBeenCalled(); + }); + it("returns existing company-scoped run before collection and does not record creation", async () => { + const existing = run("existing"); const { handler, collector, createRunWithDispatch, deps } = setup({ repository: { getByCompanyAndRequestKey: jest.fn().mockResolvedValue(existing) } }); + const response = await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) })); + expect(response.status).toBe(202); expect((await response.json()).run.id).toBe("existing"); expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); expect(createRunWithDispatch).not.toHaveBeenCalled(); expect(deps.recordRunCreated).not.toHaveBeenCalled(); + }); + it("persists durable collection inputs, creates a queued run, and records exactly one creation", async () => { + const { handler, collector, createRunWithDispatch, deps } = setup(); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(202); + expect(collector.collectFounderWeeklyReviewEvidence).not.toHaveBeenCalled(); + expect(createRunWithDispatch).toHaveBeenCalledWith(expect.objectContaining({ requestKey: "key", reportingPeriod: body.reportingPeriod, collectionInput: { workspaceTimezone: "UTC", founderContext: "Context", actorExternalUserId: "u" } })); + expect(createRunWithDispatch).toHaveBeenCalledTimes(1); expect(deps.recordRunCreated).toHaveBeenCalledTimes(1); + }); + it("does not synchronously collect evidence", async () => { + const { handler, createRunWithDispatch, deps } = setup({ evidenceCollector: { collectFounderWeeklyReviewEvidence: jest.fn().mockRejectedValue(new Error("nope")) } }); + expect((await handler(new Request("http://test", { method: "POST", body: JSON.stringify(body) }))).status).toBe(202); expect(createRunWithDispatch).toHaveBeenCalledTimes(1); expect(deps.recordRunCreated).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/runner.test.ts b/apps/web/__tests__/founderWeeklyReview/runner.test.ts new file mode 100644 index 000000000..017cef211 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/runner.test.ts @@ -0,0 +1,139 @@ +import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; +import { FounderWeeklyReviewEvidenceSnapshotSchema } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewV2PayloadSchema } from "@launchstack/features/founder-weekly-review"; + +import type { FounderWeeklyReviewGenerateFn } from "@launchstack/features/founder-weekly-review"; + +describe("Founder Weekly Review evaluation pipeline", () => { + const fakeEvidenceSnapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-07-13T00:00:00.000Z", + reportingPeriod: { + start: "2026-07-06", + end: "2026-07-12", + }, + workspaceTimezone: "UTC", + items: [ + { + sourceType: "document_change", + sourceId: "doc:release", + title: "Release notes", + sourceTimestamp: "2026-07-10T00:00:00.000Z", + excerpt: "Export shipped.", + metadata: {}, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback:export", + title: "Customer feedback", + sourceTimestamp: "2026-07-11T00:00:00.000Z", + excerpt: "Customers requested export support.", + metadata: {}, + }, + ], + sourceWarnings: [], + }); + + const fakeReport = FounderWeeklyReviewV2PayloadSchema.parse({ + schemaVersion: "founder-weekly-review/v2", + sections: { + whatChanged: { + state: "no_evidence", + noEvidence: { + code: "no_changes", + message: "No changes were reported.", + cta: "Discuss recent updates.", + }, + }, + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Export shipped.", + sourceIds: ["doc:release"], + confidence: 0.8, + }, + ], + }, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customers requested export support.", + sourceIds: ["feedback:export"], + confidence: 0.8, + }, + ], + }, + currentBlockers: { + state: "no_evidence", + noEvidence: { + code: "no_blockers", + message: "No blockers were reported.", + cta: "Reach out if blockers appear.", + }, + }, + nextPriorities: { + state: "no_evidence", + noEvidence: { + code: "no_priorities", + message: "No next priorities were specified.", + cta: "Discuss upcoming priorities.", + }, + }, + }, + }); + it("runs deterministic evaluation and LLM grading with structured output", async () => { + const mockGrader: FounderWeeklyReviewGenerateFn = async (input) => { + expect(input.schemaName).toBe("founder_weekly_review_grader"); + expect(input.prompt).toContain("Evidence:"); + expect(input.prompt).toContain("Report:"); + expect(input.system).toContain("groundedness"); + + return { + object: { + overallScore: 0.8, + dimensions: { + groundedness: 0.9, + materiality: 0.8, + temporalAccuracy: 0.7, + synthesisQuality: 0.75, + actionability: 0.85, + }, + findings: [], + summary: "Looks good", + }, + metadata: { + provider: "mock", + model: "mock-model", + }, + }; + }; + + const result = await evaluateGeneratedFounderWeeklyReview( + fakeEvidenceSnapshot, + fakeReport, + mockGrader + ); + + expect(result.deterministic).toBeDefined(); + expect(result.llmGrader?.overallScore).toBe(0.8); + expect(result.llmGrader?.metadata?.provider).toBe("mock"); + }); + + it("rejects malformed LLM grader output", async () => { + const badGrader: FounderWeeklyReviewGenerateFn = async () => ({ + overallScore: 5, + } as any); + + await expect( + evaluateGeneratedFounderWeeklyReview( + fakeEvidenceSnapshot, + fakeReport, + badGrader + ) + ).rejects.toThrow(); + }); +}); \ No newline at end of file diff --git a/apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts b/apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts new file mode 100644 index 000000000..d87f5a022 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/scenario-collection.integration.test.ts @@ -0,0 +1,83 @@ +import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewEvidenceSnapshotSchema } from "@launchstack/features/founder-weekly-review"; +import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; + +import { loadScenario } from "../../scripts/founder-weekly-review-scenario-loader"; +import { seedScenario } from "../../scripts/founder-weekly-review-scenario-seeder"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeIfDatabase = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ? describe : describe.skip; +const vector = () => Array.from({ length: 1536 }, (_, index) => index === 0 ? 1 : 0); +const fixturePath = (name: string) => `${__dirname}/../../test-fixtures/founder-weekly-review/scenarios/${name}/scenario.json`; +type V2Snapshot = Extract, { schemaVersion: "founder-weekly-review-evidence/v2" }>; + +async function collect(name: string, deterministicWorkspaceEmbeddings = false) { + const scenario = await loadScenario(fixturePath(name)); + const testDb = await createFounderWeeklyReviewTestDatabase(); + try { + const seeded = await seedScenario(testDb.db, scenario, { deterministicWorkspaceEmbeddings }); + const collector = new FounderWeeklyReviewEvidenceService( + testDb.db, + () => new Date("2026-07-27T00:00:00.000Z"), + { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(testDb.db) }, + deterministicWorkspaceEmbeddings ? new StrictCurrentWorkspaceDocumentStore(testDb.db, { embedQuery: async () => vector() }) : undefined, + ); + const snapshot = await collector.collectFounderWeeklyReviewEvidence({ + companyId: seeded.underReviewCompanyId, + reportingPeriod: scenario.reportingPeriod, + workspaceTimezone: scenario.workspaceTimezone, + founderContext: scenario.founderContext, + actor: scenario.founderContext ? { externalUserId: "scenario-test" } : undefined, + requestKey: `scenario-${scenario.name}`, + capturedAt: new Date("2026-07-27T00:00:00.000Z"), + }); + return { scenario, snapshot: FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot) as V2Snapshot }; + } finally { + await testDb.close(); + } +} + +function assertExpectations(scenario: Awaited>, snapshot: V2Snapshot) { + expect(snapshot.schemaVersion).toBe("founder-weekly-review-evidence/v2"); + const expected = scenario.expect; + for (const [sourceType, bounds] of Object.entries(expected?.evidence?.sourceTypeCounts ?? {})) { + const count = snapshot.items.filter((item) => item.sourceType === sourceType).length; + if (bounds?.exact !== undefined) expect(count).toBe(bounds.exact); + if (bounds?.min !== undefined) expect(count).toBeGreaterThanOrEqual(bounds.min); + if (bounds?.max !== undefined) expect(count).toBeLessThanOrEqual(bounds.max); + } + for (const warningCode of expected?.evidence?.warningCodes ?? []) expect(snapshot.sourceWarnings.map((warning) => warning.code)).toContain(warningCode); + if (expected?.documentChanges) { + const groups = snapshot.documentChangeAudit.groups; + if (expected.documentChanges.minGroups !== undefined) expect(groups.length).toBeGreaterThanOrEqual(expected.documentChanges.minGroups); + if (expected.documentChanges.maxGroups !== undefined) expect(groups.length).toBeLessThanOrEqual(expected.documentChanges.maxGroups); + if (expected.documentChanges.requiredCategories) for (const category of expected.documentChanges.requiredCategories) expect(groups.some((group) => group.category === category)).toBe(true); + if (expected.documentChanges.requireNoInventedBaseline) expect(groups).toHaveLength(0); + } +} + +describeIfDatabase("provider-free scenario collection against current LAU-9", () => { + jest.setTimeout(120_000); + + it.each([ + ["01-empty-evidence", false], + ["02-founder-context-only", false], + ["03-relevant-workspace-document", true], + ["04-first-ever-version-no-invented-diff", false], + ["05-multiple-versions-in-period", false], + ] as const)("collects %s with current snapshot v2 semantics", async (name, deterministicWorkspaceEmbeddings) => { + const { scenario, snapshot } = await collect(name, deterministicWorkspaceEmbeddings); + assertExpectations(scenario, snapshot); + if (name === "03-relevant-workspace-document") { + expect(snapshot.items.some((item) => item.sourceType === "workspace_document")).toBe(true); + expect(snapshot.items.some((item) => item.sourceType === "document_change")).toBe(false); + expect(snapshot.items.filter((item) => item.sourceType === "workspace_document").every((item) => item.metadata.documentVersionId !== undefined)).toBe(true); + } + if (name === "05-multiple-versions-in-period") { + const pairKeys = new Set(snapshot.documentChangeAudit.groups.map((group) => `${group.previousVersionId}->${group.currentVersionId}`)); + expect(pairKeys.size).toBeGreaterThanOrEqual(2); + expect(snapshot.documentChangeAudit.rawChanges.length).toBeGreaterThan(0); + expect(snapshot.documentChangeAudit.groups.every((group) => group.previousVersionId !== group.currentVersionId)).toBe(true); + } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts b/apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts new file mode 100644 index 000000000..c453b7717 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/scenario-contract.test.ts @@ -0,0 +1,33 @@ +import { FounderWeeklyReviewScenarioSchema } from "../../test-fixtures/founder-weekly-review/scenarios/contracts"; + +const baseScenario = () => ({ + name: "contract-test", + reportingPeriod: { start: "2026-07-20", end: "2026-07-26" }, + workspaceTimezone: "UTC", + companies: [{ name: "Northstar", underReview: true, documents: [] }], +}); + +describe("FounderWeeklyReviewScenarioSchema", () => { + it("accepts bounded invariant expectations", () => { + const parsed = FounderWeeklyReviewScenarioSchema.parse({ + ...baseScenario(), + expect: { + evidence: { sourceTypeCounts: { document_change: { min: 1 }, founder_context: { exact: 0 } } }, + documentChanges: { minGroups: 1, maxGroups: 8, requireNoInventedBaseline: true }, + sourceSemantics: { customerFeedbackOnly: true, currentWorkspaceOnly: true }, + review: { sectionStates: { whatShipped: "no_evidence" }, requiredThemes: ["ownership"] }, + }, + }); + expect(parsed.expect?.documentChanges?.requireNoInventedBaseline).toBe(true); + }); + + it("rejects contradictory count bounds and duplicate versions", () => { + expect(() => FounderWeeklyReviewScenarioSchema.parse({ ...baseScenario(), expect: { evidence: { sourceTypeCounts: { document_change: { min: 3, max: 1 } } } } })).toThrow(/min must not exceed max/); + const scenario = baseScenario() as any; + scenario.companies[0]!.documents = [{ title: "Plan", category: "Planning", versions: [ + { versionNumber: 1, timestamp: "2026-07-20T10:00:00.000Z", chunks: [] }, + { versionNumber: 1, timestamp: "2026-07-21T10:00:00.000Z", chunks: [] }, + ] }]; + expect(() => FounderWeeklyReviewScenarioSchema.parse(scenario)).toThrow(/versionNumber must be unique/); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts b/apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts new file mode 100644 index 000000000..7e965e387 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/scenario-loader.test.ts @@ -0,0 +1,19 @@ +import { resolve } from "node:path"; + +import { loadScenario, parseScenario } from "../../scripts/founder-weekly-review-scenario-loader"; + +const scenarioPath = (name: string) => resolve(__dirname, `../../test-fixtures/founder-weekly-review/scenarios/${name}/scenario.json`); + +describe("Founder Weekly Review scenario loader", () => { + it("loads the first-ever-version regression fixture", async () => { + const scenario = await loadScenario(scenarioPath("04-first-ever-version-no-invented-diff")); + expect(scenario.name).toBe("first-ever-version-no-invented-diff"); + expect(scenario.expect?.documentChanges?.requireNoInventedBaseline).toBe(true); + }); + + it("applies bounded defaults while rejecting unknown fields", () => { + const parsed = parseScenario({ ...JSON.parse('{"name":"empty","reportingPeriod":{"start":"2026-07-20","end":"2026-07-26"},"workspaceTimezone":"UTC","companies":[{"name":"Northstar","underReview":true,"documents":[]}]}') }); + expect(parsed.schemaVersion).toBe("founder-weekly-review-scenario/v1"); + expect(() => parseScenario({ ...parsed, unexpected: true })).toThrow(); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/testDb.ts b/apps/web/__tests__/founderWeeklyReview/testDb.ts new file mode 100644 index 000000000..b25e60f9b --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/testDb.ts @@ -0,0 +1,238 @@ +import { randomUUID } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import postgres, { type Sql } from "postgres"; +import { drizzle } from "drizzle-orm/postgres-js"; + +import * as coreSchema from "@launchstack/core/db/schema"; +import type { DbClient } from "@launchstack/core/db"; + +const rootDir = join(__dirname, "..", ".."); +const migrationsDir = join(rootDir, "drizzle"); + +function buildSearchPathSql(schemaName: string): string { + return `SET search_path TO "${schemaName}", public`; +} + +async function listMigrationFiles(): Promise { + const entries = await readdir(migrationsDir, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".sql")) + .map((entry) => entry.name) + .sort(); +} + +function normalizeSqlMigrationBody(body: string): string { + return body.replace(/^\uFEFF/, ""); +} + +async function bootstrapIsolatedSchema( + client: Sql, + schemaName: string +): Promise { + await client.unsafe(buildSearchPathSql(schemaName)); + await client.unsafe(` + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_company" ( + "id" serial PRIMARY KEY, + "name" varchar(256) NOT NULL, + "slug" varchar(64), + "description" text, + "industry" varchar(256), + "swatch" integer NOT NULL DEFAULT 1, + "embedding_index_key" varchar(128), + "active_embedding_index_key" varchar(128), + "pending_embedding_index_key" varchar(128), + "reindex_status" varchar(16) NOT NULL DEFAULT 'STABLE', + "reindex_job_id" text, + "reindex_started_at" timestamptz, + "reindex_completed_at" timestamptz, + "reindex_error" text, + "embedding_openai_api_key" text, + "embedding_huggingface_api_key" text, + "embedding_ollama_base_url" varchar(1024), + "embedding_ollama_model" varchar(256), + "employerPasskey" varchar(256) NOT NULL DEFAULT '', + "employeePasskey" varchar(256) NOT NULL DEFAULT '', + "numberOfEmployees" varchar(256) NOT NULL, + "use_uploadthing" boolean NOT NULL DEFAULT true, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_users" ( + "id" serial PRIMARY KEY, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "role" varchar(256) NOT NULL, + "last_active_at" timestamptz, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document" ( + "id" serial PRIMARY KEY, + "company_id" bigint REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "url" varchar(256), + "category" varchar(256), + "title" varchar(256), + "ocr_enabled" boolean DEFAULT false, + "ocr_processed" boolean DEFAULT false, + "ocr_metadata" jsonb, + "ocr_job_id" varchar(256), + "ocr_provider" varchar(50), + "ocr_confidence_score" integer, + "ocr_cost_cents" integer, + "mime_type" varchar(128), + "source_archive_name" varchar(256), + "file_type" varchar(128), + "current_version_id" bigint, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_versions" ( + "id" serial PRIMARY KEY, + "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, + "version_number" integer NOT NULL, + "url" varchar(512) NOT NULL, + "mime_type" varchar(128) NOT NULL, + "file_size" bigint, + "uploaded_by" varchar(256), + "changelog" text, + "ocr_job_id" varchar(256), + "ocr_provider" varchar(50), + "ocr_processed" boolean DEFAULT false, + "ocr_metadata" jsonb, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE ("document_id", "version_number") + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_context_chunks" ( + "id" serial PRIMARY KEY, + "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, + "version_id" bigint REFERENCES "pdr_ai_v2_document_versions"("id") ON DELETE CASCADE, + "structure_id" bigint, + "content" text NOT NULL, + "token_count" integer NOT NULL DEFAULT 0, + "char_count" integer NOT NULL DEFAULT 0, + "embedding" vector(1536), + "content_hash" varchar(64), + "semantic_type" varchar(50), + "page_number" integer, + "line_start" integer, + "line_end" integer, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_structure" ( + "id" serial PRIMARY KEY, + "document_id" bigint NOT NULL REFERENCES "pdr_ai_v2_document"("id") ON DELETE CASCADE, + "version_id" bigint, + "parent_id" bigint, + "level" integer NOT NULL DEFAULT 0, + "ordering" integer NOT NULL DEFAULT 0, + "title" text, + "content_type" varchar(50) NOT NULL DEFAULT 'section', + "path" varchar(256), + "start_page" integer, + "end_page" integer, + "child_count" integer NOT NULL DEFAULT 0, + "token_count" integer DEFAULT 0, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz + ); + + CREATE TABLE IF NOT EXISTS "pdr_ai_v2_document_retrieval_chunks" ( + "id" bigint PRIMARY KEY + ); + + CREATE TABLE IF NOT EXISTS "company" ( + "id" serial PRIMARY KEY, + "name" varchar(256) NOT NULL + ); + + CREATE TABLE IF NOT EXISTS "document" ( + "id" serial PRIMARY KEY + ); + + CREATE TABLE IF NOT EXISTS "ocr_jobs" ( + "id" varchar(256) PRIMARY KEY + ); + + CREATE TABLE IF NOT EXISTS "file_uploads" ( + "id" serial PRIMARY KEY, + "user_id" varchar(256) NOT NULL, + "filename" varchar(256) NOT NULL, + "mime_type" varchar(128) NOT NULL, + "file_data" text NOT NULL, + "file_size" integer NOT NULL, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + `); +} + +export interface FounderWeeklyReviewTestSession { + db: DbClient; + close(): Promise; +} + +export interface FounderWeeklyReviewTestDatabase { + schemaName: string; + db: DbClient; + createSession(): Promise; + close(): Promise; +} + +async function createSession( + connectionString: string, + schemaName: string +): Promise { + const client = postgres(connectionString, { max: 1 }); + await client.unsafe(buildSearchPathSql(schemaName)); + return { + db: drizzle(client, { schema: coreSchema }), + close: () => client.end({ timeout: 5 }), + }; +} + +export async function createFounderWeeklyReviewTestDatabase(): Promise { + const connectionString = + process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL; + + if (!connectionString) { + throw new Error( + "LAUNCHSTACK_TEST_DATABASE_URL or DATABASE_URL is required for founder weekly review integration tests." + ); + } + + const adminClient = postgres(connectionString, { max: 1 }); + const schemaName = `lau5_${randomUUID().replace(/-/g, "")}`; + await adminClient.unsafe(`CREATE SCHEMA "${schemaName}"`); + await adminClient.unsafe(`CREATE EXTENSION IF NOT EXISTS vector`); + await adminClient.unsafe(buildSearchPathSql(schemaName)); + await bootstrapIsolatedSchema(adminClient, schemaName); + + const migrationFiles = await listMigrationFiles(); + for (const fileName of migrationFiles) { + const body = normalizeSqlMigrationBody( + await readFile(join(migrationsDir, fileName), "utf8") + ); + await adminClient.begin(async (tx) => { + await tx.unsafe(buildSearchPathSql(schemaName)); + await tx.unsafe(body); + }); + } + + const primary = await createSession(connectionString, schemaName); + + return { + schemaName, + db: primary.db, + createSession: () => createSession(connectionString, schemaName), + async close() { + await primary.close(); + await adminClient.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`); + await adminClient.end({ timeout: 5 }); + }, + }; +} diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts new file mode 100644 index 000000000..13a6c1d2c --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/workspace-collector.test.ts @@ -0,0 +1,19 @@ +import { FounderWeeklyReviewEvidenceService } from "@launchstack/features/founder-weekly-review"; + +describe("workspace document collection", () => { + it("skips retrieval for blank Founder Context", async () => { + const store = { retrieveRelevantCurrentDocumentChunks: jest.fn() }; + const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "unconfigured" }, store); + await expect(service.collectWorkspaceDocumentEvidence(1n, " ")).resolves.toEqual({ items: [], warnings: [] }); + expect(store.retrieveRelevantCurrentDocumentChunks).not.toHaveBeenCalled(); + }); + + it("maps relevant current hits and preserves unavailability as a bounded warning", async () => { + const store = { retrieveRelevantCurrentDocumentChunks: jest.fn() + .mockResolvedValueOnce({ state: "success", hits: [{ documentId: 1n, documentTitle: "Current plan", versionId: 2n, contextChunkId: 3, content: "Current blocker", similarityScore: 0.9 }] }) + .mockResolvedValueOnce({ state: "unavailable", hits: [], warnings: ["workspace_document_retrieval_unavailable"] }) }; + const service = new FounderWeeklyReviewEvidenceService({} as never, undefined, { kind: "unconfigured" }, store); + await expect(service.collectWorkspaceDocumentEvidence(1n, " blocker ")).resolves.toMatchObject({ items: [{ sourceType: "workspace_document", metadata: { retrievalReason: "founder_context_relevance" } }], warnings: [] }); + await expect(service.collectWorkspaceDocumentEvidence(1n, " blocker ")).resolves.toEqual({ items: [], warnings: [{ code: "workspace_document_retrieval_unavailable", message: "Founder Context workspace retrieval was unavailable.", sourceType: "workspace_document" }] }); + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts new file mode 100644 index 000000000..a2a74a4a3 --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/workspace-document-store.test.ts @@ -0,0 +1,88 @@ +import { sql } from "drizzle-orm"; +import { company, document, documentContextChunks, documentVersions } from "@launchstack/core/db/schema"; +import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; +import { FounderWeeklyReviewEvidenceService } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewTestDatabase } from "./testDb"; + +const describeDb = process.env.LAUNCHSTACK_TEST_DATABASE_URL || process.env.DATABASE_URL ? describe : describe.skip; +const vector = (index: number) => Array.from({ length: 1536 }, (_, i) => i === index ? 1 : 0); +const vectorSql = (index: number) => sql`${JSON.stringify(vector(index))}::vector(1536)`; + +describeDb("strict current workspace document store", () => { + it("returns only company-owned current-version embedded chunks with deterministic ranking", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [owner] = await test.db.insert(company).values({ name: "Owner", numberOfEmployees: "1" }).returning(); + const [other] = await test.db.insert(company).values({ name: "Other", numberOfEmployees: "1" }).returning(); + const [doc] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://a", category: "Product", title: "A" }).returning(); + const [noCurrent] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://none", category: "Product", title: "No current" }).returning(); + const [foreign] = await test.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://foreign", category: "Product", title: "Foreign" }).returning(); + const [v1] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: "local://a1", mimeType: "text/plain" }).returning(); + const [v2] = await test.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 2, url: "local://a2", mimeType: "text/plain" }).returning(); + const [noneVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(noCurrent!.id), versionNumber: 1, url: "local://n", mimeType: "text/plain" }).returning(); + const [foreignVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(foreign!.id), versionNumber: 1, url: "local://f", mimeType: "text/plain" }).returning(); + await test.db.update(document).set({ currentVersionId: BigInt(v2!.id) }).where(sql`${document.id} = ${doc!.id}`); + await test.db.insert(documentContextChunks).values([ + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "current first", tokenCount: 1, charCount: 13, embedding: vectorSql(0) }, + { documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), content: "current second", tokenCount: 1, charCount: 14, embedding: vectorSql(0) }, + { documentId: BigInt(doc!.id), versionId: BigInt(v1!.id), content: "historical", tokenCount: 1, charCount: 10, embedding: vectorSql(0) }, + { documentId: BigInt(doc!.id), versionId: null, content: "legacy", tokenCount: 1, charCount: 6, embedding: vectorSql(0) }, + { documentId: BigInt(noCurrent!.id), versionId: BigInt(noneVersion!.id), content: "no current", tokenCount: 1, charCount: 10, embedding: vectorSql(0) }, + { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: "foreign", tokenCount: 1, charCount: 7, embedding: vectorSql(0) }, + ]); + const store = new StrictCurrentWorkspaceDocumentStore(test.db, { embedQuery: jest.fn().mockResolvedValue(vector(0)) }); + const result = await store.retrieveRelevantCurrentDocumentChunks({ companyId: BigInt(owner!.id), founderContext: "current", topK: 1 }); + expect(result).toMatchObject({ state: "success", hits: [{ documentId: BigInt(doc!.id), versionId: BigInt(v2!.id), documentTitle: "A", content: "current first" }] }); + if (result.state === "success") expect(result.hits).toHaveLength(1); + } finally { await test.close(); } + }); + + it("collects a computed v1-to-v2 diff plus only a current relevant workspace document", async () => { + const test = await createFounderWeeklyReviewTestDatabase(); + try { + const [owner] = await test.db.insert(company).values({ name: "Owner", numberOfEmployees: "1" }).returning(); + const [other] = await test.db.insert(company).values({ name: "Other", numberOfEmployees: "1" }).returning(); + const [a] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://a", category: "Product", title: "Release plan" }).returning(); + const [b] = await test.db.insert(document).values({ companyId: BigInt(owner!.id), url: "local://b", category: "Product", title: "Current blocker" }).returning(); + const [foreign] = await test.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://f", category: "Product", title: "Foreign" }).returning(); + const [a1] = await test.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 1, url: "local://a1", mimeType: "text/plain", createdAt: new Date("2026-01-01T00:00:00.000Z") }).returning(); + const [a2] = await test.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 2, url: "local://a2", mimeType: "text/plain", createdAt: new Date("2026-02-02T00:00:00.000Z") }).returning(); + const [a3] = await test.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 3, url: "local://a3", mimeType: "text/plain", createdAt: new Date("2026-03-01T00:00:00.000Z") }).returning(); + const [b1] = await test.db.insert(documentVersions).values({ documentId: BigInt(b!.id), versionNumber: 1, url: "local://b1", mimeType: "text/plain", createdAt: new Date("2026-01-01T00:00:00.000Z") }).returning(); + const [foreignVersion] = await test.db.insert(documentVersions).values({ documentId: BigInt(foreign!.id), versionNumber: 1, url: "local://f1", mimeType: "text/plain" }).returning(); + await test.db.update(document).set({ currentVersionId: BigInt(a3!.id) }).where(sql`${document.id} = ${a!.id}`); + await test.db.update(document).set({ currentVersionId: BigInt(b1!.id) }).where(sql`${document.id} = ${b!.id}`); + await test.db.update(document).set({ currentVersionId: BigInt(foreignVersion!.id) }).where(sql`${document.id} = ${foreign!.id}`); + await test.db.insert(documentContextChunks).values([ + { documentId: BigInt(a!.id), versionId: BigInt(a1!.id), content: "product release before", contentHash: "a".repeat(64), tokenCount: 1, charCount: 22 }, + { documentId: BigInt(a!.id), versionId: BigInt(a2!.id), content: "product release after", contentHash: "b".repeat(64), tokenCount: 1, charCount: 21 }, + { documentId: BigInt(a!.id), versionId: BigInt(a3!.id), content: "v3 must not diff", contentHash: "c".repeat(64), tokenCount: 1, charCount: 15 }, + { documentId: BigInt(b!.id), versionId: BigInt(b1!.id), content: "current blocker context", contentHash: "d".repeat(64), tokenCount: 1, charCount: 23, embedding: vectorSql(0) }, + { documentId: BigInt(b!.id), versionId: null, content: "legacy relevant", contentHash: "e".repeat(64), tokenCount: 1, charCount: 15, embedding: vectorSql(0) }, + { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: "foreign relevant", contentHash: "f".repeat(64), tokenCount: 1, charCount: 16, embedding: vectorSql(0) }, + ]); + const service = new FounderWeeklyReviewEvidenceService(test.db, () => new Date("2026-02-10T00:00:00.000Z"), { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(test.db) }, new StrictCurrentWorkspaceDocumentStore(test.db, { embedQuery: jest.fn().mockResolvedValue(vector(0)) })); + const input = { companyId: BigInt(owner!.id), reportingPeriod: { start: "2026-02-01", end: "2026-02-07" }, workspaceTimezone: "UTC", founderContext: "blocker", actor: { externalUserId: "u" }, requestKey: "computed-integration" }; + const first = await service.collectFounderWeeklyReviewEvidence(input); + const second = await service.collectFounderWeeklyReviewEvidence(input); + expect(first.schemaVersion).toBe("founder-weekly-review-evidence/v2"); + expect(first.items.filter((item) => item.sourceType === "document_change")).toHaveLength(1); + expect(first.items).toEqual(second.items); + expect(first.schemaVersion === "founder-weekly-review-evidence/v2" && first.documentChangeAudit).toEqual( + second.schemaVersion === "founder-weekly-review-evidence/v2" ? second.documentChangeAudit : undefined + ); + if (first.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Expected computed v2 evidence"); + expect(first.documentChangeAudit.rawChanges).toHaveLength(1); + expect(first.documentChangeAudit.groups).toHaveLength(1); + expect(first.documentChangeAudit.groups[0]!.evidenceSourceId).toBe(first.items.find((item) => item.sourceType === "document_change")!.sourceId); + expect(first.items).toEqual(expect.arrayContaining([ + expect.objectContaining({ sourceType: "document_change", metadata: expect.objectContaining({ previousVersionId: a1!.id, currentVersionId: a2!.id }) }), + ])); + const workspaceItem = first.items.find((item) => item.sourceType === "workspace_document"); + expect(workspaceItem).toMatchObject({ sourceType: "workspace_document", title: "Current blocker" }); + expect(workspaceItem).not.toHaveProperty("sourceTimestamp"); + expect(first.items.some((item) => item.excerpt.includes("v3 must not diff") || item.excerpt.includes("legacy relevant") || item.excerpt.includes("foreign relevant"))).toBe(false); + } finally { await test.close(); } + }); +}); diff --git a/apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts b/apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts new file mode 100644 index 000000000..973abea2a --- /dev/null +++ b/apps/web/__tests__/founderWeeklyReview/workspace-document.test.ts @@ -0,0 +1,40 @@ +import { + buildWorkspaceDocumentEvidence, + normalizeFounderContextRetrievalQuery, + selectWorkspaceDocumentHits, + type WorkspaceDocumentHit, +} from "@launchstack/features/founder-weekly-review"; + +const hit = (documentId: bigint, chunkId: number, score: number, content = `content ${chunkId}`): WorkspaceDocumentHit => ({ + documentId, documentTitle: `Document ${documentId}`, versionId: 9n, contextChunkId: chunkId, content, similarityScore: score, + structureId: 4n, structurePath: "/1", structureTitle: "Section", pageNumber: 1, lineStart: 1, lineEnd: 2, +}); + +describe("workspace document evidence", () => { + it("normalizes bounded founder context and skips blank queries", () => { + expect(normalizeFounderContextRetrievalQuery(" founder priority ")).toBe("founder priority"); + expect(normalizeFounderContextRetrievalQuery(" ")).toBeNull(); + }); + + it("selects diverse stable hits and normalizes JSON-safe evidence", () => { + const selected = selectWorkspaceDocumentHits([hit(2n, 3, 0.8), hit(1n, 2, 0.9), hit(1n, 1, 0.9), hit(1n, 1, 0.7)]); + expect(selected.map((item) => [item.documentId, item.contextChunkId])).toEqual([[1n, 1], [1n, 2], [2n, 3]]); + const evidence = buildWorkspaceDocumentEvidence(selected); + expect(evidence).toEqual(buildWorkspaceDocumentEvidence(selected)); + expect(evidence[0]).toMatchObject({ sourceType: "workspace_document", sourceId: "workspace_document:doc:1:version:9:chunk:1", metadata: { documentId: "1", documentVersionId: "9", retrievalReason: "founder_context_relevance" } }); + expect(() => JSON.stringify(evidence)).not.toThrow(); + }); + + it("uses exact BigInt ordering to break equal-score ties", () => { + const selected = selectWorkspaceDocumentHits([ + hit(9007199254740993n, 1, 0.9), + hit(9007199254740992n, 1, 0.9), + ]); + expect(selected.map((item) => item.documentId)).toEqual([9007199254740992n, 9007199254740993n]); + }); + + it("bounds excerpts", () => { + const evidence = buildWorkspaceDocumentEvidence([hit(1n, 1, 0.5, "x".repeat(5000))]); + expect(evidence[0]!.excerpt.length).toBe(4000); + }); +}); diff --git a/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts new file mode 100644 index 000000000..043e12f1f --- /dev/null +++ b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.live.test.ts @@ -0,0 +1,53 @@ +/** + * Live DB verification for #297 against a migration-shaped company table. + * + * Requires DATABASE_URL pointing at a database where migration 0011 has + * already dropped the plaintext embedding columns. Skips otherwise so CI + * without Postgres still passes the unit suite. + */ + +import postgres from "postgres"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { company } from "@launchstack/core/db/schema"; + +const databaseUrl = process.env.DATABASE_URL; + +const describeIfDb = databaseUrl ? describe : describe.skip; + +describeIfDb("company select after migration 0011 (#297 live)", () => { + const client = postgres(databaseUrl!, { max: 1 }); + const db = drizzle(client); + + afterAll(async () => { + await client.end({ timeout: 5 }); + }); + + it("selects and inserts company rows without referencing dropped columns", async () => { + const missing = await client` + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'pdr_ai_v2_company' + AND column_name = ANY(${[ + "embedding_openai_api_key", + "embedding_huggingface_api_key", + "embedding_ollama_base_url", + "embedding_ollama_model", + ]}) + `; + expect(missing).toHaveLength(0); + + await expect(db.select().from(company).limit(1)).resolves.toBeDefined(); + + const inserted = await db + .insert(company) + .values({ + name: `issue-297-live-${Date.now()}`, + numberOfEmployees: "1", + }) + .returning(); + + expect(inserted).toHaveLength(1); + expect(inserted[0]).not.toHaveProperty("embeddingOpenAIApiKey"); + expect(inserted[0]?.id).toEqual(expect.any(Number)); + }); +}); diff --git a/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts new file mode 100644 index 000000000..e30ccafe8 --- /dev/null +++ b/apps/web/__tests__/lib/ai/company-embedding-schema-alignment.test.ts @@ -0,0 +1,85 @@ +/** + * Regression for https://github.com/Deodat-Lawson/LaunchStack/issues/297 + * + * Migration 0011 drops plaintext embedding credential columns from + * `pdr_ai_v2_company`. The Drizzle company schema and credential read path + * must stay aligned with that final shape — otherwise clean migration-replayed + * databases fail on `select()` / `.returning()` of the company row. + */ + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { getTableColumns } from "drizzle-orm"; +import { company } from "@launchstack/core/db/schema"; + +const DROPPED_EMBEDDING_COLUMNS = [ + "embedding_openai_api_key", + "embedding_huggingface_api_key", + "embedding_ollama_base_url", + "embedding_ollama_model", +] as const; + +/** Jest runs with cwd = apps/web; fall back to climbing from this file. */ +function resolveRepoRoot(): string { + const fromCwd = join(process.cwd(), "../.."); + if (existsSync(join(fromCwd, "packages/core/package.json"))) { + return fromCwd; + } + const fromFile = join(__dirname, "../../../../.."); + if (existsSync(join(fromFile, "packages/core/package.json"))) { + return fromFile; + } + throw new Error( + `Unable to resolve monorepo root from cwd=${process.cwd()} __dirname=${__dirname}`, + ); +} + +const repoRoot = resolveRepoRoot(); + +describe("company embedding credential schema alignment (#297)", () => { + it("does not declare columns dropped by migration 0011 on the company table", () => { + const columns = getTableColumns(company); + const sqlNames = Object.values(columns).map((column) => column.name); + + for (const dropped of DROPPED_EMBEDDING_COLUMNS) { + expect(sqlNames).not.toContain(dropped); + } + + expect(columns).not.toHaveProperty("embeddingOpenAIApiKey"); + expect(columns).not.toHaveProperty("embeddingHuggingFaceApiKey"); + expect(columns).not.toHaveProperty("embeddingOllamaBaseUrl"); + expect(columns).not.toHaveProperty("embeddingOllamaModel"); + }); + + it("keeps migration 0011 as the drop of those plaintext columns", () => { + const migrationPath = join( + repoRoot, + "apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql", + ); + const body = readFileSync(migrationPath, "utf8"); + + expect(body).toMatch(/ALTER TABLE\s+"pdr_ai_v2_company"/i); + for (const dropped of DROPPED_EMBEDDING_COLUMNS) { + expect(body).toContain(`DROP COLUMN IF EXISTS "${dropped}"`); + } + }); + + it("reads credentials only from the encrypted credentials table", () => { + const sourcePath = join( + repoRoot, + "packages/core/src/embeddings/company-credentials.ts", + ); + const source = readFileSync(sourcePath, "utf8"); + + expect(source).toContain("companyEmbeddingCredentials"); + expect(source).not.toMatch(/company\.embeddingOpenAIApiKey/); + expect(source).not.toMatch(/company\.embeddingHuggingFaceApiKey/); + expect(source).not.toMatch(/company\.embeddingOllamaBaseUrl/); + expect(source).not.toMatch(/company\.embeddingOllamaModel/); + expect(source).not.toMatch(/embedding_openai_api_key/); + expect(source).not.toMatch(/embedding_huggingface_api_key/); + expect(source).not.toMatch(/embedding_ollama_base_url/); + expect(source).not.toMatch(/embedding_ollama_model/); + }); +}); diff --git a/apps/web/config/llm-models.json b/apps/web/config/llm-models.json index 42b8f90f9..0bcaf07c1 100644 --- a/apps/web/config/llm-models.json +++ b/apps/web/config/llm-models.json @@ -3,6 +3,28 @@ "providerPriority": ["openai", "anthropic", "google", "ollama"], "capabilities": { "smallExtraction": { + "kimi": { + "model": "kimi-k2.6", + "temperature": 0 + }, + "openai": { + "model": "gpt-4o-mini", + "temperature": 0 + }, + "anthropic": { + "model": "claude-3-5-haiku-latest", + "temperature": 0 + }, + "google": { + "model": "gemini-2.0-flash", + "temperature": 0 + }, + "ollama": { + "model": "llama3.2:3b", + "temperature": 0 + } + }, + "founderWeeklyReview": { "openai": { "model": "gpt-4o-mini", "temperature": 0 diff --git a/apps/web/drizzle/0010_company_embedding_credentials.sql b/apps/web/drizzle/0010_company_embedding_credentials.sql index c77b6792d..17a9d6bac 100644 --- a/apps/web/drizzle/0010_company_embedding_credentials.sql +++ b/apps/web/drizzle/0010_company_embedding_credentials.sql @@ -10,9 +10,9 @@ -- Rollout plan: -- 1. Apply this migration (the new table is created empty). -- 2. Deploy application code that reads/writes via the new table. --- 3. Run `pnpm tsx scripts/backfill-embedding-credentials.ts` to encrypt --- existing plaintext values from `company` into this table and null --- out the old columns. +-- 3. Run `pnpm --filter @launchstack/web db:backfill:embedding-credentials` +-- to encrypt existing plaintext values from `company` into this table and +-- null out the old columns. -- 4. Once verified, apply 0011_drop_plaintext_embedding_credentials.sql -- to remove the legacy columns. diff --git a/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql b/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql index 9aa67f7bd..c37f08d0a 100644 --- a/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql +++ b/apps/web/drizzle/0011_drop_plaintext_embedding_credentials.sql @@ -3,15 +3,15 @@ -- **Do not apply this migration until** the backfill script has run -- successfully against this environment: -- --- pnpm tsx scripts/backfill-embedding-credentials.ts +-- pnpm --filter @launchstack/web db:backfill:embedding-credentials -- -- The backfill copies every non-null plaintext key into -- `pdr_ai_v2_company_embedding_credentials` (encrypted) and then NULLs out -- the legacy columns. Applying this migration before the backfill will -- permanently destroy un-migrated company API keys. -- --- The application read path already falls back to the new credentials --- table, so once the backfill has run these columns are unused. +-- The application reads credentials only from the encrypted table; these +-- legacy columns must not remain in the Drizzle company schema. ALTER TABLE "pdr_ai_v2_company" DROP COLUMN IF EXISTS "embedding_openai_api_key", diff --git a/apps/web/drizzle/0016_founder_weekly_review_runs.sql b/apps/web/drizzle/0016_founder_weekly_review_runs.sql new file mode 100644 index 000000000..b8115e02e --- /dev/null +++ b/apps/web/drizzle/0016_founder_weekly_review_runs.sql @@ -0,0 +1,74 @@ +-- Founder Weekly Review persistence and lifecycle foundation (LAU-5). +-- +-- Creates: +-- 1. pdr_ai_v2_founder_weekly_review_runs +-- 2. pdr_ai_v2_founder_weekly_review_operations +-- +-- V1 decision: multiple runs are allowed per company and reporting period. +-- Idempotency is enforced by: +-- - create: UNIQUE(company_id, request_key) +-- - retry: UNIQUE(run_id, operation_type, request_key) + +CREATE TABLE IF NOT EXISTS "pdr_ai_v2_founder_weekly_review_runs" ( + "id" varchar(64) PRIMARY KEY, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "request_key" varchar(128) NOT NULL, + "reporting_period_start" date NOT NULL, + "reporting_period_end" date NOT NULL, + "status" varchar(32) NOT NULL DEFAULT 'queued', + "review_payload" jsonb, + "review_schema_version" varchar(64) NOT NULL, + "evidence_snapshot" jsonb NOT NULL, + "evidence_schema_version" varchar(64) NOT NULL, + "model_metadata" jsonb, + "created_by_actor_id" varchar(256) NOT NULL, + "retry_count" integer NOT NULL DEFAULT 0, + "failure_sequence" integer NOT NULL DEFAULT 0, + "generation_attempt" integer NOT NULL DEFAULT 0, + "generation_claim_id" varchar(128), + "generation_job_id" varchar(256), + "queued_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "claimed_at" timestamptz, + "generation_started_at" timestamptz, + "generated_at" timestamptz, + "published_at" timestamptz, + "error_code" varchar(128), + "error_message" varchar(1024), + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS "pdr_ai_v2_founder_weekly_review_operations" ( + "id" varchar(64) PRIMARY KEY, + "run_id" varchar(64) NOT NULL REFERENCES "pdr_ai_v2_founder_weekly_review_runs"("id") ON DELETE CASCADE, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "operation_type" varchar(32) NOT NULL, + "request_key" varchar(128) NOT NULL, + "source_failure_sequence" integer NOT NULL, + "actor_id" varchar(256) NOT NULL, + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_request_key_unique" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "request_key"); + +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_operations_run_type_request_key_unique" + ON "pdr_ai_v2_founder_weekly_review_operations" ("run_id", "operation_type", "request_key"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "created_at"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_status_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "status", "created_at"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_company_period_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "reporting_period_start", "reporting_period_end"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_runs_claim_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "id", "status", "generation_claim_id"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_operations_company_run_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_operations" ("company_id", "run_id", "created_at"); + +CREATE INDEX IF NOT EXISTS "founder_weekly_review_operations_run_type_created_at_idx" + ON "pdr_ai_v2_founder_weekly_review_operations" ("run_id", "operation_type", "created_at"); diff --git a/apps/web/drizzle/0017_founder_weekly_review_dispatches.sql b/apps/web/drizzle/0017_founder_weekly_review_dispatches.sql new file mode 100644 index 000000000..e23a31388 --- /dev/null +++ b/apps/web/drizzle/0017_founder_weekly_review_dispatches.sql @@ -0,0 +1,26 @@ +-- Durable outbox for Founder Weekly Review generation events (LAU-9). +CREATE TABLE IF NOT EXISTS "pdr_ai_v2_founder_weekly_review_dispatches" ( + "id" varchar(64) PRIMARY KEY, + "company_id" bigint NOT NULL REFERENCES "pdr_ai_v2_company"("id") ON DELETE CASCADE, + "run_id" varchar(64) NOT NULL REFERENCES "pdr_ai_v2_founder_weekly_review_runs"("id") ON DELETE CASCADE, + "operation_type" varchar(16) NOT NULL, + "operation_key" varchar(128) NOT NULL, + "event_id" varchar(128) NOT NULL, + "generation_job_id" varchar(128) NOT NULL, + "generation_claim_id" varchar(128) NOT NULL, + "status" varchar(16) NOT NULL DEFAULT 'pending', + "attempt_count" integer NOT NULL DEFAULT 0, + "available_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "dispatched_at" timestamptz, + "last_error_code" varchar(128), + "created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_run_operation_key_unique" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("run_id", "operation_type", "operation_key"); +CREATE UNIQUE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_event_id_unique" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("event_id"); +CREATE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_pending_idx" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("status", "available_at", "created_at"); +CREATE INDEX IF NOT EXISTS "founder_weekly_review_dispatches_company_run_idx" + ON "pdr_ai_v2_founder_weekly_review_dispatches" ("company_id", "run_id"); diff --git a/apps/web/drizzle/0018_founder_weekly_review_async_collection.sql b/apps/web/drizzle/0018_founder_weekly_review_async_collection.sql new file mode 100644 index 000000000..1b738b23e --- /dev/null +++ b/apps/web/drizzle/0018_founder_weekly_review_async_collection.sql @@ -0,0 +1,22 @@ +-- Move Founder Weekly Review evidence collection into the durable workflow. +ALTER TABLE "pdr_ai_v2_founder_weekly_review_runs" + ALTER COLUMN "evidence_snapshot" DROP NOT NULL; +ALTER TABLE "pdr_ai_v2_founder_weekly_review_runs" + ADD COLUMN "collection_input" jsonb, + ADD COLUMN "collection_claim_id" varchar(128), + ADD COLUMN "collection_started_at" timestamptz, + ADD COLUMN "evidence_collected_at" timestamptz; + +-- Existing pre-workflow rows already have a persisted snapshot. Their actor is +-- retained only as a bounded durable collection input; it is never exposed. +UPDATE "pdr_ai_v2_founder_weekly_review_runs" +SET "collection_input" = jsonb_build_object( + 'workspaceTimezone', COALESCE("evidence_snapshot"->>'workspaceTimezone', 'UTC'), + 'actorExternalUserId', regexp_replace("created_by_actor_id", '^user:', '') +) +WHERE "collection_input" IS NULL; + +ALTER TABLE "pdr_ai_v2_founder_weekly_review_runs" + ALTER COLUMN "collection_input" SET NOT NULL; +CREATE INDEX "founder_weekly_review_runs_collection_claim_idx" + ON "pdr_ai_v2_founder_weekly_review_runs" ("company_id", "id", "status", "collection_claim_id"); diff --git a/apps/web/package.json b/apps/web/package.json index 2e7761587..36bd459f7 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,6 +12,7 @@ "db:migrate": "node ./scripts/ensure-pgvector.mjs && node ./scripts/migrate.mjs", "db:backfill:versions": "tsx ./scripts/backfill-document-versions.ts", "db:backfill:note-embeddings": "tsx ./scripts/backfill-note-embeddings.ts", + "db:backfill:embedding-credentials": "tsx ./scripts/backfill-embedding-credentials.ts", "db:studio": "drizzle-kit studio", "dev": "concurrently --names next,inngest --prefix-colors blue,magenta \"next dev --turbo\" \"pnpm dlx inngest-cli@latest dev -u http://localhost:3000/api/inngest\"", "dev:next": "next dev --turbo", @@ -21,7 +22,8 @@ "start": "next start", "test": "jest --passWithNoTests", "typecheck": "tsc --noEmit", - "inngest:dev": "pnpm dlx inngest-cli@latest dev -u http://localhost:3000/api/inngest" + "inngest:dev": "pnpm dlx inngest-cli@latest dev -u http://localhost:3000/api/inngest", + "founder-weekly-review:demo": "tsx scripts/run-founder-weekly-review-realistic-e2e.ts --demo" }, "dependencies": { "@ai-sdk/anthropic": "^3.0.69", diff --git a/apps/web/scripts/backfill-embedding-credentials.ts b/apps/web/scripts/backfill-embedding-credentials.ts new file mode 100644 index 000000000..69a71fbcf --- /dev/null +++ b/apps/web/scripts/backfill-embedding-credentials.ts @@ -0,0 +1,144 @@ +/** + * Backfill plaintext embedding credentials from the legacy columns on + * `pdr_ai_v2_company` into the encrypted `pdr_ai_v2_company_embedding_credentials` + * table, then NULL out the legacy columns. + * + * Migration 0011 drops those legacy columns. This script is only useful for + * databases that still have them (for example older `db:push`-shaped + * environments that never ran the SQL migration path). It uses raw SQL so it + * does not depend on the removed Drizzle schema fields. + * + * Safe to re-run: `upsertCompanyCredentials` upserts, and the legacy NULL + * step is idempotent. Requires `EMBEDDING_SECRETS_KEY` to be set. + * + * Run with: + * pnpm --filter @launchstack/web exec tsx ./scripts/backfill-embedding-credentials.ts + * + * Flags: + * --dry-run Log what would change without writing anything. + */ + +import "dotenv/config"; + +import { sql } from "drizzle-orm"; + +import { toRows } from "@launchstack/core/db"; +import { upsertCompanyCredentials } from "@launchstack/core/embeddings"; +import { db } from "../src/server/db"; + +const DRY_RUN = process.argv.includes("--dry-run"); + +type LegacyRow = { + id: number; + name: string; + openAIApiKey: string | null; + huggingFaceApiKey: string | null; + ollamaBaseUrl: string | null; + ollamaModel: string | null; +}; + +async function legacyColumnsExist(): Promise { + const rows = toRows<{ exists: number }>( + await db.execute(sql` + SELECT 1 AS exists + FROM information_schema.columns + WHERE table_name = 'pdr_ai_v2_company' + AND column_name = 'embedding_openai_api_key' + LIMIT 1 + `), + ); + return rows.length > 0; +} + +async function main() { + if (!process.env.EMBEDDING_SECRETS_KEY) { + console.error( + "Refusing to run: EMBEDDING_SECRETS_KEY is not set. Generate one with:\n" + + " node -e \"console.log(require('crypto').randomBytes(32).toString('base64'))\"", + ); + process.exit(1); + } + + if (!(await legacyColumnsExist())) { + console.log( + "Legacy plaintext embedding columns are already gone " + + "(migration 0011 applied). Nothing to backfill.", + ); + return; + } + + const rows = toRows( + await db.execute(sql` + SELECT + id, + name, + embedding_openai_api_key AS "openAIApiKey", + embedding_huggingface_api_key AS "huggingFaceApiKey", + embedding_ollama_base_url AS "ollamaBaseUrl", + embedding_ollama_model AS "ollamaModel" + FROM pdr_ai_v2_company + WHERE embedding_openai_api_key IS NOT NULL + OR embedding_huggingface_api_key IS NOT NULL + OR embedding_ollama_base_url IS NOT NULL + OR embedding_ollama_model IS NOT NULL + `), + ); + + console.log( + `Found ${rows.length} company row(s) with legacy embedding credentials${DRY_RUN ? " (dry-run)" : ""}.`, + ); + + let migrated = 0; + for (const row of rows) { + const input: { + openAIApiKey?: string | null; + huggingFaceApiKey?: string | null; + ollamaBaseUrl?: string | null; + ollamaModel?: string | null; + } = {}; + if (row.openAIApiKey) input.openAIApiKey = row.openAIApiKey; + if (row.huggingFaceApiKey) input.huggingFaceApiKey = row.huggingFaceApiKey; + if (row.ollamaBaseUrl) input.ollamaBaseUrl = row.ollamaBaseUrl; + if (row.ollamaModel) input.ollamaModel = row.ollamaModel; + + if (Object.keys(input).length === 0) continue; + + console.log( + ` company #${row.id} (${row.name}): fields = [${Object.keys(input).join(", ")}]`, + ); + + if (DRY_RUN) continue; + + await upsertCompanyCredentials(row.id, input); + + // Null out the legacy columns so this row won't be picked up on re-run. + await db.execute(sql` + UPDATE pdr_ai_v2_company + SET + embedding_openai_api_key = NULL, + embedding_huggingface_api_key = NULL, + embedding_ollama_base_url = NULL, + embedding_ollama_model = NULL + WHERE id = ${row.id} + `); + + migrated += 1; + } + + if (DRY_RUN) { + console.log( + `Dry run complete. ${rows.length} row(s) would be migrated. No changes written.`, + ); + } else { + console.log( + `Migrated ${migrated} row(s). Once verified, apply drizzle/0011_drop_plaintext_embedding_credentials.sql to remove the legacy columns.`, + ); + } +} + +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error("Backfill failed:", err); + process.exit(1); + }); diff --git a/apps/web/scripts/founder-weekly-review-demo.ts b/apps/web/scripts/founder-weekly-review-demo.ts new file mode 100644 index 000000000..28faab480 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-demo.ts @@ -0,0 +1,136 @@ +import type { FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET } from "@launchstack/features/founder-weekly-review"; + +type Budget = typeof FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET; + +export type FounderWeeklyReviewDemoInput = { + rawChanges: readonly Record[]; + groups: readonly Record[]; + promptItems: readonly Record[]; + envelopeDiagnostics: Record; + analyzerCalls: readonly Record[]; + eligibleGroups: number; + warnings: readonly string[]; + reviewMarkdown: string; + provider: string; + model: string; + promptVersion: string; + outputCeiling: number; + repairCount: number; + snapshotVersion: string; + evidenceDigest: string; + persistencePassed: boolean; + readBackPassed: boolean; + noOpRemoved: number; + budget: Budget; + scaleResults?: readonly Record[]; +}; + +const text = (value: unknown) => typeof value === "string" ? value : ""; +const number = (value: unknown) => typeof value === "number" ? value : 0; +const label = (value: unknown) => text(value) || "unknown"; + +function line(key: string, value: string | number) { + return `${key.padEnd(34)} ${value}`; +} + +function sourceCounts(items: readonly Record[]) { + const counts = new Map(); + for (const item of items) counts.set(label(item.sourceType), (counts.get(label(item.sourceType)) ?? 0) + 1); + return [...counts.entries()].sort(([a], [b]) => a.localeCompare(b)); +} + +function sampleChanges(rawChanges: readonly Record[]) { + const preferred = ["Launch timing", "Retry ownership", "Requirement"]; + const selected = preferred.map(title => rawChanges.find(change => text(change.previousStructureTitle) === title)).filter((change): change is Record => Boolean(change)); + return selected.length ? selected : rawChanges.slice(0, 3); +} + +export function formatFounderWeeklyReviewDemo(input: FounderWeeklyReviewDemoInput): string { + const raw = input.rawChanges; + const auditGroups = input.groups; + const analyzed = input.analyzerCalls; + const nonMaterial = auditGroups.filter(group => { + const analysis = group.analysis as Record | undefined; + return analysis?.disposition === "non_material"; + }); + const alignmentCounts = new Map(); + for (const change of raw) { + const method = label(change.alignmentMethod); + alignmentCounts.set(method, (alignmentCounts.get(method) ?? 0) + 1); + } + const unmatched = raw.filter(change => ["added", "removed"].includes(text(change.changeType))).length; + const selectedChanges = input.promptItems.filter(item => item.sourceType === "document_change"); + const diagnostics = input.envelopeDiagnostics; + const scale = input.scaleResults ?? []; + const out: string[] = []; + out.push("FOUNDER WEEKLY REVIEW — LIVE E2E", "=".repeat(34), ""); + out.push("INPUT SCENARIO", "-------------"); + for (const change of sampleChanges(raw)) { + out.push(`${text(change.currentStructureTitle) || "Document change"}`); + out.push(` Before: ${text(change.previousExcerpt)}`); + out.push(` After: ${text(change.currentExcerpt)}`); + } + out.push("", "VERSION / ALIGNMENT", "-------------------"); + out.push(line("Documents compared", new Set(raw.map(change => text(change.documentId))).size)); + out.push(line("Version pairs", new Set(raw.map(change => `${change.documentId}:${change.previousVersionId}:${change.currentVersionId}`)).size)); + out.push(line("Historical chunks represented", raw.length * 2)); + for (const [method, count] of alignmentCounts) out.push(line(`Matched by ${method}`, count)); + out.push(line("Unmatched", unmatched)); + out.push("", "CHANGE PROCESSING", "-----------------"); + out.push(line("Raw changes", raw.length)); + out.push(line("No-op changes removed", input.noOpRemoved)); + out.push(line("DocumentChangeGroups", auditGroups.length)); + out.push("", "MATERIALITY"); + out.push(line("Analyzer eligible", input.eligibleGroups)); + out.push(line("Kimi analyzed", `${analyzed.length} / 4 cap`)); + out.push(line("Kimi non-material", analyzed.filter(call => call.disposition === "non_material").length)); + out.push(line("Analyzer failures", analyzed.filter(call => call.errorCode).length)); + out.push(line("Budget-excluded", Math.max(0, input.eligibleGroups - analyzed.length))); + out.push(line("Deterministic fallback", Math.max(0, input.eligibleGroups - analyzed.length))); + out.push(line("Prompt-facing document changes", selectedChanges.length)); + out.push("", "SELECTED MATERIAL CHANGES", "-------------------------"); + for (const item of selectedChanges) out.push(`[${label((item.metadata as Record | undefined)?.category)}] ${text(item.title)} — ${text(item.excerpt).split("\n")[0]}`); + out.push("", "FILTERED SEMANTIC REWRITES", "--------------------------"); + for (const group of nonMaterial) { + const analysis = group.analysis as Record; + out.push(`- ${text(group.structureTitle)}: ${text(analysis.beforeKeyPoint)} → ${text(analysis.afterKeyPoint)}`); + } + out.push("", "IMMUTABLE EVIDENCE SNAPSHOT v2", "-----------------------------"); + for (const [type, count] of sourceCounts(input.promptItems)) out.push(line(type, count)); + out.push(line("Prompt-facing items", input.promptItems.length)); + out.push(line("Evidence digest", `${input.evidenceDigest.slice(0, 16)}...`)); + out.push(line("Immutable", "✓")); + out.push("", "GENERATION ENVELOPE", "-------------------"); + out.push("Snapshot = complete frozen provenance"); + out.push("Envelope = bounded projection for generation", ""); + out.push(line("Selected evidence items", number(diagnostics.selectedItemCount))); + out.push(line("Serialized characters", number(diagnostics.serializedCharacterCount))); + out.push(line("Estimated input tokens", number(diagnostics.estimatedTokenCount))); + out.push(line("Envelope truncation", diagnostics.truncated ? "yes" : "no")); + out.push("", "Evidence budgets"); + out.push(line("Global", `${input.budget.totalSerializedCharacters.toLocaleString()} chars`)); + out.push(line("Document changes", `${input.budget.documentChangeSerializedCharacters.toLocaleString()} chars / max ${input.budget.documentChangeItems}`)); + out.push(line("Per document", `max ${input.budget.documentChangeItemsPerDocument} document changes`)); + out.push(line("Workspace reserve", `${input.budget.workspaceDocumentReservedCharacters.toLocaleString()} chars`)); + out.push(line("Customer feedback", `${input.budget.customerFeedbackReservedCharacters.toLocaleString()} chars`)); + out.push(line("Founder Context", `${input.budget.founderContextReservedCharacters.toLocaleString()} chars / max 1`)); + out.push("", "GENERATION", "----------"); + out.push(line("Provider / model", `${input.provider} / ${input.model}`)); + out.push(line("Prompt version", input.promptVersion)); + out.push(line("Output ceiling", `${input.outputCeiling} tokens`)); + out.push(line("Schema validation", "✓")); + out.push(line("Citation validation", "✓")); + out.push(line("Source semantics", "✓")); + out.push(line("Semantic repair", input.repairCount)); + out.push(line("Persistence", input.persistencePassed ? "✓" : "✗")); + out.push(line("Repository read-back", input.readBackPassed ? "✓" : "✗")); + out.push("", "FINAL FOUNDER WEEKLY REVIEW", "---------------------------", input.reviewMarkdown.trim()); + if (scale.length) { + out.push("", "SCALE BEHAVIOR", "-------------"); + out.push("Raw changes Prompt envelope Audit snapshot"); + for (const row of scale) out.push(`${number(row.rawChanged).toString().padEnd(16)}${number(row.envelopeCharacters).toString().padEnd(20)}${number(row.auditCharacters)}`); + out.push("", "Generation input remains bounded.", "Immutable audit grows with source volume.", "Kimi materiality calls remain capped at 4."); + } + out.push("", `Warnings: ${input.warnings.join(", ") || "none"}`); + return out.join("\n"); +} diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts new file mode 100644 index 000000000..22dcc9b9a --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-analyzers.ts @@ -0,0 +1,74 @@ +import type { + DocumentChangeMaterialityAnalysisInput, + DocumentChangeMaterialityAnalyzer, +} from "@launchstack/features/founder-weekly-review"; + +function words(value: string): Set { + const aliases: Record = { + owned: "owns", + ownership: "owns", + third: "q3", + quarter: "q3", + remains: "remain", + still: "remain", + expected: "planned", + expect: "planned", + }; + return new Set((value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []) + .filter(token => !["the", "a", "an", "is", "are", "by", "during", "we"].includes(token)) + .map(token => aliases[token] ?? token)); +} + +function overlap(previous: string, current: string): number { + const left = words(previous); + const right = words(current); + const union = new Set([...left, ...right]); + return union.size === 0 ? 1 : [...left].filter(token => right.has(token)).length / union.size; +} + +/** + * Offline fixture analyzer for plumbing evaluation only. It uses no scenario IDs + * or ground truth and is intentionally imperfect; it is not a quality claim. + */ +export class OfflineFixtureDocumentChangeMaterialityAnalyzer implements DocumentChangeMaterialityAnalyzer { + async analyze(input: DocumentChangeMaterialityAnalysisInput) { + const combined = input.changes.map(change => ({ + previous: change.previousExcerpt ?? "", + current: change.currentExcerpt ?? "", + })); + const averageOverlap = combined.length === 0 ? 0 : combined.reduce( + (total, change) => total + overlap(change.previous, change.current), 0, + ) / combined.length; + const replacement = input.changes.some(change => change.changeType === "added") + && input.changes.some(change => change.changeType === "removed"); + const deterministicCategory = input.deterministicAssessment.category; + const result = deterministicCategory !== "uncertain" && deterministicCategory !== "editorial_rewrite" + ? { + disposition: "material" as const, + category: deterministicCategory, + summary: "The supplied fragments contain a deterministic factual change.", + confidence: 0.86, + } + : !replacement && averageOverlap >= 0.45 + ? { + disposition: "non_material" as const, + category: "editorial_rewrite" as const, + summary: "The supplied fragments appear to preserve the same business meaning.", + confidence: 0.78, + } + : { + disposition: "uncertain" as const, + category: "uncertain" as const, + summary: "The bounded fragments do not support a confident semantic disposition.", + confidence: 0.45, + }; + return { + result, + metadata: { + provider: "offline-fixture", + model: "semantic-overlap-v1", + promptVersion: "document-change-materiality/v1", + }, + }; + } +} diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts new file mode 100644 index 000000000..94bb0ee31 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-baseline.ts @@ -0,0 +1,75 @@ +export const FOUNDER_WEEKLY_REVIEW_MATERIALITY_DETERMINISTIC_BASELINE = Object.freeze({ + baselineVersion: "founder-weekly-review-materiality-evaluation-baseline/v1", + baselineCommit: "630660fc861e77c464eaffc26ed6f04a7e38e7c7", + strategy: "deterministic-v1", + fixtureVersion: "founder-weekly-review-materiality-evaluation/v1", + scenarioCount: 46, + multiChunkScenarioCount: 10, + largeDocumentScenarioCount: 6, + metrics: { + categoryAccuracy: 0.9032, + materialRecall: 1, + materialPrecision: 0.7381, + uncertainRate: 0.2381, + falseMaterialRate: 0.7857, + missedMaterialRate: 0, + uncertainMaterialRate: 0.0323, + uncertainNonMaterialRate: 0.5714, + alignmentMissRate: 0.0405, + alignmentFalseMatchRate: 0, + }, + counts: { + rawChangedRecords: 70, + groups: 42, + structurallySelectedEvidenceItems: 24, + generationEnvelopeItems: 22, + rawCopiedCharacters: 5_144, + condensedCharacters: 2_430, + serializedPromptEvidenceCharacters: 13_937, + reductionRatio: 2.1169, + }, + metricDefinitions: { + falseMaterialRate: "prompt-facing groups mapped to non-material expectations, plus unexpected groups, divided by non-material ground-truth opportunities", + uncertainRate: "uncertain groups divided by all observed groups", + alignmentMissRate: "unrecovered intended modified/unchanged/split/merge relations divided by all such intended relations", + reductionRatio: "bounded raw audit excerpt characters divided by structurally selected condensed evidence characters", + }, +} as const); + +/** Historical analyzer reference points; these are not regenerated or tuned by v2 evaluation. */ +export const FOUNDER_WEEKLY_REVIEW_MATERIALITY_ANALYZER_V1_BASELINES = Object.freeze({ + offline: { + strategy: "materiality-analyzer-offline-v1", + materialRecall: 0.9677, + materialPrecision: 0.7692, + falseMaterialRate: 0.6429, + uncertainRate: 0.2381, + missedMaterialRate: 0.0323, + }, + openAiLive: { + strategy: "materiality-analyzer-live-v1", + provider: "openai", + model: "gpt-4o-mini", + semanticQualityMeasured: false, + calls: 18, + successfulCalls: 0, + timeouts: 18, + }, + kimiLive: { + strategy: "materiality-analyzer-kimi-live-v1", + provider: "kimi", + model: "kimi-k2.6", + promptVersion: "document-change-materiality/v1", + categoryAccuracy: 0.9355, + materialRecall: 1, + materialPrecision: 0.7949, + falseMaterialRate: 0.5714, + uncertainRate: 0.1667, + missedMaterialRate: 0, + uncertainMaterialRate: 0, + uncertainNonMaterialRate: 0.2143, + validAnalyzerCalls: 15, + invalidAnalyzerResponses: 3, + timeouts: 0, + }, +} as const); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts new file mode 100644 index 000000000..f34a41c80 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation-fixtures.ts @@ -0,0 +1,344 @@ +import type { + DocumentChangeCategory, + VersionChunk, +} from "@launchstack/features/founder-weekly-review"; + +export type ExpectedAlignmentRelation = { + id: string; + relation: "modified" | "unchanged" | "added" | "removed" | "split" | "merge"; + previousChunkIds: readonly number[]; + currentChunkIds: readonly number[]; +}; + +export type ExpectedChange = { + id: string; + material: boolean; + category: DocumentChangeCategory | null; + previousChunkIds: readonly number[]; + currentChunkIds: readonly number[]; +}; + +export type ExpectedNoOp = { + id: string; + previousChunkId: number; + currentChunkId: number; +}; + +export type MaterialityEvaluationScenario = { + id: string; + description: string; + documentKind: string; + multiChunk: boolean; + largeDocument: boolean; + previousChunks: readonly VersionChunk[]; + currentChunks: readonly VersionChunk[]; + expected: { + meaningfulChanges: readonly ExpectedChange[]; + nonMaterialChanges: readonly ExpectedChange[]; + expectedNoOps: readonly ExpectedNoOp[]; + expectedAlignmentRelations: readonly ExpectedAlignmentRelation[]; + }; +}; + +type ChunkSpec = { + key: string; + content: string; + path?: string | null; + title?: string | null; + order?: number; + page?: number; +}; + +type ChangeSpec = { + id: string; + material: boolean; + category: DocumentChangeCategory | null; + previousKeys: readonly string[]; + currentKeys: readonly string[]; +}; + +type RelationSpec = { + id: string; + relation: ExpectedAlignmentRelation["relation"]; + previousKeys: readonly string[]; + currentKeys: readonly string[]; +}; + +type ScenarioSpec = { + id: string; + description: string; + documentKind: string; + previous: readonly ChunkSpec[]; + current: readonly ChunkSpec[]; + changes?: readonly ChangeSpec[]; + noOps?: readonly { id: string; previousKey: string; currentKey: string }[]; + relations: readonly RelationSpec[]; + largeDocument?: boolean; +}; + +function chunk( + scenarioOrdinal: number, + version: 1 | 2, + index: number, + spec: ChunkSpec +): VersionChunk { + const documentId = BigInt(10_000 + scenarioOrdinal); + return { + chunkId: scenarioOrdinal * 1_000 + (version === 1 ? index + 1 : index + 501), + content: spec.content, + contentHash: null, + structureId: null, + structurePath: spec.path ?? `/section/${spec.key}`, + structureTitle: spec.title ?? spec.key.replace(/-/g, " "), + structureOrdering: spec.order ?? index, + pageNumber: spec.page ?? Math.floor(index / 2) + 1, + lineStart: index * 10 + 1, + lineEnd: index * 10 + 8, + documentId, + versionId: BigInt(version), + }; +} + +function buildScenario(spec: ScenarioSpec, ordinal: number): MaterialityEvaluationScenario { + const previousChunks = spec.previous.map((value, index) => chunk(ordinal, 1, index, value)); + const currentChunks = spec.current.map((value, index) => chunk(ordinal, 2, index, value)); + const previousIds = new Map(spec.previous.map((value, index) => [value.key, previousChunks[index]!.chunkId])); + const currentIds = new Map(spec.current.map((value, index) => [value.key, currentChunks[index]!.chunkId])); + const ids = (keys: readonly string[], values: Map) => keys.map(key => { + const id = values.get(key); + if (id === undefined) throw new Error(`Unknown fixture chunk key ${key} in ${spec.id}.`); + return id; + }); + const changes = (spec.changes ?? []).map(change => ({ + id: `${spec.id}:${change.id}`, + material: change.material, + category: change.category, + previousChunkIds: ids(change.previousKeys, previousIds), + currentChunkIds: ids(change.currentKeys, currentIds), + })); + return { + id: spec.id, + description: spec.description, + documentKind: spec.documentKind, + multiChunk: previousChunks.length > 1 || currentChunks.length > 1, + largeDocument: spec.largeDocument ?? false, + previousChunks, + currentChunks, + expected: { + meaningfulChanges: changes.filter(change => change.material), + nonMaterialChanges: changes.filter(change => !change.material), + expectedNoOps: (spec.noOps ?? []).map(noOp => ({ + id: `${spec.id}:${noOp.id}`, + previousChunkId: ids([noOp.previousKey], previousIds)[0]!, + currentChunkId: ids([noOp.currentKey], currentIds)[0]!, + })), + expectedAlignmentRelations: spec.relations.map(relation => ({ + id: `${spec.id}:${relation.id}`, + relation: relation.relation, + previousChunkIds: ids(relation.previousKeys, previousIds), + currentChunkIds: ids(relation.currentKeys, currentIds), + })), + }, + }; +} + +function single( + id: string, + documentKind: string, + before: string, + after: string, + expectation: { material: boolean; category: DocumentChangeCategory | null } | "noop", + description: string +): ScenarioSpec { + return { + id, + description, + documentKind, + previous: [{ key: "main", content: before, path: "/main", title: "Main" }], + current: [{ key: "main", content: after, path: "/main", title: "Main" }], + ...(expectation === "noop" + ? { noOps: [{ id: "noop", previousKey: "main", currentKey: "main" }] } + : { changes: [{ id: "change", ...expectation, previousKeys: ["main"], currentKeys: ["main"] }] }), + relations: [{ id: "alignment", relation: "modified", previousKeys: ["main"], currentKeys: ["main"] }], + }; +} + +const SIMPLE_SCENARIOS: readonly ScenarioSpec[] = [ + single("noop-whitespace", "operating plan", "Platform owns telemetry.", " Platform owns telemetry. ", "noop", "Repeated and boundary whitespace in an operating plan."), + single("noop-line-wrap", "launch plan", "The enterprise launch remains on schedule.", "The enterprise launch\r\nremains on schedule.", "noop", "Extraction line wrapping in a launch plan."), + single("noop-nbsp", "pricing plan", "Enterprise price review", "Enterprise\u00a0price review", "noop", "A non-breaking-space extraction difference."), + single("noop-unicode", "founder operating notes", "The caf\u00e9 review is complete.", "The cafe\u0301 review is complete.", "noop", "Unicode NFC-equivalent founder notes."), + single("noop-format-artifact", "product roadmap", "Roadmap assumptions remain unchanged.", "**Roadmap assumptions remain unchanged.**", { material: false, category: null }, "A Markdown emphasis extraction artifact that current no-op normalization does not remove."), + + single("paraphrase-ownership", "ownership document", "Product owns telemetry.", "Telemetry is owned by Product.", { material: false, category: null }, "A pure ownership paraphrase with the same DRI."), + single("paraphrase-quarter", "launch plan", "The launch remains planned for Q3.", "We still expect the launch during the third quarter.", { material: false, category: null }, "A deadline paraphrase with unchanged timing."), + single("paraphrase-customer", "sales strategy", "We will begin with design partners before broad availability.", "Broad availability will follow the initial design-partner cohort.", { material: false, category: null }, "A rollout paraphrase with unchanged business meaning."), + + single("ownership-team", "technical rollout plan", "Product owns retry telemetry.", "Platform owns retry telemetry.", { material: true, category: "ownership_change" }, "Ownership transfers from Product to Platform."), + single("ownership-person", "operating plan", "Owner: Alice", "Owner: Bob", { material: true, category: "ownership_change" }, "The named operating-plan owner changes."), + single("ownership-function", "sales strategy", "Marketing owns partner enablement.", "Sales owns partner enablement.", { material: true, category: "ownership_change" }, "Partner enablement ownership transfers functions."), + + single("status-launched", "product roadmap", "The migration is planned.", "The migration is launched.", { material: true, category: "status_change" }, "A roadmap item progresses from planned to launched."), + single("status-blocked", "technical rollout plan", "The rollout is in progress.", "The rollout is blocked.", { material: true, category: "status_change" }, "An in-progress rollout becomes blocked."), + single("status-resolved", "risk register", "The integration is blocked.", "The integration is resolved.", { material: true, category: "risk_or_blocker_change" }, "A documented blocker is resolved."), + single("status-cancelled", "launch plan", "The beta is active.", "The beta is cancelled.", { material: true, category: "status_change" }, "An active beta is cancelled."), + + single("deadline-quarter", "product roadmap", "Launch target: Q3", "Launch target: Q4", { material: true, category: "deadline_change" }, "The launch target moves one quarter."), + single("deadline-date", "customer commitment", "Customer go-live is June 1.", "Customer go-live is July 15.", { material: true, category: "deadline_change" }, "An enterprise customer go-live date moves."), + single("deadline-year", "operating plan", "International expansion is targeted for 2026.", "International expansion is targeted for 2027.", { material: true, category: "deadline_change" }, "Expansion moves to the following year."), + + single("metric-percent", "pricing plan", "Expected conversion is 10%.", "Expected conversion is 25%.", { material: true, category: "metric_change" }, "The expected conversion metric changes."), + single("metric-currency", "operating plan", "ARR target is $1M.", "ARR target is $750k.", { material: true, category: "metric_change" }, "The annual recurring revenue target declines."), + single("metric-customers", "sales strategy", "The cohort includes 20 customers.", "The cohort includes 50 customers.", { material: true, category: "metric_change" }, "The customer cohort expands."), + + single("requirement-may-must", "enterprise onboarding", "Admins may enable SSO.", "Admins must enable SSO.", { material: true, category: "requirement_change" }, "An onboarding capability becomes mandatory."), + single("requirement-optional", "technical rollout plan", "Audit logging is optional.", "Audit logging is required.", { material: true, category: "requirement_change" }, "Audit logging changes from optional to required."), + single("requirement-should", "customer commitment", "Support should acknowledge P0 cases.", "Support must acknowledge P0 cases.", { material: true, category: "requirement_change" }, "A support recommendation becomes an obligation."), + single("requirement-mandatory", "operating plan", "Security review is recommended.", "Security review is mandatory.", { material: true, category: "requirement_change" }, "A security review becomes mandatory."), + + single("negation-support", "enterprise onboarding", "The plan supports SSO.", "The plan does not support SSO.", { material: true, category: "requirement_change" }, "SSO support is explicitly negated."), + single("negation-availability", "pricing plan", "The add-on is available.", "The add-on is unavailable.", { material: true, category: "requirement_change" }, "An add-on becomes unavailable."), + + single("risk-added", "risk register", "No delivery concern is recorded.", "Delivery risk is recorded for the data migration.", { material: true, category: "risk_or_blocker_change" }, "A delivery risk is introduced."), + single("risk-removed", "risk register", "Vendor risk remains open.", "The vendor concern has been cleared.", { material: true, category: "risk_or_blocker_change" }, "A risk is removed using non-lexical resolution language."), + single("blocker-introduced", "technical rollout plan", "Identity testing is in progress.", "Identity testing has a blocker.", { material: true, category: "status_change" }, "A blocker interrupts work that was in progress; status precedence is expected."), + single("blocker-resolved", "founder operating notes", "Billing migration is blocked.", "Billing migration is unblocked.", { material: true, category: "risk_or_blocker_change" }, "A billing blocker is cleared."), + + single("scope-global", "launch plan", "Availability: US only", "Availability: global", { material: true, category: "scope_change" }, "Launch geography expands globally."), + single("scope-enterprise", "sales strategy", "Access is for pilot customers.", "Access is for all enterprise customers.", { material: true, category: "scope_change" }, "A pilot expands to all enterprise customers."), + single("priority-p0", "product roadmap", "Migration priority: P2", "Migration priority: P0", { material: true, category: "priority_change" }, "A roadmap item is escalated to P0."), + single("priority-immediate", "founder operating notes", "The pricing review is deferred.", "The pricing review is immediate.", { material: true, category: "priority_change" }, "A deferred pricing review becomes immediate."), +]; + +function sectionChunks(prefix: string, contents: readonly string[], path: string, pageStart = 1): ChunkSpec[] { + return contents.map((content, index) => ({ + key: `${prefix}-${index + 1}`, + content, + path, + title: prefix.replace(/-/g, " "), + order: index, + page: pageStart + index, + })); +} + +const MULTI_SCENARIOS: readonly ScenarioSpec[] = [ + { + id: "rewrite-roadmap-same-meaning", + description: "A six-fragment roadmap section is reorganized without changing ownership, timing, or scope.", + documentKind: "product roadmap", + previous: sectionChunks("roadmap", [ + "The team will validate demand with design partners.", "Product continues to own the launch checklist.", + "General availability remains targeted for the third quarter.", "The initial cohort remains limited to enterprise design partners.", + "Security review continues before activation.", "No change is made to the rollout decision gate.", + ], "/roadmap"), + current: sectionChunks("roadmap", [ + "Demand validation will continue through conversations with design partners.", "The launch checklist remains under Product ownership.", + "The third quarter remains the target for general availability.", "Enterprise design partners still make up the initial cohort.", + "Activation continues to follow the security review.", "The rollout decision gate remains as previously documented.", + ], "/roadmap"), + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["roadmap-1", "roadmap-2", "roadmap-3", "roadmap-4", "roadmap-5", "roadmap-6"], currentKeys: ["roadmap-1", "roadmap-2", "roadmap-3", "roadmap-4", "roadmap-5", "roadmap-6"] }], + relations: [1, 2, 3, 4, 5, 6].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`roadmap-${index}`], currentKeys: [`roadmap-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-onboarding-same-meaning", + description: "An enterprise onboarding section is rewritten in a more narrative style with unchanged controls.", + documentKind: "enterprise onboarding document", + previous: sectionChunks("onboarding", ["Customer success coordinates kickoff.", "Security receives the questionnaire before configuration.", "The customer validates SSO in staging.", "Production access follows customer sign-off.", "Support monitors the first business day."], "/onboarding"), + current: sectionChunks("onboarding", ["Kickoff coordination remains with customer success.", "The questionnaire continues to reach security ahead of configuration.", "SSO validation still occurs in the staging environment.", "Customer sign-off continues to precede production access.", "The first business day remains under support monitoring."], "/onboarding"), + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["onboarding-1", "onboarding-2", "onboarding-3", "onboarding-4", "onboarding-5"], currentKeys: ["onboarding-1", "onboarding-2", "onboarding-3", "onboarding-4", "onboarding-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`onboarding-${index}`], currentKeys: [`onboarding-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-pricing-same-meaning", + description: "A pricing narrative changes voice and ordering while retaining the same approval and rollout policy.", + documentKind: "pricing plan", + previous: sectionChunks("pricing", ["Finance reviews discount exceptions.", "Sales submits the business rationale.", "Legal reviews non-standard terms.", "The pricing council makes the final decision.", "Approved changes enter the next catalog release."], "/pricing"), + current: sectionChunks("pricing", ["Discount exceptions continue to receive finance review.", "The business rationale still comes from sales.", "Non-standard terms continue through legal review.", "Final decisions remain with the pricing council.", "The next catalog release continues to carry approved changes."], "/pricing"), + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["pricing-1", "pricing-2", "pricing-3", "pricing-4", "pricing-5"], currentKeys: ["pricing-1", "pricing-2", "pricing-3", "pricing-4", "pricing-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`pricing-${index}`], currentKeys: [`pricing-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-rollout-material", + description: "A six-fragment technical rollout rewrite transfers ownership, advances status, and expands scope.", + documentKind: "technical rollout plan", + previous: sectionChunks("rollout", ["Product owns deployment telemetry.", "The rollout is planned.", "Availability is US only.", "Audit logging is optional.", "Identity integration risk remains open.", "The pilot includes 20 customers."], "/rollout"), + current: sectionChunks("rollout", ["Platform owns deployment telemetry.", "The rollout is launched.", "Availability is global.", "Audit logging is required.", "Identity integration risk is resolved.", "The launch includes 50 customers."], "/rollout"), + changes: [{ id: "rewrite", material: true, category: "ownership_change", previousKeys: ["rollout-1", "rollout-2", "rollout-3", "rollout-4", "rollout-5", "rollout-6"], currentKeys: ["rollout-1", "rollout-2", "rollout-3", "rollout-4", "rollout-5", "rollout-6"] }], + relations: [1, 2, 3, 4, 5, 6].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`rollout-${index}`], currentKeys: [`rollout-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-customer-commitment-material", + description: "A customer commitment section changes its deadline and makes implementation support mandatory.", + documentKind: "customer commitment", + previous: sectionChunks("commitment", ["Customer go-live is June 1.", "Implementation support is recommended.", "The pilot covers US accounts.", "Product owns escalation review.", "The security risk remains open."], "/commitment"), + current: sectionChunks("commitment", ["Customer go-live is July 15.", "Implementation support is mandatory.", "The pilot covers global accounts.", "Platform owns escalation review.", "The security risk is resolved."], "/commitment"), + changes: [{ id: "rewrite", material: true, category: "ownership_change", previousKeys: ["commitment-1", "commitment-2", "commitment-3", "commitment-4", "commitment-5"], currentKeys: ["commitment-1", "commitment-2", "commitment-3", "commitment-4", "commitment-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`commitment-${index}`], currentKeys: [`commitment-${index}`] })), + largeDocument: true, + }, + { + id: "rewrite-risk-register-material", + description: "A multi-page risk-register section records resolution of one blocker and introduction of another.", + documentKind: "risk register", + previous: sectionChunks("risks", ["Identity migration is blocked.", "Billing reconciliation has no recorded risk.", "The rollout is in progress.", "The response owner is Product.", "The review target is Q3."], "/risks"), + current: sectionChunks("risks", ["Identity migration is resolved.", "Billing reconciliation has a blocker.", "The rollout is paused.", "The response owner is Platform.", "The review target is Q4."], "/risks"), + changes: [{ id: "rewrite", material: true, category: "ownership_change", previousKeys: ["risks-1", "risks-2", "risks-3", "risks-4", "risks-5"], currentKeys: ["risks-1", "risks-2", "risks-3", "risks-4", "risks-5"] }], + relations: [1, 2, 3, 4, 5].map(index => ({ id: `pair-${index}`, relation: "modified" as const, previousKeys: [`risks-${index}`], currentKeys: [`risks-${index}`] })), + largeDocument: true, + }, + { + id: "alignment-section-moved", + description: "An unchanged founder-notes section moves from the opening to the end of the document.", + documentKind: "founder operating notes", + previous: [{ key: "moved", content: "The operating cadence remains weekly with a Friday review.", path: "/opening", title: "Cadence", order: 0, page: 1 }, { key: "stable", content: "The hiring plan remains unchanged.", path: "/hiring", title: "Hiring", order: 1, page: 2 }], + current: [{ key: "stable", content: "The hiring plan remains unchanged.", path: "/hiring", title: "Hiring", order: 0, page: 1 }, { key: "moved", content: "The operating cadence remains weekly with a Friday review.", path: "/closing", title: "Operating rhythm", order: 1, page: 6 }], + relations: [{ id: "moved", relation: "unchanged", previousKeys: ["moved"], currentKeys: ["moved"] }, { id: "stable", relation: "unchanged", previousKeys: ["stable"], currentKeys: ["stable"] }], + }, + { + id: "alignment-renamed-heavy-rewrite", + description: "A renamed sales-strategy section preserves intent through a heavy lexical rewrite.", + documentKind: "sales strategy", + previous: [{ key: "strategy", content: "Regional account executives qualify expansion opportunities during quarterly portfolio reviews.", path: "/commercial-motion", title: "Commercial motion", order: 2, page: 4 }], + current: [{ key: "strategy", content: "Every quarter, territory sellers examine existing customers to identify places where adoption can broaden.", path: "/growth-playbook", title: "Growth playbook", order: 7, page: 12 }], + changes: [{ id: "rewrite", material: false, category: null, previousKeys: ["strategy"], currentKeys: ["strategy"] }], + relations: [{ id: "rewrite", relation: "modified", previousKeys: ["strategy"], currentKeys: ["strategy"] }], + }, + { + id: "alignment-one-to-many", + description: "One enterprise-onboarding paragraph is split into two structurally adjacent paragraphs.", + documentKind: "enterprise onboarding document", + previous: [{ key: "combined", content: "Security reviews the questionnaire and customer success schedules kickoff after approval.", path: "/onboarding", title: "Onboarding", order: 1, page: 3 }], + current: [{ key: "security", content: "Security reviews the questionnaire before approval.", path: "/onboarding", title: "Onboarding", order: 1, page: 3 }, { key: "kickoff", content: "Customer success schedules kickoff after approval.", path: "/onboarding", title: "Onboarding", order: 2, page: 3 }], + changes: [{ id: "split", material: false, category: null, previousKeys: ["combined"], currentKeys: ["security", "kickoff"] }], + relations: [{ id: "split", relation: "split", previousKeys: ["combined"], currentKeys: ["security", "kickoff"] }], + }, + { + id: "alignment-many-to-one", + description: "Two adjacent pricing-control fragments merge into one paragraph without changing policy.", + documentKind: "pricing plan", + previous: [{ key: "finance", content: "Finance reviews discount exceptions.", path: "/controls", title: "Controls", order: 1, page: 2 }, { key: "council", content: "The pricing council makes the final decision.", path: "/controls", title: "Controls", order: 2, page: 2 }], + current: [{ key: "combined", content: "Finance reviews discount exceptions before the pricing council makes the final decision.", path: "/controls", title: "Controls", order: 1, page: 2 }], + changes: [{ id: "merge", material: false, category: null, previousKeys: ["finance", "council"], currentKeys: ["combined"] }], + relations: [{ id: "merge", relation: "merge", previousKeys: ["finance", "council"], currentKeys: ["combined"] }], + }, + { + id: "alignment-repeated-headings", + description: "Repeated boilerplate headings surround one changed customer-commitment clause.", + documentKind: "customer commitment", + previous: [{ key: "north", content: "Implementation plan: Customer success coordinates the North account kickoff.", path: "/north/plan", title: "Implementation plan", order: 1 }, { key: "south", content: "Implementation plan: Customer success coordinates the South account kickoff.", path: "/south/plan", title: "Implementation plan", order: 2 }], + current: [{ key: "south", content: "Implementation plan: Customer success coordinates the South account kickoff.", path: "/south/plan", title: "Implementation plan", order: 1 }, { key: "north", content: "Implementation plan: Platform coordinates the North account kickoff.", path: "/north/plan", title: "Implementation plan", order: 2 }], + changes: [{ id: "north-owner", material: true, category: "ownership_change", previousKeys: ["north"], currentKeys: ["north"] }], + relations: [{ id: "north", relation: "modified", previousKeys: ["north"], currentKeys: ["north"] }, { id: "south", relation: "unchanged", previousKeys: ["south"], currentKeys: ["south"] }], + }, +]; + +export const MATERIALITY_EVALUATION_FIXTURE_VERSION = "founder-weekly-review-materiality-evaluation/v1" as const; + +export const MATERIALITY_EVALUATION_SCENARIOS: readonly MaterialityEvaluationScenario[] = [ + ...SIMPLE_SCENARIOS, + ...MULTI_SCENARIOS, +].map(buildScenario); diff --git a/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts new file mode 100644 index 000000000..e5619d94d --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-materiality-evaluation.ts @@ -0,0 +1,871 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + FounderWeeklyReviewEvidenceSnapshotSchema, + alignVersionChunks, + buildGenerationEvidenceEnvelope, + materializeDocumentChanges, + materializeDocumentChangesWithAnalyzer, + type AnalyzedDocumentChangeGroup, + type ChunkAlignment, + type DocumentChangeCategory, + type DocumentChangePairInput, + type DocumentChangeMaterialityAnalyzer, + type VersionPair, +} from "@launchstack/features/founder-weekly-review"; + +import { + MATERIALITY_EVALUATION_FIXTURE_VERSION, + MATERIALITY_EVALUATION_SCENARIOS, + type ExpectedAlignmentRelation, + type ExpectedChange, + type MaterialityEvaluationScenario, +} from "./founder-weekly-review-materiality-evaluation-fixtures"; +import { OfflineFixtureDocumentChangeMaterialityAnalyzer } from "./founder-weekly-review-materiality-evaluation-analyzers"; + +export const MATERIALITY_EVALUATION_RUN_ID = "deterministic-v1" as const; +export const MATERIALITY_ANALYZER_EVALUATION_RUN_ID = "materiality-analyzer-v1" as const; +export const MATERIALITY_ANALYZER_KIMI_LIVE_V2_RUN_ID = "materiality-analyzer-kimi-live-v2" as const; +export const MATERIALITY_EVALUATION_ARTIFACT_ROOT = + ".artifacts/founder-weekly-review/materiality-evaluation" as const; + +type FailureKind = + | "materiality_false_positive" + | "materiality_false_negative" + | "uncertain_material" + | "uncertain_non_material" + | "alignment_miss" + | "alignment_false_match" + | "grouping_issue" + | "budget_issue"; + +type MissingCapability = + | "future_llm_materiality_analyzer" + | "future_embedding_alignment" + | "deterministic_improvement" + | "acceptable_limitation"; + +export type MaterialityEvaluationFailure = { + kind: FailureKind; + scenarioId: string; + expectationId: string; + description: string; + observed: string; + likelyMissingCapability: MissingCapability; + syntheticFixture: { + previous: readonly string[]; + current: readonly string[]; + }; +}; + +export type MaterialityEvaluationSummary = { + fixtureVersion: string; + scenarioCount: number; + multiChunkScenarioCount: number; + largeDocumentScenarioCount: number; + groundTruthCategoryDistribution: Record; + observedCategoryDistribution: Record; + materiality: { + groundTruthMaterialChanges: number; + groundTruthNonMaterialChanges: number; + categoryCorrect: number; + categoryAccuracy: number; + materialRecall: number; + materialPrecision: number; + uncertainRate: number; + falseMaterialCount: number; + falseMaterialRate: number; + missedMaterialCount: number; + missedMaterialRate: number; + uncertainMaterialCount: number; + uncertainMaterialRate: number; + uncertainNonMaterialCount: number; + uncertainNonMaterialRate: number; + }; + alignment: { + intendedSemanticRelations: number; + correctModifiedPairings: number; + incorrectModifiedPairings: number; + intendedModifiedAsAddedRemoved: number; + falsePairings: number; + unmatchedOldChunks: number; + unmatchedNewChunks: number; + alignmentMisses: number; + alignmentMissRate: number; + falseMatchRate: number; + }; + condensation: { + rawChangedRecords: number; + groups: number; + condensedEvidenceItems: number; + rawCopiedCharacters: number; + condensedEvidenceCharacters: number; + serializedPromptCharacters: number; + reductionRatio: number; + }; + budget: { + groupBudgetTruncated: boolean; + truncatedGroupCount: number; + generationEnvelopeTruncated: boolean; + generationEnvelopeSelectedItems: number; + generationEnvelopeExcludedItems: number; + documentDiversityPreserved: boolean; + selectedDocumentCount: number; + }; + failuresByKind: Record; + recommendation: "A" | "B" | "C" | "D" | "E"; + recommendationLabel: string; +}; + +export type MaterialityEvaluationResult = { + summary: MaterialityEvaluationSummary; + failures: readonly MaterialityEvaluationFailure[]; + scenarioResults: readonly { + id: string; + alignmentCount: number; + rawChangeCount: number; + groupCount: number; + categories: readonly DocumentChangeCategory[]; + failureKinds: readonly FailureKind[]; + }[]; +}; + +const FAILURE_KINDS: readonly FailureKind[] = [ + "materiality_false_positive", + "materiality_false_negative", + "uncertain_material", + "uncertain_non_material", + "alignment_miss", + "alignment_false_match", + "grouping_issue", + "budget_issue", +]; + +function ratio(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : Number((numerator / denominator).toFixed(4)); +} + +function pairFor(scenario: MaterialityEvaluationScenario, ordinal: number): VersionPair { + const documentId = scenario.previousChunks[0]?.documentId ?? scenario.currentChunks[0]!.documentId; + return { + documentId, + documentTitle: `${scenario.documentKind}: ${scenario.id}`, + documentCategory: null, + previousVersionId: 1, + previousVersionNumber: 1, + previousCreatedAt: new Date("2026-01-01T00:00:00.000Z"), + currentVersionId: 2, + currentVersionNumber: 2, + currentCreatedAt: new Date(Date.UTC(2026, 1, 1 + ordinal)), + currentChangelog: null, + }; +} + +function chunkIds(group: AnalyzedDocumentChangeGroup): { previous: Set; current: Set } { + return { + previous: new Set(group.group.rawChanges.flatMap(change => change.previousChunk ? [change.previousChunk.chunkId] : [])), + current: new Set(group.group.rawChanges.flatMap(change => change.currentChunk ? [change.currentChunk.chunkId] : [])), + }; +} + +function overlap(change: ExpectedChange, group: AnalyzedDocumentChangeGroup): number { + const ids = chunkIds(group); + return change.previousChunkIds.filter(id => ids.previous.has(id)).length + + change.currentChunkIds.filter(id => ids.current.has(id)).length; +} + +function bestGroup(change: ExpectedChange, groups: readonly AnalyzedDocumentChangeGroup[]): AnalyzedDocumentChangeGroup | undefined { + return [...groups] + .map(group => ({ group, score: overlap(change, group) })) + .filter(candidate => candidate.score > 0) + .sort((a, b) => b.score - a.score || (a.group.group.groupId < b.group.group.groupId ? -1 : 1))[0]?.group; +} + +function relationContainsPair(relation: ExpectedAlignmentRelation, previousId: number, currentId: number): boolean { + return relation.previousChunkIds.includes(previousId) && relation.currentChunkIds.includes(currentId); +} + +function actualPair(alignments: readonly ChunkAlignment[], previousId: number, currentId: number): ChunkAlignment | undefined { + return alignments.find(alignment => alignment.previousChunk?.chunkId === previousId && alignment.currentChunk?.chunkId === currentId); +} + +function relationRecovered(relation: ExpectedAlignmentRelation, alignments: readonly ChunkAlignment[]): boolean { + if (relation.relation === "added") { + return relation.currentChunkIds.every(id => alignments.some(alignment => alignment.changeType === "added" && alignment.currentChunk?.chunkId === id)); + } + if (relation.relation === "removed") { + return relation.previousChunkIds.every(id => alignments.some(alignment => alignment.changeType === "removed" && alignment.previousChunk?.chunkId === id)); + } + if (relation.relation === "modified" || relation.relation === "unchanged") { + const alignment = actualPair(alignments, relation.previousChunkIds[0]!, relation.currentChunkIds[0]!); + return alignment?.changeType === relation.relation; + } + const edges = alignments.filter(alignment => alignment.previousChunk && alignment.currentChunk + && relationContainsPair(relation, alignment.previousChunk.chunkId, alignment.currentChunk.chunkId)); + return relation.previousChunkIds.every(id => edges.some(edge => edge.previousChunk!.chunkId === id)) + && relation.currentChunkIds.every(id => edges.some(edge => edge.currentChunk!.chunkId === id)); +} + +function failure( + scenario: MaterialityEvaluationScenario, + kind: FailureKind, + expectationId: string, + observed: string, + likelyMissingCapability: MissingCapability +): MaterialityEvaluationFailure { + return { + kind, + scenarioId: scenario.id, + expectationId, + description: scenario.description, + observed, + likelyMissingCapability, + syntheticFixture: { + previous: scenario.previousChunks.map(chunk => chunk.content), + current: scenario.currentChunks.map(chunk => chunk.content), + }, + }; +} + +function increment(record: Record, key: string): void { + record[key] = (record[key] ?? 0) + 1; +} + +function recommendationFor( + uncertainRate: number, + falseMaterialRate: number, + uncertainMaterialCount: number, + alignmentMissRate: number, + materialityFailureCount: number, + alignmentFailureCount: number +): Pick { + const materiality = uncertainRate > 0.15 || falseMaterialRate > 0.05 || uncertainMaterialCount >= 3; + const alignment = alignmentMissRate > 0.075; + if (materiality && alignment) { + return materialityFailureCount >= alignmentFailureCount + ? { recommendation: "C", recommendationLabel: "Implement both, materiality first" } + : { recommendation: "D", recommendationLabel: "Implement both, alignment first" }; + } + if (materiality) return { recommendation: "A", recommendationLabel: "Implement Phase 3 optional LLM materiality analyzer next" }; + if (alignment) return { recommendation: "B", recommendationLabel: "Implement Phase 4 embedding-assisted alignment next" }; + return { recommendation: "E", recommendationLabel: "Implement neither yet" }; +} + +/** Evaluates frozen production logic only; it never invokes a provider or mutates production behavior. */ +export function runMaterialityEvaluation( + inputScenarios: readonly MaterialityEvaluationScenario[] = MATERIALITY_EVALUATION_SCENARIOS +): MaterialityEvaluationResult { + const scenarios = [...inputScenarios].sort((a, b) => a.id.localeCompare(b.id)); + const failures: MaterialityEvaluationFailure[] = []; + const scenarioResults: MaterialityEvaluationResult["scenarioResults"][number][] = []; + const inputs: DocumentChangePairInput[] = []; + const groundTruthCategoryDistribution: Record = {}; + const observedCategoryDistribution: Record = {}; + let groundTruthMaterialChanges = 0; + let groundTruthNonMaterialChanges = 0; + let categoryCorrect = 0; + let surfacedTrueMaterial = 0; + let falseMaterialCount = 0; + let missedMaterialCount = 0; + let uncertainMaterialCount = 0; + let uncertainNonMaterialCount = 0; + let correctModifiedPairings = 0; + let incorrectModifiedPairings = 0; + let intendedModifiedAsAddedRemoved = 0; + let falsePairings = 0; + let unmatchedOldChunks = 0; + let unmatchedNewChunks = 0; + let alignmentMisses = 0; + let intendedSemanticRelations = 0; + + for (const [ordinal, scenario] of scenarios.entries()) { + const pair = pairFor(scenario, ordinal); + const alignments = alignVersionChunks(scenario.previousChunks, scenario.currentChunks); + inputs.push({ pair, alignments }); + const materialized = materializeDocumentChanges([{ pair, alignments }]); + for (const group of materialized.analyzedGroups) increment(observedCategoryDistribution, group.materiality.category); + const assignedGroupIds = new Set(); + + for (const expected of scenario.expected.meaningfulChanges) { + groundTruthMaterialChanges++; + increment(groundTruthCategoryDistribution, expected.category ?? "material_unspecified"); + const group = bestGroup(expected, materialized.analyzedGroups); + if (group) assignedGroupIds.add(group.group.groupId); + if (!group || group.materiality.category === "editorial_rewrite") { + missedMaterialCount++; + failures.push(failure(scenario, "materiality_false_negative", expected.id, + group ? "Material source was classified as editorial_rewrite." : "No retained group represented the material source.", + group ? "deterministic_improvement" : "future_embedding_alignment")); + continue; + } + surfacedTrueMaterial++; + if (group.materiality.category === expected.category) categoryCorrect++; + if (group.materiality.category === "uncertain") { + uncertainMaterialCount++; + failures.push(failure(scenario, "uncertain_material", expected.id, + "Material source was retained but classified uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const expected of scenario.expected.nonMaterialChanges) { + groundTruthNonMaterialChanges++; + const group = bestGroup(expected, materialized.analyzedGroups); + if (!group) continue; + assignedGroupIds.add(group.group.groupId); + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", expected.id, + `Non-material source became prompt-facing ${group.materiality.category} evidence.`, + group.materiality.category === "uncertain" ? "future_llm_materiality_analyzer" : "acceptable_limitation")); + if (group.materiality.category === "uncertain") { + uncertainNonMaterialCount++; + failures.push(failure(scenario, "uncertain_non_material", expected.id, + "Non-material rewrite was classified uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const noOp of scenario.expected.expectedNoOps) { + groundTruthNonMaterialChanges++; + const group = materialized.analyzedGroups.find(candidate => { + const ids = chunkIds(candidate); + return ids.previous.has(noOp.previousChunkId) || ids.current.has(noOp.currentChunkId); + }); + if (group) { + assignedGroupIds.add(group.group.groupId); + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", noOp.id, + `Expected no-op became prompt-facing ${group.materiality.category} evidence.`, "deterministic_improvement")); + } + } + + for (const group of materialized.analyzedGroups) { + if (assignedGroupIds.has(group.group.groupId)) continue; + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", group.group.groupId, + `An unexpected ${group.materiality.category} group became prompt-facing evidence.`, "deterministic_improvement")); + failures.push(failure(scenario, "grouping_issue", group.group.groupId, + `An extra ${group.materiality.category} group did not map to an expected semantic change.`, "deterministic_improvement")); + } + + const expectedRelations = scenario.expected.expectedAlignmentRelations; + for (const relation of expectedRelations) { + if (["modified", "unchanged", "split", "merge"].includes(relation.relation)) intendedSemanticRelations++; + const recovered = relationRecovered(relation, alignments); + if (!recovered && ["modified", "unchanged", "split", "merge"].includes(relation.relation)) { + alignmentMisses++; + const removed = relation.previousChunkIds.every(id => alignments.some(item => item.changeType === "removed" && item.previousChunk?.chunkId === id)); + const added = relation.currentChunkIds.every(id => alignments.some(item => item.changeType === "added" && item.currentChunk?.chunkId === id)); + if (removed && added) intendedModifiedAsAddedRemoved++; + failures.push(failure(scenario, "alignment_miss", relation.id, + removed && added ? "Intended correspondence was represented as added plus removed." : "Intended semantic correspondence was not completely recovered.", + "future_embedding_alignment")); + } + if (relation.relation === "modified" && relation.previousChunkIds.length === 1 && relation.currentChunkIds.length === 1) { + const actual = actualPair(alignments, relation.previousChunkIds[0]!, relation.currentChunkIds[0]!); + if (actual?.changeType === "modified") correctModifiedPairings++; + } + } + for (const alignment of alignments) { + if (alignment.changeType === "removed") unmatchedOldChunks++; + if (alignment.changeType === "added") unmatchedNewChunks++; + if (alignment.changeType !== "modified" || !alignment.previousChunk || !alignment.currentChunk) continue; + const expected = expectedRelations.some(relation => relationContainsPair(relation, alignment.previousChunk!.chunkId, alignment.currentChunk!.chunkId)); + if (!expected) { + incorrectModifiedPairings++; + falsePairings++; + failures.push(failure(scenario, "alignment_false_match", `${alignment.previousChunk.chunkId}:${alignment.currentChunk.chunkId}`, + "The aligner paired chunks outside the fixture's intended semantic relations.", "future_embedding_alignment")); + } + } + const scenarioFailureKinds = [...new Set(failures.filter(item => item.scenarioId === scenario.id).map(item => item.kind))].sort(); + scenarioResults.push({ + id: scenario.id, + alignmentCount: alignments.length, + rawChangeCount: materialized.rawChanges.length, + groupCount: materialized.analyzedGroups.length, + categories: materialized.analyzedGroups.map(group => group.materiality.category), + failureKinds: scenarioFailureKinds, + }); + } + + const aggregate = materializeDocumentChanges(inputs); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-03-31T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-03-31" }, + workspaceTimezone: "UTC", + items: aggregate.items, + sourceWarnings: aggregate.warnings.map(warning => ({ ...warning, sourceType: "document_change" as const })), + documentChangeAudit: aggregate.audit, + }); + const envelope = buildGenerationEvidenceEnvelope(snapshot); + const uncertainGroups = Object.entries(observedCategoryDistribution).find(([category]) => category === "uncertain")?.[1] ?? 0; + const observedGroups = Object.values(observedCategoryDistribution).reduce((total, count) => total + count, 0); + const nonMaterialDenominator = groundTruthNonMaterialChanges; + const uncertainRate = ratio(uncertainGroups, observedGroups); + const falseMaterialRate = ratio(falseMaterialCount, nonMaterialDenominator); + const alignmentMissRate = ratio(alignmentMisses, intendedSemanticRelations); + const failuresByKind = Object.fromEntries(FAILURE_KINDS.map(kind => [kind, failures.filter(failure => failure.kind === kind).length])) as Record; + const materialityFailureCount = failuresByKind.materiality_false_positive + + failuresByKind.materiality_false_negative + failuresByKind.uncertain_material + failuresByKind.uncertain_non_material; + const alignmentFailureCount = failuresByKind.alignment_miss + failuresByKind.alignment_false_match; + const recommendation = recommendationFor(uncertainRate, falseMaterialRate, uncertainMaterialCount, alignmentMissRate, materialityFailureCount, alignmentFailureCount); + const selectedDocumentCount = new Set(aggregate.selectedGroups.map(group => group.group.documentId.toString())).size; + const availableDocumentCount = new Set(aggregate.analyzedGroups.map(group => group.group.documentId.toString())).size; + const condensedEvidenceCharacters = aggregate.diagnostics.condensedPromptFacingCharacters; + const summary: MaterialityEvaluationSummary = { + fixtureVersion: MATERIALITY_EVALUATION_FIXTURE_VERSION, + scenarioCount: scenarios.length, + multiChunkScenarioCount: scenarios.filter(scenario => scenario.multiChunk).length, + largeDocumentScenarioCount: scenarios.filter(scenario => scenario.largeDocument).length, + groundTruthCategoryDistribution: Object.fromEntries(Object.entries(groundTruthCategoryDistribution).sort()), + observedCategoryDistribution: Object.fromEntries(Object.entries(observedCategoryDistribution).sort()), + materiality: { + groundTruthMaterialChanges, + groundTruthNonMaterialChanges, + categoryCorrect, + categoryAccuracy: ratio(categoryCorrect, groundTruthMaterialChanges), + materialRecall: ratio(surfacedTrueMaterial, groundTruthMaterialChanges), + materialPrecision: ratio(surfacedTrueMaterial, surfacedTrueMaterial + falseMaterialCount), + uncertainRate, + falseMaterialCount, + falseMaterialRate, + missedMaterialCount, + missedMaterialRate: ratio(missedMaterialCount, groundTruthMaterialChanges), + uncertainMaterialCount, + uncertainMaterialRate: ratio(uncertainMaterialCount, groundTruthMaterialChanges), + uncertainNonMaterialCount, + uncertainNonMaterialRate: ratio(uncertainNonMaterialCount, groundTruthNonMaterialChanges), + }, + alignment: { + intendedSemanticRelations, + correctModifiedPairings, + incorrectModifiedPairings, + intendedModifiedAsAddedRemoved, + falsePairings, + unmatchedOldChunks, + unmatchedNewChunks, + alignmentMisses, + alignmentMissRate, + falseMatchRate: ratio(falsePairings, correctModifiedPairings + incorrectModifiedPairings), + }, + condensation: { + rawChangedRecords: aggregate.rawChanges.length, + groups: aggregate.analyzedGroups.length, + condensedEvidenceItems: aggregate.items.length, + rawCopiedCharacters: aggregate.diagnostics.rawExcerptCharacters, + condensedEvidenceCharacters, + serializedPromptCharacters: envelope.diagnostics.serializedCharacterCount, + reductionRatio: ratio(aggregate.diagnostics.rawExcerptCharacters, Math.max(1, condensedEvidenceCharacters)), + }, + budget: { + groupBudgetTruncated: aggregate.diagnostics.truncatedGroupCount > 0, + truncatedGroupCount: aggregate.diagnostics.truncatedGroupCount, + generationEnvelopeTruncated: envelope.diagnostics.truncated, + generationEnvelopeSelectedItems: envelope.diagnostics.selectedItemCount, + generationEnvelopeExcludedItems: envelope.diagnostics.excludedItemCount, + documentDiversityPreserved: selectedDocumentCount === Math.min(aggregate.selectedGroups.length, availableDocumentCount), + selectedDocumentCount, + }, + failuresByKind, + ...recommendation, + }; + return { + summary, + failures: failures.sort((a, b) => a.kind.localeCompare(b.kind) || a.scenarioId.localeCompare(b.scenarioId) || a.expectationId.localeCompare(b.expectationId)), + scenarioResults: scenarioResults.sort((a, b) => a.id.localeCompare(b.id)), + }; +} + +/** Same frozen fixtures and metric formulas, with only the materiality strategy replaced. */ +export async function runMaterialityAnalyzerEvaluation( + analyzer: DocumentChangeMaterialityAnalyzer = new OfflineFixtureDocumentChangeMaterialityAnalyzer(), + inputScenarios: readonly MaterialityEvaluationScenario[] = MATERIALITY_EVALUATION_SCENARIOS +): Promise { + const scenarios = [...inputScenarios].sort((a, b) => a.id.localeCompare(b.id)); + const deterministicBaseline = runMaterialityEvaluation(scenarios); + const failures: MaterialityEvaluationFailure[] = deterministicBaseline.failures.filter(item => + item.kind === "alignment_miss" || item.kind === "alignment_false_match" || item.kind === "budget_issue"); + const scenarioResults: MaterialityEvaluationResult["scenarioResults"][number][] = []; + const inputs: DocumentChangePairInput[] = []; + const groundTruthCategoryDistribution: Record = {}; + const observedCategoryDistribution: Record = {}; + let groundTruthMaterialChanges = 0; + let groundTruthNonMaterialChanges = 0; + let categoryCorrect = 0; + let surfacedTrueMaterial = 0; + let falseMaterialCount = 0; + let missedMaterialCount = 0; + let uncertainMaterialCount = 0; + let uncertainNonMaterialCount = 0; + + for (const [ordinal, scenario] of scenarios.entries()) { + const pair = pairFor(scenario, ordinal); + const alignments = alignVersionChunks(scenario.previousChunks, scenario.currentChunks); + const pairInput = { pair, alignments }; + inputs.push(pairInput); + const materialized = await materializeDocumentChangesWithAnalyzer([pairInput], analyzer); + const auditByGroup = new Map(materialized.audit.groups.map(group => [group.groupId, group])); + const assignedGroupIds = new Set(); + const effectiveCategory = (group: AnalyzedDocumentChangeGroup): DocumentChangeCategory => { + const analysis = auditByGroup.get(group.group.groupId)?.analysis; + if (analysis?.disposition === "uncertain") return "uncertain"; + if (analysis?.disposition === "material") return analysis.category; + return group.materiality.category; + }; + const isSurfaced = (group: AnalyzedDocumentChangeGroup): boolean => + auditByGroup.get(group.group.groupId)?.evidenceSourceId !== null + && auditByGroup.get(group.group.groupId)?.analysis?.disposition !== "non_material"; + for (const group of materialized.analyzedGroups) increment(observedCategoryDistribution, effectiveCategory(group)); + + for (const expected of scenario.expected.meaningfulChanges) { + groundTruthMaterialChanges++; + increment(groundTruthCategoryDistribution, expected.category ?? "material_unspecified"); + const group = bestGroup(expected, materialized.analyzedGroups); + if (group) assignedGroupIds.add(group.group.groupId); + const category = group ? effectiveCategory(group) : undefined; + if (!group || !isSurfaced(group) || category === "editorial_rewrite") { + missedMaterialCount++; + failures.push(failure(scenario, "materiality_false_negative", expected.id, + group ? "Semantic analysis removed or editorialized a material source." : "No retained group represented the material source.", + group ? "future_llm_materiality_analyzer" : "future_embedding_alignment")); + continue; + } + surfacedTrueMaterial++; + if (category === expected.category) categoryCorrect++; + if (category === "uncertain") { + uncertainMaterialCount++; + failures.push(failure(scenario, "uncertain_material", expected.id, + "Material source was retained but semantic analysis remained uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const expected of scenario.expected.nonMaterialChanges) { + groundTruthNonMaterialChanges++; + const group = bestGroup(expected, materialized.analyzedGroups); + if (!group) continue; + assignedGroupIds.add(group.group.groupId); + if (!isSurfaced(group)) continue; + falseMaterialCount++; + const category = effectiveCategory(group); + failures.push(failure(scenario, "materiality_false_positive", expected.id, + `Non-material source remained prompt-facing ${category} evidence.`, "future_llm_materiality_analyzer")); + if (category === "uncertain") { + uncertainNonMaterialCount++; + failures.push(failure(scenario, "uncertain_non_material", expected.id, + "Non-material rewrite remained uncertain.", "future_llm_materiality_analyzer")); + } + } + + for (const noOp of scenario.expected.expectedNoOps) { + groundTruthNonMaterialChanges++; + const group = materialized.analyzedGroups.find(candidate => { + const ids = chunkIds(candidate); + return ids.previous.has(noOp.previousChunkId) || ids.current.has(noOp.currentChunkId); + }); + if (group && isSurfaced(group)) { + assignedGroupIds.add(group.group.groupId); + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", noOp.id, + `Expected no-op remained prompt-facing ${effectiveCategory(group)} evidence.`, "deterministic_improvement")); + } + } + + for (const group of materialized.analyzedGroups) { + if (assignedGroupIds.has(group.group.groupId) || !isSurfaced(group)) continue; + falseMaterialCount++; + failures.push(failure(scenario, "materiality_false_positive", group.group.groupId, + `An unexpected ${effectiveCategory(group)} group remained prompt-facing evidence.`, "future_llm_materiality_analyzer")); + failures.push(failure(scenario, "grouping_issue", group.group.groupId, + `An extra ${effectiveCategory(group)} group did not map to an expected semantic change.`, "deterministic_improvement")); + } + const scenarioFailures = failures.filter(item => item.scenarioId === scenario.id); + scenarioResults.push({ + id: scenario.id, + alignmentCount: alignments.length, + rawChangeCount: materialized.rawChanges.length, + groupCount: materialized.analyzedGroups.length, + categories: materialized.analyzedGroups.map(effectiveCategory), + failureKinds: [...new Set(scenarioFailures.map(item => item.kind))].sort(), + }); + } + + const aggregate = await materializeDocumentChangesWithAnalyzer(inputs, analyzer); + const snapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v2", + capturedAt: "2026-03-31T00:00:00.000Z", + reportingPeriod: { start: "2026-02-01", end: "2026-03-31" }, + workspaceTimezone: "UTC", + items: aggregate.items, + sourceWarnings: aggregate.warnings.map(warning => ({ ...warning, sourceType: "document_change" as const })), + documentChangeAudit: aggregate.audit, + }); + const envelope = buildGenerationEvidenceEnvelope(snapshot); + const observedGroups = Object.values(observedCategoryDistribution).reduce((total, count) => total + count, 0); + const uncertainRate = ratio(observedCategoryDistribution.uncertain ?? 0, observedGroups); + const falseMaterialRate = ratio(falseMaterialCount, groundTruthNonMaterialChanges); + const failuresByKind = Object.fromEntries(FAILURE_KINDS.map(kind => [kind, failures.filter(item => item.kind === kind).length])) as Record; + const materialityFailureCount = failuresByKind.materiality_false_positive + failuresByKind.materiality_false_negative + + failuresByKind.uncertain_material + failuresByKind.uncertain_non_material; + const alignmentFailureCount = failuresByKind.alignment_miss + failuresByKind.alignment_false_match; + const recommendation = recommendationFor( + uncertainRate, + falseMaterialRate, + uncertainMaterialCount, + deterministicBaseline.summary.alignment.alignmentMissRate, + materialityFailureCount, + alignmentFailureCount, + ); + const selectedDocumentCount = new Set(aggregate.selectedGroups.map(group => group.group.documentId.toString())).size; + const availableDocumentCount = new Set(aggregate.analyzedGroups.map(group => group.group.documentId.toString())).size; + const condensedEvidenceCharacters = aggregate.diagnostics.condensedPromptFacingCharacters; + return { + summary: { + fixtureVersion: MATERIALITY_EVALUATION_FIXTURE_VERSION, + scenarioCount: scenarios.length, + multiChunkScenarioCount: scenarios.filter(scenario => scenario.multiChunk).length, + largeDocumentScenarioCount: scenarios.filter(scenario => scenario.largeDocument).length, + groundTruthCategoryDistribution: Object.fromEntries(Object.entries(groundTruthCategoryDistribution).sort()), + observedCategoryDistribution: Object.fromEntries(Object.entries(observedCategoryDistribution).sort()), + materiality: { + groundTruthMaterialChanges, + groundTruthNonMaterialChanges, + categoryCorrect, + categoryAccuracy: ratio(categoryCorrect, groundTruthMaterialChanges), + materialRecall: ratio(surfacedTrueMaterial, groundTruthMaterialChanges), + materialPrecision: ratio(surfacedTrueMaterial, surfacedTrueMaterial + falseMaterialCount), + uncertainRate, + falseMaterialCount, + falseMaterialRate, + missedMaterialCount, + missedMaterialRate: ratio(missedMaterialCount, groundTruthMaterialChanges), + uncertainMaterialCount, + uncertainMaterialRate: ratio(uncertainMaterialCount, groundTruthMaterialChanges), + uncertainNonMaterialCount, + uncertainNonMaterialRate: ratio(uncertainNonMaterialCount, groundTruthNonMaterialChanges), + }, + alignment: deterministicBaseline.summary.alignment, + condensation: { + rawChangedRecords: aggregate.rawChanges.length, + groups: aggregate.analyzedGroups.length, + condensedEvidenceItems: aggregate.items.length, + rawCopiedCharacters: aggregate.diagnostics.rawExcerptCharacters, + condensedEvidenceCharacters, + serializedPromptCharacters: envelope.diagnostics.serializedCharacterCount, + reductionRatio: ratio(aggregate.diagnostics.rawExcerptCharacters, Math.max(1, condensedEvidenceCharacters)), + }, + budget: { + groupBudgetTruncated: aggregate.diagnostics.truncatedGroupCount > 0, + truncatedGroupCount: aggregate.diagnostics.truncatedGroupCount, + generationEnvelopeTruncated: envelope.diagnostics.truncated, + generationEnvelopeSelectedItems: envelope.diagnostics.selectedItemCount, + generationEnvelopeExcludedItems: envelope.diagnostics.excludedItemCount, + documentDiversityPreserved: selectedDocumentCount === Math.min(aggregate.selectedGroups.length, availableDocumentCount), + selectedDocumentCount, + }, + failuresByKind, + ...recommendation, + }, + failures: failures.sort((a, b) => a.kind.localeCompare(b.kind) || a.scenarioId.localeCompare(b.scenarioId) || a.expectationId.localeCompare(b.expectationId)), + scenarioResults: scenarioResults.sort((a, b) => a.id.localeCompare(b.id)), + }; +} + +export function evaluationArtifactDirectory(runId: string = MATERIALITY_EVALUATION_RUN_ID): string { + if (!/^[a-z0-9][a-z0-9._-]*$/i.test(runId)) throw new Error("Evaluation run ID must be filesystem-safe."); + return resolve(process.cwd(), MATERIALITY_EVALUATION_ARTIFACT_ROOT, runId); +} + +function percentage(value: number): string { + return `${(value * 100).toFixed(1)}%`; +} + +export function renderMaterialityEvaluationArtifacts( + result: MaterialityEvaluationResult, + options: { providerInvoked?: boolean } = {}, +): { + summaryJson: string; + failuresJson: string; + evaluationMarkdown: string; +} { + const { summary } = result; + const topFailures = Object.entries(summary.failuresByKind).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1]); + const evaluationMarkdown = [ + "# Founder Weekly Review Materiality Evaluation", + "", + `Fixture: \`${summary.fixtureVersion}\``, + `Scenarios: ${summary.scenarioCount} (${summary.multiChunkScenarioCount} multi-chunk; ${summary.largeDocumentScenarioCount} large-document)`, + "", + "## What worked", + "", + `- Material recall: ${percentage(summary.materiality.materialRecall)}; exact category accuracy: ${percentage(summary.materiality.categoryAccuracy)}.`, + `- Condensation: ${summary.condensation.rawChangedRecords} raw records -> ${summary.condensation.groups} groups -> ${summary.condensation.condensedEvidenceItems} structurally selected items.`, + `- Document diversity preserved: ${summary.budget.documentDiversityPreserved ? "yes" : "no"}.`, + "", + "## Where deterministic materiality fails", + "", + `- Uncertain rate: ${percentage(summary.materiality.uncertainRate)}.`, + `- False-material rate: ${percentage(summary.materiality.falseMaterialRate)}.`, + `- Missed-material rate: ${percentage(summary.materiality.missedMaterialRate)}.`, + `- Uncertain material/non-material: ${summary.materiality.uncertainMaterialCount}/${summary.materiality.uncertainNonMaterialCount}.`, + "", + "## Where alignment fails", + "", + `- Alignment miss rate: ${percentage(summary.alignment.alignmentMissRate)}.`, + `- False-match rate: ${percentage(summary.alignment.falseMatchRate)}.`, + `- Intended correspondences represented as added + removed: ${summary.alignment.intendedModifiedAsAddedRemoved}.`, + "", + "## Budget behavior", + "", + `- Structural group budget truncated: ${summary.budget.groupBudgetTruncated ? "yes" : "no"} (${summary.budget.truncatedGroupCount} groups).`, + `- Generation envelope selected/excluded: ${summary.budget.generationEnvelopeSelectedItems}/${summary.budget.generationEnvelopeExcludedItems}.`, + `- Serialized prompt evidence: ${summary.condensation.serializedPromptCharacters} characters; envelope truncation: ${summary.budget.generationEnvelopeTruncated ? "yes" : "no"}.`, + "", + "## Failure concentration", + "", + ...(topFailures.length ? topFailures.map(([kind, count]) => `- ${kind}: ${count}`) : ["- No recorded failures."]), + "", + "## Recommendation", + "", + `**${summary.recommendation}. ${summary.recommendationLabel}.**`, + "", + "Materiality exceeds the uncertainty/false-material guidance threshold, while alignment misses remain below the 5% lower guidance threshold and cluster in split, merge, and one renamed heavy rewrite. The next investment should therefore address semantic interpretation first.", + "", + "## Metric definitions", + "", + "- False material counts any retained prompt-facing group mapped to a non-material expectation, plus unexpected groups, divided by non-material ground-truth opportunities.", + "- Uncertain rate is uncertain groups divided by all observed groups; uncertain material and uncertain non-material remain separate buckets.", + "- Alignment miss rate is unrecovered intended modified/unchanged/split/merge relations divided by all such intended relations.", + "- Reduction ratio is bounded raw audit excerpt characters divided by structurally selected condensed evidence characters.", + "", + options.providerInvoked + ? "Generated artifacts contain synthetic fixture text only; the explicitly enabled materiality analyzer provider was invoked." + : "Generated artifacts contain synthetic fixture text only; no provider was invoked.", + "", + ].join("\n"); + return { + summaryJson: `${JSON.stringify({ summary, scenarios: result.scenarioResults }, null, 2)}\n`, + failuresJson: `${JSON.stringify({ fixtureVersion: summary.fixtureVersion, failures: result.failures }, null, 2)}\n`, + evaluationMarkdown, + }; +} + +export async function writeMaterialityEvaluationArtifacts( + result: MaterialityEvaluationResult, + runId: string = MATERIALITY_EVALUATION_RUN_ID, + options: { providerInvoked?: boolean } = {}, +): Promise<{ directory: string; summary: string; failures: string; evaluation: string }> { + const directory = evaluationArtifactDirectory(runId); + const artifacts = renderMaterialityEvaluationArtifacts(result, options); + await mkdir(directory, { recursive: true }); + const paths = { + directory, + summary: resolve(directory, "summary.json"), + failures: resolve(directory, "failures.json"), + evaluation: resolve(directory, "evaluation.md"), + }; + await Promise.all([ + writeFile(paths.summary, artifacts.summaryJson, "utf8"), + writeFile(paths.failures, artifacts.failuresJson, "utf8"), + writeFile(paths.evaluation, artifacts.evaluationMarkdown, "utf8"), + ]); + return paths; +} + +async function main(): Promise { + const mode = process.env.FWR_MATERIALITY_EVAL_MODE ?? "offline"; + let result: MaterialityEvaluationResult; + let defaultRunId: string; + let liveCallRecords: Array<{ + durationMs: number; + status: "success" | "failure"; + provider?: string; + model?: string; + promptVersion?: string; + errorCode?: string; + }> | undefined; + if (mode === "deterministic") { + result = runMaterialityEvaluation(); + defaultRunId = MATERIALITY_EVALUATION_RUN_ID; + } else if (mode === "offline") { + result = await runMaterialityAnalyzerEvaluation(); + defaultRunId = MATERIALITY_ANALYZER_EVALUATION_RUN_ID; + } else if (mode === "live") { + const { config: loadDotenv } = await import("dotenv"); + loadDotenv({ path: resolve(process.cwd(), "../../.env"), quiet: true }); + const { createConfiguredDocumentChangeMaterialityAnalyzer } = await import("../src/server/founder-weekly-review/document-change-materiality-analyzer"); + const maximumCalls = Math.max(1, Math.min(64, Number(process.env.FWR_MATERIALITY_EVAL_MAX_CALLS ?? 64) || 64)); + const live = createConfiguredDocumentChangeMaterialityAnalyzer(); + if (!live) throw new Error("Live materiality evaluation requires FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED=true."); + let calls = 0; + liveCallRecords = []; + const bounded: DocumentChangeMaterialityAnalyzer = { + async analyze(input) { + if (calls >= maximumCalls) throw new Error("Live materiality evaluation reached its explicit global call limit."); + calls++; + const startedAt = Date.now(); + try { + const response = await live.analyze(input); + liveCallRecords!.push({ + durationMs: Date.now() - startedAt, + status: "success", + ...(response.metadata?.provider ? { provider: response.metadata.provider } : {}), + ...(response.metadata?.model ? { model: response.metadata.model } : {}), + ...(response.metadata?.promptVersion ? { promptVersion: response.metadata.promptVersion } : {}), + }); + return response; + } catch (error) { + const errorCode = error && typeof error === "object" && "code" in error && typeof error.code === "string" + ? error.code : "unknown"; + liveCallRecords!.push({ durationMs: Date.now() - startedAt, status: "failure", errorCode }); + throw error; + } + }, + }; + result = await runMaterialityAnalyzerEvaluation(bounded); + defaultRunId = MATERIALITY_ANALYZER_KIMI_LIVE_V2_RUN_ID; + } else { + throw new Error("FWR_MATERIALITY_EVAL_MODE must be deterministic, offline, or live."); + } + const artifacts = await writeMaterialityEvaluationArtifacts( + result, + process.env.FWR_MATERIALITY_EVALUATION_RUN_ID ?? defaultRunId, + { providerInvoked: mode === "live" }, + ); + if (liveCallRecords) { + const durations = liveCallRecords.map(record => record.durationMs).sort((a, b) => a - b); + const percentile = (fraction: number) => durations.length === 0 ? 0 : durations[Math.min(durations.length - 1, Math.ceil(durations.length * fraction) - 1)]!; + const firstSuccess = liveCallRecords.find(record => record.status === "success"); + await writeFile(resolve(artifacts.directory, "call-summary.json"), `${JSON.stringify({ + strategy: defaultRunId, + provider: firstSuccess?.provider ?? null, + model: firstSuccess?.model ?? null, + promptVersion: firstSuccess?.promptVersion ?? null, + totalCalls: liveCallRecords.length, + successfulCalls: liveCallRecords.filter(record => record.status === "success").length, + failedCalls: liveCallRecords.filter(record => record.status === "failure").length, + failuresByCode: Object.fromEntries([...new Set(liveCallRecords.flatMap(record => record.errorCode ? [record.errorCode] : []))] + .sort().map(code => [code, liveCallRecords!.filter(record => record.errorCode === code).length])), + latencyMs: { + minimum: durations[0] ?? 0, + median: percentile(0.5), + p95: percentile(0.95), + maximum: durations.at(-1) ?? 0, + under5Seconds: durations.filter(value => value < 5_000).length, + from5To10Seconds: durations.filter(value => value >= 5_000 && value < 10_000).length, + from10To15Seconds: durations.filter(value => value >= 10_000 && value < 15_000).length, + atLeast15Seconds: durations.filter(value => value >= 15_000).length, + total: durations.reduce((total, value) => total + value, 0), + }, + providerTokenUsageAvailable: false, + }, null, 2)}\n`, "utf8"); + } + console.log(JSON.stringify({ ...result.summary, artifactDirectory: artifacts.directory })); +} + +if (process.argv[1]?.replace(/\\/g, "/").endsWith("/founder-weekly-review-materiality-evaluation.ts")) { + main().catch(error => { + console.error(error instanceof Error ? error.message : "Materiality evaluation failed."); + process.exitCode = 1; + }); +} diff --git a/apps/web/scripts/founder-weekly-review-scenario-loader.ts b/apps/web/scripts/founder-weekly-review-scenario-loader.ts new file mode 100644 index 000000000..fa04ee07d --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-scenario-loader.ts @@ -0,0 +1,11 @@ +import { readFile } from "node:fs/promises"; + +import { FounderWeeklyReviewScenarioSchema, type FounderWeeklyReviewScenario } from "../test-fixtures/founder-weekly-review/scenarios/contracts"; + +export function parseScenario(input: unknown): FounderWeeklyReviewScenario { + return FounderWeeklyReviewScenarioSchema.parse(input); +} + +export async function loadScenario(path: string): Promise { + return parseScenario(JSON.parse(await readFile(path, "utf8"))); +} diff --git a/apps/web/scripts/founder-weekly-review-scenario-seeder.ts b/apps/web/scripts/founder-weekly-review-scenario-seeder.ts new file mode 100644 index 000000000..cb35b2ed0 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-scenario-seeder.ts @@ -0,0 +1,48 @@ +import { createHash } from "node:crypto"; + +import type { DbClient } from "@launchstack/core/db"; +import { company, document, documentContextChunks, documentStructure, documentVersions } from "@launchstack/core/db/schema"; +import { eq, sql } from "drizzle-orm"; + +import type { FounderWeeklyReviewScenario } from "../test-fixtures/founder-weekly-review/scenarios/contracts"; + +export interface SeededScenario { + companyNamesToId: Map; + underReviewCompanyId: bigint; +} + +export type ScenarioSeederOptions = { deterministicWorkspaceEmbeddings?: boolean }; + +const workspaceVector = () => Array.from({ length: 1536 }, (_, index) => index === 0 ? 1 : 0); +const contentHash = (content: string) => createHash("sha256").update(content, "utf8").digest("hex"); +const structurePath = (section: string, index: number) => `/${section.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "section"}-${index + 1}`; + +/** Seeds business state only; the current collector computes evidence from these rows. */ +export async function seedScenario(db: DbClient, scenario: FounderWeeklyReviewScenario, options: ScenarioSeederOptions = {}): Promise { + const companyNamesToId = new Map(); + let underReviewCompanyId: bigint | undefined; + for (const companyEntry of scenario.companies) { + const [companyRow] = await db.insert(company).values({ name: companyEntry.name, numberOfEmployees: "10" }).returning(); + const companyId = BigInt(companyRow!.id); + companyNamesToId.set(companyEntry.name, companyId); + if (companyEntry.underReview) underReviewCompanyId = companyId; + for (const doc of companyEntry.documents) { + const [documentRow] = await db.insert(document).values({ companyId, url: `local://${doc.title}`, category: doc.category, title: doc.title }).returning(); + const documentId = BigInt(documentRow!.id); + let latestVersion: { id: bigint; versionNumber: number } | undefined; + for (const version of doc.versions) { + const [versionRow] = await db.insert(documentVersions).values({ documentId, versionNumber: version.versionNumber, url: `local://${doc.title}/v${version.versionNumber}`, mimeType: "text/plain", uploadedBy: version.uploadedBy ?? "scenario-seed", changelog: version.changelog ?? null, createdAt: new Date(version.timestamp) }).returning(); + const versionId = BigInt(versionRow!.id); + if (!latestVersion || version.versionNumber > latestVersion.versionNumber) latestVersion = { id: versionId, versionNumber: version.versionNumber }; + for (const [index, chunk] of version.chunks.entries()) { + const [structure] = await db.insert(documentStructure).values({ documentId, versionId, ordering: index + 1, title: chunk.section, path: structurePath(chunk.section, index), startPage: chunk.pageNumber ?? index + 1, endPage: chunk.pageNumber ?? index + 1 }).returning(); + const values = { documentId, versionId, structureId: BigInt(structure!.id), content: chunk.content, contentHash: contentHash(chunk.content), tokenCount: chunk.content.split(/\s+/).length, charCount: chunk.content.length, pageNumber: chunk.pageNumber ?? index + 1, lineStart: chunk.lineStart ?? 1, lineEnd: chunk.lineEnd ?? (chunk.lineStart ?? 1) }; + await db.insert(documentContextChunks).values(options.deterministicWorkspaceEmbeddings ? { ...values, embedding: sql`${JSON.stringify(workspaceVector())}::vector(1536)` } : values); + } + } + if (latestVersion) await db.update(document).set({ currentVersionId: latestVersion.id }).where(eq(document.id, documentRow!.id)); + } + } + if (!underReviewCompanyId) throw new Error("scenario has no under-review company (expected exactly one)."); + return { companyNamesToId, underReviewCompanyId }; +} diff --git a/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts b/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts new file mode 100644 index 000000000..50b396dc4 --- /dev/null +++ b/apps/web/scripts/founder-weekly-review-synthetic-fixtures.ts @@ -0,0 +1,24 @@ +import { FounderWeeklyReviewEvidenceSnapshotSchema } from "@launchstack/features/founder-weekly-review"; + +const period = { start: "2026-02-16", end: "2026-02-28" }; +const make = (items: unknown[]) => FounderWeeklyReviewEvidenceSnapshotSchema.parse({ + schemaVersion: "founder-weekly-review-evidence/v1", + capturedAt: "2026-03-01T00:00:00.000Z", + reportingPeriod: period, + workspaceTimezone: "UTC", + items, + sourceWarnings: [], +}); + +export const syntheticFounderWeeklyReviewFixtures = { + partial: make([ + { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1", evidenceStatus: "shipped" } }, + { sourceType: "founder_context", sourceId: "synthetic:context:priority", title: "Founder context", excerpt: "Prioritize onboarding reliability.", metadata: { fixture: "synthetic-v1" } }, + ]), + full: make([ + { sourceType: "document_change", sourceId: "synthetic:doc:release", title: "Release notes", sourceTimestamp: "2026-02-20T12:00:00.000Z", excerpt: "Export filtering was released.", metadata: { fixture: "synthetic-v1", evidenceStatus: "shipped" } }, + { sourceType: "customer_feedback", sourceId: "synthetic:feedback:export", title: "Customer feedback", sourceTimestamp: "2026-02-21T12:00:00.000Z", excerpt: "A customer requested saved export filters.", metadata: { fixture: "synthetic-v1" } }, + { sourceType: "founder_context", sourceId: "synthetic:context:priority", title: "Founder context", excerpt: "Prioritize onboarding reliability.", metadata: { fixture: "synthetic-v1" } }, + ]), + empty: make([]), +} as const; diff --git a/apps/web/scripts/migrate.mjs b/apps/web/scripts/migrate.mjs index 6d1a6b883..a86c5f3f0 100644 --- a/apps/web/scripts/migrate.mjs +++ b/apps/web/scripts/migrate.mjs @@ -35,6 +35,10 @@ const migrationsDir = join(__dirname, "..", "drizzle"); const sql = postgres(url, { max: 1 }); +function normalizeSqlMigrationBody(body) { + return body.replace(/^\uFEFF/, ""); +} + async function ensureMigrationsTable() { await sql` CREATE TABLE IF NOT EXISTS _launchstack_migrations ( @@ -59,7 +63,9 @@ async function listMigrationFiles() { } async function applyMigration(name) { - const body = await readFile(join(migrationsDir, name), "utf8"); + const body = normalizeSqlMigrationBody( + await readFile(join(migrationsDir, name), "utf8"), + ); const checksum = createHash("sha256").update(body).digest("hex"); console.log(`[migrate] applying ${name}`); @@ -83,7 +89,9 @@ async function main() { let drift = 0; for (const name of files) { - const body = await readFile(join(migrationsDir, name), "utf8"); + const body = normalizeSqlMigrationBody( + await readFile(join(migrationsDir, name), "utf8"), + ); const checksum = createHash("sha256").update(body).digest("hex"); const recorded = applied.get(name); diff --git a/apps/web/scripts/run-founder-weekly-review-benchmark.ts b/apps/web/scripts/run-founder-weekly-review-benchmark.ts new file mode 100644 index 000000000..cacf7fcdd --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-benchmark.ts @@ -0,0 +1,16 @@ +import dotenv from "dotenv"; +import path from "path"; + +dotenv.config({ + path: path.resolve(import.meta.dirname, "../../../.env"), +}); + +import { generateFounderWeeklyReviewStructured } from "../src/server/founder-weekly-review/generation-adapter"; +import { main } from "@launchstack/features/founder-weekly-review/benchmarks/run"; + +console.log( + "OPENAI:", + process.env.OPENAI_API_KEY ? "loaded" : "missing" +); + +await main(generateFounderWeeklyReviewStructured); \ No newline at end of file diff --git a/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts b/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts new file mode 100644 index 000000000..2e5b25ddb --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-inngest-transport.ts @@ -0,0 +1,639 @@ +import "dotenv/config"; +import { createHash, randomUUID } from "node:crypto"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { createWriteStream, type WriteStream } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import net from "node:net"; +import { relative, resolve } from "node:path"; + +import { eq } from "drizzle-orm"; +import { + company, + document, + documentContextChunks, + documentVersions, + founderWeeklyReviewDispatches, +} from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewRepository } from "@launchstack/features/founder-weekly-review"; + +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { renderFounderWeeklyReviewEvaluationMarkdown } from "~/server/founder-weekly-review/evaluation-markdown"; +import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; + +const require = createRequire(import.meta.url); +const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); +const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/realistic-company/seed.json"); +const STARTUP_TIMEOUT_MS = 45_000; +const SOCKET_RELEASE_WAIT_MS = 1_500; +const STARTUP_SMOKE_ONLY = process.env.FWR_TRANSPORT_STARTUP_SMOKE_ONLY === "1"; +const VERBOSE_CHILD_LOGS = process.env.FWR_TRANSPORT_VERBOSE_LOGS === "1"; +const RECENT_DIAGNOSTIC_LIMIT = 160; +const SHUTDOWN_DRAIN_CAP_MS = 5_000; +const CALLBACK_QUIET_MS = 1_000; + +type Fixture = { + reportingPeriod: { start: string; end: string }; + workspaceTimezone: string; + founderContext: string; + documents: Array<{ + title: string; + category: string; + changelog: string; + timestamp: string; + chunks?: string[]; + }>; +}; + +interface ManagedChild { + name: "next" | "inngest" | string; + process: ChildProcess; +} + +interface ShutdownDrainState { + terminalStatus: "draft" | "failed" | null; + finalDiagnosticsCaptured: boolean; + cleanupStarted: boolean; + callbackDrained: boolean; + lastCallbackActivityAt: number; + functionFinishedAfterTerminal: boolean; + knownMissingBodyDuringShutdown: boolean; +} + +function digest(value: unknown) { + return createHash("sha256").update(JSON.stringify(value), "utf8").digest("hex"); +} + +function sleep(ms: number) { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + +function lifecycleTimeoutMs(): number { + const raw = process.env.FWR_TRANSPORT_TIMEOUT_MS; + if (!raw) return 720_000; + if (!/^\d+$/.test(raw)) { + throw new Error("FWR_TRANSPORT_TIMEOUT_MS must be a positive safe integer."); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error("FWR_TRANSPORT_TIMEOUT_MS must be a positive safe integer."); + } + return value; +} + +async function allocateLoopbackPort(): Promise { + const server = net.createServer(); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen({ host: "127.0.0.1", port: 0, exclusive: true }, () => { + server.off("error", rejectListen); + resolveListen(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("Unable to allocate a loopback TCP port."); + } + await new Promise((resolveClose, rejectClose) => server.close((error) => error ? rejectClose(error) : resolveClose())); + return address.port; +} + +async function canConnect(port: number, host: string): Promise { + return new Promise((resolveConnect) => { + const socket = net.createConnection({ host, port }); + const settle = (listening: boolean) => { + socket.removeAllListeners(); + socket.destroy(); + resolveConnect(listening); + }; + socket.once("connect", () => settle(true)); + socket.once("error", () => settle(false)); + socket.setTimeout(500, () => settle(false)); + }); +} + +async function isPortListening(port: number): Promise { + // `next dev -H localhost` may choose IPv6 loopback on Windows. Check both + // loopback families so stale listeners are never mistaken for a free port. + return (await canConnect(port, "127.0.0.1")) || await canConnect(port, "::1"); +} + +async function assertPortClosed(port: number, label: string): Promise { + if (await isPortListening(port)) { + throw new Error(`${label} port ${port} is already listening; refusing to attach to an existing process.`); + } +} + +function childIsAlive(child: ManagedChild | undefined): boolean { + return Boolean(child && child.process.exitCode === null && child.process.signalCode === null); +} + +function boundedLogs(logs: string[]): string { + return logs.slice(-100).join("").slice(-8_000); +} + +function appendRecentDiagnostic(logs: string[], line: string): void { + logs.push(line); + if (logs.length > RECENT_DIAGNOSTIC_LIMIT) logs.splice(0, logs.length - RECENT_DIAGNOSTIC_LIMIT); +} + +function isInngestDevUiNoise(line: string): boolean { + try { + const parsed = JSON.parse(line) as { level?: unknown; event?: unknown; event_name?: unknown }; + return parsed.level === "INFO" && (parsed.event === "cli/dev_ui.loaded" || parsed.event_name === "cli/dev_ui.loaded"); + } catch { + return false; + } +} + +function isUsefulDiagnostic(name: string, line: string): boolean { + return /warn|error|fail|exception|stack|retry|founder-weekly-review|claim-evidence|collect-evidence|persist-evidence|\bclaim\b|\bgenerate\b|\bpersist\b|database|relation|column|non-2\d\d|PUT \/api\/inngest 200/i.test(line); +} + +function isTerminalNoise(name: string, line: string): boolean { + if (name === "inngest") return isInngestDevUiNoise(line); + // Keep registration failures and non-2xx callbacks visible, but omit the + // steady-state successful callback poll noise from the terminal. + return /^(GET|PUT) \/api\/inngest.*\s2\d\d\s/i.test(line) + || /^responseBody:/i.test(line); +} + +function isKnownShutdownMissingBody( + name: "next" | "inngest", + line: string, + state: ShutdownDrainState +): boolean { + return name === "next" + && line.includes("[Inngest] error - Missing body when executing, possibly due to missing request body middleware") + && state.terminalStatus !== null + && state.finalDiagnosticsCaptured + && state.cleanupStarted + && state.callbackDrained; +} + +function printRecentDiagnostics(name: "next" | "inngest", logs: string[]): void { + console.log(`===== RECENT ${name.toUpperCase()} DIAGNOSTICS =====`); + for (const line of logs) console.log(line); + console.log(`===== END ${name.toUpperCase()} DIAGNOSTICS =====`); +} + +async function closeLogStream(stream: WriteStream): Promise { + await new Promise((resolveClose) => stream.end(resolveClose)); +} + +function startManagedChild( + managedChildren: ManagedChild[], + command: string, + args: string[], + cwd: string, + env: NodeJS.ProcessEnv, + logs: string[], + rawLog: WriteStream, + name: "next" | "inngest", + shutdownDrain: ShutdownDrainState +): ManagedChild { + // pnpm.cmd requires cmd.exe on Windows. taskkill /T in cleanup targets this shell + // PID and its pnpm/Node descendants as one scoped process tree. + const child = spawn(command, args, { + cwd, + env, + shell: process.platform === "win32", + windowsHide: true, + }); + const managed: ManagedChild = { name, process: child }; + managedChildren.push(managed); + const emit = (stream: "stdout" | "stderr", data: Buffer) => { + const output = data.toString(); + rawLog.write(output); + for (const line of output.split(/\r?\n/)) { + if (!line.trim()) continue; + if (name === "next" && /\/api\/inngest/i.test(line)) shutdownDrain.lastCallbackActivityAt = Date.now(); + if (name === "inngest" && shutdownDrain.terminalStatus && line.includes("inngest/function.finished")) shutdownDrain.functionFinishedAfterTerminal = true; + const formatted = `[${name}:${stream}] ${line}`; + const expectedShutdownNoise = isKnownShutdownMissingBody(name, line, shutdownDrain); + const suppressedNoise = isTerminalNoise(name, line) || expectedShutdownNoise; + if (isUsefulDiagnostic(name, line) && !suppressedNoise) appendRecentDiagnostic(logs, formatted); + if (expectedShutdownNoise) { + shutdownDrain.knownMissingBodyDuringShutdown = true; + console.log("[next] known shutdown-only Inngest callback race observed; retained in raw log"); + } + if (VERBOSE_CHILD_LOGS || !suppressedNoise) console.log(formatted); + } + }; + child.stdout?.on("data", (data: Buffer) => emit("stdout", data)); + child.stderr?.on("data", (data: Buffer) => emit("stderr", data)); + child.once("exit", (code, signal) => { + const line = `[${name}] exited code=${code ?? "null"} signal=${signal ?? "none"}`; + appendRecentDiagnostic(logs, line); + console.log(line); + }); + return managed; +} + +async function drainTerminalCallbackActivity(state: ShutdownDrainState): Promise { + const deadline = Date.now() + SHUTDOWN_DRAIN_CAP_MS; + console.log(`[cleanup] draining terminal callback activity capMs=${SHUTDOWN_DRAIN_CAP_MS}`); + while (Date.now() < deadline) { + const callbackQuiet = Date.now() - state.lastCallbackActivityAt >= CALLBACK_QUIET_MS; + if (state.functionFinishedAfterTerminal && callbackQuiet) { + state.callbackDrained = true; + console.log("[cleanup] terminal callback drain settled"); + return; + } + await sleep(100); + } + console.warn(`[cleanup] terminal callback drain cap reached functionFinished=${state.functionFinishedAfterTerminal}`); +} + +function terminateProcessTree(child: ManagedChild): void { + const pid = child.process.pid; + if (!pid || !childIsAlive(child)) return; + if (process.platform === "win32") { + const result = spawnSync("taskkill", ["/PID", String(pid), "/T", "/F"], { + encoding: "utf8", + windowsHide: true, + }); + if (result.error || (result.status !== 0 && childIsAlive(child))) { + console.warn(`[cleanup] ${child.name} pid=${pid} taskkill did not confirm termination (already exited is non-fatal)`); + } + return; + } + child.process.kill("SIGTERM"); +} + +async function waitForChildExit(child: ManagedChild): Promise { + if (!childIsAlive(child)) return; + await Promise.race([ + new Promise((resolveExit) => child.process.once("exit", () => resolveExit())), + sleep(5_000), + ]); +} + +function owningPid(port: number): string | undefined { + if (process.platform !== "win32") return undefined; + const result = spawnSync("netstat", ["-ano", "-p", "tcp"], { encoding: "utf8", windowsHide: true }); + const match = result.stdout?.split(/\r?\n/).find((line) => new RegExp(`127\\.0\\.0\\.1:${port}\\s+.*LISTENING`, "i").test(line)); + return match?.trim().split(/\s+/).at(-1); +} + +async function waitForCallback( + callbackUrl: string, + managedChildren: ManagedChild[], + logs: string[], + shouldAbort: () => boolean +): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (shouldAbort()) throw new Error("Startup interrupted before callback readiness."); + const next = managedChildren.find((child) => child.name === "next"); + if (!childIsAlive(next)) throw new Error(`Next exited before callback readiness: ${boundedLogs(logs)}`); + try { + const response = await fetch(callbackUrl); + const registration = await response.text(); + const functionsInGetResponse = registration.includes("founder-weekly-review-dispatcher") && registration.includes("founder-weekly-review-generation"); + // The dev CLI performs the authoritative registration with PUT. A + // bare GET is still required to respond successfully, but its body + // is protocol-dependent and does not always echo function IDs. + const functionsRegisteredByCli = logs.some((line) => line.includes("PUT /api/inngest 200")); + if (response.ok && (functionsInGetResponse || functionsRegisteredByCli)) { + console.log(`[harness] callback ready url=${callbackUrl}; Founder Weekly Review functions registered=${functionsInGetResponse ? "GET" : "CLI PUT"}`); + return; + } + } catch { + // Next is still booting; child liveness is checked every iteration. + } + await sleep(500); + } + throw new Error(`Timed out waiting for registered callback ${callbackUrl}: ${boundedLogs(logs)}`); +} + +async function waitForHttp(url: string, child: ManagedChild, logs: string[], shouldAbort: () => boolean): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (shouldAbort()) throw new Error("Startup interrupted before Inngest readiness."); + if (!childIsAlive(child)) throw new Error(`Inngest exited before readiness: ${boundedLogs(logs)}`); + try { + if ((await fetch(url)).ok) return; + } catch { + // Inngest is still booting. + } + await sleep(500); + } + throw new Error(`Timed out waiting for ${url}: ${boundedLogs(logs)}`); +} + +async function waitForPort(port: number, child: ManagedChild, logs: string[], shouldAbort: () => boolean): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + while (Date.now() < deadline) { + if (shouldAbort()) throw new Error("Startup interrupted before Next was listening."); + if (!childIsAlive(child)) throw new Error(`Next exited before listening: ${boundedLogs(logs)}`); + if (await isPortListening(port)) return; + await sleep(250); + } + throw new Error(`Timed out waiting for Next port ${port}: ${boundedLogs(logs)}`); +} + +function createCleanupController( + managedChildren: ManagedChild[], + nextPort: number, + inngestPort: number, + shutdownDrain: ShutdownDrainState +): () => Promise { + let cleanupPromise: Promise | undefined; + return () => cleanupPromise ??= (async () => { + shutdownDrain.cleanupStarted = true; + console.log("[cleanup] stopping managed child process trees"); + for (const child of [...managedChildren].reverse()) { + console.log(`[cleanup] stopping ${child.name} pid=${child.process.pid ?? "unknown"}`); + terminateProcessTree(child); + await waitForChildExit(child); + } + await sleep(SOCKET_RELEASE_WAIT_MS); + let portsClosed = true; + for (const [name, port] of [["next", nextPort], ["inngest", inngestPort]] as const) { + if (await isPortListening(port)) { + portsClosed = false; + const child = managedChildren.find((candidate) => candidate.name === name); + console.warn(`[cleanup] ${name}Port=${port} still listening pid=${owningPid(port) ?? "unknown"} childAlive=${childIsAlive(child)}`); + } else { + console.log(`[cleanup] ${name}Port=closed`); + } + } + if (shutdownDrain.knownMissingBodyDuringShutdown && portsClosed) { + console.log("[cleanup] known Missing body message classified as shutdown-only after terminal callback drain and port closure"); + } + })(); +} + +async function main(): Promise { + if (process.env.SYNTHETIC_FWR_LOCAL !== "1" || process.env.NODE_ENV === "production") { + throw new Error("Refusing transport E2E outside explicit local mode."); + } + const localUrl = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? ""; + if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(localUrl)) { + throw new Error("Refusing non-local database."); + } + if (!STARTUP_SMOKE_ONLY && !process.env.OPENAI_API_KEY?.trim()) { + throw new Error("OPENAI_API_KEY is required for the real callback generation."); + } + + const testDb = await createFounderWeeklyReviewTestDatabase(); + const managedChildren: ManagedChild[] = []; + const nextLogs: string[] = []; + const inngestLogs: string[] = []; + const artifactDirectory = resolve(process.cwd(), ".artifacts/founder-weekly-review"); + await mkdir(artifactDirectory, { recursive: true }); + const nextLogPath = resolve(artifactDirectory, "transport-next.log"); + const inngestLogPath = resolve(artifactDirectory, "transport-inngest.log"); + const nextRawLog = createWriteStream(nextLogPath, { flags: "w" }); + const inngestRawLog = createWriteStream(inngestLogPath, { flags: "w" }); + const nextPort = await allocateLoopbackPort(); + let inngestPort = await allocateLoopbackPort(); + while (inngestPort === nextPort) inngestPort = await allocateLoopbackPort(); + const shutdownDrain: ShutdownDrainState = { + terminalStatus: null, + finalDiagnosticsCaptured: false, + cleanupStarted: false, + callbackDrained: false, + lastCallbackActivityAt: Date.now(), + functionFinishedAfterTerminal: false, + knownMissingBodyDuringShutdown: false, + }; + const cleanup = createCleanupController(managedChildren, nextPort, inngestPort, shutdownDrain); + let shutdownRequested = false; + const requestShutdown = (reason: string) => { + shutdownRequested = true; + console.warn(`[cleanup] ${reason} received`); + void cleanup().catch((error: unknown) => console.warn(`[cleanup] ${String(error)}`)); + }; + const onSigint = () => requestShutdown("SIGINT"); + const onSigterm = () => requestShutdown("SIGTERM"); + const onUncaughtException = (error: Error) => { + console.error(`[cleanup] uncaughtException: ${error.message}`); + process.exitCode = 1; + requestShutdown("uncaughtException"); + }; + const onUnhandledRejection = (reason: unknown) => { + console.error(`[cleanup] unhandledRejection: ${String(reason)}`); + process.exitCode = 1; + requestShutdown("unhandledRejection"); + }; + process.once("SIGINT", onSigint); + process.once("SIGTERM", onSigterm); + process.once("uncaughtException", onUncaughtException); + process.once("unhandledRejection", onUnhandledRejection); + + try { + const schemaUrl = new URL(localUrl); + schemaUrl.searchParams.set("options", `-c search_path=${testDb.schemaName},public`); + // The current Next/Inngest adapter normalizes its callback host to + // localhost. Explicitly bind there (loopback-only) so the adapter and + // CLI address the same listener; allocation remains 127.0.0.1-only. + const callbackUrl = `http://localhost:${nextPort}/api/inngest`; + const inngestUrl = `http://127.0.0.1:${inngestPort}`; + const runtimeEnv: NodeJS.ProcessEnv = { + ...process.env, + DATABASE_URL: schemaUrl.toString(), + LAUNCHSTACK_TEST_DATABASE_URL: schemaUrl.toString(), + // The current Inngest SDK treats a URL-valued INNGEST_DEV as the + // explicit local dev-server target. INNGEST_BASE_URL configures the + // API/event base and is not a replacement for this transport URL. + INNGEST_DEV: inngestUrl, + INNGEST_EVENT_KEY: "local-transport", + }; + console.log(`[harness] database host=${schemaUrl.hostname} port=${schemaUrl.port || "5432"} schema=${testDb.schemaName}`); + console.log(`[harness] selected nextPort=${nextPort} inngestPort=${inngestPort}`); + console.log(`[harness] verbose child logs: ${VERBOSE_CHILD_LOGS ? "enabled" : "disabled"}`); + console.log(`[harness] Next log: ${relative(process.cwd(), nextLogPath)}`); + console.log(`[harness] Inngest log: ${relative(process.cwd(), inngestLogPath)}`); + await assertPortClosed(nextPort, "Next"); + await assertPortClosed(inngestPort, "Inngest"); + const next = startManagedChild(managedChildren, "pnpm.cmd", ["exec", "next", "dev", "--turbo", "-H", "localhost", "-p", String(nextPort)], process.cwd(), runtimeEnv, nextLogs, nextRawLog, "next", shutdownDrain); + console.log(`[next] expected schema=${testDb.schemaName}`); + // Inngest's serve handler proxies its registration response through the + // dev server. Before that server exists, a GET can reset even though the + // current child owns and is listening on the allocated callback port. + await waitForPort(nextPort, next, nextLogs, () => shutdownRequested); + await assertPortClosed(inngestPort, "Inngest"); + const inngest = startManagedChild(managedChildren, "pnpm.cmd", ["dlx", "inngest-cli@latest", "dev", "--no-discovery", "-u", callbackUrl, "--port", String(inngestPort)], process.cwd(), runtimeEnv, inngestLogs, inngestRawLog, "inngest", shutdownDrain); + await waitForHttp(inngestUrl, inngest, inngestLogs, () => shutdownRequested); + await waitForCallback(callbackUrl, managedChildren, nextLogs, () => shutdownRequested); + console.log(`[harness] Inngest ready url=${inngestUrl} callback=${callbackUrl}`); + if (STARTUP_SMOKE_ONLY) { + console.log("[harness] startup/cleanup smoke completed; no data seeded, event dispatched, or provider called"); + return; + } + + const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as Fixture; + const [target] = await testDb.db.insert(company).values({ name: "Northstar Analytics", numberOfEmployees: "24" }).returning(); + const [other] = await testDb.db.insert(company).values({ name: "Other Company", numberOfEmployees: "4" }).returning(); + for (const entry of fixture.documents) { + const [doc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: `local://${entry.title}`, category: entry.category, title: entry.title }).returning(); + const [version] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${entry.title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: entry.changelog, createdAt: new Date(entry.timestamp) }).returning(); + for (const [index, content] of (entry.chunks ?? []).entries()) { + await testDb.db.insert(documentContextChunks).values({ documentId: BigInt(doc!.id), versionId: BigInt(version!.id), content, tokenCount: content.split(/\s+/).length, charCount: content.length, pageNumber: index + 1 }); + } + } + for (const [companyId, title, timestamp] of [[BigInt(target!.id), "Outside period", "2026-03-01T00:00:00.000Z"], [BigInt(other!.id), "Other company control", "2026-02-21T00:00:00.000Z"]] as const) { + const [doc] = await testDb.db.insert(document).values({ companyId, url: `local://${title}`, category: "Product", title }).returning(); + await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: title, createdAt: new Date(timestamp) }); + } + const [seededDocuments, seededVersions, seededChunks] = await Promise.all([ + testDb.db.select().from(document), + testDb.db.select().from(documentVersions), + testDb.db.select().from(documentContextChunks), + ]); + console.log(`[harness] fixture counts companies=2 documents=${seededDocuments.length} versions=${seededVersions.length} contextChunks=${seededChunks.length}`); + const actor = { externalUserId: "realistic-owner", internalUserId: 1n, companyId: BigInt(target!.id), role: "owner" as const }; + const created = await createFounderWeeklyReviewDispatchService(testDb.db).createRunWithDispatch({ actor, requestKey: `transport-${randomUUID()}`, reportingPeriod: fixture.reportingPeriod, collectionInput: { workspaceTimezone: fixture.workspaceTimezone, founderContext: fixture.founderContext, actorExternalUserId: actor.externalUserId } }); + if (created.run.status !== "queued" || created.run.evidenceSnapshot) throw new Error("Initial queued-without-snapshot invariant failed."); + const initialDispatches = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, created.run.id)); + if (initialDispatches.length !== 1) throw new Error("Expected exactly one initial outbox dispatch."); + process.env.INNGEST_DEV = inngestUrl; + process.env.INNGEST_EVENT_KEY = "local-transport"; + const { inngest: client } = await import("~/server/inngest/client"); + const requested = await client.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); + console.log(`[harness] dispatch event sent at=${new Date().toISOString()}`); + const repository = new FounderWeeklyReviewRepository(testDb.db); + const deadline = Date.now() + lifecycleTimeoutMs(); + const startedAt = Date.now(); + let heartbeatAt = 0; + const seen: string[] = ["queued without snapshot"]; + let previousState = seen[0]!; + let final: Awaited> | null = null; + while (Date.now() < deadline) { + if (shutdownRequested) throw new Error("Transport run interrupted during lifecycle polling."); + if (!childIsAlive(next) || !childIsAlive(inngest)) { + throw new Error("A managed child exited before the Founder Weekly Review reached a terminal state."); + } + const current = await repository.getByCompanyAndRunId(actor.companyId, created.run.id); + const currentDispatch = (await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, created.dispatch.id)))[0]; + if (current) { + const state = `${current.status}${current.evidenceSnapshot ? " with snapshot" : " without snapshot"}`; + if (state !== previousState) { + previousState = state; + seen.push(state); + console.log(`[harness] lifecycle ${state} at=${new Date().toISOString()}`); + } + if (Date.now() - heartbeatAt >= 10_000) { + heartbeatAt = Date.now(); + console.log(`[harness +${Math.floor((Date.now() - startedAt) / 1000)}s] nextAlive=${childIsAlive(next)} inngestAlive=${childIsAlive(inngest)} callbackReady=true runStatus=${current.status} snapshot=${current.evidenceSnapshot ? "present" : "absent"} dispatchStatus=${currentDispatch?.status ?? "unknown"} collectionClaim=${current.collectionClaimId ? "present" : "absent"} generationClaim=${current.generationClaimId ? "present" : "absent"}`); + } + if (current.status === "draft") { + final = current; + shutdownDrain.terminalStatus = "draft"; + console.log("[harness] terminal status=draft observed; stopping lifecycle polling and starting cleanup"); + break; + } + if (current.status === "failed") { + console.error(JSON.stringify({ + runId: current.id, + status: current.status, + failureCode: current.errorCode, + failureMessage: current.errorMessage?.slice(0, 1_024) ?? null, + snapshot: current.evidenceSnapshot ? "present" : "absent", + collectionClaim: current.collectionClaimId ? "present" : "absent", + collectionStartedAt: current.collectionStartedAt?.toISOString() ?? null, + evidenceCollectedAt: current.evidenceCollectedAt?.toISOString() ?? null, + generationClaim: current.generationClaimId ? "present" : "absent", + retryCount: current.retryCount, + dispatchStatus: currentDispatch?.status ?? null, + dispatchAttempts: currentDispatch?.attemptCount ?? null, + lastLifecycleTransition: current.updatedAt?.toISOString() ?? current.createdAt.toISOString(), + })); + shutdownDrain.terminalStatus = "failed"; + shutdownDrain.finalDiagnosticsCaptured = true; + console.log("[harness] terminal status=failed observed; stopping lifecycle polling and starting cleanup"); + throw new Error(`Founder Weekly Review run ${current.id} failed before draft persistence.`); + } + } + await sleep(500); + } + if (!final?.reviewPayload || !final.evidenceSnapshot) throw new Error("Timed out before persisted draft read-back."); + const evaluation = await evaluateGeneratedFounderWeeklyReview( + final.evidenceSnapshot, + final.reviewPayload as any, + generateFounderWeeklyReviewStructured + ); + const dispatch = (await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, created.dispatch.id)))[0]; + const rendered = renderFounderWeeklyReviewMarkdown(final); + const directory = resolve(process.cwd(), `.artifacts/founder-weekly-review/transport/${final.id}`); + await mkdir(directory, { recursive: true }); + const reportPath = resolve(directory, "report.json"); + const markdownPath = resolve(directory, "report.md"); + const evaluationJsonPath = resolve(directory, "evaluation.json"); + const evaluationMarkdownPath = resolve(directory, "evaluation.md"); + + await writeFile( + reportPath, + JSON.stringify( + { + runId: final.id, + status: final.status, + provider: final.modelMetadata?.provider, + model: final.modelMetadata?.model, + reportingPeriod: final.reportingPeriod, + review: final.reviewPayload, + }, + null, + 2 + ), + "utf8" + ); + + await writeFile(markdownPath, rendered, "utf8"); + + await writeFile( + evaluationJsonPath, + JSON.stringify(evaluation, null, 2), + "utf8" + ); + + await writeFile( + evaluationMarkdownPath, + renderFounderWeeklyReviewEvaluationMarkdown(evaluation), + "utf8" + ); + if (process.env.FWR_PRINT_REPORT === "1") { + console.log("===== FOUNDER WEEKLY REVIEW ====="); + console.log(rendered); + console.log("===== END FOUNDER WEEKLY REVIEW ====="); + } + const generationEventKeys = ["runId", "companyId", "generationJobId", "generationClaimId"]; + const forbiddenEventKeys = ["evidenceSnapshot", "founderContext", "documentContent", "customerFeedback", "prompt", "reviewPayload", "providerResponse", "databaseUrl", "credentials", "token"]; + console.log(JSON.stringify({ runId: final.id, lifecycle: seen, dispatch: { initialStatus: initialDispatches[0]!.status, finalStatus: dispatch?.status, attempts: dispatch?.attemptCount }, ingressEventIds: requested.ids, callbackUrl, functionId: "founder-weekly-review-generation", transportEvent: { name: "founder-weekly-review/generation.requested", keys: generationEventKeys, companyIdSerializedAsString: true, forbiddenKeysAbsent: forbiddenEventKeys.every((key) => !generationEventKeys.includes(key)) }, steps: ["claim-evidence", "collect-evidence", "persist-evidence", "claim", "generate", "persist"], evidenceCounts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, final!.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])), snapshotDigest: digest(final.evidenceSnapshot), retryCount: final.retryCount, generationAttempt: final.generationAttempt, provider: final.modelMetadata?.provider, model: final.modelMetadata?.model, validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, evaluation: { + deterministicScore: evaluation.deterministic?.overallScore ?? null, + llmScore: evaluation.llmGrader?.overallScore ?? null, + failures: evaluation.failures, + }, + evaluationPaths: { + report: reportPath, + markdown: markdownPath, + evaluation: evaluationJsonPath, + evaluationMarkdown: evaluationMarkdownPath, + }, terminalEqualsExport: (await readFile(markdownPath, "utf8")) === rendered, devLogsMentionFunction: inngestLogs.join("").includes("founder-weekly-review-generation") })); + shutdownDrain.finalDiagnosticsCaptured = true; + } catch (error) { + printRecentDiagnostics("next", nextLogs); + printRecentDiagnostics("inngest", inngestLogs); + throw error; + } finally { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + process.off("uncaughtException", onUncaughtException); + process.off("unhandledRejection", onUnhandledRejection); + if (shutdownDrain.terminalStatus && shutdownDrain.finalDiagnosticsCaptured) { + await drainTerminalCallbackActivity(shutdownDrain); + } + await cleanup(); + await closeLogStream(nextRawLog); + await closeLogStream(inngestRawLog); + // testDb.close closes client connections and drops the schema only after callback processes and ports are handled. + await testDb.close(); + } +} + +await main(); diff --git a/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts new file mode 100644 index 000000000..e951e449d --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-realistic-e2e.ts @@ -0,0 +1,272 @@ +import "dotenv/config"; +import { createHash, randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import { readFile, mkdir, rename, writeFile, access } from "node:fs/promises"; +import { resolve } from "node:path"; +import { eq, sql } from "drizzle-orm"; +import { z } from "zod"; +import { company, document, documentContextChunks, documentStructure, documentVersions, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewEvidenceService, FounderWeeklyReviewEvidenceSnapshotSchema, FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, buildFounderWeeklyReviewEvaluationPrompt, buildGenerationEvidenceEnvelope, evaluateFounderWeeklyReview, FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION, generateFounderWeeklyReview, validateFounderWeeklyReviewV2Citations, type DeterministicMaterialChangeDiagnostics, type DocumentChangeMaterialityAnalyzer } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewDocumentVersionStore } from "~/server/founder-weekly-review/document-version-chunks"; +import { StrictCurrentWorkspaceDocumentStore } from "~/server/founder-weekly-review/workspace-document-store"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { createConfiguredDocumentChangeMaterialityAnalyzer } from "~/server/founder-weekly-review/document-change-materiality-analyzer"; +import { generateStructuredWithMetadata } from "~/lib/llm"; +import { founderWeeklyReviewRealisticExportRoot, parseFounderWeeklyReviewRealisticEvidenceMode } from "~/server/founder-weekly-review/realistic-e2e-mode"; +import { formatFounderWeeklyReviewDemo } from "./founder-weekly-review-demo"; +import { evaluateGeneratedFounderWeeklyReview } from "@launchstack/features/founder-weekly-review/benchmarks"; + +const require = createRequire(import.meta.url); +const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); +const fixturePath = resolve(process.cwd(), "test-fixtures/founder-weekly-review/realistic-company/seed.json"); +type Fixture = { reportingPeriod: { start: string; end: string }; workspaceTimezone: string; founderContext: string; documents: Array<{ title: string; category: string; changelog: string; timestamp: string; chunks?: string[] }> }; +type ComputedTexts = { before: string; after: string; v3: string; bHistorical: string; nullVersion: string; foreign: string; unrelated: string }; +type ArtifactPaths = { evidence: string; report: string; markdown: string; evaluation: string; evaluationMarkdown: string; summary: string; envelope: string; validation: string; e2eReport: string; analyzerCalls: string }; + +const SemanticEvaluationSchema = z.object({ + overallScore: z.number().min(0).max(1), + dimensions: z.object({ groundedness: z.number().min(0).max(1), materiality: z.number().min(0).max(1), temporalAccuracy: z.number().min(0).max(1), synthesisQuality: z.number().min(0).max(1), actionability: z.number().min(0).max(1) }), + findings: z.array(z.object({ category: z.string().min(1).max(80), severity: z.enum(["info", "minor", "major"]), explanation: z.string().min(1).max(2000) })).max(8), + summary: z.string().min(1).max(2000), +}); + +function canonicalize(value: unknown): unknown { if (value === null || ["string", "boolean"].includes(typeof value)) return value; if (typeof value === "number" && Number.isFinite(value)) return value; if (Array.isArray(value)) return value.map(canonicalize); if (typeof value === "object") return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalize((value as Record)[key])])); throw new Error("Cannot canonicalize snapshot."); } +function digest(value: unknown) { return createHash("sha256").update(JSON.stringify(canonicalize(value)), "utf8").digest("hex"); } +async function writeAtomic(path: string, body: string) { try { await access(path); throw new Error("Refusing to overwrite export."); } catch (error) { if (error instanceof Error && error.message.includes("overwrite")) throw error; } const temp = `${path}.${randomUUID()}.tmp`; await writeFile(temp, body, "utf8"); await rename(temp, path); } +const vector = (index: number) => Array.from({ length: 1536 }, (_, i) => i === index ? 1 : 0); +const vectorSql = (index: number) => sql`${JSON.stringify(vector(index))}::vector(1536)`; + +function assertComputedSnapshot(snapshot: unknown, ids: Record, texts: ComputedTexts) { + const parsed = FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot); + const items = parsed.items; + const changes = items.filter((item) => item.sourceType === "document_change"); + const workspace = items.filter((item) => item.sourceType === "workspace_document"); + const feedback = items.filter((item) => item.sourceType === "customer_feedback"); + const founder = items.filter((item) => item.sourceType === "founder_context"); + const change = changes.find((item) => item.metadata.previousVersionId === ids.a1 && item.metadata.currentVersionId === ids.a2); + const workspaceItem = workspace.find((item) => item.metadata.documentId === String(ids.b)); + if (!change || !workspaceItem || !feedback.length || founder.length !== 1) throw new Error("Computed snapshot is missing required evidence types."); + if (parsed.schemaVersion !== "founder-weekly-review-evidence/v2") throw new Error("Computed collection did not produce evidence snapshot v2."); + const auditGroup = parsed.documentChangeAudit.groups.find((group) => group.evidenceSourceId === change.sourceId); + const auditedRaw = auditGroup?.rawChangeIds.map((rawChangeId) => parsed.documentChangeAudit.rawChanges.find((raw) => raw.rawChangeId === rawChangeId)); + if (!auditGroup || auditedRaw?.some((raw) => !raw) || !auditedRaw?.some((raw) => raw!.previousChunkId !== null && raw!.currentChunkId !== null && raw!.previousExcerpt?.includes(texts.before) && raw!.currentExcerpt?.includes(texts.after)) || change.metadata.category !== "ownership_change" || !change.excerpt.includes(texts.before) || !change.excerpt.includes(texts.after) || change.sourceTimestamp !== "2026-02-20T10:00:00.000Z" || change.excerpt.includes(texts.v3) || change.excerpt.startsWith("Version ")) throw new Error("Computed document-change assertions failed."); + if (workspaceItem.metadata.documentVersionId !== String(ids.b2) || workspaceItem.metadata.chunkId !== ids.bCurrentChunk || workspaceItem.metadata.retrievalReason !== "founder_context_relevance" || typeof workspaceItem.metadata.similarityScore !== "number" || "sourceTimestamp" in workspaceItem) throw new Error("Strict workspace-document assertions failed."); + if (items.some((item) => [texts.v3, texts.bHistorical, texts.nullVersion, texts.foreign, texts.unrelated].some((value) => item.excerpt.includes(value)))) throw new Error("Computed control evidence leaked into the snapshot."); + if (new Set(items.map((item) => item.sourceId)).size !== items.length) throw new Error("Computed snapshot contains duplicate source IDs."); + return { parsed, counts: Object.fromEntries(["document_change", "workspace_document", "customer_feedback", "founder_context"].map((type) => [type, items.filter((item) => item.sourceType === type).length])) }; +} +function assertComputedReport(payload: any, snapshot: ReturnType) { + validateFounderWeeklyReviewV2Citations(payload, snapshot); + const sourceTypeById = new Map(snapshot.items.map((item) => [item.sourceId, item.sourceType])); + const citedTypes = (section: any) => section.state === "evidence" ? section.items.flatMap((item: any) => item.sourceIds.map((id: string) => sourceTypeById.get(id))) : []; + const changed = payload.sections.whatChanged; + if (changed.state !== "evidence" || !citedTypes(changed).includes("document_change")) throw new Error("Generated report did not cite the computed document change in whatChanged."); + const shipped = payload.sections.whatShipped; + if (shipped.state === "evidence" && citedTypes(shipped).length > 0 && citedTypes(shipped).every((type: unknown) => type === "workspace_document")) throw new Error("Generated report used workspace evidence alone for whatShipped."); + if (citedTypes(payload.sections.whatCustomersSaid).some((type: unknown) => type !== "customer_feedback")) throw new Error("Generated report violated customer-feedback source semantics."); + if (payload.sections.currentBlockers.state === "evidence" && !citedTypes(payload.sections.currentBlockers).includes("workspace_document")) throw new Error("Generated report did not use current workspace context for blockers."); +} + +if (process.env.SYNTHETIC_FWR_LOCAL !== "1" || process.env.NODE_ENV === "production") throw new Error("Refusing realistic E2E outside explicit local mode."); +const localUrl = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? "postgresql://postgres:password@localhost:5433/pdr_ai_v2"; +if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(localUrl)) throw new Error("Refusing non-local database."); + +const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as Fixture; +const mode = parseFounderWeeklyReviewRealisticEvidenceMode(process.env.FWR_EVIDENCE_MODE); +const demoMode = process.argv.includes("--demo") || process.env.FWR_DEMO_MODE === "true"; +const originalConsoleLog = console.log; +const originalConsoleInfo = console.info; +if (demoMode) { console.log = () => undefined; console.info = () => undefined; } +const testDb = await createFounderWeeklyReviewTestDatabase(); +try { + const [target] = await testDb.db.insert(company).values({ name: "Northstar Analytics", numberOfEmployees: "24" }).returning(); + const [other] = await testDb.db.insert(company).values({ name: "Other Company", numberOfEmployees: "4" }).returning(); + const actor = { externalUserId: "realistic-owner", internalUserId: 1n, companyId: BigInt(target!.id), role: "owner" }; + let collector: FounderWeeklyReviewEvidenceService; + const configuredAnalyzer = mode === "computed" ? createConfiguredDocumentChangeMaterialityAnalyzer() : undefined; + if (mode === "computed" && process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED === "true" && !configuredAnalyzer) throw new Error("Analyzer was explicitly enabled but not configured."); + const analyzerCalls: Array> = []; + let changeDiagnostics: DeterministicMaterialChangeDiagnostics | undefined; + const analyzer: DocumentChangeMaterialityAnalyzer | undefined = configuredAnalyzer ? { + async analyze(input) { + const started = Date.now(); + try { + const response = await configuredAnalyzer.analyze(input); + const result = response.result as Record; + analyzerCalls.push({ groupId: input.groupId, structurePath: input.structurePath, structureTitle: input.structureTitle, deterministicCategory: input.deterministicAssessment.category, semanticRisk: input.deterministicAssessment.semanticRisk, durationMs: Date.now() - started, disposition: result.disposition, category: result.category, confidence: result.confidence, provider: response.metadata?.provider, model: response.metadata?.model, promptVersion: response.metadata?.promptVersion }); + return response; + } catch (error) { + analyzerCalls.push({ groupId: input.groupId, structurePath: input.structurePath, structureTitle: input.structureTitle, deterministicCategory: input.deterministicAssessment.category, semanticRisk: input.deterministicAssessment.semanticRisk, durationMs: Date.now() - started, errorCode: error instanceof Error ? error.name : "unknown" }); + throw error; + } + }, + } : undefined; + let computedIds: Record | undefined; + const texts: ComputedTexts = { before: "Product owns retry telemetry.", after: "Platform owns retry telemetry and recovery monitoring.", v3: "v3 future-only operational note", bHistorical: "historical workspace reliability text", nullVersion: "null-version workspace reliability text", foreign: "foreign workspace reliability text", unrelated: "unrelated current cafeteria menu" }; + + if (mode === "legacy") { + for (const entry of fixture.documents) { const [doc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: `local://${entry.title}`, category: entry.category, title: entry.title }).returning(); const [version] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(doc!.id), versionNumber: 1, url: `local://${entry.title}/v1`, mimeType: "text/plain", uploadedBy: "seed", changelog: entry.changelog, createdAt: new Date(entry.timestamp) }).returning(); for (const [index, content] of (entry.chunks ?? []).entries()) await testDb.db.insert(documentContextChunks).values({ documentId: BigInt(doc!.id), versionId: BigInt(version!.id), content, tokenCount: content.split(/\s+/).length, charCount: content.length, pageNumber: index + 1 }); } + const [outsideDoc] = await testDb.db.insert(document).values({ companyId: BigInt(target!.id), url: "local://outside", category: "Product", title: "Outside period" }).returning(); await testDb.db.insert(documentVersions).values({ documentId: BigInt(outsideDoc!.id), versionNumber: 1, url: "local://outside/v1", mimeType: "text/plain", uploadedBy: "seed", changelog: "Outside-period control.", createdAt: new Date("2026-03-01T00:00:00.000Z") }); + collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-01T00:00:00.000Z"), { kind: "legacy" }); + } else { + const [a] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://release-ownership", category: "Product", title: "Retry ownership" }).returning(); + const [b] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://onboarding", category: "Planning", title: "Onboarding reliability" }).returning(); + const [feedbackDoc] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://feedback", category: "Customer Feedback", title: "Customer Interviews" }).returning(); + const [unrelated] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: "local://unrelated", category: "Product", title: "Unrelated" }).returning(); + const [foreign] = await testDb.db.insert(document).values({ companyId: BigInt(other!.id), url: "local://foreign", category: "Planning", title: "Foreign" }).returning(); + const [a1] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 1, url: "local://a/v1", mimeType: "text/plain", changelog: "Initial ownership.", createdAt: new Date("2026-02-10T10:00:00.000Z") }).returning(); + const [a2] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 2, url: "local://a/v2", mimeType: "text/plain", changelog: "Platform ownership was updated.", createdAt: new Date("2026-02-20T10:00:00.000Z") }).returning(); + const [a3] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(a!.id), versionNumber: 3, url: "local://a/v3", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [b1] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(b!.id), versionNumber: 1, url: "local://b/v1", mimeType: "text/plain", createdAt: new Date("2026-02-01T10:00:00.000Z") }).returning(); + const [b2] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(b!.id), versionNumber: 2, url: "local://b/v2", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [feedbackVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(feedbackDoc!.id), versionNumber: 1, url: "local://feedback/v1", mimeType: "text/plain", createdAt: new Date("2026-02-22T12:00:00.000Z") }).returning(); + const [unrelatedVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(unrelated!.id), versionNumber: 1, url: "local://unrelated/v1", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [foreignVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(foreign!.id), versionNumber: 1, url: "local://foreign/v1", mimeType: "text/plain", createdAt: new Date("2026-03-02T10:00:00.000Z") }).returning(); + const [s1] = await testDb.db.insert(documentStructure).values({ documentId: BigInt(a!.id), versionId: BigInt(a1!.id), ordering: 1, title: "Retry ownership", path: "1" }).returning(); + const [s2] = await testDb.db.insert(documentStructure).values({ documentId: BigInt(a!.id), versionId: BigInt(a2!.id), ordering: 1, title: "Retry ownership", path: "1" }).returning(); + await testDb.db.update(document).set({ currentVersionId: BigInt(a3!.id) }).where(eq(document.id, a!.id)); + await testDb.db.update(document).set({ currentVersionId: BigInt(b2!.id) }).where(eq(document.id, b!.id)); + await testDb.db.update(document).set({ currentVersionId: BigInt(unrelatedVersion!.id) }).where(eq(document.id, unrelated!.id)); + await testDb.db.update(document).set({ currentVersionId: BigInt(foreignVersion!.id) }).where(eq(document.id, foreign!.id)); + const inserted = await testDb.db.insert(documentContextChunks).values([ + { documentId: BigInt(a!.id), versionId: BigInt(a1!.id), structureId: BigInt(s1!.id), content: texts.before, contentHash: "a".repeat(64), tokenCount: 4, charCount: texts.before.length, pageNumber: 1, lineStart: 1, lineEnd: 1 }, + { documentId: BigInt(a!.id), versionId: BigInt(a2!.id), structureId: BigInt(s2!.id), content: texts.after, contentHash: "b".repeat(64), tokenCount: 7, charCount: texts.after.length, pageNumber: 1, lineStart: 1, lineEnd: 1 }, + { documentId: BigInt(a!.id), versionId: BigInt(a3!.id), content: texts.v3, contentHash: "c".repeat(64), tokenCount: 4, charCount: texts.v3.length }, + { documentId: BigInt(b!.id), versionId: BigInt(b1!.id), content: texts.bHistorical, contentHash: "d".repeat(64), tokenCount: 4, charCount: texts.bHistorical.length, embedding: vectorSql(0) }, + { documentId: BigInt(b!.id), versionId: BigInt(b2!.id), content: "Onboarding reliability and retry monitoring are blocking enterprise expansion.", contentHash: "e".repeat(64), tokenCount: 8, charCount: 72, embedding: vectorSql(0) }, + { documentId: BigInt(b!.id), versionId: null, content: texts.nullVersion, contentHash: "f".repeat(64), tokenCount: 4, charCount: texts.nullVersion.length, embedding: vectorSql(0) }, + { documentId: BigInt(feedbackDoc!.id), versionId: BigInt(feedbackVersion!.id), content: "Enterprise buyers need reliable onboarding recovery before expansion.", contentHash: "1".repeat(64), tokenCount: 8, charCount: 67 }, + { documentId: BigInt(unrelated!.id), versionId: BigInt(unrelatedVersion!.id), content: texts.unrelated, contentHash: "2".repeat(64), tokenCount: 4, charCount: texts.unrelated.length }, + { documentId: BigInt(foreign!.id), versionId: BigInt(foreignVersion!.id), content: texts.foreign, contentHash: "3".repeat(64), tokenCount: 4, charCount: texts.foreign.length, embedding: vectorSql(0) }, + ]).returning(); + const bCurrentChunk = inserted.find((chunk) => chunk.documentId === BigInt(b!.id) && chunk.versionId === BigInt(b2!.id))!; + const enterpriseScenarios: Array<{ title: string; category: string; sections: Array<[string, string, string]> }> = [ + { title: "Roadmap and launch plan", category: "Planning", sections: [["Launch timing", "The analytics launch remains planned for Q3.", "We still expect the analytics launch during the third quarter."], ["Launch narrative", "The roadmap tracks launch readiness, dependencies, and adoption preparation.", "Launch readiness, dependencies, and adoption preparation remain tracked in the roadmap."], ["Customer rollout", "The pilot serves selected enterprise customers in the United States.", "The pilot serves selected enterprise customers in the United States."], ["Decision notes", "The team will review launch readiness with the founder next week.", "The founder will review launch readiness with the team next week."], ["Owner", "Product owns roadmap coordination.", "Platform owns roadmap coordination."]], }, + { title: "Enterprise rollout plan", category: "Operations", sections: [["Release status", "The recovery workflow is planned for the April release.", "The recovery workflow launched in the April release."], ["Requirement", "SAML support is optional for the pilot.", "SAML support is required for the pilot."], ["Scope", "The rollout is limited to US enterprise customers.", "The rollout is global for enterprise customers."], ["Rollout narrative", "The rollout plan covers enablement, support readiness, and customer communications.", "Enablement, support readiness, and customer communications are covered by the rollout plan."], ["Risk", "A rollout risk remains documented for support capacity.", "Support capacity remains a documented rollout risk."]], }, + { title: "Operating metrics", category: "Operating Plan", sections: [["Activation target", "The activation target is 10% for the quarter.", "The activation target is 25% for the quarter."], ["Deadline", "The onboarding milestone is due June 1.", "The onboarding milestone is due July 15."], ["Metric narrative", "The operating plan tracks activation, retention, and onboarding throughput.", "Activation, retention, and onboarding throughput are tracked in the operating plan."], ["Customer count", "The target is 20 enterprise customers by year end.", "The target is twenty enterprise customers by year end."], ["Review cadence", "The team reviews operating metrics every Friday.", "Operating metrics are reviewed by the team every Friday."]], }, + { title: "Risk and ownership register", category: "Risk", sections: [["Blocker status", "The data migration is blocked by missing export access.", "The data migration is resolved after export access was granted."], ["Risk rewrite", "The main risk is delayed onboarding for enterprise accounts.", "Enterprise onboarding delay remains the primary operating risk."], ["Priority", "Migration follow-up is P2 and deferred.", "Migration follow-up is P0 and immediate."], ["Ownership narrative", "Marketing owns the customer readiness checklist.", "The customer readiness checklist is owned by Marketing."], ["Decision context", "The register summarizes active risks and their next review dates.", "Active risks and next review dates are summarized in the register."]], }, + ]; + for (const [scenarioIndex, scenario] of enterpriseScenarios.slice(0, 2).entries()) { + const sections = scenario.sections.slice(0, 4); + const [scenarioDocument] = await testDb.db.insert(document).values({ companyId: actor.companyId, url: `local://scenario-${scenarioIndex}`, category: scenario.category, title: scenario.title }).returning(); + const [previousVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(scenarioDocument!.id), versionNumber: 1, url: `local://scenario-${scenarioIndex}/v1`, mimeType: "text/plain", createdAt: new Date("2026-02-10T10:00:00.000Z") }).returning(); + const [currentVersion] = await testDb.db.insert(documentVersions).values({ documentId: BigInt(scenarioDocument!.id), versionNumber: 2, url: `local://scenario-${scenarioIndex}/v2`, mimeType: "text/plain", changelog: "Operating plan sections updated for this reporting period.", createdAt: new Date("2026-02-20T10:00:00.000Z") }).returning(); + const previousStructures = await testDb.db.insert(documentStructure).values(sections.map(([title], index) => ({ documentId: BigInt(scenarioDocument!.id), versionId: BigInt(previousVersion!.id), ordering: index + 1, title, path: `${index + 1}` }))).returning(); + const currentStructures = await testDb.db.insert(documentStructure).values(sections.map(([title], index) => ({ documentId: BigInt(scenarioDocument!.id), versionId: BigInt(currentVersion!.id), ordering: index + 1, title, path: `${index + 1}` }))).returning(); + await testDb.db.insert(documentContextChunks).values(sections.flatMap(([, before, after], index) => [ + { documentId: BigInt(scenarioDocument!.id), versionId: BigInt(previousVersion!.id), structureId: BigInt(previousStructures[index]!.id), content: before, contentHash: `${scenarioIndex}${index}a`.padEnd(64, "a"), tokenCount: before.split(/\s+/).length, charCount: before.length, pageNumber: index + 1, lineStart: 1, lineEnd: 1, ...(index < 3 ? { embedding: vectorSql(0) } : {}) }, + { documentId: BigInt(scenarioDocument!.id), versionId: BigInt(currentVersion!.id), structureId: BigInt(currentStructures[index]!.id), content: after, contentHash: `${scenarioIndex}${index}b`.padEnd(64, "b"), tokenCount: after.split(/\s+/).length, charCount: after.length, pageNumber: index + 1, lineStart: 1, lineEnd: 1, ...(index < 3 ? { embedding: vectorSql(0) } : {}) }, + ])); + await testDb.db.update(document).set({ currentVersionId: BigInt(currentVersion!.id) }).where(eq(document.id, scenarioDocument!.id)); + } + computedIds = { a: BigInt(a!.id), a1: a1!.id, a2: a2!.id, a3: a3!.id, b: BigInt(b!.id), b2: BigInt(b2!.id), bCurrentChunk: bCurrentChunk.id }; + collector = new FounderWeeklyReviewEvidenceService(testDb.db, () => new Date("2026-03-03T00:00:00.000Z"), { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore(testDb.db) }, new StrictCurrentWorkspaceDocumentStore(testDb.db, { embedQuery: async () => vector(0) }), analyzer, (diagnostics) => { changeDiagnostics = diagnostics; }); + } + + const dispatchService = createFounderWeeklyReviewDispatchService(testDb.db); + const created = await dispatchService.createRunWithDispatch({ actor, requestKey: `realistic-${mode}-${randomUUID()}`, reportingPeriod: fixture.reportingPeriod, collectionInput: { workspaceTimezone: fixture.workspaceTimezone, founderContext: mode === "computed" ? "Assess whether onboarding reliability and retry monitoring are blocking enterprise expansion." : fixture.founderContext, actorExternalUserId: actor.externalUserId } }); + if (created.run.evidenceSnapshot) throw new Error("Workflow run unexpectedly has an initial snapshot."); + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); const collectionContext = { companyId: actor.companyId, runId: created.run.id, collectionClaimId: created.dispatch.generationClaimId }; + const collecting = await worker.claimEvidenceCollection(collectionContext); + const input = { companyId: actor.companyId, reportingPeriod: fixture.reportingPeriod, workspaceTimezone: fixture.workspaceTimezone, founderContext: mode === "computed" ? "Assess whether onboarding reliability and retry monitoring are blocking enterprise expansion." : fixture.founderContext, actor: { externalUserId: actor.externalUserId }, requestKey: created.run.requestKey }; + const snapshot = await collector.collectFounderWeeklyReviewEvidence(input); const repeated = analyzer ? snapshot : await collector.collectFounderWeeklyReviewEvidence(input); + if (!analyzer && (JSON.stringify(snapshot.items) !== JSON.stringify(repeated.items) || JSON.stringify(snapshot.sourceWarnings) !== JSON.stringify(repeated.sourceWarnings))) throw new Error("Evidence collection was not deterministic."); + const beforeDigest = digest(snapshot); const attached = await worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot); const afterDigest = digest(attached.evidenceSnapshot); + const checked = mode === "computed" ? assertComputedSnapshot(attached.evidenceSnapshot, computedIds!, texts) : { parsed: attached.evidenceSnapshot!, counts: Object.fromEntries(["document_change", "customer_feedback", "founder_context"].map((type) => [type, attached.evidenceSnapshot!.items.filter((item) => item.sourceType === type).length])) }; + if (attached.status !== "queued" || beforeDigest !== afterDigest || !checked.counts.document_change || !checked.counts.customer_feedback || checked.counts.founder_context !== 1) throw new Error("Realistic collector assertions failed."); + const generationContext = { companyId: actor.companyId, runId: attached.id, generationJobId: created.dispatch.generationJobId, generationClaimId: created.dispatch.generationClaimId }; const generating = await worker.claimQueuedRun(generationContext); if (!generating.evidenceSnapshot) throw new Error("Generation began without snapshot."); + let generationCalls = 0; + const generated = await generateFounderWeeklyReview({ evidenceSnapshot: generating.evidenceSnapshot, generate: async (request) => { generationCalls++; return generateFounderWeeklyReviewStructured(request); } }); + validateFounderWeeklyReviewV2Citations(generated.reviewPayload as never, generating.evidenceSnapshot); + const saved = await worker.saveGeneratedDraft(generationContext, generated.reviewPayload, generated.modelMetadata); const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, saved.id); if (!readBack?.reviewPayload || readBack.status !== "draft" || !readBack.evidenceSnapshot || digest(readBack.evidenceSnapshot) !== beforeDigest) throw new Error("Validated draft read-back or snapshot immutability failed."); + + const benchmarkEvaluation = await evaluateGeneratedFounderWeeklyReview( + readBack.evidenceSnapshot, + readBack.reviewPayload as unknown as Parameters[1], + generateFounderWeeklyReviewStructured + ); + + console.log(JSON.stringify({ + evaluation: { + deterministicScore: benchmarkEvaluation.deterministic?.overallScore, + failures: benchmarkEvaluation.failures, + llmScore: benchmarkEvaluation.llmGrader?.overallScore, + } + })); + + if (mode === "computed") assertComputedReport(readBack.reviewPayload, readBack.evidenceSnapshot); + const deterministicEvaluation = evaluateFounderWeeklyReview(readBack.evidenceSnapshot, readBack.reviewPayload as any); + let semanticEvaluation: { result: z.infer; metadata: Record } | null = null; + let evaluationCalls = 0; + if (process.env.FWR_FOUNDER_REVIEW_EVALUATION_ENABLED === "true") { + evaluationCalls++; + try { + const graded = await generateStructuredWithMetadata({ + capability: "smallExtraction", + forceProvider: "kimi", + maxOutputTokens: 2048, + system: "You are a strict Founder Weekly Review quality grader. Return one JSON object only. Do not repeat the review or evidence. Score each dimension from 0 to 1. Return at most 6 concise findings and a summary under 900 characters.", + prompt: buildFounderWeeklyReviewEvaluationPrompt(readBack.evidenceSnapshot, readBack.reviewPayload as any), + schema: SemanticEvaluationSchema, + schemaName: "founder_weekly_review_evaluation", + }); + semanticEvaluation = { result: SemanticEvaluationSchema.parse(graded.object), metadata: { provider: graded.metadata.provider, model: graded.metadata.model, promptVersion: FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION } }; + } catch (error) { + semanticEvaluation = null; + console.error(`[fwr-evaluation] semantic grader unavailable: ${error instanceof Error ? error.name : "unknown"}`); + } + } + const rendered = renderFounderWeeklyReviewMarkdown(readBack); + const dispatchRows = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.runId, saved.id)); const runRows = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); + let artifactPaths: ArtifactPaths | null = null; + if (process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1") { const exportRoot = founderWeeklyReviewRealisticExportRoot(mode, process.env.SYNTHETIC_FWR_EXPORT_DIR); const directory = resolve(process.cwd(), exportRoot, saved.id); await mkdir(directory, { recursive: true }); artifactPaths = { evidence: resolve(directory, "evidence-snapshot.json"), report: resolve(directory, "review.json"), markdown: resolve(directory, "review.md"), summary: resolve(directory, "summary.json"), envelope: resolve(directory, "generation-envelope.json"), validation: resolve(directory, "validation.json"), e2eReport: resolve(directory, "e2e-report.md"), analyzerCalls: resolve(directory, "analyzer-calls.json"), evaluation: resolve(directory, "evaluation.json"), evaluationMarkdown: resolve(directory, "evaluation.md") }; const envelope = buildGenerationEvidenceEnvelope(readBack.evidenceSnapshot); const validation = { schema: true, citations: true, sourceSemantics: true }; const evaluation = { deterministic: deterministicEvaluation, semantic: semanticEvaluation }; await writeAtomic(artifactPaths.evidence, JSON.stringify(readBack.evidenceSnapshot, null, 2)); await writeAtomic(artifactPaths.report, JSON.stringify({ runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, reportingPeriod: readBack.reportingPeriod, review: readBack.reviewPayload }, null, 2)); await writeAtomic(artifactPaths.markdown, rendered); await writeAtomic(artifactPaths.envelope, JSON.stringify(envelope, null, 2)); await writeAtomic(artifactPaths.validation, JSON.stringify(validation, null, 2)); await writeAtomic(artifactPaths.analyzerCalls, JSON.stringify({ callCount: analyzerCalls.length, calls: analyzerCalls }, null, 2)); await writeAtomic(artifactPaths.evaluation, JSON.stringify(evaluation, null, 2)); await writeAtomic(artifactPaths.evaluationMarkdown, `# Founder Weekly Review Evaluation\n\nDeterministic: ${deterministicEvaluation.passed ? "PASS" : "FAIL"}\n\n${semanticEvaluation ? `Overall: ${semanticEvaluation.result.overallScore.toFixed(2)}\n\n${semanticEvaluation.result.summary}` : "Semantic grading was not enabled."}\n`); await writeAtomic(artifactPaths.e2eReport, `# Analyzer-enabled realistic E2E\n\n- Materiality calls: ${analyzerCalls.length}\n- Evaluation calls: ${evaluationCalls}\n- Materiality provider: ${analyzerCalls.find(call => call.provider)?.provider ?? "none"}\n- Materiality model: ${analyzerCalls.find(call => call.model)?.model ?? "none"}\n- Generation provider: ${generated.modelMetadata.provider}\n- Generation model: ${generated.modelMetadata.model}\n- Snapshot schema: ${readBack.evidenceSnapshot.schemaVersion}\n- Evidence digest: ${digest(readBack.evidenceSnapshot)}\n- Prompt hash: ${generated.modelMetadata.promptHash}\n`); await writeAtomic(artifactPaths.summary, JSON.stringify({ runId: saved.id, scenario: "realistic-company", mode, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), repairCount: generationCalls - 1, retryCount: saved.retryCount, validation, snapshotDigestBefore: beforeDigest, snapshotDigestAfter: digest(readBack.evidenceSnapshot), analyzerCallCount: analyzerCalls.length, reviewGenerationCallCount: generationCalls, evaluationCallCount: evaluationCalls, analyzerCalls, evaluation, artifactPaths }, null, 2)); } + if (demoMode) { + console.log = originalConsoleLog; + console.info = originalConsoleInfo; + const demoSnapshot = FounderWeeklyReviewEvidenceSnapshotSchema.parse(readBack.evidenceSnapshot) as any; + const envelope = buildGenerationEvidenceEnvelope(demoSnapshot); + const rawChanges = demoSnapshot.documentChangeAudit.rawChanges as unknown as Record[]; + const groups = demoSnapshot.documentChangeAudit.groups as unknown as Record[]; + const eligibleGroups = analyzerCalls.length + (readBack.evidenceSnapshot.sourceWarnings.some((warning) => warning.code === "materiality_analysis_budget_truncated") ? 1 : 0); + let scaleResults: Record[] = []; + try { + const scale = JSON.parse(await readFile(resolve(process.cwd(), ".artifacts/founder-weekly-review/scale-analysis/scale-analysis.json"), "utf8")) as { results?: Record[] }; + scaleResults = scale.results ?? []; + } catch { /* Scale artifacts are optional for the presentation. */ } + console.log(formatFounderWeeklyReviewDemo({ + rawChanges, + groups, + promptItems: demoSnapshot.items as unknown as Record[], + envelopeDiagnostics: envelope.diagnostics as unknown as Record, + analyzerCalls, + warnings: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), + reviewMarkdown: rendered, + provider: String(generated.modelMetadata.provider), + model: String(generated.modelMetadata.model), + promptVersion: String(generated.modelMetadata.promptVersion ?? "founder-weekly-review-generation/v2"), + outputCeiling: 2400, + repairCount: generationCalls - 1, + snapshotVersion: demoSnapshot.schemaVersion, + evidenceDigest: digest(demoSnapshot), + persistencePassed: saved.status === "draft", + readBackPassed: Boolean(readBack.reviewPayload), + noOpRemoved: changeDiagnostics?.deterministicNoOpCount ?? 0, + eligibleGroups, + budget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + scaleResults, + })); + if (semanticEvaluation) { + console.log("\nEVALUATION\n----------"); + console.log(`Deterministic checks ${deterministicEvaluation.passed ? "PASS" : "FAIL"}`); + for (const [dimension, score] of Object.entries(semanticEvaluation.result.dimensions)) console.log(`${dimension.padEnd(28)} ${score.toFixed(2)}`); + console.log(`${"Overall".padEnd(28)} ${semanticEvaluation.result.overallScore.toFixed(2)}`); + for (const finding of semanticEvaluation.result.findings) console.log(`- ${finding.explanation}`); + } else { + console.log("\nEVALUATION\n----------\nSemantic grading disabled; set FWR_FOUNDER_REVIEW_EVALUATION_ENABLED=true."); + } + console.log(`\nArtifacts:\n${artifactPaths ? Object.values(artifactPaths).join("\n") : "export disabled"}`); + } else if (process.env.FWR_PRINT_REPORT === "1") { console.log("===== FOUNDER WEEKLY REVIEW ====="); console.log(rendered); console.log("===== END FOUNDER WEEKLY REVIEW ====="); } + if (!demoMode) console.log(JSON.stringify({ runId: saved.id, mode, lifecycle: [created.run.status, collecting.status, attached.status, generating.status, saved.status], evidenceCounts: checked.counts, warningCodes: readBack.evidenceSnapshot.sourceWarnings.map((warning) => warning.code), snapshotDigestUnchanged: beforeDigest === digest(readBack.evidenceSnapshot), validation: { canonicalSchema: true, citations: true, sourceSemantics: true }, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model, repairCount: generationCalls - 1, dispatchCount: dispatchRows.length, runRowCount: runRows.length, artifactPaths })); +} finally { console.log = originalConsoleLog; console.info = originalConsoleInfo; await testDb.close(); } diff --git a/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts b/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts new file mode 100644 index 000000000..5f182d466 --- /dev/null +++ b/apps/web/scripts/run-founder-weekly-review-synthetic-baseline.ts @@ -0,0 +1,205 @@ +import "dotenv/config"; +import { createHash, randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import { mkdir, rename, writeFile, access } from "node:fs/promises"; +import { resolve } from "node:path"; +import { ZodError } from "zod"; +import { zodSchema } from "ai"; +import { eq } from "drizzle-orm"; +import { company, founderWeeklyReviewDispatches, founderWeeklyReviewRuns } from "@launchstack/core/db/schema"; +import { FounderWeeklyReviewRepository, FounderWeeklyReviewWorkerService, FounderWeeklyReviewV2PayloadSchema, generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { createFounderWeeklyReviewDispatchService } from "~/server/founder-weekly-review/dispatch-service"; +import { renderFounderWeeklyReviewMarkdown } from "~/server/founder-weekly-review/markdown"; +import { syntheticFounderWeeklyReviewFixtures } from "./founder-weekly-review-synthetic-fixtures"; + +const require = createRequire(import.meta.url); +const { createFounderWeeklyReviewTestDatabase } = require("../__tests__/founderWeeklyReview/testDb") as typeof import("../__tests__/founderWeeklyReview/testDb"); + +function canonicalizeJson(value: unknown): unknown { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Cannot canonicalize non-finite number."); + return value; + } + if (Array.isArray(value)) return value.map(canonicalizeJson); + if (typeof value === "object") { + return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [key, canonicalizeJson((value as Record)[key])])); + } + throw new Error("Cannot canonicalize unsupported JSON value."); +} + +function digestJson(value: unknown): string { + return createHash("sha256").update(JSON.stringify(canonicalizeJson(value)), "utf8").digest("hex"); +} + +async function writeAtomically(path: string, content: string) { + try { await access(path); throw new Error("Refusing to overwrite an existing export."); } catch (error) { if (!(error instanceof Error) || !error.message.includes("ENOENT")) { if (error instanceof Error && error.message.includes("overwrite")) throw error; } } + const temporary = `${path}.${randomUUID()}.tmp`; + await writeFile(temporary, content, "utf8"); + await rename(temporary, path); +} + +function markdownFor(run: { reportingPeriod: { start: string; end: string }; modelMetadata: { provider?: string; model?: string } | null; reviewPayload: any; evidenceSnapshot: { items: Array<{ sourceId: string; sourceType: string; title: string; sourceTimestamp?: string }> } }) { + const labels: Record = { whatChanged: "Other Meaningful Changes", whatShipped: "Shipped This Period", whatCustomersSaid: "Customer Signals", currentBlockers: "Risks & Blockers", nextPriorities: "Priorities for the Next Period" }; + const none: Record = { whatChanged: "No other evidence-backed changes were identified during this reporting period.", whatShipped: "No completed product releases were identified during this reporting period.", whatCustomersSaid: "No customer feedback was available for this reporting period.", currentBlockers: "No evidence-backed blockers were identified during this reporting period.", nextPriorities: "No evidence-backed priorities were identified for the next reporting period." }; + const reference = new Map(); const source = new Map(run.evidenceSnapshot.items.map((item) => [item.sourceId, item])); const references: string[] = []; + const cite = (ids: string[] = []) => ids.map((id) => { let n = reference.get(id); if (!n) { n = reference.size + 1; reference.set(id, n); const item = source.get(id); references.push(`[${n}] ${item?.title ?? item?.sourceType ?? "Evidence"}`); } return `[${n}]`; }).join(""); + const period = `${new Date(`${run.reportingPeriod.start}T00:00:00Z`).toLocaleDateString("en-US", { month: "long", day: "numeric" })}–${new Date(`${run.reportingPeriod.end}T00:00:00Z`).toLocaleDateString("en-US", { day: "numeric", year: "numeric" })}`; + const shippedTexts = new Set((run.reviewPayload.sections.whatShipped?.items ?? []).map((item: any) => String(item.text ?? "").trim().replace(/\s+/g, " "))); + const lines = ["# Founder Weekly Review", "", `**Reporting period:** ${period}`, "", "## Key Outcomes", ""]; + for (const key of ["whatShipped", "whatChanged", "whatCustomersSaid", "currentBlockers", "nextPriorities"]) { const section = run.reviewPayload.sections[key]; lines.push(`## ${labels[key]!}`); if (section?.state === "no_evidence") lines.push(none[key]!); else { const items = (section?.items ?? []).filter((item: any) => key !== "whatChanged" || !shippedTexts.has(String(item.text ?? "").trim().replace(/\s+/g, " "))); if (!items.length) lines.push(none[key]!); else items.forEach((item: any, index: number) => lines.push(key === "nextPriorities" ? `${index + 1}. ${item.text}${cite(item.sourceIds)}` : `- ${item.text}${cite(item.sourceIds)}`)); } lines.push(""); } + if (references.length) lines.push("## Evidence References", "", ...references, ""); + lines.push("---", "", `*Generated with ${run.modelMetadata?.model ?? "the configured model"}*`); return lines.join("\n"); +} + +if (process.env.SYNTHETIC_FWR_LOCAL !== "1") throw new Error("Refusing synthetic baseline: set SYNTHETIC_FWR_LOCAL=1."); +const syntheticProvider = process.env.SYNTHETIC_FWR_PROVIDER; +const deterministicMode = (process.argv[2]?.startsWith("negative-") ?? false) || process.argv[2] === "retry-snapshot-immutability"; +if (!deterministicMode && syntheticProvider !== "ollama" && syntheticProvider !== "kimi") throw new Error("Refusing synthetic baseline: set SYNTHETIC_FWR_PROVIDER=ollama or kimi."); +if (process.env.NODE_ENV === "production") throw new Error("Refusing synthetic baseline in production."); +const url = process.env.LAUNCHSTACK_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? ""; +if (!/^postgres(?:ql)?:\/\/(?:[^@]+@)?(?:127\.0\.0\.1|localhost)(?::\d+)?\//i.test(url)) throw new Error("Refusing non-local database."); +if (syntheticProvider === "ollama") { + process.env.OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434/v1"; + process.env.LLM_PROVIDER_PRIORITY = "ollama"; + process.env.LLM_MODEL_FOUNDERWEEKLYREVIEW_OLLAMA = process.env.SYNTHETIC_FWR_MODEL ?? "llama3.1:8b"; +} +if (syntheticProvider === "kimi" && !process.env.MOONSHOT_API_KEY?.trim()) throw new Error("Refusing Kimi synthetic baseline: MOONSHOT_API_KEY is required."); + +async function generateWithLocalOllama(input: { system?: string; prompt: string; schema: TSchema; schemaName?: string }) { + console.log(JSON.stringify({ stage: "ollama_request_constructed", result: "pass" })); + const response = await fetch(`${process.env.OLLAMA_BASE_URL!.replace(/\/$/, "")}/chat/completions`, { + method: "POST", headers: { "content-type": "application/json", authorization: "Bearer ollama" }, + body: JSON.stringify({ model: process.env.SYNTHETIC_FWR_MODEL ?? "llama3.1:8b", temperature: 0, stream: false, messages: [{ role: "system", content: input.system ?? "" }, { role: "user", content: input.prompt }], response_format: { type: "json_schema", json_schema: { name: input.schemaName ?? "founder_weekly_review", strict: true, schema: zodSchema(input.schema).jsonSchema } } }), + }); + console.log(JSON.stringify({ stage: "ollama_http_response", result: response.ok ? "pass" : "fail", httpStatus: response.status })); + if (!response.ok) throw new Error(`ollama_http_error:${response.status}`); + const body = await response.json() as { choices?: Array<{ message?: { content?: string } }> }; + const content = body.choices?.[0]?.message?.content; + if (!content?.trim()) throw new Error("ollama_missing_content"); + console.log(JSON.stringify({ stage: "ollama_content_extracted", result: "pass" })); + let parsed: unknown; + try { parsed = JSON.parse(content); console.log(JSON.stringify({ stage: "ollama_json_parsed", result: "pass" })); } + catch { throw new Error("ollama_json_parse_failed"); } + try { const object = input.schema.parse(parsed); console.log(JSON.stringify({ stage: "review_schema_valid", result: "pass" })); return { object, metadata: { provider: "ollama", model: process.env.SYNTHETIC_FWR_MODEL ?? "llama3.1:8b", capability: "founderWeeklyReview", temperature: 0 } }; } + catch (error) { const issues = error instanceof ZodError ? error.issues.map((issue) => ({ path: issue.path, code: issue.code, ...("expected" in issue ? { expected: issue.expected } : {}), ...("received" in issue ? { received: issue.received } : {}) })) : []; console.log(JSON.stringify({ stage: "review_schema_valid", result: "fail", topLevelKeys: parsed && typeof parsed === "object" && !Array.isArray(parsed) ? Object.keys(parsed as object) : [], issues })); throw new Error("review_schema_invalid"); } +} + +async function generateWithKimi(input: { system?: string; prompt: string; schema: TSchema; schemaName?: string }) { + const base = (process.env.MOONSHOT_BASE_URL ?? "https://api.moonshot.ai/v1").replace(/\/$/, ""); + const schemaGuide = JSON.stringify(zodSchema(input.schema).jsonSchema); + console.log(JSON.stringify({ stage: "kimi_request_constructed", result: "pass" })); + const response = await fetch(`${base}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${process.env.MOONSHOT_API_KEY}` }, body: JSON.stringify({ model: process.env.SYNTHETIC_FWR_MODEL ?? "kimi-k2.6", max_tokens: 1800, messages: [{ role: "system", content: `${input.system ?? ""}\nReturn one JSON object only. Its complete required structural schema is: ${schemaGuide}` }, { role: "user", content: input.prompt }], stream: false, thinking: { type: "disabled" }, response_format: { type: "json_object" } }), signal: AbortSignal.timeout(90_000) }); + console.log(JSON.stringify({ stage: "kimi_http_response", result: response.ok ? "pass" : "fail", httpStatus: response.status })); + if (!response.ok) throw new Error(`kimi_http_error:${response.status}`); + const body = await response.json() as { model?: string; usage?: Record; choices?: Array<{ message?: { content?: string }; finish_reason?: string }> }; + const content = body.choices?.[0]?.message?.content; + if (!content?.trim()) throw new Error("kimi_missing_content"); + console.log(JSON.stringify({ stage: "kimi_content_extracted", result: "pass" })); + let parsed: unknown; try { parsed = JSON.parse(content); console.log(JSON.stringify({ stage: "kimi_json_parsed", result: "pass" })); } catch { throw new Error("kimi_json_parse_failed"); } + return { object: parsed as ReturnType, metadata: { provider: "kimi", model: body.model ?? (process.env.SYNTHETIC_FWR_MODEL ?? "kimi-k2.6"), capability: "founderWeeklyReview", temperature: undefined as unknown as number, ...(body.choices?.[0]?.finish_reason ? { finishReason: body.choices[0].finish_reason } : {}), ...(body.usage ? { usage: body.usage } : {}) } }; +} + +const fixtureName = process.argv[2] ?? "partial"; +const isNegativeUnknownCitation = fixtureName === "negative-unknown-citation"; +const isNegativeFounderContextCustomer = fixtureName === "negative-founder-context-customer"; +const isDeterministicNegative = isNegativeUnknownCitation || isNegativeFounderContextCustomer; +const isRetrySnapshotImmutability = fixtureName === "retry-snapshot-immutability"; +const snapshot = fixtureName === "negative-founder-context-customer" ? syntheticFounderWeeklyReviewFixtures.partial : (fixtureName.startsWith("negative-") || isRetrySnapshotImmutability) ? syntheticFounderWeeklyReviewFixtures.full : syntheticFounderWeeklyReviewFixtures[fixtureName as keyof typeof syntheticFounderWeeklyReviewFixtures]; +if (!snapshot) throw new Error("Fixture must be partial, full, empty, negative-unknown-citation, negative-founder-context-customer, or retry-snapshot-immutability."); +const testDb = await createFounderWeeklyReviewTestDatabase(); +try { + await testDb.db.insert(company).values({ id: 3, name: "Synthetic FWR", numberOfEmployees: "1" }); + const actor = { companyId: 3n, userId: 1n, externalUserId: "synthetic-owner", role: "owner" as const, workspaceTimezone: "UTC" }; + const requestKey = `synthetic-${fixtureName}-${randomUUID()}`; + const dispatchService = createFounderWeeklyReviewDispatchService(testDb.db); + const { run, dispatch } = await dispatchService.createRunWithDispatch({ actor, requestKey, reportingPeriod: snapshot.reportingPeriod, evidenceSnapshot: snapshot }); + const event = { runId: dispatch.runId, companyId: dispatch.companyId.toString(), generationJobId: dispatch.generationJobId, generationClaimId: dispatch.generationClaimId }; + const worker = new FounderWeeklyReviewWorkerService(new FounderWeeklyReviewRepository(testDb.db)); + const claimed = await worker.claimQueuedRun({ companyId: BigInt(event.companyId), runId: event.runId, generationJobId: event.generationJobId, generationClaimId: event.generationClaimId }); + let generated: Awaited> | null = null; + try { + const none = { state: "no_evidence" as const, noEvidence: { code: "none", message: "No evidence", cta: "Add evidence" } }; + const validPayload = FounderWeeklyReviewV2PayloadSchema.parse({ schemaVersion: "founder-weekly-review/v2", sections: { whatChanged: { state: "evidence", items: [{ kind: "observed_fact", text: "Release evidence.", sourceIds: ["synthetic:doc:release"], confidence: 0.8 }] }, whatShipped: none, whatCustomersSaid: { state: "evidence", items: [{ kind: "observed_fact", text: "Customer signal.", sourceIds: ["synthetic:feedback:export"], confidence: 0.8 }] }, currentBlockers: none, nextPriorities: none } }); + const payload = structuredClone(validPayload); + if (isNegativeUnknownCitation || isRetrySnapshotImmutability) payload.sections.whatChanged = { state: "evidence", items: [{ kind: "observed_fact", text: "Release evidence.", sourceIds: ["synthetic:missing:unknown-citation"], confidence: 0.8 }] }; + if (isNegativeFounderContextCustomer) payload.sections.whatCustomersSaid = { state: "evidence", items: [{ kind: "observed_fact", text: "Customer signal.", sourceIds: ["synthetic:context:priority"], confidence: 0.8 }] }; + if (!claimed.evidenceSnapshot) throw new Error("Synthetic baseline requires a persisted evidence snapshot."); + generated = await generateFounderWeeklyReview({ evidenceSnapshot: claimed.evidenceSnapshot, generate: (isDeterministicNegative || isRetrySnapshotImmutability) ? async () => ({ object: payload, metadata: { provider: "synthetic", model: "deterministic", capability: "founderWeeklyReview", temperature: 0 } }) : syntheticProvider === "kimi" ? generateWithKimi : generateWithLocalOllama }); + } catch (error) { + const message = error instanceof Error ? error.message : "unknown_generation_failure"; + const failureKind = message.includes("absent from the evidence snapshot") ? "citation_validation_failed" : message.includes("must never be presented as customer feedback") ? "source_semantic_validation_failed" : ["ollama_http_error", "ollama_missing_content", "ollama_json_parse_failed", "review_schema_invalid"].find((kind) => message.startsWith(kind)) ?? "unknown_generation_failure"; + const failed = await worker.markGenerationFailed({ companyId: BigInt(event.companyId), runId: event.runId, generationJobId: event.generationJobId, generationClaimId: event.generationClaimId }, { errorCode: "generation_failed", errorMessage: "Synthetic baseline provider generation failed." }); + const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(3n, failed.id); + const unchanged = readBack ? digestJson(readBack.evidenceSnapshot) === digestJson(snapshot) : false; + const citation = failureKind === "citation_validation_failed" ? "failed" : failureKind === "source_semantic_validation_failed" ? "passed" : "not_reached"; + const semantic = failureKind === "source_semantic_validation_failed" ? "failed" : "not_reached"; + if (isRetrySnapshotImmutability) { + if (!readBack || failureKind !== "citation_validation_failed" || readBack.status !== "failed" || readBack.retryCount !== 0 || readBack.errorCode !== "generation_failed" || readBack.reviewPayload || !unchanged) throw new Error("Retry baseline precondition failed."); + const before = readBack; + const retryRequestKey = `synthetic-retry-${randomUUID()}`; + const firstRetry = await dispatchService.retryRunWithDispatch({ actor, runId: before.id, requestKey: retryRequestKey }); + const afterFirstRetry = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, before.id); + let secondRetryBehavior: "idempotent_queued_return" | "lifecycle_conflict"; + let secondRetry: Awaited> | null = null; + try { + secondRetry = await dispatchService.retryRunWithDispatch({ actor, runId: before.id, requestKey: retryRequestKey }); + secondRetryBehavior = "idempotent_queued_return"; + } catch { + secondRetryBehavior = "lifecycle_conflict"; + } + const afterSecondRetry = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, before.id); + const allRuns = await new FounderWeeklyReviewRepository(testDb.db).listByCompany(actor.companyId); + const matchingRuns = allRuns.filter((candidate) => candidate.requestKey === before.requestKey); + const retryDispatches = (await testDb.db.select().from(founderWeeklyReviewDispatches)).filter((candidate) => candidate.runId === before.id && candidate.operationType === "retry"); + const retryClaim = await worker.claimQueuedRun({ companyId: actor.companyId, runId: firstRetry.run.id, generationJobId: firstRetry.dispatch.generationJobId, generationClaimId: firstRetry.dispatch.generationClaimId }); + const afterClaim = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(actor.companyId, before.id); + const sameIdentity = Boolean(afterFirstRetry && afterSecondRetry && afterClaim) && before.id === afterFirstRetry!.id && before.id === afterSecondRetry!.id && before.id === afterClaim!.id; + const snapshotUnchanged = Boolean(afterFirstRetry && afterSecondRetry && afterClaim) && digestJson(before.evidenceSnapshot) === digestJson(afterFirstRetry!.evidenceSnapshot) && digestJson(before.evidenceSnapshot) === digestJson(afterSecondRetry!.evidenceSnapshot) && digestJson(before.evidenceSnapshot) === digestJson(afterClaim!.evidenceSnapshot); + const requestKeyUnchanged = Boolean(afterClaim) && before.requestKey === afterClaim!.requestKey; + const companyUnchanged = Boolean(afterClaim) && before.companyId === afterClaim!.companyId; + const periodUnchanged = Boolean(afterClaim) && before.reportingPeriod.start === afterClaim!.reportingPeriod.start && before.reportingPeriod.end === afterClaim!.reportingPeriod.end; + const schemaVersionsUnchanged = Boolean(afterClaim) && before.evidenceSchemaVersion === afterClaim!.evidenceSchemaVersion && before.reviewSchemaVersion === afterClaim!.reviewSchemaVersion; + const reviewPayloadAbsent = !afterClaim?.reviewPayload; + const retryCounts = [before.retryCount, afterFirstRetry?.retryCount, afterSecondRetry?.retryCount]; + const retryDispatchCreated = firstRetry.dispatch.operationType === "retry" && retryDispatches.length === 1 && (!secondRetry || secondRetry.dispatch.id === firstRetry.dispatch.id); + console.log(JSON.stringify({ fixture: fixtureName, runId: before.id, lifecycle: [run.status, claimed.status, failed.status, firstRetry.run.status, retryClaim.status], retryCounts, sameRow: sameIdentity, matchingRunRowCount: matchingRuns.length, snapshotUnchanged, requestKeyUnchanged, companyUnchanged, periodUnchanged, schemaVersionsUnchanged, reviewPayloadAbsent, duplicateRetryBehavior: secondRetryBehavior, outboxBehavior: retryDispatchCreated ? "creates_one_retry_dispatch_and_reuses_it_for_duplicate_request_key" : "unexpected_retry_dispatch_state", postRetryClaimSucceeded: retryClaim.status === "generating", externalProviderCalled: false })); + if (!sameIdentity || matchingRuns.length !== 1 || afterFirstRetry?.retryCount !== 1 || afterSecondRetry?.retryCount !== 1 || afterClaim?.retryCount !== 1 || !snapshotUnchanged || !requestKeyUnchanged || !companyUnchanged || !periodUnchanged || !schemaVersionsUnchanged || !reviewPayloadAbsent || !retryDispatchCreated || retryClaim.status !== "generating") throw new Error("Retry snapshot immutability invariant failed."); + } else { + console.log(JSON.stringify({ fixture: fixtureName, runId: failed.id, lifecycle: [run.status, claimed.status, failed.status], canonicalSchemaValidation: "passed", citationValidation: citation, sourceSemanticValidation: semantic, firstFailingBoundary: failureKind, finalStatus: readBack?.status, errorCode: readBack?.errorCode, draftPersisted: Boolean(readBack?.reviewPayload), draftRetrievable: Boolean(readBack?.reviewPayload), snapshotUnchanged: unchanged })); + if (!isDeterministicNegative || !readBack || failureKind !== (isNegativeUnknownCitation ? "citation_validation_failed" : "source_semantic_validation_failed") || readBack.status !== "failed" || readBack.errorCode !== "generation_failed" || readBack.reviewPayload || !unchanged) throw new Error("Negative safety invariant failed."); + } + } + if (!isDeterministicNegative && !isRetrySnapshotImmutability) { + const saved = await worker.saveGeneratedDraft({ companyId: BigInt(event.companyId), runId: event.runId, generationJobId: event.generationJobId, generationClaimId: event.generationClaimId }, generated!.reviewPayload, generated!.modelMetadata); + const [persisted] = await testDb.db.select().from(founderWeeklyReviewRuns).where(eq(founderWeeklyReviewRuns.id, saved.id)); + const [persistedDispatch] = await testDb.db.select().from(founderWeeklyReviewDispatches).where(eq(founderWeeklyReviewDispatches.id, dispatch.id)); + if (!persisted || !persistedDispatch) throw new Error("Synthetic baseline persistence verification failed."); + const ids = new Set(snapshot.items.map((item) => item.sourceId)); + for (const section of Object.values(saved.reviewPayload!.sections)) if (section.state === "evidence") for (const item of section.items) for (const id of item.sourceIds) if (!ids.has(id)) throw new Error("Persisted citation is not in snapshot."); + const customerSection = saved.reviewPayload!.sections.whatCustomersSaid; + const readBack = await new FounderWeeklyReviewRepository(testDb.db).getByCompanyAndRunId(3n, saved.id); + if (!readBack?.reviewPayload || readBack.status !== "draft") throw new Error("Validated draft read-back failed."); + const shouldExport = process.env.SYNTHETIC_FWR_EXPORT_REPORT === "1"; + const shouldPrint = process.env.FWR_PRINT_REPORT === "1"; + if (shouldExport || shouldPrint) { + if (!readBack.evidenceSnapshot) throw new Error("Validated draft has no evidence snapshot."); + const renderedMarkdown = renderFounderWeeklyReviewMarkdown(readBack as typeof readBack & { evidenceSnapshot: NonNullable }); + if (shouldPrint) { + console.log("===== FOUNDER WEEKLY REVIEW ====="); + console.log(renderedMarkdown); + console.log("===== END FOUNDER WEEKLY REVIEW ====="); + } + if (shouldExport) { + const directory = resolve(process.cwd(), process.env.SYNTHETIC_FWR_EXPORT_DIR ?? ".artifacts/founder-weekly-review"); + await mkdir(directory, { recursive: true }); + const fileId = saved.id.replace(/[^A-Za-z0-9_-]/g, "_"); + const envelope = { runId: readBack.id, status: readBack.status, provider: readBack.modelMetadata?.provider, model: readBack.modelMetadata?.model, periodStart: readBack.reportingPeriod.start, periodEnd: readBack.reportingPeriod.end, review: readBack.reviewPayload }; + const markdownPath = resolve(directory, `${fileId}.md`); const jsonPath = resolve(directory, `${fileId}.json`); + await writeAtomically(markdownPath, renderedMarkdown); await writeAtomically(jsonPath, JSON.stringify(envelope, null, 2)); + console.log(JSON.stringify({ runId: readBack.id, status: readBack.status, markdownPath, jsonPath, filesWritten: true })); + } + } + console.log(JSON.stringify({ label: "Synthetic-evidence integration baseline", fixture: fixtureName, runId: saved.id, lifecycle: [run.status, claimed.status, saved.status], event, provider: generated!.modelMetadata.provider, model: generated!.modelMetadata.model, snapshotUnchanged: digestJson(persisted.evidenceSnapshot) === digestJson(snapshot), outbox: { status: persistedDispatch.status, hasEvidence: false }, customerSection: "state" in customerSection ? customerSection.state : "legacy", draftReturnedByRepository: saved.status === "draft" })); + } +} finally { await testDb.close(); } diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts new file mode 100644 index 000000000..3b4ba8dba --- /dev/null +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/retry/route.ts @@ -0,0 +1,29 @@ +import { NextResponse } from "next/server"; +import { auth } from "@clerk/nextjs/server"; +import { z } from "zod"; +import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { retryRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; +import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; +import { inngest } from "~/server/inngest/client"; +import { founderWeeklyReviewRetries, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; +const RetrySchema = z.object({ requestKey: z.string().min(1).max(128) }); +export interface FounderWeeklyReviewRetryRouteDependencies { + actorResolver: Pick; + retryRunWithDispatch: typeof retryRunWithDispatch; + sendDispatchRequested: () => Promise; + incrementRetry: () => void; +} +export function createFounderWeeklyReviewRetryPostHandler(deps: FounderWeeklyReviewRetryRouteDependencies) { + return async function POST(request: Request, { params }: { params: Promise<{ runId: string }> }) { + const { userId } = await auth(); if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + const parsed = RetrySchema.safeParse(await request.json().catch(() => null)); if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + try { const actor = await deps.actorResolver.resolve(userId); const { runId } = await params; + logFounderWeeklyReview({ runId, companyId: actor.companyId.toString(), stage: "retry_requested", status: "failed" }); + const { run, transitionApplied } = await deps.retryRunWithDispatch({ actor, runId, requestKey: parsed.data.requestKey }); + if (transitionApplied) { deps.incrementRetry(); logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "retry_queued", status: run.status, retryCount: run.retryCount }); } + await deps.sendDispatchRequested(); return NextResponse.json({ run: safeRun(run) }, { status: 202 }); + } catch (error) { return safeFounderWeeklyReviewError(error); } + }; +} +export const POST = createFounderWeeklyReviewRetryPostHandler({ actorResolver: productionFounderWeeklyReviewActorResolver, retryRunWithDispatch, sendDispatchRequested: () => inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }), incrementRetry: () => founderWeeklyReviewRetries.inc() }); diff --git a/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts new file mode 100644 index 000000000..6d970ef71 --- /dev/null +++ b/apps/web/src/app/api/founder-weekly-reviews/[runId]/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; +import { auth } from "@clerk/nextjs/server"; +import { FounderWeeklyReviewUserService } from "@launchstack/features/founder-weekly-review"; +import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; + +export interface FounderWeeklyReviewGetRouteDependencies { + actorResolver: Pick; + getRun: (actor: Parameters[0], runId: string) => ReturnType; +} + +export function createFounderWeeklyReviewGetHandler(deps: FounderWeeklyReviewGetRouteDependencies) { + return async function GET(_request: Request, { params }: { params: Promise<{ runId: string }> }) { + const { userId } = await auth(); + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + try { + const actor = await deps.actorResolver.resolve(userId); + const { runId } = await params; + const run = await deps.getRun(actor, runId); + return NextResponse.json({ run: safeRun(run) }); + } catch (error) { return safeFounderWeeklyReviewError(error); } + }; +} +export const GET = createFounderWeeklyReviewGetHandler({ + actorResolver: productionFounderWeeklyReviewActorResolver, + getRun: (actor, runId) => new FounderWeeklyReviewUserService().getRun(actor, runId), +}); diff --git a/apps/web/src/app/api/founder-weekly-reviews/route.ts b/apps/web/src/app/api/founder-weekly-reviews/route.ts new file mode 100644 index 000000000..580929f5e --- /dev/null +++ b/apps/web/src/app/api/founder-weekly-reviews/route.ts @@ -0,0 +1,56 @@ +import { NextResponse } from "next/server"; +import { auth } from "@clerk/nextjs/server"; +import { z } from "zod"; +import { FounderWeeklyReviewRepository } from "@launchstack/features/founder-weekly-review"; +import type { FounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; +import type { FounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { productionFounderWeeklyReviewActorResolver } from "~/server/founder-weekly-review/actor-resolver"; +import { createRunWithDispatch } from "~/server/founder-weekly-review/dispatch-service"; +import { safeFounderWeeklyReviewError, safeRun } from "~/server/founder-weekly-review/http"; +import { inngest } from "~/server/inngest/client"; +import { founderWeeklyReviewRunsCreated, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; + +const CreateSchema = z.object({ requestKey: z.string().min(1).max(128), reportingPeriod: z.object({ start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), end: z.string().regex(/^\d{4}-\d{2}-\d{2}$/) }), workspaceTimezone: z.string().min(1).max(128), founderContext: z.string().max(4000).optional() }); + +export interface FounderWeeklyReviewRouteDependencies { + actorResolver: Pick; + /** Retained only for test construction compatibility; collection is worker-owned. */ + evidenceCollector?: FounderWeeklyReviewEvidenceCollector; + repository: Pick; + createRunWithDispatch: typeof createRunWithDispatch; + sendDispatchRequested: () => Promise; + recordRunCreated: () => void; +} + +export function createFounderWeeklyReviewPostHandler(deps: FounderWeeklyReviewRouteDependencies) { + return async function POST(request: Request) { + const { userId } = await auth(); + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + try { + // Authorization precedes all workflow persistence: invalid callers must + // never create a company-scoped collection job. + const actor = await deps.actorResolver.resolve(userId); + const parsed = CreateSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + const existing = await deps.repository.getByCompanyAndRequestKey(actor.companyId, parsed.data.requestKey); + if (existing) return NextResponse.json({ run: safeRun(existing) }, { status: 202 }); + const { run, created } = await deps.createRunWithDispatch({ actor, requestKey: parsed.data.requestKey, reportingPeriod: parsed.data.reportingPeriod, collectionInput: { workspaceTimezone: parsed.data.workspaceTimezone, founderContext: parsed.data.founderContext, actorExternalUserId: actor.externalUserId } }); + if (created) deps.recordRunCreated(); + logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "run_created", status: run.status, retryCount: run.retryCount }); + logFounderWeeklyReview({ runId: run.id, companyId: run.companyId.toString(), stage: "dispatch_created", status: run.status }); + await deps.sendDispatchRequested(); + return NextResponse.json({ run: safeRun(run) }, { status: 202 }); + } catch (error) { + return safeFounderWeeklyReviewError(error); + } + }; +} + +const productionDependencies: FounderWeeklyReviewRouteDependencies = { + actorResolver: productionFounderWeeklyReviewActorResolver, + repository: { getByCompanyAndRequestKey: (companyId, requestKey) => new FounderWeeklyReviewRepository().getByCompanyAndRequestKey(companyId, requestKey) }, + createRunWithDispatch, + sendDispatchRequested: () => inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }), + recordRunCreated: () => founderWeeklyReviewRunsCreated.inc(), +}; +export const POST = createFounderWeeklyReviewPostHandler(productionDependencies); diff --git a/apps/web/src/app/api/inngest/route.ts b/apps/web/src/app/api/inngest/route.ts index d51fb96ed..2ba7707dd 100644 --- a/apps/web/src/app/api/inngest/route.ts +++ b/apps/web/src/app/api/inngest/route.ts @@ -18,6 +18,7 @@ import { reindexCompanyEmbeddingsJob } from "~/server/inngest/functions/reindexC import { modifyDocument } from "~/server/inngest/functions/modifyDocument"; import { crawlWebsite } from "~/server/inngest/functions/crawlWebsite"; import { rehydrateNoteAnchorsJob } from "~/server/inngest/functions/rehydrateNoteAnchors"; +import { founderWeeklyReviewDispatcher, founderWeeklyReviewDispatchReconciler, founderWeeklyReviewGenerationJob } from "~/server/inngest/functions/founderWeeklyReview"; // Register all Inngest functions const handler = serve({ @@ -32,6 +33,9 @@ const handler = serve({ modifyDocument, crawlWebsite, rehydrateNoteAnchorsJob, + founderWeeklyReviewDispatcher, + founderWeeklyReviewDispatchReconciler, + founderWeeklyReviewGenerationJob, ], }); diff --git a/apps/web/src/lib/llm/config.ts b/apps/web/src/lib/llm/config.ts index 2f17b84c8..21a653a99 100644 --- a/apps/web/src/lib/llm/config.ts +++ b/apps/web/src/lib/llm/config.ts @@ -70,6 +70,13 @@ const HARDCODED_DEFAULTS: LlmConfig = { // whose whole point is "cheap and fast." The original project hardcoded // gpt-5-nano in company-metadata/extractor.ts; do not restore it here. smallExtraction: { + kimi: { model: "kimi-k2.6", temperature: 0 }, + openai: { model: "gpt-4o-mini", temperature: 0 }, + anthropic: { model: "claude-3-5-haiku-latest", temperature: 0 }, + google: { model: "gemini-2.0-flash", temperature: 0 }, + ollama: { model: "llama3.2:3b", temperature: 0 }, + }, + founderWeeklyReview: { openai: { model: "gpt-4o-mini", temperature: 0 }, anthropic: { model: "claude-3-5-haiku-latest", temperature: 0 }, google: { model: "gemini-2.0-flash", temperature: 0 }, diff --git a/apps/web/src/lib/llm/generate.ts b/apps/web/src/lib/llm/generate.ts index be91ded83..0c07c529a 100644 --- a/apps/web/src/lib/llm/generate.ts +++ b/apps/web/src/lib/llm/generate.ts @@ -11,11 +11,14 @@ * as call sites need them. */ -import { generateObject } from "ai"; +import { generateObject, zodSchema } from "ai"; import type { ZodType } from "zod"; import { resolveModel } from "./providers"; -import type { GenerateStructuredInput } from "./types"; +import type { + GenerateStructuredInput, + StructuredGenerationResult, +} from "./types"; /** * Run a structured JSON-output LLM call against whichever provider is @@ -40,6 +43,18 @@ import type { GenerateStructuredInput } from "./types"; export async function generateStructured( input: GenerateStructuredInput, ): Promise> { + const result = await generateStructuredWithMetadata(input); + return result.object; +} + +/** + * Structured generation plus the single resolved model's reproducibility + * metadata. This is a companion API; generateStructured remains source and + * behavior compatible for existing callers. + */ +export async function generateStructuredWithMetadata( + input: GenerateStructuredInput, +): Promise>> { const resolved = resolveModel(input.capability, input.forceProvider); // Diagnostic logging: capture the chosen provider/model, prompt size, and @@ -54,35 +69,76 @@ export async function generateStructured( `provider=${resolved.provider} model=${resolved.modelId} ` + `prompt=${promptChars} chars`, ); + if (input.capability === "founderWeeklyReview") { + console.log(`[llm] FWR generation phase=${input.generationPhase ?? "initial"} provider=${resolved.provider} model=${resolved.modelId} protocol=${resolved.structuredOutputMode === "json_object" ? "chat-completions" : "responses"}`); + } try { - const result = await generateObject({ + const common = { model: resolved.model, - temperature: resolved.temperature, - schema: input.schema, - schemaName: input.schemaName, - system: input.system, + ...(resolved.temperature === undefined ? {} : { temperature: resolved.temperature }), + ...(input.maxOutputTokens !== undefined + ? { maxOutputTokens: input.maxOutputTokens } + : input.capability === "founderWeeklyReview" ? { maxOutputTokens: 2400 } : {}), + ...((input.timeoutMs !== undefined || resolved.structuredOutputMode === "json_object") + ? { abortSignal: AbortSignal.timeout(input.timeoutMs ?? 90_000) } + : {}), prompt: input.prompt, - }); + }; + // Moonshot/Kimi supports Chat Completions JSON-object mode, not the + // Responses API JSON-schema protocol. Keep schema validation local and + // deterministic after parsing the provider's JSON object. + const result = resolved.structuredOutputMode === "json_object" + ? await generateObject({ + ...common, + output: "no-schema", + system: `${input.system ?? ""}\nReturn one JSON object only. Required structural schema: ${JSON.stringify(zodSchema(input.schema).jsonSchema)}`, + }) + : await generateObject({ + ...common, + schema: input.schema, + schemaName: input.schemaName, + system: input.system, + }); const elapsed = Date.now() - startedAt; console.log( `[llm] generateStructured ok capability=${input.capability} ` + - `provider=${resolved.provider} model=${resolved.modelId} ` + - `${elapsed}ms`, + `provider=${resolved.provider} model=${resolved.modelId} ` + + `phase=${input.generationPhase ?? "initial"} ${elapsed}ms`, ); // Cast is safe: `generateObject` returns `{ object: z.infer }` // when given a Zod schema. The `ZodType` generic constraint is a // pragmatic choice — it trades a small amount of type precision for // simpler call-site ergonomics. - return result.object as ReturnType; + const responseResult = result as unknown as { + finishReason?: string; + usage?: Record; + response?: { id?: string }; + }; + return { + object: resolved.structuredOutputMode === "json_object" + ? input.schema.parse(result.object) as ReturnType + : result.object as ReturnType, + metadata: { + provider: resolved.provider, + model: resolved.modelId, + capability: input.capability, + ...(resolved.temperature === undefined ? {} : { temperature: resolved.temperature }), + ...(responseResult.finishReason ? { finishReason: responseResult.finishReason } : {}), + ...(responseResult.usage ? { usage: responseResult.usage } : {}), + ...(responseResult.response?.id + ? { providerRequestId: responseResult.response.id } + : {}), + }, + }; } catch (err) { const elapsed = Date.now() - startedAt; console.error( `[llm] generateStructured FAIL capability=${input.capability} ` + - `provider=${resolved.provider} model=${resolved.modelId} ` + - `${elapsed}ms err=${err instanceof Error ? err.message : String(err)}`, + `provider=${resolved.provider} model=${resolved.modelId} ` + + `phase=${input.generationPhase ?? "initial"} ${elapsed}ms err=${err instanceof Error ? err.message : String(err)}`, ); throw err; } diff --git a/apps/web/src/lib/llm/index.ts b/apps/web/src/lib/llm/index.ts index 1ecad9804..45a42d25a 100644 --- a/apps/web/src/lib/llm/index.ts +++ b/apps/web/src/lib/llm/index.ts @@ -11,7 +11,7 @@ * to this library or file an issue explaining why it can't be abstracted. */ -export { generateStructured } from "./generate"; +export { generateStructured, generateStructuredWithMetadata } from "./generate"; export { getAvailableProviders, resolveModel, @@ -29,4 +29,6 @@ export { type LlmConfig, type CapabilityModelConfig, type GenerateStructuredInput, + type StructuredGenerationMetadata, + type StructuredGenerationResult, } from "./types"; diff --git a/apps/web/src/lib/llm/providers.ts b/apps/web/src/lib/llm/providers.ts index b17c870d6..09a589830 100644 --- a/apps/web/src/lib/llm/providers.ts +++ b/apps/web/src/lib/llm/providers.ts @@ -105,6 +105,11 @@ export function getAvailableProviders(): ProviderAvailability[] { /** Reset cached availability. Test-only. */ export function __resetProviderCacheForTests(): void { cachedAvailability = null; + openaiInstance = null; + anthropicInstance = null; + googleInstance = null; + ollamaInstance = null; + kimiInstance = null; } /** @@ -123,6 +128,10 @@ function checkCredentials( return process.env.OPENAI_API_KEY ? { ok: true } : { ok: false, reason: "OPENAI_API_KEY not set" }; + case "kimi": + return process.env.MOONSHOT_API_KEY + ? { ok: true } + : { ok: false, reason: "MOONSHOT_API_KEY not set" }; case "anthropic": return process.env.ANTHROPIC_API_KEY ? { ok: true } @@ -149,6 +158,7 @@ let openaiInstance: OpenAIProvider | null = null; let anthropicInstance: AnthropicProvider | null = null; let googleInstance: GoogleGenerativeAIProvider | null = null; let ollamaInstance: OpenAIProvider | null = null; +let kimiInstance: OpenAIProvider | null = null; function getOpenAIProvider(): OpenAIProvider { openaiInstance ??= createOpenAI({ @@ -157,6 +167,36 @@ function getOpenAIProvider(): OpenAIProvider { return openaiInstance; } +/** + * Moonshot's OpenAI-compatible API accepts a top-level `thinking` field that + * the OpenAI adapter's typed provider options intentionally do not expose. + * Keep this narrow to Kimi Chat Completions; OpenAI requests never use it. + */ +async function fetchKimiChatCompletions( + input: Parameters[0], + init?: Parameters[1], +): Promise { + const url = input instanceof Request ? input.url : String(input); + if (!url.endsWith("/chat/completions") || typeof init?.body !== "string") { + return fetch(input, init); + } + + const body = JSON.parse(init.body) as Record; + return fetch(input, { + ...init, + body: JSON.stringify({ ...body, thinking: { type: "disabled" } }), + }); +} + +function getKimiProvider(): OpenAIProvider { + kimiInstance ??= createOpenAI({ + apiKey: process.env.MOONSHOT_API_KEY, + baseURL: (process.env.MOONSHOT_BASE_URL ?? "https://api.moonshot.ai/v1").replace(/\/+$/, ""), + fetch: fetchKimiChatCompletions, + }); + return kimiInstance; +} + function getAnthropicProvider(): AnthropicProvider { anthropicInstance ??= createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY, @@ -238,8 +278,32 @@ export interface ResolvedModel { provider: Provider; modelId: string; model: LanguageModel; - /** Carries through the temperature from config so `generate.ts` can pass it. */ - temperature: number; + /** + * Provider-supported sampling option. Undefined means the request must omit + * temperature rather than applying a cross-provider default. + * TODO: add provider-specific request capabilities as more providers need them. + */ + temperature?: number; + structuredOutputMode?: "json_object"; +} + +export type FounderWeeklyReviewGenerationProvider = "openai" | "kimi"; + +function resolveFounderWeeklyReviewModel(): ResolvedModel { + const selected = process.env.FWR_GENERATION_PROVIDER ?? "openai"; + if (selected !== "openai" && selected !== "kimi") { + throw new Error(`FWR_GENERATION_PROVIDER must be "openai" or "kimi"; received "${selected}".`); + } + const config = getLlmConfig(); + if (selected === "openai") { + if (!process.env.OPENAI_API_KEY) throw new Error("FWR_GENERATION_PROVIDER=openai requires OPENAI_API_KEY."); + const modelId = process.env.OPENAI_MODEL_ID ?? config.capabilities.founderWeeklyReview.openai?.model; + if (!modelId) throw new Error("FWR_GENERATION_PROVIDER=openai requires OPENAI_MODEL_ID or a configured founderWeeklyReview OpenAI model."); + return { provider: "openai", modelId, model: getOpenAIProvider()(modelId), temperature: config.capabilities.founderWeeklyReview.openai?.temperature ?? 0 }; + } + if (!process.env.MOONSHOT_API_KEY) throw new Error("FWR_GENERATION_PROVIDER=kimi requires MOONSHOT_API_KEY."); + const modelId = process.env.KIMI_MODEL_ID ?? "kimi-k2.6"; + return { provider: "kimi", modelId, model: getKimiProvider().chat(modelId), structuredOutputMode: "json_object" }; } /** @@ -258,6 +322,7 @@ export function resolveModel( capability: Capability, forceProvider?: Provider, ): ResolvedModel { + if (capability === "founderWeeklyReview" && !forceProvider) return resolveFounderWeeklyReviewModel(); const config = getLlmConfig(); const availability = getAvailableProviders(); const availabilityByProvider = new Map( @@ -324,6 +389,8 @@ function instantiate( model: getOpenAIProvider()(modelConfig.model), temperature, }; + case "kimi": + return { provider, modelId: modelConfig.model, model: getKimiProvider().chat(modelConfig.model), structuredOutputMode: "json_object" }; case "anthropic": return { provider, diff --git a/apps/web/src/lib/llm/types.ts b/apps/web/src/lib/llm/types.ts index 62f3a4085..1b048c548 100644 --- a/apps/web/src/lib/llm/types.ts +++ b/apps/web/src/lib/llm/types.ts @@ -22,7 +22,7 @@ * Must support JSON schema output. Does NOT need vision, long context, * or high-quality free-form generation. */ -export const CAPABILITIES = ["smallExtraction"] as const; +export const CAPABILITIES = ["smallExtraction", "founderWeeklyReview"] as const; export type Capability = (typeof CAPABILITIES)[number]; @@ -35,7 +35,7 @@ export type Capability = (typeof CAPABILITIES)[number]; * - `ollama`: local inference via Ollama's OpenAI-compatible endpoint. * Requires `OLLAMA_BASE_URL` to be set. Free but quality varies by model. */ -export const PROVIDERS = ["openai", "anthropic", "google", "ollama"] as const; +export const PROVIDERS = ["openai", "kimi", "anthropic", "google", "ollama"] as const; export type Provider = (typeof PROVIDERS)[number]; @@ -104,4 +104,26 @@ export interface GenerateStructuredInput { * / tool name. Purely cosmetic but helps with provider-side logging. */ schemaName?: string; + /** Bounded FWR-only attempt label for safe operational logging. */ + generationPhase?: "initial" | "semantic-repair"; + /** Optional caller-owned local timeout. Provider defaults remain unchanged when omitted. */ + timeoutMs?: number; + /** Optional caller-owned output ceiling. Provider defaults remain unchanged when omitted. */ + maxOutputTokens?: number; +} + +/** Resolved model and provider response details that are safe to persist for replay. */ +export interface StructuredGenerationMetadata { + provider: Provider; + model: string; + capability: Capability; + temperature?: number; + finishReason?: string; + usage?: Record; + providerRequestId?: string; +} + +export interface StructuredGenerationResult { + object: T; + metadata: StructuredGenerationMetadata; } diff --git a/apps/web/src/server/founder-weekly-review/actor-resolver.ts b/apps/web/src/server/founder-weekly-review/actor-resolver.ts new file mode 100644 index 000000000..e8a2ef9be --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/actor-resolver.ts @@ -0,0 +1,40 @@ +import { and, eq } from "drizzle-orm"; +import { users, userCompanyMemberships } from "@launchstack/core/db/schema"; +import { db } from "~/server/db"; +import { getActiveCompanyId } from "~/lib/active-workspace"; + +const GENERATION_ROLES = new Set(["owner", "admin", "editor"]); + +export class FounderWeeklyReviewAuthorizationError extends Error { + readonly code = "forbidden"; +} + +export interface FounderWeeklyReviewActor { + externalUserId: string; + internalUserId: bigint; + companyId: bigint; + role: "owner" | "admin" | "editor"; +} + +/** + * This is intentionally stricter than the legacy active-workspace context: + * a legacy users.role is never a substitute for an actual membership row. + */ +export class FounderWeeklyReviewActorResolver { + async resolve(externalUserId: string): Promise { + const companyId = await getActiveCompanyId(externalUserId); + const [user] = await db.select({ id: users.id }).from(users) + .where(eq(users.userId, externalUserId)).limit(1); + if (!user) throw new FounderWeeklyReviewAuthorizationError(); + const [membership] = await db.select({ role: userCompanyMemberships.role }) + .from(userCompanyMemberships) + .where(and(eq(userCompanyMemberships.userId, BigInt(user.id)), eq(userCompanyMemberships.companyId, companyId))) + .limit(1); + if (!membership || !GENERATION_ROLES.has(membership.role)) { + throw new FounderWeeklyReviewAuthorizationError(); + } + return { externalUserId, internalUserId: BigInt(user.id), companyId, role: membership.role as FounderWeeklyReviewActor["role"] }; + } +} + +export const productionFounderWeeklyReviewActorResolver = new FounderWeeklyReviewActorResolver(); diff --git a/apps/web/src/server/founder-weekly-review/dispatch-service.ts b/apps/web/src/server/founder-weekly-review/dispatch-service.ts new file mode 100644 index 000000000..0fa5edcb2 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/dispatch-service.ts @@ -0,0 +1,141 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, lte, or, sql } from "drizzle-orm"; +import { db } from "~/server/db"; +import type { DbClient } from "@launchstack/core/db"; +import { + founderWeeklyReviewDispatches, + type FounderWeeklyReviewDispatchRow, +} from "@launchstack/core/db/schema"; +import { + FounderWeeklyReviewRepository, + FounderWeeklyReviewUserService, + type FounderWeeklyReviewRunRecord, + type FounderWeeklyReviewUserActor, + type ReportingPeriod, + type FounderWeeklyReviewEvidenceSnapshot, +} from "@launchstack/features/founder-weekly-review"; + +export type FounderWeeklyReviewDispatch = Pick; + +function toDispatch(row: FounderWeeklyReviewDispatchRow): FounderWeeklyReviewDispatch { + return row; +} + +function identifiers(runId: string, operationType: "create" | "retry", operationKey: string) { + const suffix = `${operationType}:${operationKey}`; + return { + id: `fwrd_${randomUUID()}`, + eventId: `fwr-event:${runId}:${suffix}`.slice(0, 128), + generationJobId: `fwr-job:${runId}:${suffix}`.slice(0, 128), + generationClaimId: `fwr-claim:${runId}:${suffix}`.slice(0, 128), + }; +} + +export type FounderWeeklyReviewTransactionClient = Pick; +export interface CreateFounderWeeklyReviewDispatchInput { + run: FounderWeeklyReviewRunRecord; + operationType: "create" | "retry"; + operationKey: string; +} +export interface FounderWeeklyReviewDispatchServiceDependencies { + createDispatch?: (transaction: FounderWeeklyReviewTransactionClient, input: CreateFounderWeeklyReviewDispatchInput) => Promise; +} + +async function createDispatch( + tx: Pick, + run: FounderWeeklyReviewRunRecord, + operationType: "create" | "retry", + operationKey: string, +): Promise { + const ids = identifiers(run.id, operationType, operationKey); + const [inserted] = await tx.insert(founderWeeklyReviewDispatches).values({ + id: ids.id, + companyId: run.companyId, + runId: run.id, + operationType, + operationKey, + eventId: ids.eventId, + generationJobId: ids.generationJobId, + generationClaimId: ids.generationClaimId, + status: "pending", + }).onConflictDoNothing().returning(); + if (inserted) return toDispatch(inserted); + const [existing] = await tx.select().from(founderWeeklyReviewDispatches).where(and( + eq(founderWeeklyReviewDispatches.runId, run.id), + eq(founderWeeklyReviewDispatches.operationType, operationType), + eq(founderWeeklyReviewDispatches.operationKey, operationKey), + )).limit(1); + if (!existing) throw new Error("Founder weekly review dispatch was not created"); + return toDispatch(existing); +} + +export type CreateRunWithDispatchInput = { + actor: FounderWeeklyReviewUserActor; + requestKey: string; + reportingPeriod: ReportingPeriod; + evidenceSnapshot?: FounderWeeklyReviewEvidenceSnapshot; + collectionInput?: import("@launchstack/features/founder-weekly-review").FounderWeeklyReviewCollectionInput; +}; +export type RetryRunWithDispatchInput = { actor: FounderWeeklyReviewUserActor; runId: string; requestKey: string }; +export type CreateRunWithDispatchResult = { run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch; created: boolean }; +export type RetryRunWithDispatchResult = { run: FounderWeeklyReviewRunRecord; dispatch: FounderWeeklyReviewDispatch; transitionApplied: boolean }; + +export function createFounderWeeklyReviewDispatchService(database: Pick, dependencies: FounderWeeklyReviewDispatchServiceDependencies = {}) { + const writeDispatch = dependencies.createDispatch ?? ((transaction, input) => createDispatch(transaction, input.run, input.operationType, input.operationKey)); + const createRunWithDispatch = async (input: CreateRunWithDispatchInput): Promise => database.transaction(async (tx) => { + const service = new FounderWeeklyReviewUserService(new FounderWeeklyReviewRepository(tx as unknown as DbClient)); + const { run, created } = await service.createOrGetRunWithMetadata(input.actor, input); + const dispatch = await writeDispatch(tx, { run, operationType: "create", operationKey: input.requestKey }); + return { run, dispatch, created }; + }); + const retryRunWithDispatch = async (input: RetryRunWithDispatchInput): Promise => database.transaction(async (tx) => { + const service = new FounderWeeklyReviewUserService(new FounderWeeklyReviewRepository(tx as unknown as DbClient)); + const { run, transitionApplied } = await service.retryFailedRunWithMetadata(input.actor, input.runId, input.requestKey); + const dispatch = await writeDispatch(tx, { run, operationType: "retry", operationKey: input.requestKey }); + return { run, dispatch, transitionApplied }; + }); + return { createRunWithDispatch, retryRunWithDispatch }; +} + +const productionDispatchService = createFounderWeeklyReviewDispatchService(db); +export const createRunWithDispatch = productionDispatchService.createRunWithDispatch; +export const retryRunWithDispatch = productionDispatchService.retryRunWithDispatch; + +export async function claimPendingDispatches(limit = 20): Promise { + const now = new Date(); + const staleDispatchingBefore = new Date(now.getTime() - 5 * 60 * 1000); + const candidates = await db.select({ id: founderWeeklyReviewDispatches.id }) + .from(founderWeeklyReviewDispatches).where(and( + or( + and(inArray(founderWeeklyReviewDispatches.status, ["pending", "failed"]), lte(founderWeeklyReviewDispatches.availableAt, now)), + and(eq(founderWeeklyReviewDispatches.status, "dispatching"), lte(founderWeeklyReviewDispatches.updatedAt, staleDispatchingBefore)), + ), + )).limit(limit); + const claimed: FounderWeeklyReviewDispatch[] = []; + for (const candidate of candidates) { + const [row] = await db.update(founderWeeklyReviewDispatches).set({ + status: "dispatching", attemptCount: sql`${founderWeeklyReviewDispatches.attemptCount} + 1`, + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewDispatches.id, candidate.id), + or(inArray(founderWeeklyReviewDispatches.status, ["pending", "failed"]), and(eq(founderWeeklyReviewDispatches.status, "dispatching"), lte(founderWeeklyReviewDispatches.updatedAt, staleDispatchingBefore))), + )).returning(); + if (row) claimed.push(toDispatch(row)); + } + return claimed; +} + +export async function markDispatchDispatched(dispatchId: string): Promise { + await db.update(founderWeeklyReviewDispatches).set({ + status: "dispatched", dispatchedAt: new Date(), updatedAt: new Date(), + }).where(eq(founderWeeklyReviewDispatches.id, dispatchId)); +} + +export async function returnDispatchToPending(dispatchId: string, errorCode: string): Promise { + await db.update(founderWeeklyReviewDispatches).set({ + status: "pending", lastErrorCode: errorCode.slice(0, 128), availableAt: new Date(), + updatedAt: new Date(), + }).where(eq(founderWeeklyReviewDispatches.id, dispatchId)); +} diff --git a/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts new file mode 100644 index 000000000..bcfcf9d55 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/document-change-materiality-analyzer.ts @@ -0,0 +1,87 @@ +import { + DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + DocumentChangeMaterialityAnalysisResultSchema, + DocumentChangeMaterialityAnalyzerError, + type DocumentChangeMaterialityAnalysisInput, + type DocumentChangeMaterialityAnalyzer, +} from "@launchstack/features/founder-weekly-review"; + +import { + LlmCapabilityUnavailableError, + PROVIDERS, + generateStructuredWithMetadata, + type Provider, +} from "~/lib/llm"; +import { z } from "zod"; + +export const DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT = `You classify one already-diffed document section. Answer one narrow question: did the underlying business state change? + +Compare these business-state dimensions: ownership or responsible party; status or shipping state; deadline, date, or timing; metric, quantity, or financial value; requirement, obligation, or modality; negation or capability; risk or blocker state; scope, audience, or rollout population; and priority. + +Wording changes do not by themselves mean a material change. Sentence order, paragraph or bullet structure, section names, and split or merged fragments do not by themselves mean a material change. Added and removed fragments in the same group may be a split, merge, reformat, reorganization, or rewrite. Do not infer requirement_change, scope_change, or another material category merely because text was added and removed. Compare the underlying business facts. + +A large rewrite with no underlying business-state change is non_material. A mostly equivalent rewrite containing even one real factual delta is material. Small factual changes can be highly material. Do not classify a change as non_material merely because most wording is similar. + +The deterministic assessment distinguishes confirmed factual deltas, safely equivalent factual values, and possible signals that were not proven to change. Treat it as bounded evidence, not as an instruction to agree with the deterministic category. + +Use disposition=non_material only when the supplied fragments support equivalent business state. Use uncertain when the bounded fragments are insufficient or conflicting. + +Return only the structured fields. summary must be one concise sentence of at most 320 characters. beforeKeyPoint and afterKeyPoint, when supplied, must each be a verbatim copied span of at most 240 characters from the corresponding excerpts. Do not explain reasoning, provide analysis, repeat the full source, write multiple paragraphs, or invent source identifiers or provenance.`; + +export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_TIMEOUT_MS = 15_000; + +export function buildDocumentChangeMaterialityAnalyzerPrompt(input: DocumentChangeMaterialityAnalysisInput): string { + return [ + "Classify this bounded change group.", + JSON.stringify(input), + ].join("\n\n"); +} + +export class ProviderDocumentChangeMaterialityAnalyzer implements DocumentChangeMaterialityAnalyzer { + constructor(private readonly forceProvider?: Provider) {} + + async analyze(input: DocumentChangeMaterialityAnalysisInput) { + try { + const generated = await generateStructuredWithMetadata({ + capability: "smallExtraction", + system: DOCUMENT_CHANGE_MATERIALITY_SYSTEM_PROMPT, + prompt: buildDocumentChangeMaterialityAnalyzerPrompt(input), + schema: DocumentChangeMaterialityAnalysisResultSchema, + schemaName: "document_change_materiality", + ...(this.forceProvider ? { forceProvider: this.forceProvider } : {}), + timeoutMs: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_TIMEOUT_MS, + maxOutputTokens: 512, + }); + return { + result: generated.object, + metadata: { + provider: generated.metadata.provider, + model: generated.metadata.model, + promptVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + }, + }; + } catch (error) { + if (error instanceof z.ZodError || error instanceof Error && /NoObjectGenerated|JSONParse|schema validation/i.test(`${error.name} ${error.message}`)) { + throw new DocumentChangeMaterialityAnalyzerError("invalid", "Document-change materiality output failed structured validation."); + } + if (error instanceof LlmCapabilityUnavailableError) { + throw new DocumentChangeMaterialityAnalyzerError("unavailable", error.message); + } + if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError" || /timed?\s*out/i.test(error.message))) { + throw new DocumentChangeMaterialityAnalyzerError("timeout", "Document-change materiality analysis timed out."); + } + throw new DocumentChangeMaterialityAnalyzerError("unavailable", "Document-change materiality analysis was unavailable."); + } + } +} + +/** Production collection opts in explicitly; credentials alone never trigger analyzer calls. */ +export function createConfiguredDocumentChangeMaterialityAnalyzer(): DocumentChangeMaterialityAnalyzer | undefined { + if (process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_ENABLED !== "true") return undefined; + const configured = process.env.FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER; + if (!configured) return new ProviderDocumentChangeMaterialityAnalyzer(); + if (!(PROVIDERS as readonly string[]).includes(configured)) { + throw new Error(`FWR_DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROVIDER must be one of: ${PROVIDERS.join(", ")}.`); + } + return new ProviderDocumentChangeMaterialityAnalyzer(configured as Provider); +} diff --git a/apps/web/src/server/founder-weekly-review/document-version-chunks.ts b/apps/web/src/server/founder-weekly-review/document-version-chunks.ts new file mode 100644 index 000000000..08b0c6d04 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/document-version-chunks.ts @@ -0,0 +1,40 @@ +import { and, asc, eq, gte, lt } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { document, documentContextChunks, documentStructure, documentVersions } from "@launchstack/core/db/schema"; +import type { DocumentVersionForComparison, VersionChunk } from "@launchstack/features/founder-weekly-review"; + +export type VersionChunkLoad = + | { state: "complete"; chunks: VersionChunk[]; warnings: string[] } + | { state: "partial"; chunks: VersionChunk[]; warnings: string[] } + | { state: "missing"; chunks: []; warnings: string[] }; + +/** App-side ownership-checked, explicit-version data access for weekly review diffs. */ +export class FounderWeeklyReviewDocumentVersionStore { + constructor(private readonly db: DbClient = getDb()) {} + + async listVersionsBeforePeriodEnd(companyId: bigint, endExclusive: Date): Promise { + return this.db.select({ documentId: documentVersions.documentId, documentTitle: document.title, documentCategory: document.category, + versionId: documentVersions.id, versionNumber: documentVersions.versionNumber, createdAt: documentVersions.createdAt, changelog: documentVersions.changelog }) + .from(documentVersions).innerJoin(document, eq(document.id, documentVersions.documentId)) + .where(and(eq(document.companyId, companyId), lt(documentVersions.createdAt, endExclusive))) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.versionNumber), asc(documentVersions.id)); + } + + async getDocumentChunksForVersion(input: { companyId: bigint; documentId: bigint; versionId: number }): Promise { + const [ownedVersion] = await this.db.select({ id: documentVersions.id }).from(documentVersions) + .innerJoin(document, eq(document.id, documentVersions.documentId)) + .where(and(eq(document.companyId, input.companyId), eq(documentVersions.id, input.versionId), eq(documentVersions.documentId, input.documentId))).limit(1); + if (!ownedVersion) return { state: "missing", chunks: [], warnings: ["document_version_not_accessible"] }; + const rows = await this.db.select({ chunkId: documentContextChunks.id, content: documentContextChunks.content, contentHash: documentContextChunks.contentHash, + structureId: documentContextChunks.structureId, structurePath: documentStructure.path, structureTitle: documentStructure.title, structureOrdering: documentStructure.ordering, + pageNumber: documentContextChunks.pageNumber, lineStart: documentContextChunks.lineStart, lineEnd: documentContextChunks.lineEnd, + documentId: documentContextChunks.documentId, versionId: documentContextChunks.versionId }) + .from(documentContextChunks).leftJoin(documentStructure, eq(documentContextChunks.structureId, documentStructure.id)) + .where(and(eq(documentContextChunks.documentId, input.documentId), eq(documentContextChunks.versionId, BigInt(input.versionId)))) + .orderBy(asc(documentStructure.path), asc(documentContextChunks.pageNumber), asc(documentContextChunks.lineStart), asc(documentStructure.ordering), asc(documentContextChunks.id)); + if (rows.length === 0) return { state: "missing", chunks: [], warnings: ["version_chunks_missing"] }; + const chunks = rows.map((row) => ({ ...row, versionId: row.versionId! })); + const partial = chunks.some((chunk) => !chunk.contentHash || !chunk.structurePath); + return partial ? { state: "partial", chunks, warnings: ["version_chunks_partial_provenance"] } : { state: "complete", chunks, warnings: [] }; + } +} diff --git a/apps/web/src/server/founder-weekly-review/evaluation-markdown.ts b/apps/web/src/server/founder-weekly-review/evaluation-markdown.ts new file mode 100644 index 000000000..de2d9ba53 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/evaluation-markdown.ts @@ -0,0 +1,67 @@ +import type { GeneratedReviewEvaluation } from "@launchstack/features/founder-weekly-review/benchmarks"; + +export function renderFounderWeeklyReviewEvaluationMarkdown( + evaluation: GeneratedReviewEvaluation +) { + return ` +# Founder Weekly Review Evaluation + +## Deterministic Evaluation + +${ + evaluation.deterministic + ? ` +Score: ${evaluation.deterministic.overallScore} + +Failures: + +${ + evaluation.deterministic.failures.length + ? evaluation.deterministic.failures + .map( + (failure) => + `- ${failure.category}: ${failure.explanation}` + ) + .join("\n") + : "- None" +} +` + : "Malformed payload" +} + + +## LLM Grader + +${ + evaluation.llmGrader + ? ` +Overall Score: ${evaluation.llmGrader.overallScore} + +## Dimensions + +- Groundedness: ${evaluation.llmGrader.dimensions.groundedness} +- Materiality: ${evaluation.llmGrader.dimensions.materiality} +- Temporal Accuracy: ${evaluation.llmGrader.dimensions.temporalAccuracy} +- Synthesis Quality: ${evaluation.llmGrader.dimensions.synthesisQuality} +- Actionability: ${evaluation.llmGrader.dimensions.actionability} + + +## Findings + +${evaluation.llmGrader.findings + .map( + (finding) => + `### ${finding.section} +${finding.severity}: ${finding.explanation}` + ) + .join("\n\n")} + + +## Summary + +${evaluation.llmGrader.summary} +` + : "No LLM grader result" +} +`; +} \ No newline at end of file diff --git a/apps/web/src/server/founder-weekly-review/evidence-collector.ts b/apps/web/src/server/founder-weekly-review/evidence-collector.ts new file mode 100644 index 000000000..732cae2e8 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/evidence-collector.ts @@ -0,0 +1,75 @@ +import { + FounderWeeklyReviewEvidenceService, + FounderWeeklyReviewEvidenceSnapshotSchema, + type FounderWeeklyReviewEvidenceSnapshot, + type ReportingPeriod, +} from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewDocumentVersionStore } from "./document-version-chunks"; +import { StrictCurrentWorkspaceDocumentStore } from "./workspace-document-store"; +import { createConfiguredDocumentChangeMaterialityAnalyzer } from "./document-change-materiality-analyzer"; + +export interface FounderWeeklyReviewEvidenceCollector { + collectFounderWeeklyReviewEvidence(input: { + companyId: bigint; + reportingPeriod: ReportingPeriod; + workspaceTimezone: string; + founderContext?: string; + /** LAU-6 records request-time founder context against this actor. */ + actor: { externalUserId: string }; + /** Stable identity makes idempotent founder context source IDs stable. */ + requestKey: string; + }): Promise; +} + +/** LAU-6 owns collection. This deliberately never creates an empty snapshot. */ +export class FounderWeeklyReviewEvidenceCollectorUnavailableError extends Error { + readonly code = "evidence_collector_unavailable"; + constructor() { + super("Founder weekly review evidence collection is not configured."); + } +} + +export const unavailableFounderWeeklyReviewEvidenceCollector: FounderWeeklyReviewEvidenceCollector = { + async collectFounderWeeklyReviewEvidence() { + throw new FounderWeeklyReviewEvidenceCollectorUnavailableError(); + }, +}; + +/** + * App adapter over LAU-6's canonical workspace collector. This is kept as an + * interface boundary deliberately: LAU-8 can compose github_activity here + * without changing routes, persistence, or the generation worker. + */ +export class CanonicalFounderWeeklyReviewEvidenceCollector implements FounderWeeklyReviewEvidenceCollector { + // Lazy construction keeps route composition importable in tests and does + // not open a DB dependency until production collection is actually used. + constructor(private service?: FounderWeeklyReviewEvidenceService) {} + + async collectFounderWeeklyReviewEvidence(input: { + companyId: bigint; + reportingPeriod: ReportingPeriod; + workspaceTimezone: string; + founderContext?: string; + actor: { externalUserId: string }; + requestKey: string; + }): Promise { + const snapshot = await (this.service ??= new FounderWeeklyReviewEvidenceService( + undefined, + undefined, + { kind: "computed", store: new FounderWeeklyReviewDocumentVersionStore() }, + new StrictCurrentWorkspaceDocumentStore(), + createConfiguredDocumentChangeMaterialityAnalyzer(), + )).collectFounderWeeklyReviewEvidence({ + companyId: input.companyId, + reportingPeriod: input.reportingPeriod, + workspaceTimezone: input.workspaceTimezone, + founderContext: input.founderContext, + actor: input.actor, + requestKey: input.requestKey, + }); + return FounderWeeklyReviewEvidenceSnapshotSchema.parse(snapshot); + } +} + +export const canonicalFounderWeeklyReviewEvidenceCollector = + new CanonicalFounderWeeklyReviewEvidenceCollector(); diff --git a/apps/web/src/server/founder-weekly-review/generation-adapter.ts b/apps/web/src/server/founder-weekly-review/generation-adapter.ts new file mode 100644 index 000000000..f591bacc4 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/generation-adapter.ts @@ -0,0 +1,27 @@ +import type { ZodType } from "zod"; + +import { + generateStructuredWithMetadata, + type StructuredGenerationMetadata, +} from "~/lib/llm"; + +/** + * Host adapter for LAU-7. The feature receives this function as an injected + * dependency and therefore remains independent of providers and web code. + */ +export function generateFounderWeeklyReviewStructured(input: { + system?: string; + prompt: string; + schema: TSchema; + schemaName?: string; + generationPhase?: "initial" | "semantic-repair"; + temperature?: number; +}): Promise<{ + object: ReturnType; + metadata: StructuredGenerationMetadata; +}> { + return generateStructuredWithMetadata({ + ...input, + capability: "founderWeeklyReview", + }); +} diff --git a/apps/web/src/server/founder-weekly-review/http.ts b/apps/web/src/server/founder-weekly-review/http.ts new file mode 100644 index 000000000..2d0eccf60 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/http.ts @@ -0,0 +1,23 @@ +import { NextResponse } from "next/server"; +import type { FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; + +export function safeRun(run: FounderWeeklyReviewRunRecord) { + return { + id: run.id, status: run.status, reportingPeriod: run.reportingPeriod, + generationAttempt: run.generationAttempt, retryCount: run.retryCount, + queuedAt: run.queuedAt.toISOString(), claimedAt: run.claimedAt?.toISOString() ?? null, + generatedAt: run.generatedAt?.toISOString() ?? null, publishedAt: run.publishedAt?.toISOString() ?? null, + errorCode: run.status === "failed" ? run.errorCode : null, + reviewPayload: run.reviewPayload, + }; +} + +export function safeFounderWeeklyReviewError(error: unknown) { + const code = error && typeof error === "object" && "code" in error ? (error as { code?: string }).code : undefined; + if (code === "forbidden") return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + if (code === "not_found") return NextResponse.json({ error: "Not found" }, { status: 404 }); + if (code === "invalid_transition" || code === "conflict") return NextResponse.json({ error: "Conflict" }, { status: 409 }); + if (code === "invalid_payload") return NextResponse.json({ error: "Invalid request" }, { status: 400 }); + if (code === "evidence_collector_unavailable" || code === "infrastructure_unavailable") return NextResponse.json({ error: "Generation unavailable" }, { status: 503 }); + return NextResponse.json({ error: "Internal server error" }, { status: 500 }); +} diff --git a/apps/web/src/server/founder-weekly-review/markdown.ts b/apps/web/src/server/founder-weekly-review/markdown.ts new file mode 100644 index 000000000..8666adab9 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/markdown.ts @@ -0,0 +1,85 @@ +import type { + FounderWeeklyReviewEvidenceItem, +} from "@launchstack/features/founder-weekly-review"; + +type RenderableRun = { + reportingPeriod: { start: string; end: string }; + modelMetadata: { model?: string } | null; + reviewPayload: any; + evidenceSnapshot: { items: FounderWeeklyReviewEvidenceItem[] } | null; +}; + +const SECTION_ORDER = ["whatShipped", "whatChanged", "whatCustomersSaid", "currentBlockers", "nextPriorities"] as const; +const SECTION_LABELS: Record<(typeof SECTION_ORDER)[number], string> = { + whatShipped: "Shipped This Period", + whatChanged: "Other Meaningful Changes", + whatCustomersSaid: "Customer Signals", + currentBlockers: "Risks & Blockers", + nextPriorities: "Priorities for the Next Period", +}; +const REFERENCE_SEPARATOR = String.fromCharCode(0x2014); + +function metadataNumber(item: FounderWeeklyReviewEvidenceItem, key: string): string | null { + const value = item.metadata[key]; + return typeof value === "number" || typeof value === "string" ? String(value) : null; +} + +export function formatFounderWeeklyReviewEvidenceReference(item: FounderWeeklyReviewEvidenceItem): string { + if (item.sourceType === "founder_context") return "Founder-provided context"; + const details: string[] = []; + const page = metadataNumber(item, "pageNumber"); + const section = metadataNumber(item, "sectionId"); + if (page) details.push(`page ${page}`); + if (section) details.push(`section ${section}`); + const title = item.title.trim() || "Untitled evidence"; + const prefix = item.sourceType === "document_change" ? `Document change ${REFERENCE_SEPARATOR} ` : ""; + return [prefix + title, ...details].join(` ${REFERENCE_SEPARATOR} `); +} + +/** Renders only a persisted, validated draft and intentionally excludes operational internals. */ +export function renderFounderWeeklyReviewMarkdown(run: RenderableRun): string { + if (!run.reviewPayload || !run.evidenceSnapshot) { + throw new Error("A persisted review payload and evidence snapshot are required for Markdown rendering."); + } + const sources = new Map(run.evidenceSnapshot.items.map((item) => [item.sourceId, item])); + const referenceNumbers = new Map(); + const references: string[] = []; + const cite = (sourceIds: readonly string[]) => sourceIds.map((sourceId) => { + let number = referenceNumbers.get(sourceId); + if (!number) { + number = referenceNumbers.size + 1; + referenceNumbers.set(sourceId, number); + const source = sources.get(sourceId); + if (source) references.push(`[${number}] ${formatFounderWeeklyReviewEvidenceReference(source)}`); + } + return `[${number}]`; + }).join(""); + const shipped = new Set(run.reviewPayload.sections.whatShipped.state === "evidence" + ? run.reviewPayload.sections.whatShipped.items.map((item: any) => item.text.trim().replace(/\s+/g, " ")) + : []); + const lines = ["# Founder Weekly Review", "", `**Reporting period:** ${run.reportingPeriod.start} to ${run.reportingPeriod.end}`, ""]; + for (const key of SECTION_ORDER) { + const section = run.reviewPayload.sections[key]; + const items = section.state === "evidence" + ? section.items.filter((item: any) => key !== "whatChanged" || !shipped.has(item.text.trim().replace(/\s+/g, " "))) + : []; + const noEvidence = section.state === "no_evidence" ? section.noEvidence : null; + if (!items.length && !noEvidence) continue; + lines.push(`## ${SECTION_LABELS[key]}`, ""); + if (noEvidence) { + lines.push(noEvidence.message, "", `Next: ${noEvidence.cta}`, ""); + continue; + } + items.forEach((item: any, index: number) => { + const marker = key === "nextPriorities" ? `${index + 1}.` : "-"; + lines.push(`${marker} ${item.text}${cite(item.sourceIds)}`); + if (item.kind === "recommendation" && item.rationale) { + lines.push(` - Why now: ${item.rationale}${cite(item.sourceIds)}`); + } + }); + lines.push(""); + } + if (references.length) lines.push("## Evidence References", "", ...references, ""); + lines.push("---", "", `*Generated with ${run.modelMetadata?.model ?? "the configured model"}*`); + return lines.join("\n"); +} diff --git a/apps/web/src/server/founder-weekly-review/observability.ts b/apps/web/src/server/founder-weekly-review/observability.ts new file mode 100644 index 000000000..2a749819a --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/observability.ts @@ -0,0 +1,19 @@ +import { createLogger } from "~/lib/logger"; +import { + founderWeeklyReviewGenerationTotal, + founderWeeklyReviewJobsEnqueued, + founderWeeklyReviewRetries, + founderWeeklyReviewStageDuration, + founderWeeklyReviewCitationFailures, + founderWeeklyReviewRunsCreated, + founderWeeklyReviewRunsCompleted, + founderWeeklyReviewRunsFailed, + founderWeeklyReviewDispatchFailures, +} from "~/server/metrics/registry"; + +const logger = createLogger("founder-weekly-review"); +export function logFounderWeeklyReview(fields: { + runId: string; companyId: string; stage: string; status: string; durationMs?: number; + generationAttempt?: number; retryCount?: number; errorClass?: string; provider?: string; model?: string; +}) { logger.info(fields, "founder weekly review stage"); } +export { founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRetries, founderWeeklyReviewStageDuration, founderWeeklyReviewCitationFailures, founderWeeklyReviewRunsCreated, founderWeeklyReviewRunsCompleted, founderWeeklyReviewRunsFailed, founderWeeklyReviewDispatchFailures }; diff --git a/apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts b/apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts new file mode 100644 index 000000000..2347a1dff --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/realistic-e2e-mode.ts @@ -0,0 +1,18 @@ +export type FounderWeeklyReviewRealisticEvidenceMode = "legacy" | "computed"; + +/** Explicit runner-only mode selection; default preserves the legacy baseline. */ +export function parseFounderWeeklyReviewRealisticEvidenceMode( + raw: string | undefined +): FounderWeeklyReviewRealisticEvidenceMode { + if (!raw || raw === "legacy") return "legacy"; + if (raw === "computed") return "computed"; + throw new Error(`Unsupported FWR_EVIDENCE_MODE: ${raw}`); +} + +/** Keeps an explicit export root from receiving the evidence mode twice. */ +export function founderWeeklyReviewRealisticExportRoot( + mode: FounderWeeklyReviewRealisticEvidenceMode, + configuredRoot: string | undefined +): string { + return configuredRoot ?? `.artifacts/founder-weekly-review/${mode}`; +} diff --git a/apps/web/src/server/founder-weekly-review/workspace-document-store.ts b/apps/web/src/server/founder-weekly-review/workspace-document-store.ts new file mode 100644 index 000000000..c983a5a45 --- /dev/null +++ b/apps/web/src/server/founder-weekly-review/workspace-document-store.ts @@ -0,0 +1,48 @@ +import { and, asc, eq, sql } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { createEmbeddingModel, resolveEmbeddingIndex, type EmbeddingsProvider } from "@launchstack/core/embeddings"; +import { document, documentContextChunks, documentStructure } from "@launchstack/core/db/schema"; +import { + MAX_WORKSPACE_RETRIEVAL_CANDIDATES, + normalizeFounderContextRetrievalQuery, + type WorkspaceDocumentHit, + type WorkspaceDocumentRetrievalInput, + type WorkspaceDocumentRetrievalResult, +} from "@launchstack/features/founder-weekly-review"; + +export interface FounderWeeklyReviewWorkspaceDocumentStore { + retrieveRelevantCurrentDocumentChunks(input: WorkspaceDocumentRetrievalInput): Promise; +} + +/** Strict current-version vector retrieval used only by Founder Weekly Review. */ +export class StrictCurrentWorkspaceDocumentStore implements FounderWeeklyReviewWorkspaceDocumentStore { + constructor(private readonly db: DbClient = getDb(), private readonly embeddings?: EmbeddingsProvider) {} + + async retrieveRelevantCurrentDocumentChunks(input: WorkspaceDocumentRetrievalInput): Promise { + const query = normalizeFounderContextRetrievalQuery(input.founderContext); + if (!query) return { state: "empty", hits: [] }; + const topK = Math.max(1, Math.min(MAX_WORKSPACE_RETRIEVAL_CANDIDATES, input.topK ?? MAX_WORKSPACE_RETRIEVAL_CANDIDATES)); + try { + // document_context_chunks.embedding is the legacy 1536-dimensional store. + // Do not resolve a configurable dimension-table index for this SQL path. + const embedding = await (this.embeddings ?? createEmbeddingModel(resolveEmbeddingIndex("legacy-openai-1536"))).embedQuery(query); + if (embedding.length !== 1536 || embedding.some((value) => !Number.isFinite(value))) { + return { state: "unavailable", hits: [], warnings: ["workspace_document_embedding_index_unavailable"] }; + } + const literal = sql.raw(`'${JSON.stringify(embedding)}'::vector(1536)`); + const rows = await this.db.select({ documentId: documentContextChunks.documentId, documentTitle: document.title, + versionId: documentContextChunks.versionId, contextChunkId: documentContextChunks.id, content: documentContextChunks.content, + structureId: documentContextChunks.structureId, structurePath: documentStructure.path, structureTitle: documentStructure.title, + pageNumber: documentContextChunks.pageNumber, lineStart: documentContextChunks.lineStart, lineEnd: documentContextChunks.lineEnd, + similarityScore: sql`1 - (${documentContextChunks.embedding} <=> ${literal})` }) + .from(documentContextChunks).innerJoin(document, eq(documentContextChunks.documentId, document.id)) + .leftJoin(documentStructure, eq(documentContextChunks.structureId, documentStructure.id)) + .where(and(eq(document.companyId, input.companyId), eq(documentContextChunks.versionId, document.currentVersionId), sql`${documentContextChunks.versionId} IS NOT NULL`, sql`${document.currentVersionId} IS NOT NULL`, sql`${documentContextChunks.embedding} IS NOT NULL`)) + .orderBy(sql`${documentContextChunks.embedding} <=> ${literal}`, asc(document.id), asc(documentContextChunks.versionId), asc(documentContextChunks.id)).limit(topK); + if (!rows.length) return { state: "empty", hits: [] }; + return { state: "success", hits: rows.map((row): WorkspaceDocumentHit => ({ ...row, versionId: row.versionId!, similarityScore: row.similarityScore })) }; + } catch { + return { state: "unavailable", hits: [], warnings: ["workspace_document_retrieval_unavailable"] }; + } + } +} diff --git a/apps/web/src/server/inngest/client.ts b/apps/web/src/server/inngest/client.ts index e7747a931..b299fa1f1 100644 --- a/apps/web/src/server/inngest/client.ts +++ b/apps/web/src/server/inngest/client.ts @@ -83,6 +83,14 @@ export type RehydrateNoteAnchorsEvent = { versionId: number; }; }; +export type FounderWeeklyReviewDispatchEvent = { + name: "founder-weekly-review/dispatch.requested"; + data: { dispatchId?: string }; +}; +export type FounderWeeklyReviewGenerationEvent = { + name: "founder-weekly-review/generation.requested"; + data: { runId: string; companyId: string; generationJobId: string; generationClaimId: string }; +}; export type Events = | ProcessDocumentEvent @@ -93,7 +101,9 @@ export type Events = | ReindexCompanyEmbeddingsEvent | DocumentModifyEvent | WebsiteCrawlEvent - | RehydrateNoteAnchorsEvent; + | RehydrateNoteAnchorsEvent + | FounderWeeklyReviewDispatchEvent + | FounderWeeklyReviewGenerationEvent; /** * Create the Inngest client. diff --git a/apps/web/src/server/inngest/functions/founderWeeklyReview.ts b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts new file mode 100644 index 000000000..ebe49ebd8 --- /dev/null +++ b/apps/web/src/server/inngest/functions/founderWeeklyReview.ts @@ -0,0 +1,125 @@ +import { z } from "zod"; +import { NonRetriableError } from "inngest"; +import { inngest } from "../client"; +import { FounderWeeklyReviewWorkerService, type FounderWeeklyReviewRunRecord } from "@launchstack/features/founder-weekly-review"; +import { generateFounderWeeklyReview } from "@launchstack/features/founder-weekly-review"; +import { FounderWeeklyReviewGenerationValidationError } from "@launchstack/features/founder-weekly-review"; +import { generateFounderWeeklyReviewStructured } from "~/server/founder-weekly-review/generation-adapter"; +import { canonicalFounderWeeklyReviewEvidenceCollector } from "~/server/founder-weekly-review/evidence-collector"; +import { claimPendingDispatches, markDispatchDispatched, returnDispatchToPending } from "~/server/founder-weekly-review/dispatch-service"; +import { founderWeeklyReviewCitationFailures, founderWeeklyReviewDispatchFailures, founderWeeklyReviewGenerationTotal, founderWeeklyReviewJobsEnqueued, founderWeeklyReviewRunsCompleted, founderWeeklyReviewRunsFailed, founderWeeklyReviewStageDuration, logFounderWeeklyReview } from "~/server/founder-weekly-review/observability"; + +const GenerationEventSchema = z.object({ runId: z.string().min(1), companyId: z.string().min(1), generationJobId: z.string().min(1), generationClaimId: z.string().min(1) }); + +export const founderWeeklyReviewDispatcher = inngest.createFunction( + { id: "founder-weekly-review-dispatcher", retries: 3 }, + { event: "founder-weekly-review/dispatch.requested" }, + async ({ step }) => step.run("dispatch-pending", async () => { + const dispatches = await claimPendingDispatches(); + for (const dispatch of dispatches) { + try { + await inngest.send({ id: dispatch.eventId, name: "founder-weekly-review/generation.requested", data: { + runId: dispatch.runId, companyId: dispatch.companyId.toString(), generationJobId: dispatch.generationJobId, generationClaimId: dispatch.generationClaimId, + }}); + await markDispatchDispatched(dispatch.id); + founderWeeklyReviewJobsEnqueued.inc({ operation: dispatch.operationType }); + logFounderWeeklyReview({ runId: dispatch.runId, companyId: dispatch.companyId.toString(), stage: "dispatch_sent", status: "dispatched" }); + } catch { + await returnDispatchToPending(dispatch.id, "dispatch_failed"); + founderWeeklyReviewDispatchFailures.inc(); + } + } + return { dispatched: dispatches.length }; + }) +); + +/** Periodic reconciliation covers a process failure after the DB outbox commit. */ +export const founderWeeklyReviewDispatchReconciler = inngest.createFunction( + { id: "founder-weekly-review-dispatch-reconciler", retries: 2 }, + { cron: "*/5 * * * *" }, + async () => { + await inngest.send({ name: "founder-weekly-review/dispatch.requested", data: {} }); + return { requested: true }; + } +); + +export const founderWeeklyReviewGenerationJob = inngest.createFunction( + { id: "founder-weekly-review-generation", retries: 3, concurrency: { key: "event.data.runId", limit: 1 }, + onFailure: async ({ event }) => { + const parsed = GenerationEventSchema.safeParse(event.data); if (!parsed.success) return; + const worker = new FounderWeeklyReviewWorkerService(); + const companyId = BigInt(parsed.data.companyId); + const current = await worker.getRun(companyId, parsed.data.runId).catch(() => null); + if (current?.status === "collecting" && !current.evidenceSnapshot) { + await worker.markCollectionFailed({ companyId, runId: parsed.data.runId, collectionClaimId: parsed.data.generationClaimId }, { errorCode: "evidence_collection_failed", errorMessage: "Evidence collection failed after retries." }).catch(() => undefined); + } else { + await worker.markGenerationFailed({ companyId, runId: parsed.data.runId, generationJobId: parsed.data.generationJobId, generationClaimId: parsed.data.generationClaimId }, { errorCode: "generation_failed", errorMessage: "Generation failed after retries." }).catch(() => undefined); + } + } }, + { event: "founder-weekly-review/generation.requested" }, + async ({ event, step }) => { + const data = GenerationEventSchema.parse(event.data); + const context = { companyId: BigInt(data.companyId), runId: data.runId, generationJobId: data.generationJobId, generationClaimId: data.generationClaimId }; + const worker = new FounderWeeklyReviewWorkerService(); + let workflowRun = await worker.getRun(context.companyId, context.runId); + if (workflowRun.status === "draft" || workflowRun.status === "published") return { skipped: true }; + if (!workflowRun.evidenceSnapshot) { + const collectionContext = { companyId: context.companyId, runId: context.runId, collectionClaimId: data.generationClaimId }; + try { + // Step results are JSON transport values; a run record contains bigint + // IDs, so read the canonical record back rather than consuming a + // lossy serialized step result in the callback. + await step.run("claim-evidence", () => worker.claimEvidenceCollection(collectionContext)); + const collecting = await worker.getRun(context.companyId, context.runId); + if (!collecting.evidenceSnapshot && collecting.status === "collecting" && collecting.collectionClaimId === collectionContext.collectionClaimId) { + logFounderWeeklyReview({ runId: collecting.id, companyId: collecting.companyId.toString(), stage: "evidence_collection_started", status: collecting.status, retryCount: collecting.retryCount }); + const snapshot = await step.run("collect-evidence", () => canonicalFounderWeeklyReviewEvidenceCollector.collectFounderWeeklyReviewEvidence({ + companyId: collecting.companyId, + reportingPeriod: collecting.reportingPeriod, + workspaceTimezone: collecting.collectionInput?.workspaceTimezone ?? "UTC", + founderContext: collecting.collectionInput?.founderContext, + actor: { externalUserId: collecting.collectionInput?.actorExternalUserId ?? "unknown" }, + requestKey: collecting.requestKey, + })); + await step.run("persist-evidence", () => worker.attachEvidenceSnapshotIfAbsent(collectionContext, snapshot)); + workflowRun = await worker.getRun(context.companyId, context.runId); + logFounderWeeklyReview({ runId: workflowRun.id, companyId: workflowRun.companyId.toString(), stage: "evidence_collection_completed", status: workflowRun.status, retryCount: workflowRun.retryCount }); + } else workflowRun = collecting; + } catch (error) { + await worker.markCollectionFailed(collectionContext, { errorCode: "evidence_collection_failed", errorMessage: "Evidence collection failed after retries." }).catch(() => undefined); + throw error; + } + } + if (!workflowRun.evidenceSnapshot) return { skipped: true }; + // Inngest's generic step inference intersects event return types; retain the + // concrete LAU-5 lifecycle record at this boundary. + await step.run("claim", () => worker.claimQueuedRun(context)); + const claimed = await worker.getRun(context.companyId, context.runId); + if (claimed.status !== "generating" || claimed.generationClaimId !== data.generationClaimId) return { skipped: true }; + logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: "worker_claimed", status: claimed.status, generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount }); + try { + const generationStartedAt = performance.now(); + logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: "generation_started", status: claimed.status, generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount }); + const evidenceSnapshot = claimed.evidenceSnapshot; + if (!evidenceSnapshot) throw new Error("Evidence snapshot is required before generation."); + const generated = await step.run("generate", () => generateFounderWeeklyReview({ evidenceSnapshot, generate: generateFounderWeeklyReviewStructured })); + await step.run("persist", () => worker.saveGeneratedDraft(context, generated.reviewPayload, generated.modelMetadata)); + const saved = await worker.getRun(context.companyId, context.runId); + founderWeeklyReviewGenerationTotal.inc({ result: "success", error_class: "none" }); + founderWeeklyReviewRunsCompleted.inc(); + founderWeeklyReviewStageDuration.observe({ stage: "generation", result: "success" }, (performance.now() - generationStartedAt) / 1000); + founderWeeklyReviewStageDuration.observe({ stage: "end_to_end", result: "success" }, (Date.now() - saved.createdAt.getTime()) / 1000); + logFounderWeeklyReview({ runId: saved.id, companyId: saved.companyId.toString(), stage: "generation_completed", status: saved.status, durationMs: Math.round(performance.now() - generationStartedAt), generationAttempt: saved.generationAttempt, retryCount: saved.retryCount, provider: generated.modelMetadata.provider, model: generated.modelMetadata.model }); + return { runId: saved.id, status: saved.status }; + } catch (error) { + if (error instanceof FounderWeeklyReviewGenerationValidationError) founderWeeklyReviewCitationFailures.inc(); + founderWeeklyReviewGenerationTotal.inc({ result: "failure", error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + founderWeeklyReviewRunsFailed.inc({ error_class: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + logFounderWeeklyReview({ runId: claimed.id, companyId: claimed.companyId.toString(), stage: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation_failed" : "generation_failed", status: "generating", generationAttempt: claimed.generationAttempt, retryCount: claimed.retryCount, errorClass: error instanceof FounderWeeklyReviewGenerationValidationError ? "citation_validation" : "generation" }); + if (error instanceof FounderWeeklyReviewGenerationValidationError) { + throw new NonRetriableError("Founder Weekly Review validation failed after bounded semantic repair."); + } + throw error; + } + } +); diff --git a/apps/web/src/server/metrics/registry.ts b/apps/web/src/server/metrics/registry.ts index f788cf0f7..62b61a038 100644 --- a/apps/web/src/server/metrics/registry.ts +++ b/apps/web/src/server/metrics/registry.ts @@ -55,6 +55,16 @@ export const qaRequestCounter = new Counter({ registers: [metricsRegistry] }); +export const founderWeeklyReviewJobsEnqueued = new Counter({ name: "pdr_founder_weekly_review_jobs_enqueued_total", help: "Founder review jobs enqueued", labelNames: ["operation"], registers: [metricsRegistry] }); +export const founderWeeklyReviewGenerationTotal = new Counter({ name: "pdr_founder_weekly_review_generation_total", help: "Founder review generations by result", labelNames: ["result", "error_class"], registers: [metricsRegistry] }); +export const founderWeeklyReviewRetries = new Counter({ name: "pdr_founder_weekly_review_retries_total", help: "Founder review retries", registers: [metricsRegistry] }); +export const founderWeeklyReviewCitationFailures = new Counter({ name: "pdr_founder_weekly_review_citation_validation_failures_total", help: "Founder review citation validation failures", registers: [metricsRegistry] }); +export const founderWeeklyReviewRunsCreated = new Counter({ name: "pdr_founder_weekly_review_runs_created_total", help: "Founder review runs created", registers: [metricsRegistry] }); +export const founderWeeklyReviewRunsCompleted = new Counter({ name: "pdr_founder_weekly_review_runs_completed_total", help: "Founder review runs completed", registers: [metricsRegistry] }); +export const founderWeeklyReviewRunsFailed = new Counter({ name: "pdr_founder_weekly_review_runs_failed_total", help: "Founder review runs failed", labelNames: ["error_class"], registers: [metricsRegistry] }); +export const founderWeeklyReviewDispatchFailures = new Counter({ name: "pdr_founder_weekly_review_dispatch_failures_total", help: "Founder review outbox dispatch failures", registers: [metricsRegistry] }); +export const founderWeeklyReviewStageDuration = new Histogram({ name: "pdr_founder_weekly_review_stage_duration_seconds", help: "Founder review stage duration", labelNames: ["stage", "result"], buckets: [0.1, 0.5, 1, 5, 15, 60, 300], registers: [metricsRegistry] }); + export async function getMetricsSnapshot(): Promise { return metricsRegistry.metrics(); } diff --git a/apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json b/apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json new file mode 100644 index 000000000..824ef0ff1 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/realistic-company/seed.json @@ -0,0 +1,10 @@ +{ + "reportingPeriod": { "start": "2026-02-16", "end": "2026-02-28" }, + "workspaceTimezone": "UTC", + "founderContext": "Prioritize onboarding reliability and follow up on saved filter adoption.", + "documents": [ + { "title": "Saved export filters release", "category": "Product", "changelog": "Released saved export filters so teams can reuse recurring reporting views.", "timestamp": "2026-02-20T10:00:00.000Z" }, + { "title": "Onboarding reliability plan", "category": "Planning", "changelog": "Documented retry telemetry and ownership for onboarding reliability work.", "timestamp": "2026-02-24T11:00:00.000Z" }, + { "title": "Customer Interviews — February 2026", "category": "Customer Feedback", "changelog": "Processed customer feedback for the reporting period.", "timestamp": "2026-02-22T12:00:00.000Z", "chunks": ["Saved export filters would remove repetitive setup from our weekly reporting.", "Onboarding occasionally retries before completing and needs clearer recovery feedback.", "The recently released approval workflow reduced our manual follow-up work."] } + ] +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json new file mode 100644 index 000000000..1ea6b28e6 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/01-empty-evidence/scenario.json @@ -0,0 +1,18 @@ +{ + "name": "empty-evidence", + "description": "No in-window versions and no founder context produce an empty evidence snapshot.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Legacy Handbook", + "category": "Reference", + "versions": [{ "versionNumber": 1, "timestamp": "2026-05-01T10:00:00.000Z", "changelog": "Outside the reporting period.", "chunks": [] }] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "document_change": { "exact": 0 }, "workspace_document": { "exact": 0 }, "customer_feedback": { "exact": 0 }, "founder_context": { "exact": 0 } } } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json new file mode 100644 index 000000000..9577612ca --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/02-founder-context-only/scenario.json @@ -0,0 +1,12 @@ +{ + "name": "founder-context-only", + "description": "Founder Context is the only collected source.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Focus on onboarding reliability and the billing revamp decision.", + "companies": [{ "name": "Northstar Analytics", "underReview": true, "documents": [] }], + "expect": { + "evidence": { "sourceTypeCounts": { "founder_context": { "exact": 1 }, "document_change": { "exact": 0 }, "workspace_document": { "exact": 0 }, "customer_feedback": { "exact": 0 } } }, + "sourceSemantics": { "customerFeedbackOnly": true, "temporalEvidenceRequired": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json new file mode 100644 index 000000000..5178df203 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/03-relevant-workspace-document/scenario.json @@ -0,0 +1,28 @@ +{ + "name": "relevant-workspace-document", + "description": "An unchanged current document is retrieved for Founder Context without becoming a temporal document change.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Focus on onboarding reliability and who owns retry telemetry.", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Onboarding Reliability Plan", + "category": "Planning", + "versions": [{ + "versionNumber": 1, + "timestamp": "2026-06-15T10:00:00.000Z", + "changelog": "Last updated before the reporting period.", + "chunks": [ + { "section": "Ownership", "content": "Platform owns retry telemetry and recovery monitoring for onboarding." }, + { "section": "Runbook", "content": "On failure, onboarding retries with backoff and pages the platform on-call." } + ] + }] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "workspace_document": { "min": 1 }, "document_change": { "exact": 0 } } }, + "sourceSemantics": { "currentWorkspaceOnly": true, "temporalEvidenceRequired": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json new file mode 100644 index 000000000..2ff8c83b3 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/04-first-ever-version-no-invented-diff/scenario.json @@ -0,0 +1,25 @@ +{ + "name": "first-ever-version-no-invented-diff", + "description": "A first-ever in-window version has no predecessor and must not create invented temporal change evidence.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Review onboarding ownership.", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Onboarding Plan", + "category": "Planning", + "versions": [{ + "versionNumber": 1, + "timestamp": "2026-07-24T10:00:00.000Z", + "changelog": "Documented retry telemetry ownership.", + "chunks": [{ "section": "Ownership", "content": "Platform owns retry telemetry and recovery monitoring." }] + }] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "document_change": { "exact": 0 } }, "warningCodes": ["document_change_baseline_missing"] }, + "documentChanges": { "requireNoInventedBaseline": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json b/apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json new file mode 100644 index 000000000..6a4136a13 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/05-multiple-versions-in-period/scenario.json @@ -0,0 +1,26 @@ +{ + "name": "multiple-versions-in-period", + "description": "A pre-window baseline and three adjacent in-window versions exercise pair chaining, alignment, grouping, and audit provenance.", + "reportingPeriod": { "start": "2026-07-20", "end": "2026-07-26" }, + "workspaceTimezone": "America/New_York", + "founderContext": "Track how ownership of retry telemetry changed this week.", + "companies": [{ + "name": "Northstar Analytics", + "underReview": true, + "documents": [{ + "title": "Onboarding Reliability Plan", + "category": "Planning", + "versions": [ + { "versionNumber": 1, "timestamp": "2026-07-10T10:00:00.000Z", "changelog": "Baseline before the reporting window.", "chunks": [{ "section": "Ownership", "content": "Product owns retry telemetry." }] }, + { "versionNumber": 2, "timestamp": "2026-07-21T10:00:00.000Z", "changelog": "Moved retry telemetry ownership to Platform.", "chunks": [{ "section": "Ownership", "content": "Platform owns retry telemetry." }] }, + { "versionNumber": 3, "timestamp": "2026-07-23T10:00:00.000Z", "changelog": "Added recovery monitoring to Platform ownership.", "chunks": [{ "section": "Ownership", "content": "Platform owns retry telemetry and recovery monitoring." }] }, + { "versionNumber": 4, "timestamp": "2026-07-25T10:00:00.000Z", "changelog": "Clarified on-call paging.", "chunks": [{ "section": "Runbook", "content": "On failure, page the Platform on-call." }] } + ] + }] + }], + "expect": { + "evidence": { "sourceTypeCounts": { "document_change": { "min": 1 } } }, + "documentChanges": { "minGroups": 1, "maxGroups": 8 }, + "sourceSemantics": { "temporalEvidenceRequired": true } + } +} diff --git a/apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts b/apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts new file mode 100644 index 000000000..18ff1f249 --- /dev/null +++ b/apps/web/test-fixtures/founder-weekly-review/scenarios/contracts.ts @@ -0,0 +1,118 @@ +import { z } from "zod"; + +import { + DOCUMENT_CHANGE_CATEGORIES, + ReportingPeriodSchema, + type FounderWeeklyReviewEvidenceItem, +} from "@launchstack/features/founder-weekly-review"; + +export const FOUNDER_WEEKLY_REVIEW_SCENARIO_SCHEMA_VERSION = "founder-weekly-review-scenario/v1" as const; + +const ScenarioSourceTypeSchema = z.enum([ + "workspace_document", + "document_change", + "customer_feedback", + "github_activity", + "manual_note", + "founder_context", + "other", +]); +export type ScenarioSourceType = z.infer; + +const ScenarioSectionNameSchema = z.enum([ + "whatChanged", + "whatShipped", + "whatCustomersSaid", + "currentBlockers", + "nextPriorities", +]); +export type ScenarioSectionName = z.infer; + +const CountExpectationSchema = z.object({ + min: z.number().int().nonnegative().optional(), + max: z.number().int().nonnegative().optional(), + exact: z.number().int().nonnegative().optional(), +}).strict().superRefine((value, ctx) => { + if (value.min !== undefined && value.max !== undefined && value.min > value.max) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "min must not exceed max" }); + if (value.exact !== undefined && ((value.min !== undefined && value.exact < value.min) || (value.max !== undefined && value.exact > value.max))) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "exact must satisfy min/max" }); +}); + +export const ScenarioExpectSchema = z.object({ + evidence: z.object({ + sourceTypeCounts: z.record(ScenarioSourceTypeSchema, CountExpectationSchema).optional(), + warningCodes: z.array(z.string().min(1).max(128)).max(32).optional(), + }).strict().optional(), + documentChanges: z.object({ + minGroups: z.number().int().nonnegative().optional(), + maxGroups: z.number().int().nonnegative().optional(), + requiredCategories: z.array(z.enum(DOCUMENT_CHANGE_CATEGORIES)).max(16).optional(), + requireNoInventedBaseline: z.boolean().optional(), + }).strict().optional(), + sourceSemantics: z.object({ + customerFeedbackOnly: z.boolean().optional(), + temporalEvidenceRequired: z.boolean().optional(), + noCrossCompanyLeakage: z.boolean().optional(), + currentWorkspaceOnly: z.boolean().optional(), + }).strict().optional(), + review: z.object({ + sectionStates: z.record(ScenarioSectionNameSchema, z.enum(["evidence", "no_evidence"])).optional(), + requiredThemes: z.array(z.string().min(1).max(256)).max(16).optional(), + forbiddenThemes: z.array(z.string().min(1).max(256)).max(16).optional(), + }).strict().optional(), +}).strict(); +export type FounderWeeklyReviewScenarioExpect = z.infer; + +export const ScenarioChunkSchema = z.object({ + section: z.string().min(1).max(256), + content: z.string().min(1).max(2000), + pageNumber: z.number().int().positive().optional(), + lineStart: z.number().int().positive().optional(), + lineEnd: z.number().int().positive().optional(), +}).strict(); +export type FounderWeeklyReviewScenarioChunk = z.infer; + +export const ScenarioVersionSchema = z.object({ + versionNumber: z.number().int().positive(), + timestamp: z.string().datetime({ offset: true }), + changelog: z.string().max(1000).optional(), + uploadedBy: z.string().min(1).max(256).optional(), + chunks: z.array(ScenarioChunkSchema).max(200).default([]), +}).strict(); +export type FounderWeeklyReviewScenarioVersion = z.infer; + +export const ScenarioDocumentSchema = z.object({ + title: z.string().min(1).max(512), + category: z.string().min(1).max(256), + versions: z.array(ScenarioVersionSchema).max(50), +}).strict(); +export type FounderWeeklyReviewScenarioDocument = z.infer; + +export const ScenarioCompanySchema = z.object({ + name: z.string().min(1).max(256), + underReview: z.boolean().default(false), + documents: z.array(ScenarioDocumentSchema).max(200).default([]), +}).strict(); +export type FounderWeeklyReviewScenarioCompany = z.infer; + +export const FounderWeeklyReviewScenarioSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_SCENARIO_SCHEMA_VERSION).default(FOUNDER_WEEKLY_REVIEW_SCENARIO_SCHEMA_VERSION), + name: z.string().min(1).max(128), + description: z.string().max(1000).optional(), + reportingPeriod: ReportingPeriodSchema, + workspaceTimezone: z.string().min(1).max(128), + founderContext: z.string().min(1).max(1000).optional(), + companies: z.array(ScenarioCompanySchema).min(1).max(10), + expect: ScenarioExpectSchema.optional(), +}).strict().superRefine((scenario, ctx) => { + const underReview = scenario.companies.filter((company) => company.underReview); + if (underReview.length !== 1) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["companies"], message: "Exactly one company must have underReview: true" }); + for (const [companyIndex, company] of scenario.companies.entries()) { + for (const [documentIndex, doc] of company.documents.entries()) { + const versionNumbers = doc.versions.map((version) => version.versionNumber); + if (new Set(versionNumbers).size !== versionNumbers.length) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["companies", companyIndex, "documents", documentIndex, "versions"], message: "versionNumber must be unique within a document" }); + } + } +}); +export type FounderWeeklyReviewScenario = z.infer; + +export type ScenarioEvidenceSourceType = FounderWeeklyReviewEvidenceItem["sourceType"]; diff --git a/docs/runbooks/founder-weekly-review-staging.md b/docs/runbooks/founder-weekly-review-staging.md new file mode 100644 index 000000000..cac044326 --- /dev/null +++ b/docs/runbooks/founder-weekly-review-staging.md @@ -0,0 +1,41 @@ +# Founder Weekly Review staging smoke test + +Status: **pending execution**. This runbook requires an accessible staging +workspace, configured Inngest, and a configured generation provider. Never use +production evidence or commit generated review contents. + +1. Create an isolated staging workspace and grant the tester an owner, admin, + or editor membership. Add a `Customer Feedback` category document with + processed sections; use synthetic data only. +2. Verify the Inngest endpoint and configured `founderWeeklyReview` provider. + Submit `POST /api/founder-weekly-reviews` with a fresh `requestKey`, local + reporting-period dates, IANA `workspaceTimezone`, and optional founder + context. Save the returned run ID. +3. Poll `GET /api/founder-weekly-reviews/{runId}` and verify + `queued -> generating -> draft`. Read the returned draft only after it is + `draft` and compare every cited ID with the persisted run EvidenceSnapshot; + customer statements may cite `customer_feedback` only. +4. Repeat with partial evidence (no Customer Feedback) and empty evidence. + Verify explicit no-evidence customer sections, no invented facts, and that + empty evidence performs no provider generation. +5. Force a safe provider failure in staging, then `POST + /api/founder-weekly-reviews/{runId}/retry` with a new retry request key. + Confirm the same run ID returns to queued, retry count increments once, and + the stored snapshot is byte-for-byte unchanged. +6. Inspect only allowlisted structured stage logs and bounded metrics + (`runs_created`, completed/failed, retries, citation failures, dispatch + failures, evidence/generation durations). Do not copy source contents from + logs or database tooling into the record. + +## Result record + +| Field | Value | +| --- | --- | +| Date / environment | | +| Synthetic workspace | | +| Run ID | | +| Status transitions | | +| Citation validation result | | +| Retry result | | +| Provider/model identifier | | +| Observer | | diff --git a/package.json b/package.json index d46d769fe..903e37424 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,12 @@ "db:migrate": "pnpm --filter @launchstack/web db:migrate", "db:studio": "pnpm --filter @launchstack/web db:studio", "db:backfill:versions": "pnpm --filter @launchstack/web db:backfill:versions", + "db:backfill:embedding-credentials": "pnpm --filter @launchstack/web db:backfill:embedding-credentials", "inngest:dev": "pnpm --filter @launchstack/web inngest:dev", "changeset": "changeset", "version": "changeset version", - "release": "pnpm --filter @launchstack/core build && changeset publish" + "release": "pnpm --filter @launchstack/core build && changeset publish", + "test:fwr:realistic": "cross-env SYNTHETIC_FWR_LOCAL=1 pnpm --filter @launchstack/web exec tsx scripts/run-founder-weekly-review-realistic-e2e.ts" }, "devDependencies": { "@changesets/cli": "^2.31.0", @@ -36,8 +38,10 @@ "@types/node": "^24.3.1", "@typescript-eslint/eslint-plugin": "^8.42.0", "@typescript-eslint/parser": "^8.42.0", + "cross-env": "^10.1.0", "eslint": "^9.34.0", "prettier": "^3.6.2", + "tsx": "^4.21.0", "typescript": "^5.9.2" }, "packageManager": "pnpm@10.15.1", diff --git a/packages/core/src/db/schema.ts b/packages/core/src/db/schema.ts index 02e3eb78e..c8e3bd36e 100644 --- a/packages/core/src/db/schema.ts +++ b/packages/core/src/db/schema.ts @@ -16,3 +16,4 @@ export * from "./schema/client-prospector"; export * from "./schema/company-metadata"; export * from "./schema/marketing-history"; export * from "./schema/credits"; +export * from "./schema/founder-weekly-review"; diff --git a/packages/core/src/db/schema/base.ts b/packages/core/src/db/schema/base.ts index f532e9b34..015b3676d 100644 --- a/packages/core/src/db/schema/base.ts +++ b/packages/core/src/db/schema/base.ts @@ -70,14 +70,9 @@ export const company = pgTable("company", { reindexStartedAt: timestamp("reindex_started_at", { withTimezone: true }), reindexCompletedAt: timestamp("reindex_completed_at", { withTimezone: true }), reindexError: text("reindex_error"), - // Legacy plaintext credential columns. Read-only during migration; - // writes go through src/lib/ai/company-credentials.ts into the - // encrypted `company_embedding_credentials` table. Dropped by - // drizzle/0011 once backfill has run. - embeddingOpenAIApiKey: text("embedding_openai_api_key"), - embeddingHuggingFaceApiKey: text("embedding_huggingface_api_key"), - embeddingOllamaBaseUrl: varchar("embedding_ollama_base_url", { length: 1024 }), - embeddingOllamaModel: varchar("embedding_ollama_model", { length: 256 }), + // Embedding provider credentials live in `company_embedding_credentials` + // (encrypted). The legacy plaintext columns on this table were dropped by + // drizzle/0011_drop_plaintext_embedding_credentials.sql. employerpasskey: varchar("employerPasskey", { length: 256 }).notNull().default(""), employeepasskey: varchar("employeePasskey", { length: 256 }).notNull().default(""), numberOfEmployees: varchar("numberOfEmployees", { length: 256 }).notNull(), diff --git a/packages/core/src/db/schema/company-credentials.ts b/packages/core/src/db/schema/company-credentials.ts index 39190809a..ad0262dc8 100644 --- a/packages/core/src/db/schema/company-credentials.ts +++ b/packages/core/src/db/schema/company-credentials.ts @@ -19,9 +19,8 @@ import { pgTable } from "./helpers"; * two real secrets (OpenAI, Hugging Face) are stored as base64 ciphertext * plus a 4-char suffix for UI feedback (`…ab12`). * - * Populated by `src/lib/ai/company-credentials.ts`. The legacy plaintext - * `company.embedding*` columns remain until backfill runs, then get dropped - * in a follow-up migration. + * Populated by `packages/core/src/embeddings/company-credentials.ts`. + * Legacy plaintext `company.embedding*` columns were dropped by migration 0011. */ export const companyEmbeddingCredentials = pgTable( "company_embedding_credentials", diff --git a/packages/core/src/db/schema/founder-weekly-review.ts b/packages/core/src/db/schema/founder-weekly-review.ts new file mode 100644 index 000000000..3fd90e187 --- /dev/null +++ b/packages/core/src/db/schema/founder-weekly-review.ts @@ -0,0 +1,187 @@ +import { sql } from "drizzle-orm"; +import type { InferSelectModel } from "drizzle-orm"; +import { + bigint, + date, + index, + integer, + jsonb, + uniqueIndex, + varchar, + timestamp, +} from "drizzle-orm/pg-core"; + +import { company } from "./base"; +import { pgTable } from "./helpers"; + +export const founderWeeklyReviewRunStatusEnum = [ + "queued", + "collecting", + "generating", + "draft", + "published", + "failed", +] as const; + +export const founderWeeklyReviewOperationTypeEnum = ["retry"] as const; +export const founderWeeklyReviewDispatchOperationTypeEnum = ["create", "retry"] as const; +export const founderWeeklyReviewDispatchStatusEnum = [ + "pending", "dispatching", "dispatched", "failed", +] as const; + +export const founderWeeklyReviewRuns = pgTable( + "founder_weekly_review_runs", + { + id: varchar("id", { length: 64 }).primaryKey(), + companyId: bigint("company_id", { mode: "bigint" }) + .notNull() + .references(() => company.id, { onDelete: "cascade" }), + requestKey: varchar("request_key", { length: 128 }).notNull(), + reportingPeriodStart: date("reporting_period_start").notNull(), + reportingPeriodEnd: date("reporting_period_end").notNull(), + status: varchar("status", { + length: 32, + enum: founderWeeklyReviewRunStatusEnum, + }) + .notNull() + .default("queued"), + reviewPayload: jsonb("review_payload").$type | null>(), + reviewSchemaVersion: varchar("review_schema_version", { length: 64 }).notNull(), + evidenceSnapshot: jsonb("evidence_snapshot") + .$type | null>(), + evidenceSchemaVersion: varchar("evidence_schema_version", { length: 64 }).notNull(), + collectionInput: jsonb("collection_input").$type>().notNull(), + collectionClaimId: varchar("collection_claim_id", { length: 128 }), + collectionStartedAt: timestamp("collection_started_at", { withTimezone: true }), + evidenceCollectedAt: timestamp("evidence_collected_at", { withTimezone: true }), + modelMetadata: jsonb("model_metadata").$type | null>(), + createdByActorId: varchar("created_by_actor_id", { length: 256 }).notNull(), + retryCount: integer("retry_count").notNull().default(0), + failureSequence: integer("failure_sequence").notNull().default(0), + generationAttempt: integer("generation_attempt").notNull().default(0), + generationClaimId: varchar("generation_claim_id", { length: 128 }), + generationJobId: varchar("generation_job_id", { length: 256 }), + queuedAt: timestamp("queued_at", { withTimezone: true }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + claimedAt: timestamp("claimed_at", { withTimezone: true }), + generationStartedAt: timestamp("generation_started_at", { withTimezone: true }), + generatedAt: timestamp("generated_at", { withTimezone: true }), + publishedAt: timestamp("published_at", { withTimezone: true }), + errorCode: varchar("error_code", { length: 128 }), + errorMessage: varchar("error_message", { length: 1024 }), + createdAt: timestamp("created_at", { withTimezone: true }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).$onUpdate( + () => new Date() + ), + }, + (table) => ({ + requestKeyUnique: uniqueIndex( + "founder_weekly_review_runs_company_request_key_unique" + ).on(table.companyId, table.requestKey), + companyCreatedIdx: index("founder_weekly_review_runs_company_created_at_idx").on( + table.companyId, + table.createdAt + ), + companyStatusCreatedIdx: index( + "founder_weekly_review_runs_company_status_created_at_idx" + ).on(table.companyId, table.status, table.createdAt), + companyPeriodIdx: index("founder_weekly_review_runs_company_period_idx").on( + table.companyId, + table.reportingPeriodStart, + table.reportingPeriodEnd + ), + claimIdx: index("founder_weekly_review_runs_claim_idx").on( + table.companyId, + table.id, + table.status, + table.generationClaimId + ), + collectionClaimIdx: index("founder_weekly_review_runs_collection_claim_idx").on( + table.companyId, table.id, table.status, table.collectionClaimId + ), + }) +); + +export const founderWeeklyReviewOperations = pgTable( + "founder_weekly_review_operations", + { + id: varchar("id", { length: 64 }).primaryKey(), + runId: varchar("run_id", { length: 64 }) + .notNull() + .references(() => founderWeeklyReviewRuns.id, { onDelete: "cascade" }), + companyId: bigint("company_id", { mode: "bigint" }) + .notNull() + .references(() => company.id, { onDelete: "cascade" }), + operationType: varchar("operation_type", { + length: 32, + enum: founderWeeklyReviewOperationTypeEnum, + }).notNull(), + requestKey: varchar("request_key", { length: 128 }).notNull(), + sourceFailureSequence: integer("source_failure_sequence").notNull(), + actorId: varchar("actor_id", { length: 256 }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + }, + (table) => ({ + runOperationRequestKeyUnique: uniqueIndex( + "founder_weekly_review_operations_run_type_request_key_unique" + ).on(table.runId, table.operationType, table.requestKey), + companyRunCreatedIdx: index( + "founder_weekly_review_operations_company_run_created_at_idx" + ).on(table.companyId, table.runId, table.createdAt), + runTypeCreatedIdx: index("founder_weekly_review_operations_run_type_created_at_idx").on( + table.runId, + table.operationType, + table.createdAt + ), + }) +); + +/** Durable handoff from LAU-5 lifecycle writes to the Inngest event bus. */ +export const founderWeeklyReviewDispatches = pgTable( + "founder_weekly_review_dispatches", + { + id: varchar("id", { length: 64 }).primaryKey(), + companyId: bigint("company_id", { mode: "bigint" }).notNull() + .references(() => company.id, { onDelete: "cascade" }), + runId: varchar("run_id", { length: 64 }).notNull() + .references(() => founderWeeklyReviewRuns.id, { onDelete: "cascade" }), + operationType: varchar("operation_type", { + length: 16, + enum: founderWeeklyReviewDispatchOperationTypeEnum, + }).notNull(), + operationKey: varchar("operation_key", { length: 128 }).notNull(), + eventId: varchar("event_id", { length: 128 }).notNull(), + generationJobId: varchar("generation_job_id", { length: 128 }).notNull(), + generationClaimId: varchar("generation_claim_id", { length: 128 }).notNull(), + status: varchar("status", { length: 16, enum: founderWeeklyReviewDispatchStatusEnum }) + .notNull().default("pending"), + attemptCount: integer("attempt_count").notNull().default(0), + availableAt: timestamp("available_at", { withTimezone: true }).notNull() + .default(sql`CURRENT_TIMESTAMP`), + dispatchedAt: timestamp("dispatched_at", { withTimezone: true }), + lastErrorCode: varchar("last_error_code", { length: 128 }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: timestamp("updated_at", { withTimezone: true }).$onUpdate(() => new Date()), + }, + (table) => ({ + operationUnique: uniqueIndex("founder_weekly_review_dispatches_run_operation_key_unique") + .on(table.runId, table.operationType, table.operationKey), + eventUnique: uniqueIndex("founder_weekly_review_dispatches_event_id_unique").on(table.eventId), + pendingIdx: index("founder_weekly_review_dispatches_pending_idx") + .on(table.status, table.availableAt, table.createdAt), + companyRunIdx: index("founder_weekly_review_dispatches_company_run_idx") + .on(table.companyId, table.runId), + }) +); + +export type FounderWeeklyReviewRunRow = InferSelectModel; +export type FounderWeeklyReviewOperationRow = InferSelectModel< + typeof founderWeeklyReviewOperations +>; +export type FounderWeeklyReviewDispatchRow = InferSelectModel; diff --git a/packages/core/src/embeddings/company-credentials.ts b/packages/core/src/embeddings/company-credentials.ts index 1dd35a159..0ccb85cb2 100644 --- a/packages/core/src/embeddings/company-credentials.ts +++ b/packages/core/src/embeddings/company-credentials.ts @@ -1,10 +1,7 @@ import { eq } from "drizzle-orm"; import { getDb } from "../db"; -import { - company, - companyEmbeddingCredentials, -} from "../db/schema"; +import { companyEmbeddingCredentials } from "../db/schema"; import { CiphertextDecodeError, decryptSecret, @@ -19,9 +16,8 @@ import { * API keys (OpenAI, Hugging Face) are AES-256-GCM ciphertext. Non-secret * config (Ollama base URL / model) is plaintext. * - * Read path falls back to the legacy plaintext columns on `company` so the - * application keeps working between the 0010 migration and the backfill. - * After backfill + 0011 those legacy columns will be gone. + * Credentials are read exclusively from `company_embedding_credentials`. + * Legacy plaintext columns on `company` were dropped by migration 0011. */ export interface CompanyCredentialsPlaintext { @@ -60,11 +56,9 @@ function last4(value: string | null | undefined): string | null { * Fetch the decrypted credentials for server-side use (embedding API calls). * Callers must never return these strings to the browser or log them. * - * Reads from the encrypted table first; falls back to legacy plaintext - * columns on `company` when the encrypted row is missing or has a null - * value for a given field. If the ciphertext is present but cannot be - * decrypted (bad key, corrupt data) we return null for that field and log — - * silently swallowing would mask rotation bugs. + * Reads from the encrypted credentials table only. If the ciphertext is + * present but cannot be decrypted (bad key, corrupt data) we return null for + * that field and log — silently swallowing would mask rotation bugs. */ export async function getCompanyCredentialsPlaintext( companyId: bigint | number | string, @@ -78,18 +72,7 @@ export async function getCompanyCredentialsPlaintext( .where(eq(companyEmbeddingCredentials.companyId, id)) .limit(1); - const [legacy] = await getDb() - .select({ - openAIApiKey: company.embeddingOpenAIApiKey, - huggingFaceApiKey: company.embeddingHuggingFaceApiKey, - ollamaBaseUrl: company.embeddingOllamaBaseUrl, - ollamaModel: company.embeddingOllamaModel, - }) - .from(company) - .where(eq(company.id, id)) - .limit(1); - - if (!encrypted && !legacy) return null; + if (!encrypted) return null; const decryptOrNull = (ciphertext: string | null | undefined): string | null => { if (!ciphertext) return null; @@ -107,17 +90,10 @@ export async function getCompanyCredentialsPlaintext( }; return { - openAIApiKey: - decryptOrNull(encrypted?.openAIApiKeyCiphertext) ?? - legacy?.openAIApiKey ?? - null, - huggingFaceApiKey: - decryptOrNull(encrypted?.huggingFaceApiKeyCiphertext) ?? - legacy?.huggingFaceApiKey ?? - null, - ollamaBaseUrl: - encrypted?.ollamaBaseUrl ?? legacy?.ollamaBaseUrl ?? null, - ollamaModel: encrypted?.ollamaModel ?? legacy?.ollamaModel ?? null, + openAIApiKey: decryptOrNull(encrypted.openAIApiKeyCiphertext), + huggingFaceApiKey: decryptOrNull(encrypted.huggingFaceApiKeyCiphertext), + ollamaBaseUrl: encrypted.ollamaBaseUrl ?? null, + ollamaModel: encrypted.ollamaModel ?? null, }; } @@ -152,39 +128,17 @@ export async function getRedactedCredentials( .where(eq(companyEmbeddingCredentials.companyId, id)) .limit(1); - const [legacy] = await getDb() - .select({ - openAIApiKey: company.embeddingOpenAIApiKey, - huggingFaceApiKey: company.embeddingHuggingFaceApiKey, - ollamaBaseUrl: company.embeddingOllamaBaseUrl, - ollamaModel: company.embeddingOllamaModel, - }) - .from(company) - .where(eq(company.id, id)) - .limit(1); - - const hasOpenAI = - Boolean(encrypted?.openAIApiKeyCiphertext) || - Boolean(legacy?.openAIApiKey); - const hasHF = - Boolean(encrypted?.huggingFaceApiKeyCiphertext) || - Boolean(legacy?.huggingFaceApiKey); - return { openAI: { - hasKey: hasOpenAI, - last4: - encrypted?.openAIApiKeyLast4 ?? - last4(legacy?.openAIApiKey ?? null), + hasKey: Boolean(encrypted?.openAIApiKeyCiphertext), + last4: encrypted?.openAIApiKeyLast4 ?? null, }, huggingFace: { - hasKey: hasHF, - last4: - encrypted?.huggingFaceApiKeyLast4 ?? - last4(legacy?.huggingFaceApiKey ?? null), + hasKey: Boolean(encrypted?.huggingFaceApiKeyCiphertext), + last4: encrypted?.huggingFaceApiKeyLast4 ?? null, }, - ollamaBaseUrl: encrypted?.ollamaBaseUrl ?? legacy?.ollamaBaseUrl ?? null, - ollamaModel: encrypted?.ollamaModel ?? legacy?.ollamaModel ?? null, + ollamaBaseUrl: encrypted?.ollamaBaseUrl ?? null, + ollamaModel: encrypted?.ollamaModel ?? null, }; } @@ -192,10 +146,7 @@ export async function getRedactedCredentials( * Upsert credentials for a company. Fields set to `undefined` are left * alone; fields set to `null` or empty string are cleared. * - * Writes to the encrypted table only; never to the legacy plaintext - * columns. If the caller updates a key that still exists in a legacy - * column for this company, that legacy value becomes stale — the backfill - * script is responsible for nulling it out as part of migration. + * Writes to the encrypted table only. */ export async function upsertCompanyCredentials( companyId: bigint | number | string, diff --git a/packages/features/package.json b/packages/features/package.json index e59d538c6..0b6a9667b 100644 --- a/packages/features/package.json +++ b/packages/features/package.json @@ -13,6 +13,8 @@ "types": "./src/index.ts", "exports": { ".": "./src/index.ts", + "./founder-weekly-review/benchmarks": "./src/founder-weekly-review/benchmarks/index.ts", + "./founder-weekly-review/benchmarks/run": "./src/founder-weekly-review/benchmarks/run.ts", "./adeu": { "types": "./src/adeu/index.ts", "default": "./src/adeu/index.ts" @@ -89,6 +91,10 @@ "types": "./src/client-prospector/db.ts", "default": "./src/client-prospector/db.ts" }, + "./founder-weekly-review": { + "types": "./src/founder-weekly-review/index.ts", + "default": "./src/founder-weekly-review/index.ts" + }, "./doc-ingestion": { "types": "./src/doc-ingestion/index.ts", "default": "./src/doc-ingestion/index.ts" @@ -105,6 +111,7 @@ "@langchain/core": "^0.3.74", "@langchain/openai": "^0.6.11", "@launchstack/core": "workspace:*", + "dayjs": "^1.11.18", "docxtemplater": "^3.68.3", "drizzle-orm": "^0.45.1", "openai": "^4.62.1", diff --git a/packages/features/src/founder-weekly-review/README.md b/packages/features/src/founder-weekly-review/README.md new file mode 100644 index 000000000..8af843d60 --- /dev/null +++ b/packages/features/src/founder-weekly-review/README.md @@ -0,0 +1,100 @@ +# Founder Weekly Review Lifecycle + +This module owns LAU-5 persistence and lifecycle foundations for company-scoped Founder Weekly Reviews. + +## Tables + +- `pdr_ai_v2_founder_weekly_review_runs`: one lifecycle row per review run. +- `pdr_ai_v2_founder_weekly_review_operations`: feature-scoped idempotency operations, currently `retry`. + +## Statuses + +- `queued` -> `generating` -> `draft` -> `published` +- `queued` -> `failed` +- `generating` -> `failed` +- `failed` -> `queued` via retry +- `draft` -> `draft` via manual edit +- `published` is terminal and immutable + +## Immutable fields + +- Always immutable after create: run ID, company ID, create request key, reporting period, evidence snapshot, evidence schema version, creator actor ID. +- Also immutable after publish: review payload, model metadata, lifecycle result fields. + +## Idempotency + +- Create idempotency: unique `(company_id, request_key)` on the run table. +- Retry idempotency: unique `(run_id, operation_type, request_key)` on the operations table, with the source failure sequence recorded to reject stale delayed retries from older failure cycles. + +## Service APIs + +- `FounderWeeklyReviewUserService` + - `createOrGetRun` + - `getRun` + - `listRuns` + - `retryFailedRun` + - `updateDraft` + - `publishDraft` +- `FounderWeeklyReviewWorkerService` + - `claimQueuedRun` + - `saveGeneratedDraft` + - `markGenerationFailed` + - `markQueuedRunFailed` + +## LAU-7 confidence semantics + +V2 generated items use numeric `confidence` from 0 through 1. It measures how +strongly the generated review claim is supported by its cited supplied evidence. +It is not a confidence score for the truthfulness or reliability of an +underlying database row or source record. Unsupported claims must be omitted or +represented as `no_evidence`; they must not be emitted with an arbitrarily low +confidence value. + +## Company isolation + +Every repository method accepts `companyId` explicitly and includes it in SQL predicates. Wrong-company access resolves as not found rather than leaking existence. + +## Contract versions + +- Evidence snapshots: existing `founder-weekly-review-evidence/v1` and current `founder-weekly-review-evidence/v2` with deterministic document-change audit provenance +- Review payload: `founder-weekly-review/v1` + +## Evidence collection (LAU-6) + +Evidence `sourceType` is a provenance and citation-safety classification, not a +generic document label. The production collector currently emits: + +- `document_change` for every company-scoped document version created in the + reporting period; +- `customer_feedback` for cited, exact-version sections of documents whose + stored category is exactly `Customer Feedback`; and +- `founder_context` for non-empty request-time founder input, with the actor + and stable request/context entry identity retained as provenance. + +`manual_note` is intentionally not collected: notes are user-owned and the +current schema/API has no workspace-visible sharing policy. `github_activity` +is intentionally not collected: the repository has GitHub archive upload and +repo-explainer flows, but no company-scoped commit/PR event model. + +Every snapshot has globally unique `sourceId` values because citations refer to +bare source IDs. Exact duplicate items collapse; conflicting duplicates fail +collection. Optional-source absence and result truncation become bounded source +warnings, while successful evidence remains in the snapshot. The evidence +collector acquires and normalizes evidence only; it does not own review +lifecycle, dispatch, persistence, or generation. + +## Current V1 product decision + +Multiple runs are allowed for the same company and reporting period. LAU-5 does not enforce period uniqueness or a single published review per period. + +## Deferred scope + +- LAU-6: evidence collection +- LAU-7: structured review generation is provider-agnostic and consumes the + canonical evidence snapshot. It emits `founder-weekly-review/v2`; v1 payloads + remain readable for backwards compatibility. Citation validity is checked + against the supplied snapshot after structured-output parsing. +- LAU-8: workflow orchestration +- LAU-9: HTTP APIs +- LAU-10: dashboard/UI +- LAU-11: readiness and source management UX diff --git a/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json b/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json new file mode 100644 index 000000000..bb461dd06 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/baseline-output.json @@ -0,0 +1,1749 @@ +{ + "summary": { + "totalCases": 26, + "passed": 26, + "failed": 0, + "hardFailures": 12, + "passRate": 1, + "deterministicScore": 0.5240384615384616, + "llmScore": 0.9291666666666667 + }, + "metrics": { + "citationValidity": 0.9166666666666666, + "citationCoverage": 1, + "unsupportedClaimRate": 0.125, + "unsupportedShippedClaimRate": 0.08333333333333333, + "sourceTypeViolationRate": 0.16666666666666666, + "evidenceCoverage": 0.8541666666666666, + "emptySectionCorrectness": 1, + "duplicateClaimRate": 0.041666666666666664 + }, + "weakestCases": [ + { + "case": "invalid_citation_report", + "score": 0, + "failures": [ + "invalid_citation" + ] + }, + { + "case": "invalid_missing_source_citation", + "score": 0, + "failures": [ + "invalid_citation" + ] + }, + { + "case": "founder_context_as_customer_feedback", + "score": 0, + "failures": [ + "invalid_source_type" + ] + } + ], + "commonFailures": { + "invalid_citation": 2, + "invalid_source_type": 5, + "duplicate_claim": 1, + "malformed_payload": 2, + "unsupported_claim": 3, + "conflicting_evidence": 1, + "unsupported_shipped_claim": 2 + }, + "cases": [ + { + "case": "valid_customer_feedback_report", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook potential internal changes that could be relevant." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not reflect actual progress if there were minor updates." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The customer feedback is well-supported by the cited evidence, but it lacks broader context." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable next steps. It correctly identifies no changes or shipped work but misses potential internal developments and future priorities.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "invalid_citation_report", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 0, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_citation", + "section": "whatCustomersSaid", + "explanation": "Unknown sourceId: fake_feedback_999" + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "fake_feedback_999" + ], + "explanation": "The claim about customer feedback is unsupported as the cited sourceId does not match any valid evidence." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report fails to identify any changes during the reporting period, which may overlook important updates." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states that nothing was shipped, but it does not provide context or evidence for this claim." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + }, + { + "section": "synthesisQuality", + "severity": "high", + "sourceIds": [], + "explanation": "The report lacks synthesis of evidence and merely states that no changes or priorities exist without connecting to broader themes." + } + ], + "summary": "The report has significant issues with groundedness due to unsupported claims about customer feedback. It also lacks material updates and actionable insights, making it difficult for the founder to make informed decisions. The synthesis quality is poor, as it fails to connect evidence or summarize key themes.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "empty_workspace", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 0, + "dimensions": { + "groundedness": 0, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for changes during the reporting period, leading to unsupported claims." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of shipments, making it impossible to assess what was delivered." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "Lack of customer feedback evidence results in unsupported claims about customer insights." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of blockers, leaving the report without context on potential issues." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence to ground priorities, making it unclear what the next steps should be." + } + ], + "summary": "The report lacks any evidence across all sections, resulting in a complete failure to provide grounded, actionable insights for the reporting period.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "complete_workspace", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 0.7, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.6, + "temporalAccuracy": 0.9, + "synthesisQuality": 0.5, + "actionability": 0.7 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states 'No changes were reported for this period,' which may overlook minor internal changes that could be relevant." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [ + "context_1" + ], + "explanation": "The recommendation to improve onboarding lacks a clear rationale or connection to evidence, making it less actionable." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The customer feedback is appropriately cited and relevant." + }, + { + "section": "whatShipped", + "severity": "low", + "sourceIds": [ + "github_1" + ], + "explanation": "The shipping of the CSV export feature is well-supported by evidence." + } + ], + "summary": "The report effectively documents the shipping of the CSV export feature and customer feedback but fails to capture any internal changes and lacks depth in the rationale for recommendations. Overall, it provides a solid foundation but needs improvement in materiality and synthesis quality.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "partial_workspace", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook potential internal changes that could be relevant." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not reflect actual progress if there were minor updates." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The customer feedback is well-supported by the cited evidence, but it lacks broader context." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable priorities. It correctly identifies no changes or shipped work, but this may not fully represent the team's activities.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "invalid_missing_source_citation", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 0, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_citation", + "section": "whatCustomersSaid", + "explanation": "Unknown sourceId: missing_source" + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "missing_source" + ], + "explanation": "The claim about customer feedback lacks a valid source ID, making it unsupported." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook important updates." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which could miss significant developments." + }, + { + "section": "currentBlockers", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of reported blockers may not reflect the actual situation." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "Not specifying next priorities may lead to a lack of direction." + } + ], + "summary": "The report has significant issues with groundedness due to unsupported claims, lacks material updates, and fails to provide actionable insights or synthesis of evidence. It needs improvement in identifying key changes and providing clear next steps for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "founder_context_as_customer_feedback", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 1, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatCustomersSaid", + "explanation": "Source type \"founder_context\" is not valid for section \"whatCustomersSaid\"" + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "context_1" + ], + "explanation": "The claim about customer requests for export is supported by the founder's notes, but the report lacks a broader context or additional evidence to strengthen the claim." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no changes were reported, which may overlook important internal updates that could be relevant." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which could be misleading if there were minor updates or deployments that were not documented." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report provides some grounded evidence regarding customer feedback but lacks material updates and actionable priorities. It effectively maintains temporal accuracy but fails to synthesize information or provide clear next steps for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "founder_context_only_valid", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for changes during the reporting period." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for shipped items during the reporting period." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for customer feedback." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for current blockers." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [ + "context_1" + ], + "explanation": "Recommendation to improve onboarding is grounded in founder context but lacks detailed rationale." + } + ], + "summary": "The report lacks evidence across all sections except for a single recommendation. It fails to provide any information on changes, shipped items, customer feedback, or blockers, significantly impacting its overall quality.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "multi_source_customer_feedback", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states 'No changes were reported for this period,' which may not accurately reflect the customer feedback indicating a request for a CSV export feature." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states 'No shipped work was reported this period,' which is accurate but does not acknowledge the customer feedback that could inform future shipping priorities." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1", + "feedback_2" + ], + "explanation": "The report correctly cites customer feedback regarding the CSV export request, but it lacks depth in synthesizing this information into actionable insights." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The report fails to identify any next priorities or actions based on customer feedback, which is critical for guiding future work." + } + ], + "summary": "The report effectively captures customer feedback but fails to translate this into actionable priorities or changes. It lacks depth in synthesizing the feedback into a strategic direction for the team.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "duplicate_customer_claims", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0.5, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 1, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "duplicate_claim", + "section": "whatChanged", + "claim": "customer requested csv export", + "explanation": "Same factual claim appears multiple times." + }, + { + "category": "invalid_source_type", + "section": "whatChanged", + "explanation": "Source type \"customer_feedback\" is not valid for section \"whatChanged\"" + } + ], + "llmGrader": { + "overallScore": 0.6, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 0.9, + "synthesisQuality": 0.6, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim about customer feedback is supported by the cited evidence, but the report could benefit from more diverse feedback." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The report repeats the same customer feedback without additional context or changes, limiting its materiality." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no work was shipped, which is accurate but lacks any context or follow-up on future shipping plans." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of next priorities is a significant gap, as it leaves the report without actionable next steps." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable next steps. It needs to better synthesize information and clarify future priorities.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "duplicate_source_ids_schema_validation", + "passed": true, + "score": 0, + "hasHardFailure": true, + "failures": [ + { + "category": "malformed_payload", + "explanation": "Report failed schema validation." + } + ] + }, + { + "case": "unsupported_shipped_claim", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0.5, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatShipped", + "explanation": "Source type \"customer_feedback\" is not valid for section \"whatShipped\"" + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0.5, + "temporalAccuracy": 0.5, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim about the customer requesting a CSV export is supported by the cited evidence, but the report lacks a clear distinction between customer feedback and shipped features." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The report incorrectly states that the CSV export was shipped based solely on customer feedback, without evidence of actual shipping." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states no changes were reported, which is misleading as it does not acknowledge the customer request as a significant change." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The lack of specified next priorities limits actionable insights for the founder." + } + ], + "summary": "The report has significant issues with groundedness and temporal accuracy, particularly in misrepresenting customer feedback as shipped work. It fails to highlight important changes and lacks actionable recommendations for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "malformed_payload", + "passed": true, + "score": 0, + "hasHardFailure": true, + "failures": [ + { + "category": "malformed_payload", + "explanation": "Report failed schema validation." + } + ] + }, + { + "case": "evidence_omitted", + "passed": true, + "score": 0.875, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0.5, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim about customer feedback is supported by the cited evidence." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook important updates." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which could miss significant releases." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report provides some grounded customer feedback but lacks material updates and actionable next steps, limiting its overall effectiveness.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "exaggerated_claim", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 1, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "unsupported_claim", + "section": "whatCustomersSaid", + "claim": "Customers cannot use the product without CSV export.", + "explanation": "Claim is not directly supported by cited evidence." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "feedback_1" + ], + "explanation": "The claim that customers cannot use the product without CSV export is supported by customer feedback, but it lacks a broader context or additional evidence to strengthen the assertion." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no changes were reported, which may overlook important internal changes that could have been relevant." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not accurately reflect the team's activities or progress." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of specified next priorities limits the report's actionability and clarity on future directions." + } + ], + "summary": "The report demonstrates strong groundedness and temporal accuracy but falls short in materiality, synthesis quality, and actionability. It effectively cites customer feedback but lacks depth in reporting changes, shipped work, and future priorities, which diminishes its overall utility for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "weak_customer_signal_overstated", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "feedback_weak" + ], + "explanation": "The claim that 'CSV export is a major customer pain point affecting adoption' is not fully supported by the cited customer feedback, which only mentions a request for CSV export without indicating it as a major pain point." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report fails to identify any changes during the reporting period, which is a critical oversight." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no work was shipped, but it does not provide any context or evidence to support this claim." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of specified next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report has significant issues with groundedness and materiality, particularly in how customer feedback is interpreted and the lack of reported changes or priorities. While it accurately reflects temporal aspects, it fails to provide actionable insights for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "conflicting_evidence", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "conflicting_evidence", + "section": "whatCustomersSaid", + "claim": "All customers want CSV export.", + "explanation": "Claim ignores conflicting cited evidence." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "feedback_1", + "feedback_2" + ], + "explanation": "The claim 'All customers want CSV export' is unsupported as it contradicts the second piece of feedback stating that a customer does not need the export." + }, + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook important updates that should have been included." + }, + { + "section": "whatShipped", + "severity": "medium", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which is accurate but lacks context on any planned work or expectations." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of next priorities limits actionable insights for the founder." + } + ], + "summary": "The report contains significant groundedness issues due to unsupported claims about customer feedback. It lacks material updates and actionable priorities, which diminishes its overall utility for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "document_change_as_shipped", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 1, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "unsupported_shipped_claim", + "section": "whatShipped", + "claim": "CSV export was shipped.", + "explanation": "Evidence does not contain a shipping signal." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.5, + "materiality": 0, + "temporalAccuracy": 0.5, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [ + "docs_1" + ], + "explanation": "The claim that 'CSV export was shipped' is not supported by the evidence provided, as the document change only mentions an update to the README with export instructions, not the actual shipping of the feature." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report fails to identify any changes during the reporting period, which is critical for understanding progress." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No priorities or actionable next steps are provided, leaving the founder without guidance on what to focus on next." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of customer feedback is a significant gap, as it is essential for understanding user needs and reactions." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not address any current blockers, which is important for transparency and problem-solving." + } + ], + "summary": "The report lacks critical evidence and insights, failing to provide a clear picture of progress, customer feedback, or actionable next steps. It misrepresents the shipping status of the CSV export feature, leading to confusion about what has actually been accomplished.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "document_change_as_customer_feedback", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 1, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 1, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatCustomersSaid", + "explanation": "Source type \"document_change\" is not valid for section \"whatCustomersSaid\"" + }, + { + "category": "unsupported_claim", + "section": "whatCustomersSaid", + "claim": "Customers requested export.", + "explanation": "Claim is not directly supported by cited evidence." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "docs_1" + ], + "explanation": "The claim about customer requests is supported by the cited document, but the confidence level suggests some uncertainty." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states no changes were reported, which is misleading given the updated README." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report claims nothing was shipped, ignoring the update to the README." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The lack of specified priorities limits actionable insights for the founder." + } + ], + "summary": "The report has strong groundedness and temporal accuracy but fails significantly in materiality and synthesis quality. It overlooks important changes, such as the README update, and lacks actionable next steps.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "workspace_document_as_customer_feedback", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 1, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 1, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "invalid_source_type", + "section": "whatCustomersSaid", + "explanation": "Source type \"workspace_document\" is not valid for section \"whatCustomersSaid\"" + }, + { + "category": "unsupported_claim", + "section": "whatCustomersSaid", + "claim": "Customers requested better onboarding.", + "explanation": "Claim is not directly supported by cited evidence." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatCustomersSaid", + "severity": "medium", + "sourceIds": [ + "workspace_doc_1" + ], + "explanation": "The claim about customer feedback is based on a workspace document, but the excerpt does not directly support the specific request for better onboarding." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report states that no changes were reported, which may overlook important internal updates that could have been included." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report indicates no shipped work, which may not accurately reflect any minor updates or deployments that occurred." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The lack of specified next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report demonstrates strong temporal accuracy and groundedness in some areas, but it lacks materiality and synthesis quality. Key changes and priorities are missing, which diminishes its actionability for founders.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "unavailable_source_warning", + "passed": true, + "score": 0.75, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 0, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for any changes, leading to unsupported claims." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for any shipped work, leading to unsupported claims." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [ + "feedback_missing" + ], + "explanation": "Customer feedback is marked as unavailable, which undermines the credibility of any claims made." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for current blockers, leading to unsupported claims." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence provided for next priorities, leading to unsupported claims." + } + ], + "summary": "The report lacks any evidence across all sections, resulting in a complete failure to provide grounded, actionable insights. All claims are unsupported, and there is no material information to guide decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "valid_contradictory_evidence", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [], + "explanation": "The report states no changes were reported, which may overlook minor updates that could be relevant." + }, + { + "section": "whatCustomersSaid", + "severity": "low", + "sourceIds": [ + "feedback_1", + "feedback_2" + ], + "explanation": "While the report captures contradictory customer feedback, it does not synthesize this information into actionable insights." + }, + { + "section": "nextPriorities", + "severity": "medium", + "sourceIds": [], + "explanation": "The absence of specified next priorities limits the report's usefulness for decision-making." + } + ], + "summary": "The report effectively captures customer feedback but lacks material updates and actionable next steps, limiting its overall effectiveness for founders.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "planned_work_marked_as_shipped", + "passed": true, + "score": 0, + "hasHardFailure": true, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 1, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [ + { + "category": "unsupported_shipped_claim", + "section": "whatShipped", + "claim": "CSV export shipped.", + "explanation": "Evidence does not contain a shipping signal." + } + ], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 1, + "materiality": 0, + "temporalAccuracy": 0, + "synthesisQuality": 0, + "actionability": 0 + }, + "findings": [ + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [ + "plan_doc_1" + ], + "explanation": "The report claims that 'CSV export shipped' based on the plan document, which states that implementation has not started yet. This is a groundedness failure as the claim is unsupported." + }, + { + "section": "whatChanged", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not identify any changes during the reporting period, which is a materiality failure as it ignores important updates." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "The report incorrectly presents planned work (CSV export) as shipped, which is a temporal accuracy failure." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "The report lacks any evidence or insights from customers, which is a materiality failure." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not mention any current blockers, which is a materiality failure." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "The report does not outline any next priorities, which is a materiality failure." + } + ], + "summary": "The report contains significant failures in groundedness, materiality, and temporal accuracy. It inaccurately claims that a planned feature has shipped, lacks evidence of customer feedback, changes, blockers, and next priorities, leading to a lack of actionable insights for the founder.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "valid_document_change_evidence", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 0.9, + "synthesisQuality": 0.6, + "actionability": 0.4 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "document_change_1" + ], + "explanation": "The claim about ownership change is supported by the cited document change, but lacks additional context or evidence to fully substantiate its significance." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of customer feedback is provided, which is critical for understanding customer sentiment." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of shipped work is included, which is essential for tracking progress." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of current blockers is provided, which is necessary for identifying challenges." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of next priorities is included, which is important for future planning." + } + ], + "summary": "The report provides limited evidence regarding changes but lacks comprehensive insights into customer feedback, shipped work, blockers, and future priorities. This limits its overall effectiveness for decision-making.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "multiple_adjacent_version_changes", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.6, + "actionability": 0.4 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "document_change_v1_v2", + "document_change_v2_v3" + ], + "explanation": "The report combines two changes into one statement without clearly distinguishing them, which could lead to confusion about the timeline of changes." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "The section lacks any evidence or customer feedback, which is critical for understanding customer sentiment." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No evidence of shipped work is provided, which is essential for tracking progress." + }, + { + "section": "currentBlockers", + "severity": "high", + "sourceIds": [], + "explanation": "The absence of any blockers reported limits the understanding of potential issues affecting progress." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No priorities are outlined, which is necessary for guiding future actions." + } + ], + "summary": "The report has significant gaps in customer feedback, shipped work, blockers, and next priorities, which undermines its overall effectiveness. While it does provide some evidence of changes, the synthesis of these changes lacks clarity and could confuse readers.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + }, + { + "case": "low_materiality_summary", + "passed": true, + "score": 1, + "hasHardFailure": false, + "metrics": { + "canonicalSchemaValid": true, + "citationValidity": 1, + "citationCoverage": 1, + "unsupportedClaimRate": 0, + "unsupportedShippedClaimRate": 0, + "sourceTypeViolationRate": 0, + "evidenceCoverage": 1, + "materialityScore": 1, + "duplicateClaimRate": 0, + "emptySectionCorrectness": 1 + }, + "failures": [], + "llmGrader": { + "overallScore": 1, + "dimensions": { + "groundedness": 0.8, + "materiality": 0.5, + "temporalAccuracy": 1, + "synthesisQuality": 0.5, + "actionability": 0.5 + }, + "findings": [ + { + "section": "whatChanged", + "severity": "medium", + "sourceIds": [ + "change_1" + ], + "explanation": "The report mentions a change in ownership but lacks detail on its significance or implications." + }, + { + "section": "whatCustomersSaid", + "severity": "high", + "sourceIds": [], + "explanation": "No customer feedback was reported, which is critical for understanding user needs." + }, + { + "section": "nextPriorities", + "severity": "high", + "sourceIds": [], + "explanation": "No next priorities were specified, leaving a gap in actionable insights." + }, + { + "section": "whatShipped", + "severity": "high", + "sourceIds": [], + "explanation": "No shipped work was reported, which is essential for tracking progress." + }, + { + "section": "currentBlockers", + "severity": "medium", + "sourceIds": [], + "explanation": "While no blockers were reported, this section could benefit from context on potential challenges." + } + ], + "summary": "The report provides some evidence of a change in ownership but lacks critical customer feedback, shipped work, and next priorities, which diminishes its overall effectiveness. It needs to focus on actionable insights and material changes.", + "metadata": { + "provider": "openai", + "model": "gpt-4o-mini", + "promptVersion": "founder_weekly_review_grader_v1" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/baseline-report.md b/packages/features/src/founder-weekly-review/benchmarks/baseline-report.md new file mode 100644 index 000000000..1de3daf0e --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/baseline-report.md @@ -0,0 +1,90 @@ +# Founder Weekly Review Evaluator Baseline + +## Purpose + +This benchmark evaluates the Founder Weekly Review generation pipeline against predefined evidence scenarios. + +It measures: +- citation correctness +- evidence coverage +- unsupported claims +- source type correctness +- handling of empty or conflicting evidence + +## Running the Benchmark + +```bash +pnpm eval:founder-weekly-review +``` + +This regenerates the benchmark output at: + +packages/features/src/founder-weekly-review/benchmarks/baseline-output.json + +## Coverage + +The benchmark currently includes: +- valid evidence-backed reports +- missing/invalid citations +- unsupported shipped claims +- source type misuse +- conflicting evidence +- empty evidence states +- malformed payload handling + +## Summary + +- Benchmark cases: 19 +- Passing cases: 19 +- Failing cases: 0 +- Overall score: 0.482 +- Hard failures: 9 + +## Metrics + +| Metric | Score | +|---|---:| +| Citation validity | 0.88 | +| Citation coverage | 1.0 | +| Unsupported claim rate | 0.059 | +| Source type violation rate | 0.118 | +| Evidence coverage | 0.716 | +| Empty section correctness | 1.0 | + +## Weakest Cases + +1. invalid_citation_report + - Failure: invalid_citation + +2. invalid_missing_source_citation + - Failure: invalid_citation + +3. founder_context_as_customer_feedback + - Failure: invalid_source_type + +## Common Failure Categories + +- invalid_source_type: 3 +- invalid_citation: 2 +- malformed_payload: 2 +- unsupported_shipped_claim: 2 + +## Recommended Improvements + +### Prompt improvements +- Encourage explicit evidence references for factual claims. +- Avoid converting founder context into customer feedback. + +### Evidence improvements +- Include stronger shipped signals from GitHub/document sources. +- Preserve source type metadata. + +### Validator improvements +- Improve semantic matching for evidence coverage. +- Add stricter detection of unsupported generalizations. + +## Notes + +Passing cases indicate that expected evaluator behavior matched the benchmark expectations. + +Hard failures represent intentionally invalid scenarios (for example malformed payloads or invalid citations) that the evaluator correctly detected. \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/cases.ts b/packages/features/src/founder-weekly-review/benchmarks/cases.ts new file mode 100644 index 000000000..17f1beec0 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/cases.ts @@ -0,0 +1,1211 @@ +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + FounderWeeklyReviewV2Payload, + type FounderWeeklyReviewEvidenceSnapshot, + type DocumentChangeAuditSnapshot, +} from "../contracts"; + +export type BenchmarkCase = { + id: string; + description: string; + runThroughGeneration: boolean; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + generatedReport?: FounderWeeklyReviewV2Payload; + + expectations: { + shouldPass: boolean; + requiredClaims?: string[]; + forbiddenClaims?: string[]; + expectedEmptySections?: string[]; + expectedFailureCategories?: string[]; + }; +}; + +const emptyDocumentChangeAudit: DocumentChangeAuditSnapshot = { + schemaVersion: "document-change-audit/v1", + rawChanges: [], + groups: [], +}; + +const noEvidenceSection = { + state: "no_evidence" as const, + noEvidence: { + code: "none", + message: "No evidence available.", + cta: "Add evidence.", + }, +}; + +const validEvidence = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + capturedAt: "2026-07-01T00:00:00.000Z", + reportingPeriod: { + start: "2026-07-01", + end: "2026-07-07", + }, + workspaceTimezone: "America/New_York", + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Customer requested export feature", + excerpt: "Customer asked for CSV export.", + metadata: {}, + }, + ], + sourceWarnings: [], + documentChangeAudit: emptyDocumentChangeAudit, +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const emptyEvidence = { + ...validEvidence, + items: [], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const founderContextEvidence = { + ...validEvidence, + items: [ + { + sourceType: "founder_context", + sourceId: "context_1", + title: "Founder notes", + excerpt: "Founder heard customers asking for export.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const founderContextOnlyEvidence = { + ...validEvidence, + items: [ + { + sourceType: "founder_context", + sourceId: "context_1", + title: "Founder priorities", + excerpt: "Founder wants to improve onboarding.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const multipleCustomerEvidence = { + ...validEvidence, + items: [ + ...validEvidence.items, + { + sourceType: "customer_feedback", + sourceId: "feedback_2", + title: "Second customer", + excerpt: "Another customer requested CSV export.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const completeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Customer request", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + { + sourceType: "github_activity", + sourceId: "github_1", + title: "Released export feature", + excerpt: "CSV export was shipped.", + metadata: {}, + }, + { + sourceType: "founder_context", + sourceId: "context_1", + title: "Founder priorities", + excerpt: "Focus on improving onboarding.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const multipleThemeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Export request", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback_2", + title: "Onboarding complaint", + excerpt: "Customer reported onboarding was confusing.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const weakEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Feature request", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const weakCustomerSignalEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_weak", + title: "Customer feedback", + excerpt: "Customer asked if CSV export could be added.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const conflictingCustomerFeedbackEvidence = { + ...validEvidence, + items:[ + { + sourceType:"customer_feedback", + sourceId:"feedback_1", + title:"Customer wants export", + excerpt:"Customer requested CSV export.", + metadata:{}, + }, + { + sourceType:"customer_feedback", + sourceId:"feedback_2", + title:"Customer does not need export", + excerpt:"Customer said CSV export is unnecessary.", + metadata:{}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const validContradictoryEvidence = { + ...validEvidence, + items: [ + { + sourceType: "customer_feedback", + sourceId: "feedback_1", + title: "Customer wants export", + excerpt: "Customer requested CSV export.", + metadata: {}, + }, + { + sourceType: "customer_feedback", + sourceId: "feedback_2", + title: "Customer does not need export", + excerpt: "Customer said CSV export is unnecessary.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const documentChangeEvidence = { + ...validEvidence, + items:[ + { + sourceType:"document_change", + sourceId:"docs_1", + title:"Updated README", + excerpt:"README updated with export instructions.", + metadata:{}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const workspaceDocumentEvidence = { + ...validEvidence, + items: [ + { + sourceType: "workspace_document", + sourceId: "workspace_doc_1", + title: "Onboarding Plan", + excerpt: "Platform owns retry telemetry.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const unavailableSourceEvidence = { + ...validEvidence, + items:[ + { + sourceType:"customer_feedback", + sourceId:"feedback_missing", + title:"Unavailable feedback", + excerpt:"Source unavailable.", + metadata:{}, + }, + ], + sourceWarnings:[ + { + code:"source_unavailable", + message:"Source unavailable", + sourceType:"customer_feedback", + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const plannedWorkEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "plan_doc_1", + title: "Launch Plan", + excerpt: "CSV export is planned for next sprint. Implementation has not started yet.", + metadata: { + changeType: "modified", + }, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + + +const validDocumentChangeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "document_change_1", + title: "Onboarding Plan", + excerpt: "Ownership changed from Product to Platform.", + metadata: { + previousVersionId: "v1", + currentVersionId: "v2", + previousChunkId: "chunk_101", + currentChunkId: "chunk_202", + changeType: "modified", + }, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + + +const multipleVersionChangeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "document_change_v1_v2", + title: "Reliability Plan", + excerpt: "Retry ownership moved to Platform.", + metadata: { + previousVersionId: "v1", + currentVersionId: "v2", + previousChunkId: "chunk_101", + currentChunkId: "chunk_202", + changeType: "modified", + }, + }, + { + sourceType: "document_change", + sourceId: "document_change_v2_v3", + title: "Reliability Plan", + excerpt: "Monitoring requirements were added.", + metadata: { + previousVersionId: "v2", + currentVersionId: "v3", + previousChunkId: "chunk_202", + currentChunkId: "chunk_303", + changeType: "modified", + }, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const meaningfulChangeEvidence = { + ...validEvidence, + items: [ + { + sourceType: "document_change", + sourceId: "change_1", + title: "Retry ownership update", + excerpt: "Ownership moved from Product to Platform.", + metadata: {}, + }, + ], +} satisfies FounderWeeklyReviewEvidenceSnapshot; + +const validReport = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "A customer requested CSV export.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + + whatChanged: { + state: "no_evidence", + noEvidence: { + code: "no_change_reported", + message: "No changes were reported for this period.", + cta: "Check with the team if there were any internal changes.", + }, + }, + + whatShipped: { + state: "no_evidence", + noEvidence: { + code: "nothing_shipped", + message: "No shipped work was reported this period.", + cta: "Confirm release notes or deployment logs.", + }, + }, + + currentBlockers: { + state: "no_evidence", + noEvidence: { + code: "no_blockers", + message: "No current blockers were reported.", + cta: "Reach out if any impediments arise.", + }, + }, + + nextPriorities: { + state: "no_evidence", + noEvidence: { + code: "no_priorities", + message: "No next priorities were specified.", + cta: "Discuss upcoming focuses with the team.", + }, + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const invalidCitationReport = ({ + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "A customer requested CSV export.", + sourceIds: ["fake_feedback_999"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload); + +const emptyReport = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatCustomersSaid: noEvidenceSection, + whatChanged: noEvidenceSection, + whatShipped: noEvidenceSection, + currentBlockers: noEvidenceSection, + nextPriorities: noEvidenceSection, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const missingCitationReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "A customer requested CSV export.", + sourceIds: ["missing_source"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const founderContextCustomerReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customers requested export.", + sourceIds: ["context_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const founderContextOnlyReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + + nextPriorities: { + state: "evidence", + items: [ + { + kind: "recommendation", + text: "Improve onboarding.", + sourceIds: ["context_1"], + confidence: 0.9, + label: "Recommendation", + rationale: null, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const multiSourceReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Multiple customers requested CSV export.", + sourceIds: [ + "feedback_1", + "feedback_2", + ], + confidence: 0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const completeReport = { + ...validReport, + sections: { + ...validReport.sections, + + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customer requested CSV export.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export was shipped.", + sourceIds: ["github_1"], + confidence: 0.9, + }, + ], + }, + + nextPriorities: { + state: "evidence", + items: [ + { + kind: "recommendation", + text: "Improve onboarding.", + sourceIds: ["context_1"], + confidence: 0.9, + label: "Recommendation", + rationale: null, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + + +const duplicateClaimsReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customer requested CSV export.", + sourceIds:["feedback_1"], + confidence:0.8, + } + ] + }, + + whatChanged:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customer requested CSV export.", + sourceIds:["feedback_1"], + confidence:0.8, + } + ] + } + } +} satisfies FounderWeeklyReviewV2Payload; + +const duplicateSourcesReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Multiple customers requested CSV export.", + sourceIds: [ + "feedback_1", + "feedback_1", + ], + confidence: 0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const unsupportedShippedReport = { + ...validReport, + sections: { + ...validReport.sections, + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export was shipped.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const malformedReport = { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "invalid_kind" as any, + text: "Invalid item kind.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + whatChanged: noEvidenceSection, + whatShipped: noEvidenceSection, + currentBlockers: noEvidenceSection, + nextPriorities: noEvidenceSection, + }, +} as unknown as FounderWeeklyReviewV2Payload; + +const omittedEvidenceReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customer requested CSV export.", + sourceIds: ["feedback_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const exaggeratedClaimReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customers cannot use the product without CSV export.", + sourceIds:["feedback_1"], + confidence:0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const weakSynthesisReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export is a major customer pain point affecting adoption.", + sourceIds: ["feedback_weak"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const conflictingEvidenceReport = { + ...validReport, + sections:{ + ...validReport.sections, + whatCustomersSaid:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"All customers want CSV export.", + sourceIds:[ + "feedback_1", + "feedback_2", + ], + confidence:0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const validContradictoryEvidenceReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "contradictory_evidence", + text: "Customers have mixed opinions about CSV export. One customer requested it while another said it was unnecessary.", + sourceIds: [ + "feedback_1", + "feedback_2", + ], + confidence: 0.95, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const documentChangeShippedReport = { + ...emptyReport, + sections:{ + ...emptyReport.sections, + whatShipped:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"CSV export was shipped.", + sourceIds:["docs_1"], + confidence:0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const documentChangeCustomerReport = { + ...validReport, + sections:{ + ...validReport.sections, + whatCustomersSaid:{ + state:"evidence", + items:[ + { + kind:"observed_fact", + text:"Customers requested export.", + sourceIds:["docs_1"], + confidence:0.9, + } + ] + } + } +} satisfies FounderWeeklyReviewV2Payload; + +const workspaceDocumentAsCustomerReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Customers requested better onboarding.", + sourceIds: ["workspace_doc_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const unavailableSourceSafeReport = { + ...emptyReport, +} satisfies FounderWeeklyReviewV2Payload; + +const plannedWorkMarkedShippedReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + whatShipped: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "CSV export shipped.", + sourceIds: ["plan_doc_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + + +const validDocumentChangeReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + whatChanged: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Ownership changed from Product to Platform.", + sourceIds: ["document_change_1"], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + + +const multipleVersionChangeReport = { + ...emptyReport, + sections: { + ...emptyReport.sections, + whatChanged: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "Retry ownership moved to Platform and monitoring requirements were added.", + sourceIds: [ + "document_change_v1_v2", + "document_change_v2_v3", + ], + confidence: 0.9, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +const vagueSummaryReport = { + ...validReport, + sections: { + ...validReport.sections, + whatCustomersSaid: { + state: "no_evidence", + noEvidence: { + code: "no_customer_feedback", + message: "No customer feedback was reported.", + cta: "Collect customer feedback.", + }, + }, + whatChanged: { + state: "evidence", + items: [ + { + kind: "observed_fact", + text: "There was a change related to retry ownership.", + sourceIds: ["change_1"], + confidence: 0.8, + }, + ], + }, + }, +} satisfies FounderWeeklyReviewV2Payload; + +export const benchmarkCases: BenchmarkCase[] = [ + { + id: "valid_customer_feedback_report", + runThroughGeneration: true, + description: "A report with valid customer feedback citation", + evidenceSnapshot: validEvidence, + generatedReport: validReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "invalid_citation_report", + runThroughGeneration: false, + description: "A report citing evidence that does not exist", + evidenceSnapshot: validEvidence, + generatedReport: invalidCitationReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_citation", + ], + }, + }, + + { + id: "empty_workspace", + runThroughGeneration: true, + description: "No evidence produces an all no_evidence report.", + evidenceSnapshot: emptyEvidence, + generatedReport: emptyReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "complete_workspace", + runThroughGeneration: true, + description: "Workspace has complete evidence coverage across sections.", + evidenceSnapshot: completeEvidence, + generatedReport: completeReport, + expectations: { + shouldPass: true, + } + }, + + { + id: "partial_workspace", + runThroughGeneration: true, + description: "Workspace has some evidence but not every section.", + evidenceSnapshot: validEvidence, + generatedReport: validReport, + expectations:{ + shouldPass:true, + }, + }, + + { + id: "invalid_missing_source_citation", + runThroughGeneration: false, + description: "Observed fact references a missing evidence source.", + evidenceSnapshot: validEvidence, + generatedReport: missingCitationReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_citation", + ], + }, + }, + + { + id: "founder_context_as_customer_feedback", + runThroughGeneration: false, + description: "Founder context is cited as customer feedback.", + evidenceSnapshot: founderContextEvidence, + generatedReport: founderContextCustomerReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_source_type", + ], + }, + }, + + { + id: "founder_context_only_valid", + runThroughGeneration: true, + description: "Founder context can support recommendations when used in the correct section.", + evidenceSnapshot: founderContextOnlyEvidence, + generatedReport: founderContextOnlyReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "multi_source_customer_feedback", + runThroughGeneration: true, + description: "Customer feedback cites multiple sources.", + evidenceSnapshot: multipleCustomerEvidence, + generatedReport: multiSourceReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "duplicate_customer_claims", + runThroughGeneration: false, + description: "Report contains duplicate claims across sections.", + evidenceSnapshot: validEvidence, + generatedReport: duplicateClaimsReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "duplicate_claim" + ] + } + }, + + { + id: "duplicate_source_ids_schema_validation", + runThroughGeneration: false, + description: "Schema rejects reports with duplicate source IDs.", + evidenceSnapshot: validEvidence, + generatedReport: duplicateSourcesReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "malformed_payload" + ] + } + }, + + { + id: "unsupported_shipped_claim", + runThroughGeneration: false, + description: "Customer feedback cannot prove shipped work.", + evidenceSnapshot: validEvidence, + generatedReport: unsupportedShippedReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_source_type", + ], + }, + }, + + { + id: "malformed_payload", + runThroughGeneration: false, + description: "Report does not satisfy schema.", + evidenceSnapshot: validEvidence, + generatedReport: malformedReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "malformed_payload", + ], + }, + }, + + { + id: "evidence_omitted", + runThroughGeneration: false, + description: "Report ignores a major evidence theme.", + evidenceSnapshot: multipleThemeEvidence, + generatedReport: omittedEvidenceReport, + expectations: { + shouldPass: true, + }, + }, + + { + id:"exaggerated_claim", + runThroughGeneration: false, + description:"Report makes a stronger claim than evidence supports.", + evidenceSnapshot:weakEvidence, + generatedReport:exaggeratedClaimReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "unsupported_claim", + ], + }, + }, + + { + id: "weak_customer_signal_overstated", + runThroughGeneration: false, + description: + "Report overstates a small customer signal into a major business conclusion.", + evidenceSnapshot: weakCustomerSignalEvidence, + generatedReport: weakSynthesisReport, + expectations: { + shouldPass: true, + }, + }, + + { + id:"conflicting_evidence", + runThroughGeneration: false, + description:"Report ignores conflicting evidence.", + evidenceSnapshot:conflictingCustomerFeedbackEvidence, + generatedReport:conflictingEvidenceReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "conflicting_evidence", + ], + }, + }, + + { + id:"document_change_as_shipped", + runThroughGeneration: false, + description:"Documentation updates cannot prove shipped features.", + evidenceSnapshot:documentChangeEvidence, + generatedReport:documentChangeShippedReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "unsupported_shipped_claim", + ], + }, + }, + + { + id:"document_change_as_customer_feedback", + runThroughGeneration:false, + description:"Document changes cannot represent customer feedback.", + evidenceSnapshot:documentChangeEvidence, + generatedReport:documentChangeCustomerReport, + expectations:{ + shouldPass:false, + expectedFailureCategories:[ + "invalid_source_type" + ] + } + }, + + { + id: "workspace_document_as_customer_feedback", + runThroughGeneration: false, + description: "Workspace documents cannot be used as customer feedback.", + evidenceSnapshot: workspaceDocumentEvidence, + generatedReport: workspaceDocumentAsCustomerReport, + expectations: { + shouldPass: false, + expectedFailureCategories: [ + "invalid_source_type", + ], + }, + }, + + { + id:"unavailable_source_warning", + runThroughGeneration: true, + description:"Unavailable evidence should not force hallucination.", + evidenceSnapshot:unavailableSourceEvidence, + generatedReport:unavailableSourceSafeReport, + expectations:{ + shouldPass:true, + }, + }, + + { + id: "valid_contradictory_evidence", + runThroughGeneration: true, + description: "Report correctly represents conflicting evidence.", + evidenceSnapshot: validContradictoryEvidence, + generatedReport: validContradictoryEvidenceReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "planned_work_marked_as_shipped", + runThroughGeneration: false, + description: "Planned document work cannot be represented as shipped work.", + evidenceSnapshot: plannedWorkEvidence, + generatedReport: plannedWorkMarkedShippedReport, + expectations: { + shouldPass: false, + forbiddenClaims: [ + "CSV export shipped", + ], + expectedFailureCategories: [ + "unsupported_shipped_claim", + ], + }, + }, + + { + id: "valid_document_change_evidence", + runThroughGeneration: false, + description: "Document changes can support what changed when properly cited.", + evidenceSnapshot: validDocumentChangeEvidence, + generatedReport: validDocumentChangeReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "multiple_adjacent_version_changes", + runThroughGeneration: false, + description: "Multiple document versions in a reporting period should preserve adjacent changes.", + evidenceSnapshot: multipleVersionChangeEvidence, + generatedReport: multipleVersionChangeReport, + expectations: { + shouldPass: true, + }, + }, + + { + id: "low_materiality_summary", + runThroughGeneration: false, + description: + "Report cites evidence but provides a vague low-value summary.", + evidenceSnapshot: meaningfulChangeEvidence, + generatedReport: vagueSummaryReport, + expectations: { + shouldPass: true, + }, + }, +]; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts b/packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts new file mode 100644 index 000000000..42c6ebfd2 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/evaluate-generated-review.ts @@ -0,0 +1,58 @@ +import { + evaluateFounderWeeklyReview, + type EvaluationFailure, +} from "../evaluation"; +import { + FounderWeeklyReviewV2PayloadSchema, + type FounderWeeklyReviewV2Payload, +} from "../contracts"; +import { + type FounderWeeklyReviewGenerateFn, + gradeFounderWeeklyReview, +} from "@launchstack/features/founder-weekly-review"; +import { LLMGraderResult } from "../llm-grader"; + +export type GeneratedReviewEvaluation = { + deterministic: ReturnType | null; + llmGrader?: LLMGraderResult; + failures: EvaluationFailure[]; +}; + +export async function evaluateGeneratedFounderWeeklyReview( + evidenceSnapshot: Parameters[0], + generatedReport: FounderWeeklyReviewV2Payload, + generate: FounderWeeklyReviewGenerateFn +): Promise { + const schemaResult = FounderWeeklyReviewV2PayloadSchema.safeParse( + generatedReport + ); + + if (!schemaResult.success) { + return { + deterministic: null, + failures: [ + { + category: "malformed_payload", + explanation: "Report failed schema validation.", + } satisfies EvaluationFailure + ], + }; + } + + const result = evaluateFounderWeeklyReview( + evidenceSnapshot, + generatedReport + ); + + const llmGrader = await gradeFounderWeeklyReview({ + evidenceSnapshot, + report: generatedReport, + generate, + }); + + return { + deterministic: result, + llmGrader, + failures: result.failures, + }; +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/index.ts b/packages/features/src/founder-weekly-review/benchmarks/index.ts new file mode 100644 index 000000000..42b597ec2 --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/index.ts @@ -0,0 +1,3 @@ +export * from "./evaluate-generated-review"; +export * from "../grader"; +export * from "../llm-grader"; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/run.ts b/packages/features/src/founder-weekly-review/benchmarks/run.ts new file mode 100644 index 000000000..b634f610b --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/run.ts @@ -0,0 +1 @@ +export { main } from "./runner"; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/benchmarks/runner.ts b/packages/features/src/founder-weekly-review/benchmarks/runner.ts new file mode 100644 index 000000000..0073944fe --- /dev/null +++ b/packages/features/src/founder-weekly-review/benchmarks/runner.ts @@ -0,0 +1,257 @@ +import { + evaluateFounderWeeklyReview, + type EvaluationFailure, +} from "../evaluation"; +import { benchmarkCases } from "./cases"; +import { + type FounderWeeklyReviewV2Payload, +} from "../contracts"; +import { + generateFounderWeeklyReview, + type FounderWeeklyReviewGenerateFn, +} from "@launchstack/features/founder-weekly-review"; +import { writeFileSync } from "fs"; +import type { LLMGraderResult } from "../llm-grader"; +import path from "path"; +import { evaluateGeneratedFounderWeeklyReview } from "./evaluate-generated-review"; + +type BenchmarkMetrics = { + citationValidity: number; + citationCoverage: number; + unsupportedClaimRate: number; + unsupportedShippedClaimRate: number; + sourceTypeViolationRate: number; + evidenceCoverage: number; + duplicateClaimRate: number; + emptySectionCorrectness: number; +}; + +type BenchmarkResult = { + case: string; + passed: boolean; + score: number; + hasHardFailure: boolean; + metrics?: BenchmarkMetrics; + failures: EvaluationFailure[]; + llmGrader?: LLMGraderResult; +}; + +type BenchmarkState = { + passedCount: number; + failedCount: number; + hardFailureCount: number; +}; + +export async function runBenchmarks( + generate: FounderWeeklyReviewGenerateFn, + results: BenchmarkResult[], + state: BenchmarkState +): Promise { + for (const testCase of benchmarkCases) { + + if (!testCase.generatedReport) { + continue; + } + + let generatedReport: FounderWeeklyReviewV2Payload; + + if(testCase.runThroughGeneration) { + const generated = await generateFounderWeeklyReview({ + evidenceSnapshot: testCase.evidenceSnapshot, + generate: async ({schema}) => ({ + object: schema.parse(testCase.generatedReport), + metadata: { + provider: "benchmark", + model: "fixture", + capability: "founderWeeklyReview", + temperature: 0, + }, + }), + }); + + generatedReport = generated.reviewPayload; + + } else { + generatedReport = testCase.generatedReport; + } + + const evaluation = await evaluateGeneratedFounderWeeklyReview( + testCase.evidenceSnapshot, + generatedReport, + generate + ); + + if (evaluation.deterministic === null) { + const passed = + testCase.expectations.expectedFailureCategories?.includes( + "malformed_payload" + ) ?? false; + + results.push({ + case: testCase.id, + passed, + score: 0, + hasHardFailure: true, + failures: evaluation.failures, + }); + + state.hardFailureCount++; + + if (passed) { + state.passedCount++; + } else { + state.failedCount++; + } + + console.log( + `${passed ? "✅" : "❌"} ${testCase.id} (schema validation failure)` + ); + + continue; + } + + const result = evaluation.deterministic; + const llmGrade = evaluation.llmGrader ?? undefined; + + const actualFailures = result.failures.map(f => f.category); + + const passed = + testCase.expectations.shouldPass + ? actualFailures.length === 0 + : testCase.expectations.expectedFailureCategories?.every( + category => actualFailures.includes(category) + ) ?? false; + + if (result.hasHardFailure) { + state.hardFailureCount++; + } + + if (passed) { + state.passedCount++; + } else { + state.failedCount++; + } + + results.push({ + case: testCase.id, + passed, + score: result.overallScore, + hasHardFailure: result.hasHardFailure, + metrics: result.deterministic, + failures: result.failures, + llmGrader: llmGrade, + }); + + console.log( + `${passed ? "✅" : "❌"} ${testCase.id}` + ); + + if (!passed) { + console.log(JSON.stringify({ + case: testCase.id, + deterministicFailures: result.failures, + llmGrader: llmGrade, + }, null, 2)); + } + } +} + +export async function main(generate: FounderWeeklyReviewGenerateFn) { + const results: BenchmarkResult[] = []; + + const state: BenchmarkState = { + passedCount: 0, + failedCount: 0, + hardFailureCount: 0 + }; + + await runBenchmarks(generate, results, state); + + const averageScore = + results.length === 0 + ? 0 + : results.reduce( + (sum, result) => sum + result.score, + 0 + ) / results.length; + + const llmScores = results + .map(r => r.llmGrader?.overallScore) + .filter((v): v is number => v !== undefined); + + const averageLLMScore = + llmScores.length === 0 + ? 0 + : llmScores.reduce((sum, v) => sum + v, 0) / llmScores.length; + + const averageMetric = (key: keyof BenchmarkMetrics) => { + const values = results + .map(result => result.metrics?.[key]) + .filter((value): value is number => value !== undefined); + + return values.length === 0 + ? 0 + : values.reduce((sum, value) => sum + value, 0) / values.length; + }; + + const weakestCases = [...results] + .sort((a,b) => a.score - b.score) + .slice(0,3) + .map(result => ({ + case: result.case, + score: result.score, + failures: result.failures.map(f => f.category), + })); + + const failureCounts = results + .flatMap(result => result.failures) + .reduce>((acc, failure) => { + acc[failure.category] = + (acc[failure.category] ?? 0) + 1; + + return acc; + }, {}); + + const benchmarkOutput = { + summary: { + totalCases: benchmarkCases.length, + passed: state.passedCount, + failed: state.failedCount, + hardFailures: state.hardFailureCount, + passRate: benchmarkCases.length === 0 + ? 0 + : state.passedCount / benchmarkCases.length, + deterministicScore: averageScore, + llmScore: averageLLMScore, + }, + metrics: { + citationValidity: averageMetric("citationValidity"), + citationCoverage: averageMetric("citationCoverage"), + unsupportedClaimRate: averageMetric("unsupportedClaimRate"), + unsupportedShippedClaimRate: averageMetric("unsupportedShippedClaimRate"), + sourceTypeViolationRate: averageMetric("sourceTypeViolationRate"), + evidenceCoverage: averageMetric("evidenceCoverage"), + emptySectionCorrectness: averageMetric("emptySectionCorrectness"), + duplicateClaimRate: averageMetric("duplicateClaimRate"), + }, + weakestCases, + commonFailures: failureCounts, + cases: results, + }; + + console.log("\n===== Benchmark Summary ====="); + + console.log(JSON.stringify(benchmarkOutput, null, 2)); + + const outputPath = path.resolve( + import.meta.dirname, + "./baseline-output.json" + ); + + writeFileSync( + outputPath, + JSON.stringify(benchmarkOutput, null, 2) + ); + + process.exitCode = state.failedCount > 0 ? 1 : 0; +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/contracts.ts b/packages/features/src/founder-weekly-review/contracts.ts new file mode 100644 index 000000000..41843d83d --- /dev/null +++ b/packages/features/src/founder-weekly-review/contracts.ts @@ -0,0 +1,511 @@ +import { z } from "zod"; + +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_V1_SCHEMA_VERSION = + "founder-weekly-review-evidence/v1" as const; +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION = + "founder-weekly-review-evidence/v2" as const; +/** Current version used for newly collected evidence. */ +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION = + FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION; +export const FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION = + "founder-weekly-review/v1" as const; +export const FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION = + "founder-weekly-review/v2" as const; + +export const FounderWeeklyReviewStatusSchema = z.enum([ + "queued", + "collecting", + "generating", + "draft", + "published", + "failed", +]); +export type FounderWeeklyReviewStatus = z.infer; + +/** Durable, request-derived inputs needed to collect evidence after the HTTP response. */ +export const FounderWeeklyReviewCollectionInputSchema = z.object({ + workspaceTimezone: z.string().min(1).max(128), + founderContext: z.string().min(1).max(4000).optional(), + actorExternalUserId: z.string().min(1).max(256), +}).strict(); +export type FounderWeeklyReviewCollectionInput = z.infer; + +export const FounderWeeklyReviewOperationTypeSchema = z.enum(["retry"]); +export type FounderWeeklyReviewOperationType = z.infer< + typeof FounderWeeklyReviewOperationTypeSchema +>; + +export const ReportingPeriodSchema = z + .object({ + start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + end: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + }) + .refine((value) => value.start <= value.end, { + message: "Reporting period start must be on or before end", + path: ["end"], + }); +export type ReportingPeriod = z.infer; + +const SerializableMetadataPrimitiveSchema = z.union([ + z.string().max(512), + z.number().finite(), + z.boolean(), + z.null(), +]); + +const SerializableMetadataValueSchema: z.ZodType< + string | number | boolean | null | Array +> = z.union([ + SerializableMetadataPrimitiveSchema, + z.array(SerializableMetadataPrimitiveSchema).max(20), +]); + +export const FounderWeeklyReviewEvidenceItemSchema = z.object({ + sourceType: z.enum([ + "workspace_document", + "document_change", + "customer_feedback", + "github_activity", + "manual_note", + "founder_context", + "other", + ]), + sourceId: z.string().min(1).max(256), + title: z.string().min(1).max(512), + sourceTimestamp: z.string().datetime({ offset: true }).optional(), + excerpt: z.string().min(1).max(4000), + canonicalUrl: z.string().url().max(2048).optional(), + workspaceDeepLink: z.string().max(2048).optional(), + metadata: z.record(z.string().max(64), SerializableMetadataValueSchema).default({}), +}); +export type FounderWeeklyReviewEvidenceItem = z.infer< + typeof FounderWeeklyReviewEvidenceItemSchema +>; + +export const FounderWeeklyReviewEvidenceWarningSchema = z.object({ + code: z.string().min(1).max(64), + message: z.string().min(1).max(512), + sourceType: FounderWeeklyReviewEvidenceItemSchema.shape.sourceType.optional(), +}); +export type FounderWeeklyReviewEvidenceWarning = z.infer< + typeof FounderWeeklyReviewEvidenceWarningSchema +>; + +const FounderWeeklyReviewEvidenceSnapshotBaseSchema = z.object({ + capturedAt: z.string().datetime({ offset: true }), + reportingPeriod: ReportingPeriodSchema, + workspaceTimezone: z.string().min(1).max(128), + items: z.array(FounderWeeklyReviewEvidenceItemSchema).max(500), + sourceWarnings: z.array(FounderWeeklyReviewEvidenceWarningSchema).max(100).default([]), +}); + +export const DocumentChangeCategorySchema = z.enum([ + "ownership_change", + "status_change", + "deadline_change", + "metric_change", + "requirement_change", + "risk_or_blocker_change", + "scope_change", + "priority_change", + "editorial_rewrite", + "uncertain", +]); +export const DeterministicMaterialityConfidenceSchema = z.enum(["strong", "moderate", "uncertain"]); + +export const DocumentChangeMaterialityDispositionSchema = z.enum([ + "material", + "non_material", + "uncertain", +]); + +export const DocumentChangeMaterialityAnalysisSnapshotSchema = z.object({ + disposition: DocumentChangeMaterialityDispositionSchema, + category: DocumentChangeCategorySchema, + confidence: z.number().min(0).max(1), + summary: z.string().min(1).max(320), + beforeKeyPoint: z.string().min(1).max(240).optional(), + afterKeyPoint: z.string().min(1).max(240).optional(), + analysisMethod: z.literal("llm"), + analysisPromptVersion: z.string().min(1).max(128), + analysisProvider: z.string().min(1).max(128).optional(), + analysisModel: z.string().min(1).max(256).optional(), + analysisInputDigest: z.string().regex(/^[a-f0-9]{64}$/), + analysisResultSchemaVersion: z.string().min(1).max(128), +}).strict().superRefine((result, context) => { + if (result.disposition === "uncertain" && result.category !== "uncertain") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Uncertain analysis requires uncertain category.", path: ["category"] }); + } + if (result.disposition === "material" && result.category === "editorial_rewrite") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Material analysis cannot use editorial_rewrite.", path: ["category"] }); + } +}); +export type DocumentChangeMaterialityAnalysisSnapshot = z.infer< + typeof DocumentChangeMaterialityAnalysisSnapshotSchema +>; + +export const RawDocumentChangeSnapshotSchema = z.object({ + rawChangeId: z.string().min(1).max(128), + changeType: z.enum(["added", "removed", "modified"]), + alignmentMethod: z.enum(["content_hash", "structure_path", "section_title", "structural_position", "text_similarity", "unmatched"]), + similarityScore: z.number().finite().optional(), + documentId: z.string().regex(/^\d+$/).max(64), + previousVersionId: z.number().int().nonnegative(), + currentVersionId: z.number().int().nonnegative(), + previousChunkId: z.number().int().nonnegative().nullable(), + currentChunkId: z.number().int().nonnegative().nullable(), + previousExcerpt: z.string().max(600).nullable(), + currentExcerpt: z.string().max(600).nullable(), + previousHash: z.string().regex(/^[a-f0-9]{64}$/).nullable(), + currentHash: z.string().regex(/^[a-f0-9]{64}$/).nullable(), + previousStructurePath: z.string().max(512).nullable(), + currentStructurePath: z.string().max(512).nullable(), + previousStructureTitle: z.string().max(512).nullable(), + currentStructureTitle: z.string().max(512).nullable(), + previousPageNumber: z.number().int().nullable(), + currentPageNumber: z.number().int().nullable(), + previousLineStart: z.number().int().nullable(), + previousLineEnd: z.number().int().nullable(), + currentLineStart: z.number().int().nullable(), + currentLineEnd: z.number().int().nullable(), + processingVersion: z.string().min(1).max(128), +}).strict(); +export type RawDocumentChangeSnapshot = z.infer; + +export const DocumentChangeGroupSnapshotSchema = z.object({ + groupId: z.string().min(1).max(128), + evidenceSourceId: z.string().min(1).max(256).nullable(), + documentId: z.string().regex(/^\d+$/).max(64), + previousVersionId: z.number().int().nonnegative(), + currentVersionId: z.number().int().nonnegative(), + structurePath: z.string().max(512).nullable(), + structureTitle: z.string().max(512).nullable(), + splitOrdinal: z.number().int().nonnegative(), + rawChangeIds: z.array(z.string().min(1).max(128)).min(1).max(16), + category: DocumentChangeCategorySchema, + priority: z.number().int().min(1).max(10), + confidence: DeterministicMaterialityConfidenceSchema, + signals: z.array(z.string().min(1).max(64)).max(20), + materialityMethod: z.literal("deterministic"), + materialityVersion: z.string().min(1).max(128), + analysis: DocumentChangeMaterialityAnalysisSnapshotSchema.optional(), +}).strict(); +export type DocumentChangeGroupSnapshot = z.infer; + +export const DocumentChangeAuditSnapshotSchema = z.object({ + schemaVersion: z.literal("document-change-audit/v1"), + rawChanges: z.array(RawDocumentChangeSnapshotSchema).max(5000), + groups: z.array(DocumentChangeGroupSnapshotSchema).max(2500), +}).strict(); +export type DocumentChangeAuditSnapshot = z.infer; + +export const FounderWeeklyReviewEvidenceSnapshotV1Schema = FounderWeeklyReviewEvidenceSnapshotBaseSchema.extend({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_EVIDENCE_V1_SCHEMA_VERSION), +}).strict(); +export const FounderWeeklyReviewEvidenceSnapshotV2Schema = FounderWeeklyReviewEvidenceSnapshotBaseSchema.extend({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION), + documentChangeAudit: DocumentChangeAuditSnapshotSchema, +}).strict(); +export const FounderWeeklyReviewEvidenceSnapshotSchema = z.discriminatedUnion("schemaVersion", [ + FounderWeeklyReviewEvidenceSnapshotV1Schema, + FounderWeeklyReviewEvidenceSnapshotV2Schema, +]).superRefine((snapshot, context) => { + if (snapshot.schemaVersion !== FOUNDER_WEEKLY_REVIEW_EVIDENCE_V2_SCHEMA_VERSION) return; + const rawIds = new Set(snapshot.documentChangeAudit.rawChanges.map(change => change.rawChangeId)); + const itemById = new Map(snapshot.items.map(item => [item.sourceId, item])); + const seenGroups = new Set(); + if (rawIds.size !== snapshot.documentChangeAudit.rawChanges.length) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Document-change audit raw IDs must be unique.", path: ["documentChangeAudit", "rawChanges"] }); + } + for (const [index, group] of snapshot.documentChangeAudit.groups.entries()) { + if (seenGroups.has(group.groupId)) context.addIssue({ code: z.ZodIssueCode.custom, message: "Document-change audit group IDs must be unique.", path: ["documentChangeAudit", "groups", index, "groupId"] }); + seenGroups.add(group.groupId); + if (group.rawChangeIds.some(rawChangeId => !rawIds.has(rawChangeId))) context.addIssue({ code: z.ZodIssueCode.custom, message: "Document-change audit group references an unknown raw change.", path: ["documentChangeAudit", "groups", index, "rawChangeIds"] }); + if (group.evidenceSourceId) { + const item = itemById.get(group.evidenceSourceId); + if (!item || item.sourceType !== "document_change" || item.metadata.groupId !== group.groupId) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Condensed document-change evidence must resolve to its audit group.", path: ["documentChangeAudit", "groups", index, "evidenceSourceId"] }); + } + } + } + for (const [index, item] of snapshot.items.entries()) { + if (item.sourceType === "document_change" && item.sourceId.startsWith("document_change:group:") && !snapshot.documentChangeAudit.groups.some(group => group.evidenceSourceId === item.sourceId)) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Condensed document-change evidence has no audit group.", path: ["items", index, "sourceId"] }); + } + } +}); +export type FounderWeeklyReviewEvidenceSnapshot = z.infer< + typeof FounderWeeklyReviewEvidenceSnapshotSchema +>; +export type FounderWeeklyReviewEvidenceSchemaVersion = FounderWeeklyReviewEvidenceSnapshot["schemaVersion"]; + +export const FounderWeeklyReviewConfidenceSchema = z.enum(["high", "medium", "low"]); +export type FounderWeeklyReviewConfidence = z.infer< + typeof FounderWeeklyReviewConfidenceSchema +>; + +export const FounderWeeklyReviewSectionItemSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("observed_fact"), + text: z.string().min(1).max(2000), + sourceIds: z.array(z.string().min(1).max(256)).min(1).max(20), + confidence: FounderWeeklyReviewConfidenceSchema, + }), + z.object({ + kind: z.literal("recommended_item"), + text: z.string().min(1).max(2000), + rationale: z.string().min(1).max(2000).optional(), + sourceIds: z.array(z.string().min(1).max(256)).max(20).default([]), + confidence: FounderWeeklyReviewConfidenceSchema.optional(), + }), + z.object({ + kind: z.literal("no_evidence"), + code: z.enum(["no_relevant_evidence", "source_unavailable", "not_assessed"]), + note: z.string().min(1).max(512).optional(), + }), + z.object({ + kind: z.literal("human_edit"), + markdown: z.string().min(1).max(12000), + }), +]); +export type FounderWeeklyReviewSectionItem = z.infer< + typeof FounderWeeklyReviewSectionItemSchema +>; + +const FounderWeeklyReviewSectionSchema = z.object({ + heading: z.string().min(1).max(128), + items: z.array(FounderWeeklyReviewSectionItemSchema).max(100), +}); + +/** + * LAU-5 payload. Keep this schema byte-for-byte compatible with persisted v1 + * drafts; LAU-7 generation emits the separate v2 schema below. + */ +export const FounderWeeklyReviewV1PayloadSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION), + sections: z.object({ + whatChanged: FounderWeeklyReviewSectionSchema, + whatShipped: FounderWeeklyReviewSectionSchema, + whatCustomersSaid: FounderWeeklyReviewSectionSchema, + currentBlockers: FounderWeeklyReviewSectionSchema, + nextPriorities: FounderWeeklyReviewSectionSchema, + }), +}); +export type FounderWeeklyReviewV1Payload = z.infer; + +const V2ConfidenceSchema = z.number().min(0).max(1); +function v2SourceIdsSchema(minimum: number) { + return z.array(z.string().min(1).max(256)).min(minimum).max(20).superRefine( + (sourceIds, context) => { + if (new Set(sourceIds).size !== sourceIds.length) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "sourceIds must be unique" }); + } + } + ); +} + +export const FounderWeeklyReviewV2ObservedFactSchema = z.object({ + kind: z.literal("observed_fact"), + text: z.string().min(1).max(2000), + sourceIds: v2SourceIdsSchema(1), + confidence: V2ConfidenceSchema, +}).strict(); + +export const FounderWeeklyReviewV2ContradictoryEvidenceSchema = z.object({ + kind: z.literal("contradictory_evidence"), + text: z.string().min(1).max(2000), + sourceIds: v2SourceIdsSchema(2), + confidence: V2ConfidenceSchema, +}).strict(); + +export const FounderWeeklyReviewV2RecommendationSchema = z.object({ + kind: z.literal("recommendation"), + label: z.literal("Recommendation"), + text: z.string().min(1).max(2000), + rationale: z.string().min(1).max(2000).nullable(), + sourceIds: v2SourceIdsSchema(1), + confidence: V2ConfidenceSchema, +}).strict(); + +export const FounderWeeklyReviewV2NoEvidenceSchema = z.object({ + code: z.string().min(1).max(64), + message: z.string().min(1).max(512), + cta: z.string().min(1).max(512), +}).strict(); + +const FounderWeeklyReviewV2NoEvidenceSectionSchema = z.object({ + state: z.literal("no_evidence"), + noEvidence: FounderWeeklyReviewV2NoEvidenceSchema, +}).strict(); + +const FounderWeeklyReviewV2FactualItemSchema = z.union([ + FounderWeeklyReviewV2ObservedFactSchema, + FounderWeeklyReviewV2ContradictoryEvidenceSchema, +]); +const FounderWeeklyReviewV2FactualSectionSchema = z.union([ + z.object({ + state: z.literal("evidence"), + items: z.array(FounderWeeklyReviewV2FactualItemSchema).min(1).max(100), + }).strict(), + FounderWeeklyReviewV2NoEvidenceSectionSchema, +]); +const FounderWeeklyReviewV2PrioritySectionSchema = z.union([ + z.object({ + state: z.literal("evidence"), + items: z.array(FounderWeeklyReviewV2RecommendationSchema).min(1).max(100), + }).strict(), + FounderWeeklyReviewV2NoEvidenceSectionSchema, +]); + +export const FounderWeeklyReviewV2PayloadSchema = z.object({ + schemaVersion: z.literal(FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION), + sections: z.object({ + whatChanged: FounderWeeklyReviewV2FactualSectionSchema, + whatShipped: FounderWeeklyReviewV2FactualSectionSchema, + whatCustomersSaid: FounderWeeklyReviewV2FactualSectionSchema, + currentBlockers: FounderWeeklyReviewV2FactualSectionSchema, + nextPriorities: FounderWeeklyReviewV2PrioritySectionSchema, + }).strict(), +}).strict(); +export type FounderWeeklyReviewV2Payload = z.infer; + +export const FounderWeeklyReviewPayloadSchema = z.union([ + FounderWeeklyReviewV1PayloadSchema, + FounderWeeklyReviewV2PayloadSchema, +]); +export type FounderWeeklyReviewPayload = z.infer; +export type FounderWeeklyReviewPayloadSchemaVersion = + | typeof FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION + | typeof FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION; + +export const FounderWeeklyReviewModelMetadataSchema = z.object({ + provider: z.string().min(1).max(128).optional(), + model: z.string().min(1).max(256).optional(), + promptVersion: z.string().min(1).max(128).optional(), + temperature: z.number().finite().optional(), + capability: z.string().min(1).max(128).optional(), + promptHash: z.string().regex(/^[a-f0-9]{64}$/).optional(), + evidenceSchemaVersion: z.string().min(1).max(128).optional(), + reviewPayloadSchemaVersion: z.string().min(1).max(128).optional(), + completionId: z.string().min(1).max(256).optional(), + notes: z.string().min(1).max(1024).optional(), + attributes: z.record(z.string().max(64), SerializableMetadataValueSchema).default({}), +}); +export type FounderWeeklyReviewModelMetadata = z.infer< + typeof FounderWeeklyReviewModelMetadataSchema +>; + +export interface FounderWeeklyReviewRunRecord { + id: string; + companyId: bigint; + requestKey: string; + reportingPeriod: ReportingPeriod; + status: FounderWeeklyReviewStatus; + reviewPayload: FounderWeeklyReviewPayload | null; + reviewSchemaVersion: FounderWeeklyReviewPayloadSchemaVersion; + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | null; + evidenceSchemaVersion: FounderWeeklyReviewEvidenceSchemaVersion; + collectionInput?: FounderWeeklyReviewCollectionInput; + collectionClaimId?: string | null; + collectionStartedAt?: Date | null; + evidenceCollectedAt?: Date | null; + modelMetadata: FounderWeeklyReviewModelMetadata | null; + createdByActorId: string; + retryCount: number; + failureSequence: number; + generationAttempt: number; + generationClaimId: string | null; + generationJobId: string | null; + queuedAt: Date; + claimedAt: Date | null; + generationStartedAt: Date | null; + generatedAt: Date | null; + publishedAt: Date | null; + errorCode: string | null; + errorMessage: string | null; + createdAt: Date; + updatedAt: Date | null; +} + +export interface FounderWeeklyReviewOperationRecord { + id: string; + runId: string; + companyId: bigint; + operationType: FounderWeeklyReviewOperationType; + requestKey: string; + sourceFailureSequence: number; + actorId: string; + createdAt: Date; +} + +export interface CreateFounderWeeklyReviewRunInput { + id: string; + companyId: bigint; + requestKey: string; + reportingPeriod: ReportingPeriod; + /** Existing callers may provide a snapshot; workflow callers intentionally do not. */ + evidenceSnapshot?: FounderWeeklyReviewEvidenceSnapshot; + collectionInput?: FounderWeeklyReviewCollectionInput; + createdByActorId: string; +} + +export interface FounderWeeklyReviewRetryInput { + operationId: string; + companyId: bigint; + runId: string; + requestKey: string; + actorId: string; +} + +export interface FounderWeeklyReviewClaimInput { + companyId: bigint; + runId: string; + generationClaimId: string; + generationJobId?: string; +} + +export interface FounderWeeklyReviewCollectionClaimInput { + companyId: bigint; + runId: string; + collectionClaimId: string; +} + +export interface FounderWeeklyReviewGenerationFailure { + errorCode: string; + errorMessage?: string; +} + +export interface FounderWeeklyReviewUserActor { + externalUserId: string; + internalUserId?: bigint; + companyId: bigint; + role: string; +} + +export function buildFounderWeeklyReviewActorId( + actor: Pick +): string { + return `user:${actor.externalUserId}`; +} + +export function parseFounderWeeklyReviewPayload( + value: unknown +): FounderWeeklyReviewPayload { + return FounderWeeklyReviewPayloadSchema.parse(value); +} + +export function parseFounderWeeklyReviewEvidenceSnapshot( + value: unknown +): FounderWeeklyReviewEvidenceSnapshot { + return FounderWeeklyReviewEvidenceSnapshotSchema.parse(value); +} + +export function parseFounderWeeklyReviewCollectionInput(value: unknown): FounderWeeklyReviewCollectionInput { + return FounderWeeklyReviewCollectionInputSchema.parse(value); +} + +export function parseFounderWeeklyReviewModelMetadata( + value: unknown +): FounderWeeklyReviewModelMetadata { + return FounderWeeklyReviewModelMetadataSchema.parse(value); +} diff --git a/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts b/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts new file mode 100644 index 000000000..dac221168 --- /dev/null +++ b/packages/features/src/founder-weekly-review/document-change-materiality-analyzer.ts @@ -0,0 +1,590 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +import type { + DocumentChangeAuditSnapshot, + DocumentChangeMaterialityAnalysisSnapshot, + FounderWeeklyReviewEvidenceItem, +} from "./contracts"; +import type { + DocumentChangeGroup, + DocumentChangePairInput, + DocumentChangeProcessingWarning, + RawDocumentChange, +} from "./document-change"; +import { + DOCUMENT_CHANGE_CATEGORIES, + DOCUMENT_CHANGE_MATERIALITY_VERSION, + analyzeDocumentChangeFactualDeltas, + buildCondensedDocumentChangeEvidence, + documentChangeCategoryPriority, + documentChangeGroupSourceId, + materializeDocumentChanges, + selectMaterialDocumentChangeGroups, + type AnalyzedDocumentChangeGroup, + type DeterministicMaterialChangeResult, + type DeterministicMaterialityConfidence, + type DocumentChangeCategory, + type DocumentChangeFactualDelta, + type DocumentChangeFactualDeltaAssessment, + type DocumentChangeFactualDeltaKind, +} from "./document-change-materiality"; + +export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION = + "document-change-materiality/v2" as const; +export const DOCUMENT_CHANGE_MATERIALITY_ANALYSIS_RESULT_VERSION = + "document-change-materiality-result/v1" as const; +export const DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS = Object.freeze({ + callsPerReview: 4, + concurrency: 2, + rawChangesPerGroup: 16, + canonicalInputCharacters: 8_000, + excerptCharactersPerSide: 150, + summaryCharacters: 320, + keyPointCharacters: 240, +}); + +export const DocumentChangeMaterialityAnalysisResultSchema = z.object({ + disposition: z.enum(["material", "non_material", "uncertain"]).describe("Whether the underlying business state materially changed."), + category: z.enum(DOCUMENT_CHANGE_CATEGORIES).describe("The primary changed business-state dimension, editorial_rewrite, or uncertain."), + summary: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.summaryCharacters) + .describe("One concise sentence of at most 320 characters. Do not include reasoning or repeat the full source."), + beforeKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters) + .describe("Optional verbatim copied before span of at most 240 characters.").optional(), + afterKeyPoint: z.string().trim().min(1).max(DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.keyPointCharacters) + .describe("Optional verbatim copied after span of at most 240 characters.").optional(), + confidence: z.number().min(0).max(1).describe("Judgment confidence from 0 through 1."), +}).strict().superRefine((result, context) => { + if (result.disposition === "uncertain" && result.category !== "uncertain") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Uncertain disposition requires uncertain category.", path: ["category"] }); + } + if (result.disposition === "material" && result.category === "editorial_rewrite") { + context.addIssue({ code: z.ZodIssueCode.custom, message: "Material disposition cannot use editorial_rewrite.", path: ["category"] }); + } +}); + +export type DocumentChangeMaterialityAnalysisResult = z.infer< + typeof DocumentChangeMaterialityAnalysisResultSchema +>; + +export type DocumentChangeGroupShape = "modified" | "added" | "removed" | "replacement" | "mixed"; +export type DocumentChangeSemanticRisk = + | "confirmed_factual_delta" + | "paraphrase_possible" + | "structural_rewrite" + | "split_merge_possible" + | "ambiguous_business_signal"; + +export type DocumentChangeMaterialityAnalysisInput = { + groupId: string; + documentTitle?: string; + structurePath?: string | null; + structureTitle?: string | null; + deterministicAssessment: { + category: DocumentChangeCategory; + confidence: DeterministicMaterialityConfidence; + detectedSignals: readonly string[]; + confirmedFactualDeltas: readonly DocumentChangeFactualDelta[]; + equivalentFactualValues: readonly DocumentChangeFactualDelta[]; + possibleSignals: readonly DocumentChangeFactualDeltaKind[]; + groupShape: DocumentChangeGroupShape; + semanticRisk: DocumentChangeSemanticRisk; + }; + changes: readonly { + changeType: RawDocumentChange["changeType"]; + previousExcerpt?: string; + currentExcerpt?: string; + alignmentMethod: RawDocumentChange["alignmentMethod"]; + }[]; +}; + +export type DocumentChangeMaterialityAnalyzerMetadata = { + provider?: string; + model?: string; + promptVersion?: string; +}; + +export interface DocumentChangeMaterialityAnalyzer { + analyze(input: DocumentChangeMaterialityAnalysisInput): Promise<{ + result: unknown; + metadata?: DocumentChangeMaterialityAnalyzerMetadata; + }>; +} + +export class DocumentChangeMaterialityAnalyzerError extends Error { + constructor( + readonly code: "unavailable" | "timeout" | "invalid", + message: string, + ) { + super(message); + this.name = "DocumentChangeMaterialityAnalyzerError"; + } +} + +export type DocumentChangeMaterialityEligibilityReason = + | "uncertain_multi_change" + | "uncertain_single_change" + | "replacement_shaped" + | "complex_multi_change" + | "large_rewrite" + | "moderate_signal" + | "unconfirmed_business_signal" + | "paraphrase_risk"; + +export type DocumentChangeMaterialityEligibility = { + eligible: boolean; + reason?: DocumentChangeMaterialityEligibilityReason; + priority: number; +}; + +export type DocumentChangeMaterialityAnalyzerDiagnostics = { + eligibleGroupCount: number; + selectedAnalysisCount: number; + skippedByCallBudgetCount: number; + inputTruncatedCount: number; + succeededAnalysisCount: number; + failedAnalysisCount: number; + materialCount: number; + nonMaterialCount: number; + uncertainCount: number; +}; + +export type AnalyzedMaterialChangeResult = DeterministicMaterialChangeResult & { + analyzerDiagnostics: DocumentChangeMaterialityAnalyzerDiagnostics; +}; + +type AnalysisCandidate = { + analyzed: AnalyzedDocumentChangeGroup; + eligibility: DocumentChangeMaterialityEligibility; +}; + +type SuccessfulAnalysis = { + analyzed: AnalyzedDocumentChangeGroup; + input: DocumentChangeMaterialityAnalysisInput; + inputDigest: string; + inputTruncated: boolean; + result: DocumentChangeMaterialityAnalysisResult; + metadata: DocumentChangeMaterialityAnalyzerMetadata; +}; + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function changedCharacters(group: DocumentChangeGroup): number { + return group.rawChanges.reduce((total, change) => total + + (change.previousNormalizedContent?.length ?? 0) + + (change.currentNormalizedContent?.length ?? 0), 0); +} + +function tokenSet(value: string): Set { + return new Set(value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []); +} + +function lexicalSimilarity(change: RawDocumentChange): number { + const previous = tokenSet(change.previousNormalizedContent ?? ""); + const current = tokenSet(change.currentNormalizedContent ?? ""); + if (previous.size === 0 && current.size === 0) return 1; + const intersection = [...previous].filter(token => current.has(token)).length; + const union = new Set([...previous, ...current]).size; + return union === 0 ? 1 : intersection / union; +} + +function groupShape(group: DocumentChangeGroup): DocumentChangeGroupShape { + const types = new Set(group.rawChanges.map(change => change.changeType)); + if (types.has("added") && types.has("removed")) return "replacement"; + if (types.size > 1) return "mixed"; + return group.rawChanges[0]?.changeType ?? "mixed"; +} + +function semanticRisk( + analyzed: AnalyzedDocumentChangeGroup, + assessment: DocumentChangeFactualDeltaAssessment, +): DocumentChangeSemanticRisk { + const shape = groupShape(analyzed.group); + if (shape === "replacement") return "split_merge_possible"; + if (analyzed.group.rawChanges.length > 1 || shape === "mixed") return "structural_rewrite"; + if (assessment.confirmedFactualDeltas.length > 0 && assessment.possibleSignals.length === 0) return "confirmed_factual_delta"; + if (assessment.possibleSignals.length > 0) return "ambiguous_business_signal"; + return "paraphrase_possible"; +} + +export function hasStrongConfirmedFactualDeltaBypass(analyzed: AnalyzedDocumentChangeGroup): boolean { + const { group, materiality } = analyzed; + if (group.rawChanges.length !== 1 || group.rawChanges[0]!.changeType !== "modified") return false; + if (changedCharacters(group) > 700 || materiality.category === "uncertain" || materiality.category === "editorial_rewrite") return false; + const assessment = analyzeDocumentChangeFactualDeltas(group, materiality.signals); + return assessment.confirmedFactualDeltas.length > 0 + && assessment.confirmedFactualDeltas.every(delta => delta.confidence === "strong") + && assessment.possibleSignals.length === 0; +} + +/** Pure semantic-risk gate. It intentionally does not use embeddings or changelog text. */ +export function shouldAnalyzeDocumentChangeGroup( + analyzed: AnalyzedDocumentChangeGroup, +): DocumentChangeMaterialityEligibility { + if (hasStrongConfirmedFactualDeltaBypass(analyzed)) return { eligible: false, priority: Number.MAX_SAFE_INTEGER }; + const { group, materiality } = analyzed; + const factualAssessment = analyzeDocumentChangeFactualDeltas(group, materiality.signals); + if (materiality.category === "uncertain" && group.rawChanges.length > 1) { + return { eligible: true, reason: "uncertain_multi_change", priority: 1 }; + } + if (materiality.category === "uncertain") { + return { eligible: true, reason: "uncertain_single_change", priority: 2 }; + } + const types = new Set(group.rawChanges.map(change => change.changeType)); + if (types.has("added") && types.has("removed")) { + return { eligible: true, reason: "replacement_shaped", priority: 3 }; + } + if (group.rawChanges.length > 1) { + return { eligible: true, reason: "complex_multi_change", priority: 4 }; + } + if (factualAssessment.possibleSignals.length > 0 && factualAssessment.confirmedFactualDeltas.length === 0) { + return { eligible: true, reason: "unconfirmed_business_signal", priority: 4 }; + } + if (changedCharacters(group) >= 1_200) { + return { eligible: true, reason: "large_rewrite", priority: 4 }; + } + if (materiality.confidence === "moderate") { + return { eligible: true, reason: "moderate_signal", priority: 4 }; + } + const only = group.rawChanges[0]; + if (only?.changeType === "modified" && changedCharacters(group) >= 240 && lexicalSimilarity(only) < 0.45) { + return { eligible: true, reason: "paraphrase_risk", priority: 5 }; + } + return { eligible: false, priority: Number.MAX_SAFE_INTEGER }; +} + +function stableCandidateOrder(a: AnalysisCandidate, b: AnalysisCandidate): number { + return a.eligibility.priority - b.eligibility.priority + || b.analyzed.pair.currentCreatedAt.getTime() - a.analyzed.pair.currentCreatedAt.getTime() + || compareOrdinal(a.analyzed.group.structurePath ?? a.analyzed.group.structureTitle ?? "", b.analyzed.group.structurePath ?? b.analyzed.group.structureTitle ?? "") + || compareOrdinal(a.analyzed.group.groupId, b.analyzed.group.groupId); +} + +/** Selects at most four calls, round-robin across documents inside each eligibility tier. */ +export function selectDocumentChangeGroupsForMaterialityAnalysis( + groups: readonly AnalyzedDocumentChangeGroup[], +): { selected: AnalyzedDocumentChangeGroup[]; skipped: AnalyzedDocumentChangeGroup[] } { + const candidates = groups + .map(analyzed => ({ analyzed, eligibility: shouldAnalyzeDocumentChangeGroup(analyzed) })) + .filter((candidate): candidate is AnalysisCandidate => candidate.eligibility.eligible) + .sort(stableCandidateOrder); + const selected: AnalyzedDocumentChangeGroup[] = []; + const selectedIds = new Set(); + for (const priority of [...new Set(candidates.map(candidate => candidate.eligibility.priority))].sort((a, b) => a - b)) { + const tier = candidates.filter(candidate => candidate.eligibility.priority === priority); + const byDocument = new Map(); + for (const candidate of tier) { + const key = candidate.analyzed.group.documentId.toString(); + const values = byDocument.get(key) ?? []; + values.push(candidate); + byDocument.set(key, values); + } + const documents = [...byDocument.keys()].sort((a, b) => BigInt(a) < BigInt(b) ? -1 : BigInt(a) > BigInt(b) ? 1 : 0); + let progress = true; + while (progress && selected.length < DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview) { + progress = false; + for (const documentId of documents) { + const candidate = byDocument.get(documentId)!.find(value => !selectedIds.has(value.analyzed.group.groupId)); + if (!candidate) continue; + selected.push(candidate.analyzed); + selectedIds.add(candidate.analyzed.group.groupId); + progress = true; + if (selected.length >= DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview) break; + } + } + if (selected.length >= DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.callsPerReview) break; + } + return { + selected, + skipped: candidates.filter(candidate => !selectedIds.has(candidate.analyzed.group.groupId)).map(candidate => candidate.analyzed), + }; +} + +function compactExcerpt(value: string | undefined): string | undefined { + if (!value) return undefined; + const normalized = value.replace(/\s+/g, " ").trim(); + return normalized.length <= DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide + ? normalized + : `${normalized.slice(0, DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide - 1)}…`; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value as Record) + .sort(([a], [b]) => compareOrdinal(a, b)) + .map(([key, entry]) => [key, canonicalize(entry)])); + } + return value; +} + +export function buildDocumentChangeMaterialityAnalysisInput(analyzed: AnalyzedDocumentChangeGroup): { + input: DocumentChangeMaterialityAnalysisInput; + inputDigest: string; + canonicalCharacterCount: number; + truncated: boolean; +} { + const changes = analyzed.group.rawChanges + .slice(0, DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.rawChangesPerGroup) + .map(change => ({ + changeType: change.changeType, + ...(compactExcerpt(change.previousChunk?.content) ? { previousExcerpt: compactExcerpt(change.previousChunk?.content) } : {}), + ...(compactExcerpt(change.currentChunk?.content) ? { currentExcerpt: compactExcerpt(change.currentChunk?.content) } : {}), + alignmentMethod: change.alignmentMethod, + })); + const factualAssessment = analyzeDocumentChangeFactualDeltas(analyzed.group, analyzed.materiality.signals); + const input: DocumentChangeMaterialityAnalysisInput = { + groupId: analyzed.group.groupId, + documentTitle: analyzed.pair.documentTitle.slice(0, 256), + structurePath: analyzed.group.structurePath?.slice(0, 256) ?? null, + structureTitle: analyzed.group.structureTitle?.slice(0, 256) ?? null, + deterministicAssessment: { + category: analyzed.materiality.category, + confidence: analyzed.materiality.confidence, + detectedSignals: analyzed.materiality.signals.slice(0, 20), + confirmedFactualDeltas: factualAssessment.confirmedFactualDeltas.slice(0, 16), + equivalentFactualValues: factualAssessment.equivalentFactualValues.slice(0, 16), + possibleSignals: factualAssessment.possibleSignals, + groupShape: groupShape(analyzed.group), + semanticRisk: semanticRisk(analyzed, factualAssessment), + }, + changes, + }; + const canonical = JSON.stringify(canonicalize(input)); + if (canonical.length > DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.canonicalInputCharacters) { + throw new Error("Bounded document-change materiality input exceeded its invariant limit."); + } + return { + input, + inputDigest: createHash("sha256").update(canonical, "utf8").digest("hex"), + canonicalCharacterCount: canonical.length, + truncated: analyzed.group.rawChanges.length > changes.length + || analyzed.group.rawChanges.some(change => + (change.previousChunk?.content.replace(/\s+/g, " ").trim().length ?? 0) > DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide + || (change.currentChunk?.content.replace(/\s+/g, " ").trim().length ?? 0) > DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.excerptCharactersPerSide), + }; +} + +function confidenceFromAnalysis(value: number): DeterministicMaterialityConfidence { + return value >= 0.8 ? "strong" : value >= 0.55 ? "moderate" : "uncertain"; +} + +function effectiveAnalyzedGroup(success: SuccessfulAnalysis): AnalyzedDocumentChangeGroup | null { + if (success.result.disposition === "non_material") return null; + const category = success.result.disposition === "uncertain" ? "uncertain" : success.result.category; + return { + ...success.analyzed, + materiality: { + category, + priority: documentChangeCategoryPriority(category), + confidence: success.result.disposition === "uncertain" ? "uncertain" : confidenceFromAnalysis(success.result.confidence), + signals: success.result.disposition === "uncertain" + ? ["semantic_analysis_uncertain"] + : ["semantic_analysis_material"], + }, + }; +} + +function normalizedCopied(value: string): string { + return value.replace(/\s+/g, " ").trim().toLocaleLowerCase(); +} + +function verifiedKeyPoint(value: string | undefined, changes: readonly RawDocumentChange[], side: "previousChunk" | "currentChunk"): string | undefined { + if (!value) return undefined; + const candidate = normalizedCopied(value); + return changes.some(change => normalizedCopied(change[side]?.content ?? "").includes(candidate)) ? value : undefined; +} + +function firstCopiedSpan(changes: readonly RawDocumentChange[], side: "previousChunk" | "currentChunk"): string | undefined { + const value = changes.map(change => change[side]?.content).find(content => content?.trim()); + if (!value) return undefined; + const compact = value.replace(/\s+/g, " ").trim(); + return compact.length <= 240 ? compact : `${compact.slice(0, 239)}…`; +} + +function buildAnalyzerEvidence(success: SuccessfulAnalysis, effective: AnalyzedDocumentChangeGroup): FounderWeeklyReviewEvidenceItem { + const deterministic = buildCondensedDocumentChangeEvidence(effective.pair, effective.group, effective.materiality); + const before = verifiedKeyPoint(success.result.beforeKeyPoint, effective.group.rawChanges, "previousChunk") + ?? firstCopiedSpan(effective.group.rawChanges, "previousChunk"); + const after = verifiedKeyPoint(success.result.afterKeyPoint, effective.group.rawChanges, "currentChunk") + ?? firstCopiedSpan(effective.group.rawChanges, "currentChunk"); + const excerpt = [ + success.result.summary, + ...(before ? [`Before: “${before}”`] : []), + ...(after ? [`After: “${after}”`] : []), + ].join("\n\n").slice(0, 1800); + return { + ...deterministic, + excerpt, + metadata: { + ...deterministic.metadata, + category: effective.materiality.category, + materialityMethod: "llm", + materialityConfidence: success.result.confidence, + materialityVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + }, + }; +} + +function analysisSnapshot(success: SuccessfulAnalysis): DocumentChangeMaterialityAnalysisSnapshot { + return { + disposition: success.result.disposition, + category: success.result.category, + confidence: success.result.confidence, + summary: success.result.summary, + ...(success.result.beforeKeyPoint ? { beforeKeyPoint: verifiedKeyPoint(success.result.beforeKeyPoint, success.analyzed.group.rawChanges, "previousChunk") } : {}), + ...(success.result.afterKeyPoint ? { afterKeyPoint: verifiedKeyPoint(success.result.afterKeyPoint, success.analyzed.group.rawChanges, "currentChunk") } : {}), + analysisMethod: "llm", + analysisPromptVersion: success.metadata.promptVersion ?? DOCUMENT_CHANGE_MATERIALITY_ANALYZER_PROMPT_VERSION, + ...(success.metadata.provider ? { analysisProvider: success.metadata.provider.slice(0, 128) } : {}), + ...(success.metadata.model ? { analysisModel: success.metadata.model.slice(0, 256) } : {}), + analysisInputDigest: success.inputDigest, + analysisResultSchemaVersion: DOCUMENT_CHANGE_MATERIALITY_ANALYSIS_RESULT_VERSION, + }; +} + +function warning(code: DocumentChangeProcessingWarning["code"], message: string): DocumentChangeProcessingWarning { + return { code, message }; +} + +async function executeBounded(values: readonly T[], concurrency: number, task: (value: T) => Promise): Promise[]> { + const results: PromiseSettledResult[] = new Array(values.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, async () => { + while (next < values.length) { + const index = next++; + try { + results[index] = { status: "fulfilled", value: await task(values[index]!) }; + } catch (reason) { + results[index] = { status: "rejected", reason }; + } + } + })); + return results; +} + +/** Optional semantic pass. Omit the analyzer to preserve the exact deterministic-v1 result. */ +export async function materializeDocumentChangesWithAnalyzer( + inputs: readonly DocumentChangePairInput[], + analyzer?: DocumentChangeMaterialityAnalyzer, +): Promise { + const deterministic = materializeDocumentChanges(inputs); + const emptyDiagnostics: DocumentChangeMaterialityAnalyzerDiagnostics = { + eligibleGroupCount: 0, + selectedAnalysisCount: 0, + skippedByCallBudgetCount: 0, + inputTruncatedCount: 0, + succeededAnalysisCount: 0, + failedAnalysisCount: 0, + materialCount: 0, + nonMaterialCount: 0, + uncertainCount: 0, + }; + if (!analyzer) return { ...deterministic, analyzerDiagnostics: emptyDiagnostics }; + + const eligible = deterministic.analyzedGroups.filter(group => shouldAnalyzeDocumentChangeGroup(group).eligible); + const analysisSelection = selectDocumentChangeGroupsForMaterialityAnalysis(deterministic.analyzedGroups); + const prepared = analysisSelection.selected.map(analyzed => ({ analyzed, ...buildDocumentChangeMaterialityAnalysisInput(analyzed) })); + const warnings: DocumentChangeProcessingWarning[] = [...deterministic.warnings]; + if (analysisSelection.skipped.length > 0 || prepared.some(value => value.truncated)) { + warnings.push(warning("materiality_analysis_budget_truncated", "Optional materiality analysis used bounded call or input capacity; remaining groups used deterministic fallback.")); + } + const settled = await executeBounded(prepared, DOCUMENT_CHANGE_MATERIALITY_ANALYZER_LIMITS.concurrency, async preparedInput => { + const response = await analyzer.analyze(preparedInput.input); + const result = DocumentChangeMaterialityAnalysisResultSchema.parse(response.result); + return { + analyzed: preparedInput.analyzed, + input: preparedInput.input, + inputDigest: preparedInput.inputDigest, + inputTruncated: preparedInput.truncated, + result, + metadata: response.metadata ?? {}, + } satisfies SuccessfulAnalysis; + }); + const successes: SuccessfulAnalysis[] = []; + let invalid = false; + let timeout = false; + let unavailable = false; + for (const result of settled) { + if (result.status === "fulfilled") successes.push(result.value); + else if (result.reason instanceof z.ZodError) invalid = true; + else if (result.reason instanceof DocumentChangeMaterialityAnalyzerError && result.reason.code === "invalid") invalid = true; + else if (result.reason instanceof DocumentChangeMaterialityAnalyzerError && result.reason.code === "timeout") timeout = true; + else if (result.reason instanceof DocumentChangeMaterialityAnalyzerError && result.reason.code === "unavailable") unavailable = true; + else unavailable = true; + } + if (invalid) warnings.push(warning("materiality_result_invalid", "An optional materiality result failed strict validation; deterministic evidence was retained.")); + if (timeout) warnings.push(warning("materiality_analysis_timeout", "Optional materiality analysis timed out; deterministic evidence was retained.")); + if (unavailable) warnings.push(warning("materiality_analyzer_unavailable", "Optional materiality analysis was unavailable; deterministic evidence was retained.")); + if (settled.some(result => result.status === "rejected")) { + warnings.push(warning("materiality_analysis_partial", "Optional materiality analysis was partial; failed groups used deterministic fallback.")); + } + + const successByGroup = new Map(successes.map(success => [success.analyzed.group.groupId, success])); + const effectiveGroups = deterministic.analyzedGroups.flatMap(analyzed => { + const success = successByGroup.get(analyzed.group.groupId); + if (!success) return [analyzed]; + const effective = effectiveAnalyzedGroup(success); + return effective ? [effective] : []; + }); + const selection = selectMaterialDocumentChangeGroups(effectiveGroups); + warnings.push(...selection.warnings); + const selectedIds = new Set(selection.selectedGroups.map(group => group.group.groupId)); + const items = selection.selectedGroups.map(effective => { + const success = successByGroup.get(effective.group.groupId); + return success?.result.disposition === "material" + ? buildAnalyzerEvidence(success, effective) + : buildCondensedDocumentChangeEvidence(effective.pair, effective.group, effective.materiality); + }); + const audit: DocumentChangeAuditSnapshot = { + ...deterministic.audit, + groups: deterministic.audit.groups.map(group => ({ + ...group, + evidenceSourceId: selectedIds.has(group.groupId) ? documentChangeGroupSourceId( + deterministic.analyzedGroups.find(candidate => candidate.group.groupId === group.groupId)!.group, + ) : null, + ...(successByGroup.has(group.groupId) ? { analysis: analysisSnapshot(successByGroup.get(group.groupId)!) } : {}), + })), + }; + const selectedGroups = selection.selectedGroups; + const rawExcerptCharacters = audit.rawChanges.reduce((total, change) => + total + (change.previousExcerpt?.length ?? 0) + (change.currentExcerpt?.length ?? 0), 0); + const condensedPromptFacingCharacters = items.reduce((total, item) => total + item.excerpt.length, 0); + const materialCount = successes.filter(success => success.result.disposition === "material").length; + const nonMaterialCount = successes.filter(success => success.result.disposition === "non_material").length; + const uncertainCount = successes.filter(success => success.result.disposition === "uncertain").length; + return { + ...deterministic, + selectedGroups, + items, + audit, + warnings: [...new Map(warnings.map(value => [value.code, value])).values()], + diagnostics: { + ...deterministic.diagnostics, + selectedGroupCount: selectedGroups.length, + truncatedGroupCount: effectiveGroups.length - selectedGroups.length, + selectedGroupsByCategory: Object.fromEntries(DOCUMENT_CHANGE_CATEGORIES.map(category => [category, + selectedGroups.filter(group => group.materiality.category === category).length])) as Record, + truncatedGroupsByCategory: Object.fromEntries(DOCUMENT_CHANGE_CATEGORIES.map(category => [category, + effectiveGroups.filter(group => !selectedIds.has(group.group.groupId) && group.materiality.category === category).length])) as Record, + condensedEvidenceCount: items.length, + rawExcerptCharacters, + condensedPromptFacingCharacters, + estimatedReductionRatio: rawExcerptCharacters === 0 ? 1 : Number((rawExcerptCharacters / Math.max(1, condensedPromptFacingCharacters)).toFixed(3)), + }, + analyzerDiagnostics: { + eligibleGroupCount: eligible.length, + selectedAnalysisCount: prepared.length, + skippedByCallBudgetCount: analysisSelection.skipped.length, + inputTruncatedCount: prepared.filter(value => value.truncated).length, + succeededAnalysisCount: successes.length, + failedAnalysisCount: settled.length - successes.length, + materialCount, + nonMaterialCount, + uncertainCount, + }, + }; +} diff --git a/packages/features/src/founder-weekly-review/document-change-materiality.ts b/packages/features/src/founder-weekly-review/document-change-materiality.ts new file mode 100644 index 000000000..a95ae42a3 --- /dev/null +++ b/packages/features/src/founder-weekly-review/document-change-materiality.ts @@ -0,0 +1,733 @@ +import { createHash } from "node:crypto"; + +import type { + DocumentChangeAuditSnapshot, + FounderWeeklyReviewEvidenceItem, + FounderWeeklyReviewEvidenceSnapshot, + RawDocumentChangeSnapshot, +} from "./contracts"; +import { + DOCUMENT_CHANGE_GROUP_BUDGET, + RAW_DOCUMENT_CHANGE_VERSION, + buildRawDocumentChanges, + groupRawDocumentChanges, + type DocumentChangeGroup, + type DocumentChangePairInput, + type DocumentChangeProcessingWarning, + type RawDocumentChange, + type VersionChunk, + type VersionPair, +} from "./document-change"; + +export const DOCUMENT_CHANGE_MATERIALITY_VERSION = "document-change-materiality/v1" as const; +export const DOCUMENT_CHANGE_AUDIT_VERSION = "document-change-audit/v1" as const; + +export const DOCUMENT_CHANGE_CATEGORIES = [ + "ownership_change", + "status_change", + "deadline_change", + "metric_change", + "requirement_change", + "risk_or_blocker_change", + "scope_change", + "priority_change", + "uncertain", + "editorial_rewrite", +] as const; + +export type DocumentChangeCategory = typeof DOCUMENT_CHANGE_CATEGORIES[number]; +export type DeterministicMaterialityConfidence = "strong" | "moderate" | "uncertain"; + +export type DeterministicMaterialityResult = { + category: DocumentChangeCategory; + priority: number; + confidence: DeterministicMaterialityConfidence; + signals: readonly string[]; +}; + +export const DOCUMENT_CHANGE_FACTUAL_DELTA_VERSION = "document-change-factual-delta/v2" as const; + +export const DOCUMENT_CHANGE_FACTUAL_DELTA_KINDS = [ + "ownership", + "status", + "deadline", + "metric", + "requirement", + "negation", + "risk_or_blocker", + "scope", + "priority", +] as const; + +export type DocumentChangeFactualDeltaKind = typeof DOCUMENT_CHANGE_FACTUAL_DELTA_KINDS[number]; +export type DocumentChangeFactualDelta = { + kind: DocumentChangeFactualDeltaKind; + previousValue?: string; + currentValue?: string; + relation: "changed" | "equivalent" | "unknown"; + confidence: "strong" | "moderate"; +}; + +export type DocumentChangeFactualDeltaAssessment = { + version: typeof DOCUMENT_CHANGE_FACTUAL_DELTA_VERSION; + factualComparisons: readonly DocumentChangeFactualDelta[]; + confirmedFactualDeltas: readonly DocumentChangeFactualDelta[]; + equivalentFactualValues: readonly DocumentChangeFactualDelta[]; + possibleSignals: readonly DocumentChangeFactualDeltaKind[]; +}; + +export type AnalyzedDocumentChangeGroup = { + pair: VersionPair; + group: DocumentChangeGroup; + materiality: DeterministicMaterialityResult; +}; + +export type DeterministicMaterialChangeDiagnostics = { + versionPairCount: number; + alignedChunkCount: number; + rawModifiedCount: number; + rawAddedCount: number; + rawRemovedCount: number; + deterministicNoOpCount: number; + groupCount: number; + oversizedGroupSplitCount: number; + selectedGroupCount: number; + truncatedGroupCount: number; + approximateChangedCharacters: number; + groupsByMaterialityCategory: Record; + groupsByDeterministicConfidence: Record; + selectedGroupsByCategory: Record; + truncatedGroupsByCategory: Record; + rawAuditCount: number; + condensedEvidenceCount: number; + rawExcerptCharacters: number; + condensedPromptFacingCharacters: number; + estimatedReductionRatio: number; +}; + +export type DeterministicMaterialChangeResult = { + rawChanges: readonly RawDocumentChange[]; + analyzedGroups: readonly AnalyzedDocumentChangeGroup[]; + selectedGroups: readonly AnalyzedDocumentChangeGroup[]; + items: readonly FounderWeeklyReviewEvidenceItem[]; + audit: DocumentChangeAuditSnapshot; + warnings: readonly DocumentChangeProcessingWarning[]; + diagnostics: DeterministicMaterialChangeDiagnostics; +}; + +const CATEGORY_PRIORITY: Record = Object.fromEntries( + DOCUMENT_CHANGE_CATEGORIES.map((category, index) => [category, index + 1]) +) as Record; + +export function documentChangeCategoryPriority(category: DocumentChangeCategory): number { + return CATEGORY_PRIORITY[category]; +} + +const CATEGORY_LABEL: Record = { + ownership_change: "Ownership changed.", + status_change: "Status changed.", + deadline_change: "Deadline or date changed.", + metric_change: "Metric changed.", + requirement_change: "Requirement changed.", + risk_or_blocker_change: "Risk or blocker changed.", + scope_change: "Scope changed.", + priority_change: "Priority changed.", + uncertain: "Section changed.", + editorial_rewrite: "Editorial formatting changed.", +}; + +const TEAM_TERMS = new Set(["product", "platform", "marketing", "sales", "engineering", "operations", "finance", "legal", "support"]); +const STATUS_TERMS = ["planned", "launched", "shipped", "in progress", "active", "cancelled", "canceled", "completed", "paused"]; +const REQUIREMENT_TERMS = ["may", "must", "should", "optional", "required", "recommended", "mandatory"]; +const RISK_TERMS = ["risk", "risks", "blocker", "blockers", "blocked", "resolved", "unblocked", "at risk"]; +const PRIORITY_TERMS = ["p0", "p1", "p2", "p3", "deferred", "immediate", "low priority", "high priority", "critical"]; +const SCOPE_TERMS = ["us only", "global", "pilot customers", "all enterprise customers", "one team", "company-wide", "company wide"]; + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function compareBigInt(a: bigint, b: bigint): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function bound(value: string, maximum: number): string { + return value.length <= maximum ? value : `${value.slice(0, maximum - 1)}…`; +} + +function copiedText(value: string, maximum = 480): string { + return bound(value.replace(/\s+/g, " ").trim(), maximum); +} + +function valuesMatching(value: string, expressions: readonly RegExp[]): string[] { + const values = expressions.flatMap(expression => [...value.matchAll(expression)].map(match => (match[1] ?? match[0]).toLocaleLowerCase())); + return [...new Set(values)].sort(compareOrdinal); +} + +function phraseValues(value: string, phrases: readonly string[]): string[] { + const lower = value.toLocaleLowerCase(); + return phrases.filter(phrase => new RegExp(`\\b${phrase.replace(/\s+/g, "\\s+")}\\b`, "i").test(lower)); +} + +function changedValues(before: string, after: string, extract: (value: string) => readonly string[]): boolean { + const previous = [...extract(before)].sort(compareOrdinal); + const current = [...extract(after)].sort(compareOrdinal); + return (previous.length > 0 || current.length > 0) && JSON.stringify(previous) !== JSON.stringify(current); +} + +function ownershipValues(value: string): string[] { + const explicit = valuesMatching(value, [ + /\bowner\s*:\s*([\p{L}\p{N}_ -]{1,80})/giu, + /\b([\p{L}\p{N}_-]+(?:\s+[\p{L}\p{N}_-]+){0,3})\s+owns\b/giu, + /\bowned\s+by\s+([\p{L}\p{N}_-]+(?:\s+[\p{L}\p{N}_-]+){0,3})/giu, + ]); + const trimmed = value.trim().toLocaleLowerCase(); + if (TEAM_TERMS.has(trimmed)) explicit.push(trimmed); + return [...new Set(explicit)].sort(compareOrdinal); +} + +function deadlineValues(value: string): string[] { + return valuesMatching(value, [ + /\b(q[1-4])\b/giu, + /\b(20\d{2})\b/gu, + /\b((?:january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2}(?:,\s*20\d{2})?)\b/giu, + /\b(20\d{2}-\d{2}-\d{2})\b/gu, + ]); +} + +function metricValues(value: string): string[] { + return valuesMatching(value, [ + /([$€£]\s*\d+(?:\.\d+)?\s*[kmb]?)/giu, + /(\d+(?:\.\d+)?\s*%)/gu, + /\b(\d+(?:\.\d+)?\s*(?:customers?|users?|accounts?|employees?|days?|weeks?|months?|revenue|arr|mrr))\b/giu, + ]); +} + +function uniqueSorted(values: readonly string[]): string[] { + return [...new Set(values)].sort(compareOrdinal); +} + +function canonicalDeadlineValues(value: string): string[] { + const quarterAliases: Record = { + "q1": "q1", "first": "q1", + "q2": "q2", "second": "q2", + "q3": "q3", "third": "q3", + "q4": "q4", "fourth": "q4", + }; + const quarters = [ + ...value.matchAll(/\b(q[1-4])\b/giu), + ...value.matchAll(/\b(first|second|third|fourth)[ -]quarter\b/giu), + ].map(match => quarterAliases[match[1]!.toLocaleLowerCase()]!); + return uniqueSorted([ + ...quarters, + ...valuesMatching(value, [ + /\b(20\d{2})\b/gu, + /\b((?:january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2}(?:,\s*20\d{2})?)\b/giu, + /\b(20\d{2}-\d{2}-\d{2})\b/gu, + ]), + ]); +} + +const NUMBER_WORD_VALUES: Record = { + zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, + ten: 10, eleven: 11, twelve: 12, thirteen: 13, fourteen: 14, fifteen: 15, sixteen: 16, + seventeen: 17, eighteen: 18, nineteen: 19, twenty: 20, thirty: 30, forty: 40, fifty: 50, + sixty: 60, seventy: 70, eighty: 80, ninety: 90, +}; + +function parseNumberWords(value: string): number | null { + const tokens = value.toLocaleLowerCase().split(/[\s-]+/).filter(Boolean); + if (tokens.length === 0 || tokens.some(token => !(token in NUMBER_WORD_VALUES) && !["hundred", "thousand", "million", "billion", "and"].includes(token))) { + return null; + } + let total = 0; + let current = 0; + for (const token of tokens) { + if (token === "and") continue; + if (token in NUMBER_WORD_VALUES) current += NUMBER_WORD_VALUES[token]!; + else if (token === "hundred") current = Math.max(1, current) * 100; + else { + const scale = token === "thousand" ? 1_000 : token === "million" ? 1_000_000 : 1_000_000_000; + total += Math.max(1, current) * scale; + current = 0; + } + } + return total + current; +} + +function canonicalMetricValues(value: string): string[] { + const result: string[] = []; + const currencyCodes: Record = { "$": "usd", "€": "eur", "£": "gbp" }; + for (const match of value.matchAll(/([$€£])\s*(\d+(?:\.\d+)?)\s*([kmb])?\b/giu)) { + const multiplier = match[3]?.toLocaleLowerCase() === "k" ? 1_000 + : match[3]?.toLocaleLowerCase() === "m" ? 1_000_000 + : match[3]?.toLocaleLowerCase() === "b" ? 1_000_000_000 : 1; + result.push(`${currencyCodes[match[1]!]!}:${Number(match[2]) * multiplier}`); + } + const numberWords = "zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|and"; + const wordCurrency = new RegExp(`\\b((?:(?:${numberWords})[\\s-]+){1,8})(dollars?|euros?|pounds?)\\b`, "giu"); + for (const match of value.matchAll(wordCurrency)) { + const amount = parseNumberWords(match[1]!); + const unit = match[2]!.toLocaleLowerCase().startsWith("dollar") ? "usd" + : match[2]!.toLocaleLowerCase().startsWith("euro") ? "eur" : "gbp"; + if (amount !== null) result.push(`${unit}:${amount}`); + } + for (const match of value.matchAll(/(\d+(?:\.\d+)?)\s*%/gu)) result.push(`percent:${Number(match[1])}`); + for (const match of value.matchAll(/\b(\d+(?:\.\d+)?)\s*(customers?|users?|accounts?|employees?|days?|weeks?|months?|revenue|arr|mrr)\b/giu)) { + result.push(`${match[2]!.toLocaleLowerCase().replace(/s$/, "")}:${Number(match[1])}`); + } + return uniqueSorted(result); +} + +function canonicalPhraseValues(value: string, phrases: readonly string[], aliases: Record = {}): string[] { + return uniqueSorted(phraseValues(value, phrases).map(phrase => aliases[phrase] ?? phrase)); +} + +type FactualValueExtractor = (value: string) => readonly string[]; + +const FACTUAL_EXTRACTORS: ReadonlyArray<{ + kind: Exclude; + extract: FactualValueExtractor; +}> = [ + { kind: "ownership", extract: ownershipValues }, + { kind: "status", extract: value => canonicalPhraseValues(value, STATUS_TERMS, { canceled: "cancelled" }) }, + { kind: "deadline", extract: canonicalDeadlineValues }, + { kind: "metric", extract: canonicalMetricValues }, + { kind: "requirement", extract: value => canonicalPhraseValues(value, REQUIREMENT_TERMS) }, + { kind: "risk_or_blocker", extract: value => canonicalPhraseValues(value, RISK_TERMS, { risks: "risk", blockers: "blocker" }) }, + { kind: "scope", extract: value => canonicalPhraseValues(value, SCOPE_TERMS, { "company wide": "company-wide" }) }, + { kind: "priority", extract: value => canonicalPhraseValues(value, PRIORITY_TERMS) }, +]; + +const SIGNAL_KIND: Record = { + ownership_subject_changed: "ownership", + status_term_changed: "status", + date_or_deadline_changed: "deadline", + numeric_metric_changed: "metric", + requirement_or_modality_changed: "requirement", + negation_changed: "negation", + risk_or_blocker_term_changed: "risk_or_blocker", + scope_marker_changed: "scope", + priority_marker_changed: "priority", +}; + +function comparison( + change: RawDocumentChange, + kind: DocumentChangeFactualDeltaKind, + previous: readonly string[], + current: readonly string[], +): DocumentChangeFactualDelta | null { + if (previous.length === 0 && current.length === 0) return null; + const previousValue = previous.join(" | ") || undefined; + const currentValue = current.join(" | ") || undefined; + if (change.changeType !== "modified" || !change.previousChunk || !change.currentChunk || !previousValue || !currentValue) { + return { kind, ...(previousValue ? { previousValue } : {}), ...(currentValue ? { currentValue } : {}), relation: "unknown", confidence: "moderate" }; + } + return { + kind, + previousValue, + currentValue, + relation: previousValue === currentValue ? "equivalent" : "changed", + confidence: "strong", + }; +} + +/** + * Separates business-token presence from a confirmed two-sided factual delta. + * It is intentionally narrow: unknown values remain analyzer candidates rather than being normalized away. + */ +export function analyzeDocumentChangeFactualDeltas( + group: DocumentChangeGroup, + deterministicSignals: readonly string[] = [], +): DocumentChangeFactualDeltaAssessment { + const factualComparisons: DocumentChangeFactualDelta[] = []; + for (const change of group.rawChanges) { + const before = change.previousNormalizedContent ?? ""; + const after = change.currentNormalizedContent ?? ""; + for (const extractor of FACTUAL_EXTRACTORS) { + const result = comparison(change, extractor.kind, extractor.extract(before), extractor.extract(after)); + if (result) factualComparisons.push(result); + } + if (change.changeType === "modified" && change.previousChunk && change.currentChunk) { + const negations = (value: string) => phraseValues(value, ["not", "no", "never", "unavailable"]); + const beforeNegated = negations(before).length > 0; + const afterNegated = negations(after).length > 0; + if (beforeNegated || afterNegated) { + factualComparisons.push({ + kind: "negation", + previousValue: beforeNegated ? "negated" : "affirmed", + currentValue: afterNegated ? "negated" : "affirmed", + relation: beforeNegated === afterNegated ? "equivalent" : "changed", + confidence: "strong", + }); + } + } else { + const negations = phraseValues(`${before} ${after}`, ["not", "no", "never", "unavailable"]); + if (negations.length > 0) { + factualComparisons.push({ + kind: "negation", + ...(before ? { previousValue: negations.join(" | ") } : {}), + ...(after ? { currentValue: negations.join(" | ") } : {}), + relation: "unknown", + confidence: "moderate", + }); + } + } + } + const deduplicated = [...new Map(factualComparisons.map(value => [ + [value.kind, value.previousValue ?? "", value.currentValue ?? "", value.relation].join(":"), value, + ])).values()].sort((a, b) => compareOrdinal(a.kind, b.kind) + || compareOrdinal(a.previousValue ?? "", b.previousValue ?? "") + || compareOrdinal(a.currentValue ?? "", b.currentValue ?? "") + || compareOrdinal(a.relation, b.relation)); + const confirmedFactualDeltas = deduplicated.filter(value => value.relation === "changed"); + const equivalentFactualValues = deduplicated.filter(value => value.relation === "equivalent"); + const confirmedKinds = new Set(confirmedFactualDeltas.map(value => value.kind)); + const possibleSignals = uniqueSorted([ + ...deduplicated.filter(value => value.relation === "unknown").map(value => value.kind), + ...deterministicSignals.flatMap(signal => { + const kind = SIGNAL_KIND[signal]; + return kind && !confirmedKinds.has(kind) ? [kind] : []; + }), + ]) as DocumentChangeFactualDeltaKind[]; + return { + version: DOCUMENT_CHANGE_FACTUAL_DELTA_VERSION, + factualComparisons: deduplicated, + confirmedFactualDeltas, + equivalentFactualValues, + possibleSignals, + }; +} + +function punctuationOnlyEditorial(before: string, after: string): boolean { + const stripBullet = (value: string) => value.replace(/^\s*[-*+]\s+/gm, "").replace(/\s+/g, " ").trim(); + return before !== after && stripBullet(before) === stripBullet(after); +} + +function addSignal( + candidates: Map>, + category: DocumentChangeCategory, + signal: string +): void { + const signals = candidates.get(category) ?? new Set(); + signals.add(signal); candidates.set(category, signals); +} + +/** Conservative token-pattern nomination; absence of a signal remains uncertain. */ +export function analyzeDocumentChangeGroup(group: DocumentChangeGroup): DeterministicMaterialityResult { + const candidates = new Map>(); + let editorialOnly = group.rawChanges.length > 0; + for (const change of group.rawChanges) { + const before = change.previousNormalizedContent ?? ""; + const after = change.currentNormalizedContent ?? ""; + if (changedValues(before, after, ownershipValues)) addSignal(candidates, "ownership_change", "ownership_subject_changed"); + if (changedValues(before, after, value => phraseValues(value, STATUS_TERMS))) addSignal(candidates, "status_change", "status_term_changed"); + if (changedValues(before, after, deadlineValues)) addSignal(candidates, "deadline_change", "date_or_deadline_changed"); + if (changedValues(before, after, metricValues)) addSignal(candidates, "metric_change", "numeric_metric_changed"); + if (changedValues(before, after, value => phraseValues(value, REQUIREMENT_TERMS))) addSignal(candidates, "requirement_change", "requirement_or_modality_changed"); + if (changedValues(before, after, value => phraseValues(value, RISK_TERMS))) addSignal(candidates, "risk_or_blocker_change", "risk_or_blocker_term_changed"); + if (changedValues(before, after, value => phraseValues(value, SCOPE_TERMS))) addSignal(candidates, "scope_change", "scope_marker_changed"); + if (changedValues(before, after, value => phraseValues(value, PRIORITY_TERMS))) addSignal(candidates, "priority_change", "priority_marker_changed"); + if (changedValues(before, after, value => phraseValues(value, ["not", "no", "never", "unavailable"]))) { + addSignal(candidates, "requirement_change", "negation_changed"); + } + editorialOnly = editorialOnly && change.changeType === "modified" && punctuationOnlyEditorial(before, after); + } + const category = [...candidates.keys()].sort((a, b) => CATEGORY_PRIORITY[a] - CATEGORY_PRIORITY[b])[0] + ?? (editorialOnly ? "editorial_rewrite" : "uncertain"); + const signals = candidates.size > 0 + ? [...candidates.entries()].sort(([a], [b]) => CATEGORY_PRIORITY[a] - CATEGORY_PRIORITY[b]) + .flatMap(([, categorySignals]) => [...categorySignals].sort(compareOrdinal)) + : [category === "editorial_rewrite" ? "markdown_bullet_marker_changed" : "no_strong_deterministic_signal"]; + const confidence: DeterministicMaterialityConfidence = category === "uncertain" + ? "uncertain" + : category === "editorial_rewrite" || signals.includes("ownership_subject_changed") && group.rawChanges.every(change => { + const before = change.previousNormalizedContent?.trim().toLocaleLowerCase() ?? ""; + const after = change.currentNormalizedContent?.trim().toLocaleLowerCase() ?? ""; + return TEAM_TERMS.has(before) && TEAM_TERMS.has(after); + }) ? "moderate" : "strong"; + return { category, priority: CATEGORY_PRIORITY[category], confidence, signals }; +} + +function confidenceRank(value: DeterministicMaterialityConfidence): number { + return value === "strong" ? 0 : value === "moderate" ? 1 : 2; +} + +function structuralKey(group: DocumentChangeGroup): string { + const first = group.rawChanges[0]!; + const chunk = first.currentChunk ?? first.previousChunk!; + return [chunk.structureOrdering ?? "", chunk.pageNumber ?? "", chunk.lineStart ?? "", group.structurePath ?? "", group.groupId].join(":"); +} + +function compareSelectionPriority(a: AnalyzedDocumentChangeGroup, b: AnalyzedDocumentChangeGroup): number { + return a.materiality.priority - b.materiality.priority + || confidenceRank(a.materiality.confidence) - confidenceRank(b.materiality.confidence) + || b.pair.currentCreatedAt.getTime() - a.pair.currentCreatedAt.getTime() + || a.group.splitOrdinal - b.group.splitOrdinal + || compareOrdinal(structuralKey(a.group), structuralKey(b.group)); +} + +function compareFinalOrder(a: AnalyzedDocumentChangeGroup, b: AnalyzedDocumentChangeGroup): number { + return a.pair.currentCreatedAt.getTime() - b.pair.currentCreatedAt.getTime() + || compareBigInt(a.group.documentId, b.group.documentId) + || a.group.previousVersionId - b.group.previousVersionId + || a.group.currentVersionId - b.group.currentVersionId + || compareOrdinal(structuralKey(a.group), structuralKey(b.group)); +} + +export function selectMaterialDocumentChangeGroups(input: readonly AnalyzedDocumentChangeGroup[]): { + selectedGroups: AnalyzedDocumentChangeGroup[]; + truncatedGroups: AnalyzedDocumentChangeGroup[]; + warnings: DocumentChangeProcessingWarning[]; +} { + const byDocument = new Map(); + for (const analyzed of input) { + const key = analyzed.group.documentId.toString(); + const groups = byDocument.get(key) ?? []; + groups.push(analyzed); byDocument.set(key, groups); + } + const documents = [...byDocument.entries()].sort(([a], [b]) => compareBigInt(BigInt(a), BigInt(b))); + for (const [, groups] of documents) groups.sort(compareSelectionPriority); + const selectedGroups: AnalyzedDocumentChangeGroup[] = []; + const selectedIds = new Set(); + const documentCounts = new Map(); + const pairCounts = new Map(); + let madeProgress = true; + while (selectedGroups.length < DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview && madeProgress) { + madeProgress = false; + for (const [documentId, groups] of documents) { + if ((documentCounts.get(documentId) ?? 0) >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerDocument) continue; + const candidate = groups.find(analyzed => { + if (selectedIds.has(analyzed.group.groupId)) return false; + const key = `${documentId}:${analyzed.group.previousVersionId}:${analyzed.group.currentVersionId}`; + return (pairCounts.get(key) ?? 0) < DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerVersionPair; + }); + if (!candidate) continue; + const pair = `${documentId}:${candidate.group.previousVersionId}:${candidate.group.currentVersionId}`; + selectedGroups.push(candidate); selectedIds.add(candidate.group.groupId); + documentCounts.set(documentId, (documentCounts.get(documentId) ?? 0) + 1); + pairCounts.set(pair, (pairCounts.get(pair) ?? 0) + 1); + madeProgress = true; + if (selectedGroups.length >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview) break; + } + } + const truncatedGroups = input.filter(analyzed => !selectedIds.has(analyzed.group.groupId)).sort(compareFinalOrder); + return { + selectedGroups: selectedGroups.sort(compareFinalOrder), + truncatedGroups, + warnings: truncatedGroups.length > 0 ? [{ + code: "document_change_budget_truncated", + message: "Document-change groups were truncated to deterministic materiality-aware structural limits.", + }] : [], + }; +} + +export function documentChangeGroupSourceId(group: DocumentChangeGroup): string { + return `document_change:group:${group.groupId.replace(/^document_change_group:/, "")}`; +} + +/** Resolves a condensed citation through the frozen v2 group to every raw source record. */ +export function resolveDocumentChangeEvidenceAudit( + snapshot: FounderWeeklyReviewEvidenceSnapshot, + sourceId: string +): { group: DocumentChangeAuditSnapshot["groups"][number]; rawChanges: RawDocumentChangeSnapshot[] } | null { + if (snapshot.schemaVersion !== "founder-weekly-review-evidence/v2") return null; + const group = snapshot.documentChangeAudit.groups.find(candidate => candidate.evidenceSourceId === sourceId); + if (!group) return null; + const byId = new Map(snapshot.documentChangeAudit.rawChanges.map(change => [change.rawChangeId, change])); + const rawChanges = group.rawChangeIds.map(rawChangeId => byId.get(rawChangeId)); + return rawChanges.some(change => !change) ? null : { group, rawChanges: rawChanges as RawDocumentChangeSnapshot[] }; +} + +function copiedList(changes: readonly RawDocumentChange[], side: "previousChunk" | "currentChunk"): string | null { + const values = changes.map(change => change[side]?.content).filter((value): value is string => Boolean(value?.trim())); + if (values.length === 0) return null; + let result = ""; + for (const value of values) { + const line = `- ${copiedText(value, 220)}`; + if (`${result}${result ? "\n" : ""}${line}`.length > 480) break; + result += `${result ? "\n" : ""}${line}`; + } + return result || `- ${copiedText(values[0]!, 476)}`; +} + +/** One bounded group-level item containing only deterministic labels and copied source spans. */ +export function buildCondensedDocumentChangeEvidence( + pair: VersionPair, + group: DocumentChangeGroup, + materiality: DeterministicMaterialityResult +): FounderWeeklyReviewEvidenceItem { + const before = copiedList(group.rawChanges, "previousChunk"); + const after = copiedList(group.rawChanges, "currentChunk"); + const fragmentLabel = group.rawChanges.length === 1 ? "" : ` Section changed across ${group.rawChanges.length} source fragments.`; + const sections = [ + `${CATEGORY_LABEL[materiality.category]}${fragmentLabel}`, + ...(before ? [`Before:\n${before}`] : []), + ...(after ? [`After:\n${after}`] : []), + ]; + const section = group.structureTitle ?? group.structurePath; + const firstChunk = group.rawChanges[0]!.currentChunk ?? group.rawChanges[0]!.previousChunk!; + return { + sourceType: "document_change", + sourceId: documentChangeGroupSourceId(group), + title: bound(section ? `${pair.documentTitle} — ${section}` : pair.documentTitle, 512), + sourceTimestamp: pair.currentCreatedAt.toISOString(), + excerpt: bound(sections.join("\n\n"), 1800), + workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, + metadata: { + documentId: pair.documentId.toString(), + groupId: group.groupId, + category: materiality.category, + materialityMethod: "deterministic", + materialityConfidence: materiality.confidence, + materialityVersion: DOCUMENT_CHANGE_MATERIALITY_VERSION, + previousVersionId: pair.previousVersionId, + currentVersionId: pair.currentVersionId, + previousVersionNumber: pair.previousVersionNumber, + currentVersionNumber: pair.currentVersionNumber, + structurePath: group.structurePath ?? null, + structureTitle: group.structureTitle ?? null, + structureOrdering: firstChunk.structureOrdering, + pageNumber: firstChunk.pageNumber, + lineStart: firstChunk.lineStart, + rawChangeCount: group.rawChanges.length, + }, + }; +} + +function hashContent(chunk: VersionChunk | undefined): string | null { + return chunk ? createHash("sha256").update(chunk.content, "utf8").digest("hex") : null; +} + +function rawSnapshot(pair: VersionPair, change: RawDocumentChange): RawDocumentChangeSnapshot { + const previous = change.previousChunk; const current = change.currentChunk; + return { + rawChangeId: change.rawChangeId, + changeType: change.changeType, + alignmentMethod: change.alignmentMethod, + ...(change.similarityScore === undefined ? {} : { similarityScore: change.similarityScore }), + documentId: pair.documentId.toString(), + previousVersionId: pair.previousVersionId, + currentVersionId: pair.currentVersionId, + previousChunkId: previous?.chunkId ?? null, + currentChunkId: current?.chunkId ?? null, + previousExcerpt: previous ? bound(previous.content, 600) : null, + currentExcerpt: current ? bound(current.content, 600) : null, + previousHash: hashContent(previous), + currentHash: hashContent(current), + previousStructurePath: previous?.structurePath ? bound(previous.structurePath, 512) : null, + currentStructurePath: current?.structurePath ? bound(current.structurePath, 512) : null, + previousStructureTitle: previous?.structureTitle ? bound(previous.structureTitle, 512) : null, + currentStructureTitle: current?.structureTitle ? bound(current.structureTitle, 512) : null, + previousPageNumber: previous?.pageNumber ?? null, + currentPageNumber: current?.pageNumber ?? null, + previousLineStart: previous?.lineStart ?? null, + previousLineEnd: previous?.lineEnd ?? null, + currentLineStart: current?.lineStart ?? null, + currentLineEnd: current?.lineEnd ?? null, + processingVersion: RAW_DOCUMENT_CHANGE_VERSION, + }; +} + +function emptyCategoryCounts(): Record { + return Object.fromEntries(DOCUMENT_CHANGE_CATEGORIES.map(category => [category, 0])) as Record; +} + +function categoryCounts(groups: readonly AnalyzedDocumentChangeGroup[]): Record { + const counts = emptyCategoryCounts(); + for (const group of groups) counts[group.materiality.category]++; + return counts; +} + +function confidenceCounts(groups: readonly AnalyzedDocumentChangeGroup[]): Record { + const counts = { strong: 0, moderate: 0, uncertain: 0 }; + for (const group of groups) counts[group.materiality.confidence]++; + return counts; +} + +/** Builds selected condensed evidence plus a complete immutable-audit-ready projection. */ +export function materializeDocumentChanges(inputs: readonly DocumentChangePairInput[]): DeterministicMaterialChangeResult { + const rawChanges: RawDocumentChange[] = []; + const analyzedGroups: AnalyzedDocumentChangeGroup[] = []; + const warnings: DocumentChangeProcessingWarning[] = []; + let deterministicNoOpCount = 0; + let oversizedGroupSplitCount = 0; + const orderedInputs = [...inputs].sort((a, b) => + compareBigInt(a.pair.documentId, b.pair.documentId) + || a.pair.currentCreatedAt.getTime() - b.pair.currentCreatedAt.getTime() + || a.pair.currentVersionId - b.pair.currentVersionId); + for (const { pair, alignments } of orderedInputs) { + const raw = buildRawDocumentChanges(pair, alignments); + const grouped = groupRawDocumentChanges(pair, raw.rawChanges); + rawChanges.push(...raw.rawChanges); + deterministicNoOpCount += raw.deterministicNoOpCount; + oversizedGroupSplitCount += grouped.oversizedGroupSplitCount; + warnings.push(...grouped.warnings); + analyzedGroups.push(...grouped.groups.map(group => ({ pair, group, materiality: analyzeDocumentChangeGroup(group) }))); + } + const selection = selectMaterialDocumentChangeGroups(analyzedGroups); + warnings.push(...selection.warnings); + const items = selection.selectedGroups.map(({ pair, group, materiality }) => buildCondensedDocumentChangeEvidence(pair, group, materiality)); + const selectedIds = new Set(selection.selectedGroups.map(({ group }) => group.groupId)); + const rawAuditById = new Map(); + for (const analyzed of analyzedGroups) { + for (const change of analyzed.group.rawChanges) { + if (!rawAuditById.has(change.rawChangeId)) rawAuditById.set(change.rawChangeId, rawSnapshot(analyzed.pair, change)); + } + } + const audit: DocumentChangeAuditSnapshot = { + schemaVersion: DOCUMENT_CHANGE_AUDIT_VERSION, + rawChanges: [...rawAuditById.values()].sort((a, b) => compareOrdinal(a.rawChangeId, b.rawChangeId)), + groups: [...analyzedGroups].sort(compareFinalOrder).map(({ group, materiality }) => ({ + groupId: group.groupId, + evidenceSourceId: selectedIds.has(group.groupId) ? documentChangeGroupSourceId(group) : null, + documentId: group.documentId.toString(), + previousVersionId: group.previousVersionId, + currentVersionId: group.currentVersionId, + structurePath: group.structurePath ?? null, + structureTitle: group.structureTitle ?? null, + splitOrdinal: group.splitOrdinal, + rawChangeIds: group.rawChanges.map(change => change.rawChangeId), + category: materiality.category, + priority: materiality.priority, + confidence: materiality.confidence, + signals: [...materiality.signals], + materialityMethod: "deterministic", + materialityVersion: DOCUMENT_CHANGE_MATERIALITY_VERSION, + })), + }; + const count = (type: RawDocumentChange["changeType"]) => rawChanges.filter(change => change.changeType === type).length; + const rawExcerptCharacters = audit.rawChanges.reduce((total, change) => + total + (change.previousExcerpt?.length ?? 0) + (change.currentExcerpt?.length ?? 0), 0); + const condensedPromptFacingCharacters = items.reduce((total, item) => total + item.excerpt.length, 0); + return { + rawChanges, + analyzedGroups: [...analyzedGroups].sort(compareFinalOrder), + selectedGroups: selection.selectedGroups, + items, + audit, + warnings: [...new Map(warnings.map(item => [item.code, item])).values()], + diagnostics: { + versionPairCount: inputs.length, + alignedChunkCount: inputs.reduce((total, input) => total + input.alignments.length, 0), + rawModifiedCount: count("modified"), + rawAddedCount: count("added"), + rawRemovedCount: count("removed"), + deterministicNoOpCount, + groupCount: analyzedGroups.length, + oversizedGroupSplitCount, + selectedGroupCount: selection.selectedGroups.length, + truncatedGroupCount: selection.truncatedGroups.length, + approximateChangedCharacters: rawChanges.reduce((total, change) => total + (change.previousChunk?.content.length ?? 0) + (change.currentChunk?.content.length ?? 0), 0), + groupsByMaterialityCategory: categoryCounts(analyzedGroups), + groupsByDeterministicConfidence: confidenceCounts(analyzedGroups), + selectedGroupsByCategory: categoryCounts(selection.selectedGroups), + truncatedGroupsByCategory: categoryCounts(selection.truncatedGroups), + rawAuditCount: audit.rawChanges.length, + condensedEvidenceCount: items.length, + rawExcerptCharacters, + condensedPromptFacingCharacters, + estimatedReductionRatio: rawExcerptCharacters === 0 ? 1 : Number((rawExcerptCharacters / Math.max(1, condensedPromptFacingCharacters)).toFixed(3)), + }, + }; +} diff --git a/packages/features/src/founder-weekly-review/document-change.ts b/packages/features/src/founder-weekly-review/document-change.ts new file mode 100644 index 000000000..33f95b8ae --- /dev/null +++ b/packages/features/src/founder-weekly-review/document-change.ts @@ -0,0 +1,582 @@ +import { createHash } from "node:crypto"; +import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; + +export type DocumentVersionForComparison = { + documentId: bigint; + documentTitle: string; + documentCategory: string | null; + versionId: number; + versionNumber: number; + createdAt: Date; + changelog: string | null; +}; + +export type VersionPair = { + documentId: bigint; + documentTitle: string; + documentCategory: string | null; + previousVersionId: number; + previousVersionNumber: number; + previousCreatedAt: Date; + currentVersionId: number; + currentVersionNumber: number; + currentCreatedAt: Date; + currentChangelog: string | null; +}; + +export type VersionChunk = { + chunkId: number; + content: string; + contentHash: string | null; + structureId: bigint | null; + structurePath: string | null; + structureTitle: string | null; + structureOrdering: number | null; + pageNumber: number | null; + lineStart: number | null; + lineEnd: number | null; + documentId: bigint; + versionId: bigint; +}; + +export type ChunkAlignment = { + changeType: "added" | "removed" | "modified" | "unchanged"; + previousChunk?: VersionChunk; + currentChunk?: VersionChunk; + alignmentMethod: "content_hash" | "structure_path" | "section_title" | "structural_position" | "text_similarity" | "unmatched"; + similarityScore?: number; +}; + +export const RAW_DOCUMENT_CHANGE_VERSION = "raw-document-change/v1" as const; +export const DOCUMENT_CHANGE_GROUPING_VERSION = "document-change-grouping/v1" as const; +export const DOCUMENT_CHANGE_GROUP_BUDGET = Object.freeze({ + rawChangesPerGroup: 16, + groupsPerVersionPair: 8, + groupsPerDocument: 8, + groupsPerReview: 24, +}); + +export type RawDocumentChange = { + rawChangeId: string; + changeType: Exclude; + alignmentMethod: ChunkAlignment["alignmentMethod"]; + similarityScore?: number; + previousChunk?: VersionChunk; + currentChunk?: VersionChunk; + previousNormalizedContent?: string; + currentNormalizedContent?: string; +}; + +export type DocumentChangeGroup = { + groupId: string; + documentId: bigint; + previousVersionId: number; + currentVersionId: number; + structurePath?: string | null; + structureTitle?: string | null; + splitOrdinal: number; + rawChanges: readonly RawDocumentChange[]; +}; + +export type DocumentChangeProcessingWarning = { + code: + | "materiality_group_too_large" + | "document_change_budget_truncated" + | "materiality_analyzer_unavailable" + | "materiality_analysis_partial" + | "materiality_result_invalid" + | "materiality_analysis_timeout" + | "materiality_analysis_budget_truncated"; + message: string; +}; + +export type DocumentChangeCondensationDiagnostics = { + versionPairCount: number; + alignedChunkCount: number; + rawModifiedCount: number; + rawAddedCount: number; + rawRemovedCount: number; + deterministicNoOpCount: number; + groupCount: number; + oversizedGroupSplitCount: number; + selectedGroupCount: number; + truncatedGroupCount: number; + approximateChangedCharacters: number; +}; + +export type DocumentChangePairInput = { + pair: VersionPair; + alignments: readonly ChunkAlignment[]; +}; + +export type DocumentChangeCondensationResult = { + rawChanges: readonly RawDocumentChange[]; + groups: readonly DocumentChangeGroup[]; + selectedGroups: readonly DocumentChangeGroup[]; + warnings: readonly DocumentChangeProcessingWarning[]; + diagnostics: DocumentChangeCondensationDiagnostics; +}; + +const MAX_EXCERPT = 4000; +const MAX_METADATA_TEXT = 512; +const MIN_TEXT_SIMILARITY_CHARACTERS = 20; +const MIN_TEXT_SIMILARITY_TOKENS = 3; + +const compareVersions = (a: DocumentVersionForComparison, b: DocumentVersionForComparison) => + a.createdAt.getTime() - b.createdAt.getTime() || a.versionNumber - b.versionNumber || a.versionId - b.versionId; +const normalize = (value: string | null | undefined) => value?.replace(/\s+/g, " ").trim().toLocaleLowerCase() || null; +const validContentHash = (value: string | null): value is string => value !== null && /^[a-f0-9]{64}$/i.test(value); +const compareChunks = (a: VersionChunk, b: VersionChunk) => + (a.structurePath ?? "").localeCompare(b.structurePath ?? "") || + (a.pageNumber ?? -1) - (b.pageNumber ?? -1) || + (a.lineStart ?? -1) - (b.lineStart ?? -1) || + (a.structureOrdering ?? -1) - (b.structureOrdering ?? -1) || a.chunkId - b.chunkId; + +function digest(parts: readonly (string | number | null)[]): string { + return createHash("sha256").update(JSON.stringify(parts), "utf8").digest("hex"); +} + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +function compareBigInt(a: bigint, b: bigint): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** Select only adjacent pairs whose current version is in the reporting period. */ +export function selectVersionPairsForReportingPeriod( + versions: readonly DocumentVersionForComparison[], startInclusive: Date, endExclusive: Date +): VersionPair[] { + const byDocument = new Map(); + for (const version of versions) { + if (version.createdAt >= endExclusive) continue; + const key = version.documentId.toString(); + const group = byDocument.get(key) ?? []; + group.push(version); byDocument.set(key, group); + } + const pairs: VersionPair[] = []; + for (const group of byDocument.values()) { + const ordered = [...group].sort(compareVersions); + for (let index = 0; index < ordered.length; index++) { + const current = ordered[index]!; + if (current.createdAt < startInclusive) continue; + const previous = ordered[index - 1]; + if (!previous) continue; // First-ever version is an added baseline, not an invented diff. + pairs.push({ documentId: current.documentId, documentTitle: current.documentTitle, documentCategory: current.documentCategory, + previousVersionId: previous.versionId, previousVersionNumber: previous.versionNumber, previousCreatedAt: previous.createdAt, + currentVersionId: current.versionId, currentVersionNumber: current.versionNumber, currentCreatedAt: current.createdAt, + currentChangelog: current.changelog }); + } + } + return pairs.sort((a, b) => a.currentCreatedAt.getTime() - b.currentCreatedAt.getTime() || a.currentVersionNumber - b.currentVersionNumber || a.currentVersionId - b.currentVersionId); +} + +function bestMatch(previous: VersionChunk, candidates: VersionChunk[], used: Set, key: (chunk: VersionChunk) => string | null, requireUnique = false, predicate: (candidate: VersionChunk) => boolean = () => true): VersionChunk | undefined { + const value = key(previous); if (!value) return undefined; + const matches = candidates.filter((candidate) => !used.has(candidate.chunkId.toString()) && key(candidate) === value && predicate(candidate)).sort(compareChunks); + return requireUnique && matches.length !== 1 ? undefined : matches[0]; +} + +function textSimilarity(left: string, right: string): number { + const tokens = (value: string) => new Set(value.toLocaleLowerCase().match(/[\p{L}\p{N}_-]{2,}/gu) ?? []); + const a = tokens(left); const b = tokens(right); + if (left.trim().length < MIN_TEXT_SIMILARITY_CHARACTERS || right.trim().length < MIN_TEXT_SIMILARITY_CHARACTERS || a.size < MIN_TEXT_SIMILARITY_TOKENS || b.size < MIN_TEXT_SIMILARITY_TOKENS) return 0; + let intersection = 0; for (const token of a) if (b.has(token)) intersection++; + return intersection / (a.size + b.size - intersection); +} + +function bestTextMatch(previous: VersionChunk, candidates: VersionChunk[], used: Set): { chunk: VersionChunk; score: number } | undefined { + const ranked = candidates.filter((candidate) => !used.has(candidate.chunkId.toString())).sort(compareChunks).slice(0, 200) + .map((chunk) => ({ chunk, score: textSimilarity(previous.content, chunk.content) })) + .filter((candidate) => candidate.score >= 0.35) + .sort((a, b) => b.score - a.score || compareChunks(a.chunk, b.chunk)); + return ranked[0]; +} + +/** Deterministic one-to-one alignment without embeddings or model calls. */ +export function alignVersionChunks(previousChunks: readonly VersionChunk[], currentChunks: readonly VersionChunk[]): ChunkAlignment[] { + const previous = [...previousChunks].sort(compareChunks); const current = [...currentChunks].sort(compareChunks); + const used = new Set(); const results: ChunkAlignment[] = []; + const strategies: Array<{ method: ChunkAlignment["alignmentMethod"]; key: (chunk: VersionChunk) => string | null; requireUnique?: boolean; predicate?: (previous: VersionChunk, current: VersionChunk) => boolean }> = [ + { method: "content_hash", key: (c) => validContentHash(c.contentHash) ? c.contentHash.toLocaleLowerCase() : null, predicate: (previous, candidate) => previous.content === candidate.content }, + { method: "structure_path", key: (c) => normalize(c.structurePath), requireUnique: true }, + { method: "section_title", key: (c) => normalize(c.structureTitle), requireUnique: true }, + { method: "structural_position", key: (c) => { + const path = normalize(c.structurePath); + return path && c.pageNumber !== null && c.structureOrdering !== null ? `${path}|${c.pageNumber}|${c.lineStart ?? ""}|${c.structureOrdering}` : null; + } }, + ]; + for (const oldChunk of previous) { + let match: VersionChunk | undefined; let method: ChunkAlignment["alignmentMethod"] = "unmatched"; + for (const strategy of strategies) { + match = bestMatch(oldChunk, current, used, strategy.key, strategy.requireUnique, (candidate) => strategy.predicate?.(oldChunk, candidate) ?? true); + if (match) { method = strategy.method; break; } + } + if (!match) { + const textMatch = bestTextMatch(oldChunk, current, used); + if (textMatch) { match = textMatch.chunk; method = "text_similarity"; } + } + if (!match) { results.push({ changeType: "removed", previousChunk: oldChunk, alignmentMethod: "unmatched" }); continue; } + used.add(match.chunkId.toString()); + const score = method === "text_similarity" ? textSimilarity(oldChunk.content, match.content) : undefined; + results.push({ changeType: oldChunk.content === match.content ? "unchanged" : "modified", previousChunk: oldChunk, currentChunk: match, alignmentMethod: method, ...(score === undefined ? {} : { similarityScore: score }) }); + } + for (const chunk of current) if (!used.has(chunk.chunkId.toString())) results.push({ changeType: "added", currentChunk: chunk, alignmentMethod: "unmatched" }); + return results.sort((a, b) => compareChunks(a.currentChunk ?? a.previousChunk!, b.currentChunk ?? b.previousChunk!)); +} + +/** Conservative normalization used only to remove deterministic formatting no-ops. */ +export function normalizeDocumentChangeContent(value: string): string { + return value.normalize("NFC").replace(/\r\n?/g, "\n").replace(/\u00a0/g, " ").replace(/\s+/g, " ").trim(); +} + +function rawChangeContentIdentity(chunk: VersionChunk | undefined): string | null { + return chunk ? digest([chunk.contentHash, chunk.content]) : null; +} + +function compareRawChanges(a: RawDocumentChange, b: RawDocumentChange): number { + const aChunk = a.currentChunk ?? a.previousChunk!; + const bChunk = b.currentChunk ?? b.previousChunk!; + return (aChunk.structureOrdering ?? Number.MAX_SAFE_INTEGER) - (bChunk.structureOrdering ?? Number.MAX_SAFE_INTEGER) + || (aChunk.pageNumber ?? Number.MAX_SAFE_INTEGER) - (bChunk.pageNumber ?? Number.MAX_SAFE_INTEGER) + || (aChunk.lineStart ?? Number.MAX_SAFE_INTEGER) - (bChunk.lineStart ?? Number.MAX_SAFE_INTEGER) + || compareOrdinal(aChunk.structurePath ?? "", bChunk.structurePath ?? "") + || (a.previousChunk?.chunkId ?? -1) - (b.previousChunk?.chunkId ?? -1) + || (a.currentChunk?.chunkId ?? -1) - (b.currentChunk?.chunkId ?? -1) + || compareOrdinal(a.rawChangeId, b.rawChangeId); +} + +/** Converts alignments into stable raw records and drops normalized-equal modifications. */ +export function buildRawDocumentChanges( + pair: VersionPair, + alignments: readonly ChunkAlignment[] +): { rawChanges: RawDocumentChange[]; deterministicNoOpCount: number } { + const rawChanges: RawDocumentChange[] = []; + let deterministicNoOpCount = 0; + for (const alignment of alignments) { + if (alignment.changeType === "unchanged") continue; + const previousNormalizedContent = alignment.previousChunk + ? normalizeDocumentChangeContent(alignment.previousChunk.content) + : undefined; + const currentNormalizedContent = alignment.currentChunk + ? normalizeDocumentChangeContent(alignment.currentChunk.content) + : undefined; + if ( + alignment.changeType === "modified" + && previousNormalizedContent === currentNormalizedContent + ) { + deterministicNoOpCount++; + continue; + } + const rawChangeId = `raw_document_change:${digest([ + RAW_DOCUMENT_CHANGE_VERSION, + pair.documentId.toString(), + pair.previousVersionId, + pair.currentVersionId, + alignment.changeType, + alignment.previousChunk?.chunkId ?? null, + alignment.currentChunk?.chunkId ?? null, + alignment.alignmentMethod, + rawChangeContentIdentity(alignment.previousChunk), + rawChangeContentIdentity(alignment.currentChunk), + ])}`; + rawChanges.push({ + rawChangeId, + changeType: alignment.changeType, + alignmentMethod: alignment.alignmentMethod, + ...(alignment.similarityScore === undefined ? {} : { similarityScore: alignment.similarityScore }), + ...(alignment.previousChunk ? { previousChunk: alignment.previousChunk, previousNormalizedContent } : {}), + ...(alignment.currentChunk ? { currentChunk: alignment.currentChunk, currentNormalizedContent } : {}), + }); + } + return { rawChanges: rawChanges.sort(compareRawChanges), deterministicNoOpCount }; +} + +function normalizedSectionValues(change: RawDocumentChange, field: "structurePath" | "structureTitle"): string[] { + return [...new Set([change.previousChunk?.[field], change.currentChunk?.[field]].map(normalize).filter((value): value is string => value !== null))].sort(compareOrdinal); +} + +function intersects(left: readonly string[], right: readonly string[]): boolean { + return left.some(value => right.includes(value)); +} + +function hasKnownSection(change: RawDocumentChange): boolean { + return normalizedSectionValues(change, "structurePath").length > 0 + || normalizedSectionValues(change, "structureTitle").length > 0; +} + +function structurallyProximate(left: RawDocumentChange, right: RawDocumentChange): boolean { + const a = left.currentChunk ?? left.previousChunk!; + const b = right.currentChunk ?? right.previousChunk!; + if (a.structureOrdering !== null && b.structureOrdering !== null && Math.abs(a.structureOrdering - b.structureOrdering) <= 1) { + return true; + } + if (a.pageNumber !== null && b.pageNumber !== null && Math.abs(a.pageNumber - b.pageNumber) <= 1) { + if (a.pageNumber !== b.pageNumber) return true; + if (a.lineEnd !== null && b.lineStart !== null && b.lineStart - a.lineEnd <= 2) return true; + if (b.lineEnd !== null && a.lineStart !== null && a.lineStart - b.lineEnd <= 2) return true; + } + return false; +} + +function shouldGroup(left: RawDocumentChange, right: RawDocumentChange): boolean { + const leftPaths = normalizedSectionValues(left, "structurePath"); + const rightPaths = normalizedSectionValues(right, "structurePath"); + if (intersects(leftPaths, rightPaths)) return true; + if (leftPaths.length > 0 && rightPaths.length > 0) return false; + const leftTitles = normalizedSectionValues(left, "structureTitle"); + const rightTitles = normalizedSectionValues(right, "structureTitle"); + if (intersects(leftTitles, rightTitles)) return true; + // Proximity is only a fallback when neither record has a known section. This + // prevents an unlabelled record from transitively bridging two named sections. + if (hasKnownSection(left) || hasKnownSection(right)) return false; + return structurallyProximate(left, right); +} + +function canonicalSectionKey(changes: readonly RawDocumentChange[]): string { + const paths = [...new Set(changes.flatMap(change => normalizedSectionValues(change, "structurePath")))].sort(compareOrdinal); + const titles = [...new Set(changes.flatMap(change => normalizedSectionValues(change, "structureTitle")))].sort(compareOrdinal); + if (paths.length > 0) return `path:${paths.join("|")}`; + if (titles.length > 0) return `title:${titles.join("|")}`; + const first = changes[0]!.currentChunk ?? changes[0]!.previousChunk!; + return `position:${first.structureOrdering ?? ""}:${first.pageNumber ?? ""}:${first.lineStart ?? ""}`; +} + +function preferredStructureValue(changes: readonly RawDocumentChange[], field: "structurePath" | "structureTitle"): string | null { + const values = changes.flatMap(change => [change.currentChunk?.[field], change.previousChunk?.[field]]) + .filter((value): value is string => Boolean(value?.trim())); + return values.sort(compareOrdinal)[0] ?? null; +} + +function makeGroup( + pair: VersionPair, + naturalGroup: readonly RawDocumentChange[], + rawChanges: readonly RawDocumentChange[], + splitOrdinal: number +): DocumentChangeGroup { + const sectionKey = canonicalSectionKey(naturalGroup); + return { + groupId: `document_change_group:${digest([ + DOCUMENT_CHANGE_GROUPING_VERSION, + pair.documentId.toString(), + pair.previousVersionId, + pair.currentVersionId, + sectionKey, + ...rawChanges.map(change => change.rawChangeId), + splitOrdinal, + ])}`, + documentId: pair.documentId, + previousVersionId: pair.previousVersionId, + currentVersionId: pair.currentVersionId, + structurePath: preferredStructureValue(naturalGroup, "structurePath"), + structureTitle: preferredStructureValue(naturalGroup, "structureTitle"), + splitOrdinal, + rawChanges, + }; +} + +function compareGroups(a: DocumentChangeGroup, b: DocumentChangeGroup): number { + const aFirst = a.rawChanges[0]!; + const bFirst = b.rawChanges[0]!; + return compareBigInt(a.documentId, b.documentId) + || a.currentVersionId - b.currentVersionId + || a.previousVersionId - b.previousVersionId + || compareRawChanges(aFirst, bFirst) + || a.splitOrdinal - b.splitOrdinal + || compareOrdinal(a.groupId, b.groupId); +} + +/** Groups changed records within one immutable document version pair. */ +export function groupRawDocumentChanges( + pair: VersionPair, + input: readonly RawDocumentChange[] +): { groups: DocumentChangeGroup[]; oversizedGroupSplitCount: number; warnings: DocumentChangeProcessingWarning[] } { + const rawChanges = [...input].sort(compareRawChanges); + const parents = rawChanges.map((_, index) => index); + const find = (index: number): number => { + while (parents[index] !== index) { + parents[index] = parents[parents[index]!]!; + index = parents[index]!; + } + return index; + }; + const union = (left: number, right: number) => { + const a = find(left); const b = find(right); + if (a !== b) parents[Math.max(a, b)] = Math.min(a, b); + }; + for (let left = 0; left < rawChanges.length; left++) { + for (let right = left + 1; right < rawChanges.length; right++) { + if (shouldGroup(rawChanges[left]!, rawChanges[right]!)) union(left, right); + } + } + const components = new Map(); + rawChanges.forEach((change, index) => { + const root = find(index); + const component = components.get(root) ?? []; + component.push(change); components.set(root, component); + }); + const naturalGroups = [...components.values()].map(group => group.sort(compareRawChanges)) + .sort((a, b) => compareRawChanges(a[0]!, b[0]!)); + const groups: DocumentChangeGroup[] = []; + let oversizedGroupSplitCount = 0; + for (const naturalGroup of naturalGroups) { + const windowCount = Math.ceil(naturalGroup.length / DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup); + if (windowCount > 1) oversizedGroupSplitCount += windowCount - 1; + for (let splitOrdinal = 0; splitOrdinal < windowCount; splitOrdinal++) { + const start = splitOrdinal * DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup; + const window = naturalGroup.slice(start, start + DOCUMENT_CHANGE_GROUP_BUDGET.rawChangesPerGroup); + groups.push(makeGroup(pair, naturalGroup, window, splitOrdinal)); + } + } + return { + groups: groups.sort(compareGroups), + oversizedGroupSplitCount, + warnings: oversizedGroupSplitCount > 0 ? [{ + code: "materiality_group_too_large", + message: "One or more document-change groups exceeded the raw-change limit and were split into deterministic windows.", + }] : [], + }; +} + +function pairKey(group: Pick): string { + return `${group.previousVersionId}:${group.currentVersionId}`; +} + +function sectionSelectionKey(group: DocumentChangeGroup): string { + return `${group.structurePath ?? ""}|${group.structureTitle ?? ""}`; +} + +/** Neutral structural budget: documents, then version pairs, then distinct sections. */ +export function selectDocumentChangeGroups(input: readonly DocumentChangeGroup[]): { + selectedGroups: DocumentChangeGroup[]; + truncatedGroupCount: number; + warnings: DocumentChangeProcessingWarning[]; +} { + const ordered = [...input].sort(compareGroups); + const byDocument = new Map>(); + for (const group of ordered) { + const documentKey = group.documentId.toString(); + const pairs = byDocument.get(documentKey) ?? new Map(); + const key = pairKey(group); + const pairGroups = pairs.get(key) ?? []; + pairGroups.push(group); pairs.set(key, pairGroups); byDocument.set(documentKey, pairs); + } + for (const pairs of byDocument.values()) { + for (const [key, groups] of pairs) { + pairs.set(key, groups.sort((a, b) => + a.splitOrdinal - b.splitOrdinal + || compareOrdinal(sectionSelectionKey(a), sectionSelectionKey(b)) + || compareGroups(a, b))); + } + } + const documents = [...byDocument.entries()].sort(([a], [b]) => compareBigInt(BigInt(a), BigInt(b))); + const documentCounts = new Map(); + const pairCounts = new Map(); + const pairCursors = new Map(); + const selectedGroups: DocumentChangeGroup[] = []; + let madeProgress = true; + while (selectedGroups.length < DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview && madeProgress) { + madeProgress = false; + for (const [documentId, pairs] of documents) { + if ((documentCounts.get(documentId) ?? 0) >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerDocument) continue; + const pairEntries = [...pairs.entries()].sort(([, a], [, b]) => compareGroups(a[0]!, b[0]!)); + const start = pairCursors.get(documentId) ?? 0; + for (let offset = 0; offset < pairEntries.length; offset++) { + const pairIndex = (start + offset) % pairEntries.length; + const [key, groups] = pairEntries[pairIndex]!; + const countKey = `${documentId}:${key}`; + const count = pairCounts.get(countKey) ?? 0; + if (count >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerVersionPair || !groups[count]) continue; + selectedGroups.push(groups[count]!); + documentCounts.set(documentId, (documentCounts.get(documentId) ?? 0) + 1); + pairCounts.set(countKey, count + 1); + pairCursors.set(documentId, (pairIndex + 1) % pairEntries.length); + madeProgress = true; + break; + } + if (selectedGroups.length >= DOCUMENT_CHANGE_GROUP_BUDGET.groupsPerReview) break; + } + } + const truncatedGroupCount = ordered.length - selectedGroups.length; + return { + selectedGroups: selectedGroups.sort(compareGroups), + truncatedGroupCount, + warnings: truncatedGroupCount > 0 ? [{ + code: "document_change_budget_truncated", + message: "Document-change groups were truncated to deterministic structural limits.", + }] : [], + }; +} + +/** Complete pure condensation boundary used before current evidence compatibility projection. */ +export function condenseDocumentChanges(inputs: readonly DocumentChangePairInput[]): DocumentChangeCondensationResult { + const rawChanges: RawDocumentChange[] = []; + const groups: DocumentChangeGroup[] = []; + const warnings: DocumentChangeProcessingWarning[] = []; + let deterministicNoOpCount = 0; + let oversizedGroupSplitCount = 0; + for (const { pair, alignments } of [...inputs].sort((a, b) => + compareBigInt(a.pair.documentId, b.pair.documentId) + || a.pair.currentCreatedAt.getTime() - b.pair.currentCreatedAt.getTime() + || a.pair.currentVersionId - b.pair.currentVersionId)) { + const raw = buildRawDocumentChanges(pair, alignments); + const grouped = groupRawDocumentChanges(pair, raw.rawChanges); + rawChanges.push(...raw.rawChanges); + groups.push(...grouped.groups); + deterministicNoOpCount += raw.deterministicNoOpCount; + oversizedGroupSplitCount += grouped.oversizedGroupSplitCount; + warnings.push(...grouped.warnings); + } + const selected = selectDocumentChangeGroups(groups); + warnings.push(...selected.warnings); + const counts = (type: RawDocumentChange["changeType"]) => rawChanges.filter(change => change.changeType === type).length; + return { + rawChanges: rawChanges.sort(compareRawChanges), + groups: groups.sort(compareGroups), + selectedGroups: selected.selectedGroups, + warnings: [...new Map(warnings.map(item => [item.code, item])).values()], + diagnostics: { + versionPairCount: inputs.length, + alignedChunkCount: inputs.reduce((total, input) => total + input.alignments.length, 0), + rawModifiedCount: counts("modified"), + rawAddedCount: counts("added"), + rawRemovedCount: counts("removed"), + deterministicNoOpCount, + groupCount: groups.length, + oversizedGroupSplitCount, + selectedGroupCount: selected.selectedGroups.length, + truncatedGroupCount: selected.truncatedGroupCount, + approximateChangedCharacters: rawChanges.reduce((total, change) => + total + (change.previousChunk?.content.length ?? 0) + (change.currentChunk?.content.length ?? 0), 0), + }, + }; +} + +function bound(value: string, max = MAX_EXCERPT) { return value.length <= max ? value : `${value.slice(0, max - 1)}…`; } +function preview(value: string) { return bound(value.replace(/\s+/g, " ").trim(), 900); } + +function buildRawChangeEvidence(pair: VersionPair, alignment: RawDocumentChange): FounderWeeklyReviewEvidenceItem { + const previous = alignment.previousChunk; const current = alignment.currentChunk; + const previousId = previous?.chunkId.toString() ?? "none"; const currentId = current?.chunkId.toString() ?? "none"; + const sourceId = `document_change:doc:${pair.documentId}:v${pair.previousVersionId}:v${pair.currentVersionId}:chunk:${previousId}:${currentId}`; + const excerpt = alignment.changeType === "modified" + ? `Section modified. Before: ${preview(previous!.content)} After: ${preview(current!.content)}` + : alignment.changeType === "added" ? `Section added: ${preview(current!.content)}` : `Section removed: ${preview(previous!.content)}`; + return { sourceType: "document_change", sourceId, title: pair.documentTitle, + sourceTimestamp: pair.currentCreatedAt.toISOString(), excerpt: bound(excerpt), workspaceDeepLink: `/employer/documents/viewer?docId=${pair.documentId}`, + metadata: { documentId: pair.documentId.toString(), previousVersionId: pair.previousVersionId, currentVersionId: pair.currentVersionId, previousVersionNumber: pair.previousVersionNumber, currentVersionNumber: pair.currentVersionNumber, + previousChunkId: previous?.chunkId ?? null, currentChunkId: current?.chunkId ?? null, changeType: alignment.changeType, alignmentMethod: alignment.alignmentMethod, + previousContentHash: previous?.contentHash ?? null, currentContentHash: current?.contentHash ?? null, structurePath: current?.structurePath ?? previous?.structurePath ?? null, + userChangelog: pair.currentChangelog ? bound(pair.currentChangelog.replace(/\s+/g, " ").trim(), MAX_METADATA_TEXT) : null } }; +} + +/** Compatibility projection: one existing-format evidence item per selected structural group. */ +export function buildDocumentChangeGroupEvidence(pair: VersionPair, group: DocumentChangeGroup): FounderWeeklyReviewEvidenceItem { + return buildRawChangeEvidence(pair, group.rawChanges[0]!); +} + +export function buildDocumentChangeEvidence(pair: VersionPair, alignments: readonly ChunkAlignment[]): FounderWeeklyReviewEvidenceItem[] { + const condensed = condenseDocumentChanges([{ pair, alignments }]); + return condensed.selectedGroups.map(group => buildDocumentChangeGroupEvidence(pair, group)); +} diff --git a/packages/features/src/founder-weekly-review/errors.ts b/packages/features/src/founder-weekly-review/errors.ts new file mode 100644 index 000000000..74658363e --- /dev/null +++ b/packages/features/src/founder-weekly-review/errors.ts @@ -0,0 +1,57 @@ +export class FounderWeeklyReviewError extends Error { + constructor( + message: string, + readonly code: + | "not_found" + | "forbidden" + | "invalid_transition" + | "conflict" + | "invalid_payload" + | "claim_ownership_mismatch" + ) { + super(message); + this.name = new.target.name; + } +} + +export class FounderWeeklyReviewNotFoundError extends FounderWeeklyReviewError { + constructor(runId: string) { + super(`Founder weekly review run "${runId}" was not found.`, "not_found"); + } +} + +export class FounderWeeklyReviewForbiddenError extends FounderWeeklyReviewError { + constructor(message = "The active workspace role cannot mutate founder weekly reviews.") { + super(message, "forbidden"); + } +} + +export class FounderWeeklyReviewInvalidTransitionError extends FounderWeeklyReviewError { + constructor(fromStatus: string, action: string) { + super( + `Cannot ${action} founder weekly review run from status "${fromStatus}".`, + "invalid_transition" + ); + } +} + +export class FounderWeeklyReviewConflictError extends FounderWeeklyReviewError { + constructor(message: string) { + super(message, "conflict"); + } +} + +export class FounderWeeklyReviewInvalidPayloadError extends FounderWeeklyReviewError { + constructor(message: string) { + super(message, "invalid_payload"); + } +} + +export class FounderWeeklyReviewClaimOwnershipMismatchError extends FounderWeeklyReviewError { + constructor(runId: string) { + super( + `Generation claim ownership mismatch for founder weekly review run "${runId}".`, + "claim_ownership_mismatch" + ); + } +} diff --git a/packages/features/src/founder-weekly-review/evaluation-prompt.ts b/packages/features/src/founder-weekly-review/evaluation-prompt.ts new file mode 100644 index 000000000..cbf8521ed --- /dev/null +++ b/packages/features/src/founder-weekly-review/evaluation-prompt.ts @@ -0,0 +1,68 @@ +import type { FounderWeeklyReviewEvidenceSnapshot, FounderWeeklyReviewV2Payload } from "./contracts"; + +export const FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION = "founder-weekly-review-evaluation/v2" as const; +export function buildFounderWeeklyReviewEvaluationPrompt(snapshot: FounderWeeklyReviewEvidenceSnapshot, review: FounderWeeklyReviewV2Payload): string { + return JSON.stringify({ + promptVersion: FOUNDER_WEEKLY_REVIEW_EVALUATION_PROMPT_VERSION, + task: "Grade the persisted Founder Weekly Review against its immutable evidence snapshot. Return judgments, not a restatement of the review.", + rubric: { + groundedness: "Claims must be supported by cited snapshot evidence. Treat supports, indicates, and suggests as appropriately bounded language; penalize confirms/proves when the evidence is only a limited signal.", + materiality: "Prioritize meaningful business-state changes over editorial rewrites. Do not reward one-bullet-per-source enumeration.", + temporalAccuracy: "document_change is reporting-period temporal evidence. workspace_document is current context and cannot independently prove whatChanged or whatShipped. document_change plus workspace_document is valid when the document change establishes the temporal proposition and workspace only adds context. Explicitly distinguish planned, scheduled, or future releases from actually shipped during the reporting period; a future-dated release is not shipped in the current period.", + synthesisQuality: "Judge coverage of major material business themes, not source-level citation coverage. Reward concise founder-level synthesis; several related sources may be synthesized into one founder-level claim. Reusing evidence across sections is acceptable when the section-specific meaning changes; penalize only repetition that adds no new meaning.", + actionability: "Reward a clear action plus an evidence-grounded reason or dependency. Do not require or reward invented owners, dates, metrics, or sequencing that are absent from the evidence.", + }, + snapshot, + review, + }); +} +export const founderWeeklyReviewGraderRubric = ` +Evaluate the Founder Weekly Review based on: + +1. Groundedness +- Are every factual claims directly supported by cited evidence? +- Are cited sourceIds semantically appropriate for the claim? +- Does the report avoid inventing customer feedback, shipped work, blockers, or outcomes? +- Does the report avoid turning plans or documents into facts? + +2. Materiality +- Did the report identify the most important changes during the reporting period? +- Did it avoid spending attention on minor or irrelevant updates? +- Did it include meaningful evidence themes instead of ignoring important changes? + +3. Temporal Accuracy +- Does the report correctly distinguish between: + - changed this week + - shipped this week + - planned work + - recommendations + - historical context + +- Planned work must not be represented as shipped. +- Document changes should only appear as "what changed" unless there is separate shipping evidence. +- Multiple document versions should preserve the sequence of changes instead of collapsing unrelated updates incorrectly. + +- workspace_document evidence represents current workspace context. +- It must not be represented as an in-period change unless there is explicit change evidence. +- workspace_document evidence may support blockers, context, or recommendations. + +A technically valid citation may still be semantically unsupported if the cited evidence does not actually justify the claim being made. + +4. Synthesis Quality +- Does the report connect multiple evidence sources when appropriate? +- Does it summarize patterns across evidence rather than simply repeating individual events? +- Does it provide founder-level interpretation while remaining grounded in evidence? + +5. Actionability +- Are recommendations specific and useful? +- Do recommendations clearly separate evidence-backed observations from suggested actions? +- Would a founder know what decision, follow-up, or next step to take? +- Does the report help a founder make decisions? +- Does it highlight what deserves attention instead of merely summarizing activity? + +When evaluating failures: +- Mark unsupported claims as groundedness failures. +- Mark planned work presented as shipped as temporal accuracy failures. +- Mark missing important evidence as materiality failures. +- Mark weak summaries that only restate evidence as synthesis quality failures. +`; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/evaluation.ts b/packages/features/src/founder-weekly-review/evaluation.ts new file mode 100644 index 000000000..1f4d62611 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evaluation.ts @@ -0,0 +1,629 @@ +import{ + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewPayload, + FounderWeeklyReviewV2PayloadSchema, +} from "./contracts"; + +export interface EvaluationResult { + passed: boolean; + + hasHardFailure: boolean; + + deterministic: { + canonicalSchemaValid: boolean; + citationValidity: number; + citationCoverage: number; + unsupportedClaimRate: number; + unsupportedShippedClaimRate: number; + sourceTypeViolationRate: number; + evidenceCoverage: number; + materialityScore: number; + duplicateClaimRate: number; + emptySectionCorrectness: number; + }; + + overallScore: number; + + failures: EvaluationFailure[]; +} + +export type EvaluationFailure = { + category: string; + section?: string; + claim?: string; + explanation: string; +}; + +const monthNumbers: Record = { + january: 1, + february: 2, + march: 3, + april: 4, + may: 5, + june: 6, + july: 7, + august: 8, + september: 9, + october: 10, + november: 11, + december: 12 +}; + +function futureReleaseMentioned(text: string, periodEnd: string): boolean { + const lower = text.toLowerCase(); + if (!/(planned|scheduled|expected|launch|ship|release|deploy)/i.test(lower)) return false; + const end = new Date(`${periodEnd}T23:59:59.999Z`); + const year = end.getUTCFullYear(); + for (const [month, number] of Object.entries(monthNumbers)) { + if (!new RegExp(`\\b${month}\\b`, "i").test(lower)) continue; + const candidate = new Date(Date.UTC(year, number - 1, 1)); + if (candidate > end) return true; + } + const yearMention = lower.match(/\b(20\d{2})\b/); + return yearMention ? Number(yearMention[1]) > year : false; +} + +const SECTION_SOURCE_RULES: Record = { + whatCustomersSaid: ["customer_feedback"], + whatChanged: [ + "document_change", + "workspace_document", + "github_activity", + "founder_context", + ], + whatShipped: [ + "github_activity", + "document_change", + "workspace_document", + ], + currentBlockers: [ + "founder_context", + "workspace_document", + "github_activity", + ], + nextPriorities: [ + "founder_context", + "workspace_document", + ], +}; + +type EvidenceSourceType = + FounderWeeklyReviewEvidenceSnapshot["items"][number]["sourceType"]; + +function isValidSourceForSection( + sectionName: string, + sourceType: EvidenceSourceType +) { + const allowed = SECTION_SOURCE_RULES[sectionName]; + + if (!allowed) return true; + + return allowed.includes(sourceType); +} + +function getSectionItems(section: unknown): unknown[] { + if (typeof section !== "object" || section === null) { + return []; + } + + if ( + "items" in section && + Array.isArray(section.items) + ) { + return section.items; + } + + return []; +} + +function getItemKind(item: unknown): string | undefined { + if ( + typeof item === "object" && + item !== null && + "kind" in item && + typeof item.kind === "string" + ) { + return item.kind; + } + + return undefined; +} + +function normalizeClaim(text:string) { + return text + .toLowerCase() + .replace(/[.,!?]/g, "") + .replace(/\s+/g," ") + .trim(); +} + +function claimSupportedByEvidence( + claim: string, + evidenceText: string +) { + const claimWords = new Set( + normalizeClaim(claim) + .split(" ") + .filter(word => word.length > 3) + ); + + const evidenceWords = new Set( + normalizeClaim(evidenceText) + .split(" ") + .filter(word => word.length > 3) + ); + + const overlap = [...claimWords].filter(word => + evidenceWords.has(word) + ); + + return overlap.length >= 2; +} + +function evaluateEmptySections( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + report: FounderWeeklyReviewPayload, + failures: EvaluationFailure[] +): number { + if (evidenceSnapshot.items.length !== 0) return 1; + + let incorrect = 0; + + for (const [sectionName, section] of Object.entries(report.sections)) { + if (section.state !== "no_evidence") { + incorrect++; + + failures.push({ + category: "invalid_empty_section", + section: sectionName, + explanation: + "Empty evidence snapshot must produce no_evidence sections.", + }); + } + } + + return incorrect === 0 ? 1 : 0; +} + +function evidenceIndicatesShipped( + evidence: FounderWeeklyReviewEvidenceSnapshot["items"][number] +) { + const text = normalizeClaim( + `${evidence.title} ${evidence.excerpt}` + ); + + return [ + "released", + "deployed", + "launched", + "shipped", + "merged", + "implemented", + "completed" + ].some(keyword => text.includes(keyword)); +} + +function evidenceConflicts( + evidenceItems: FounderWeeklyReviewEvidenceSnapshot["items"][number][] +) { + const texts = evidenceItems.map(e => + normalizeClaim(`${e.title} ${e.excerpt}`) + ); + + const supporting = texts.some(text => + [ + "want", + "requested", + "needs", + "need", + ].some(keyword => text.includes(keyword)) + ); + + const opposing = texts.some(text => + [ + "unnecessary", + "does not need", + "doesn't need", + "don't need", + "not needed", + ].some(keyword => text.includes(keyword)) + ); + + return supporting && opposing; +} + +function isTemporalChangeClaim(text: string): boolean { + const lower = text.toLowerCase(); + + const changeLanguage = + /\b(changed|updated|modified|added|removed|introduced)\b/.test(lower); + + const temporalLanguage = + /\b(this week|this period|during the reporting period|during this period|recently)\b/.test( + lower + ); + + return changeLanguage && temporalLanguage; +} + +export function evaluateFounderWeeklyReview( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + report: FounderWeeklyReviewPayload, +): EvaluationResult { + const failures: EvaluationFailure[] = []; + + const evidenceById = new Map( + evidenceSnapshot.items.map((item) => [ + item.sourceId, + item, + ]) + ); + + const claims = new Set(); + + let totalCitations = 0; + let validCitations = 0; + + let totalClaimsWithCitationIds = 0; + let totalClaimsRequiringCitation = 0; + + let sourceTypeViolations = 0; + let totalSourceTypeChecks = 0; + + let emptySectionCorrectness = 1; + + let unsupportedClaims = 0; + let unsupportedClaimChecks = 0; + + let unsupportedShippedClaims = 0; + let shippedClaimChecks = 0; + + let canonicalSchemaValid = true; + + try { + FounderWeeklyReviewV2PayloadSchema.parse(report); + } catch { + canonicalSchemaValid = false; + + failures.push({ + category:"malformed_payload", + explanation: + "Report does not satisfy Founder Weekly Review V2 schema.", + }); + } + + if (!canonicalSchemaValid) { + return { + passed: false, + hasHardFailure: true, + deterministic: { + canonicalSchemaValid, + citationValidity: 0, + citationCoverage: 0, + unsupportedShippedClaimRate: 0, + unsupportedClaimRate: 0, + sourceTypeViolationRate: 0, + evidenceCoverage: 0, + materialityScore: 0, + duplicateClaimRate: 0, + emptySectionCorrectness, + }, + overallScore: 0, + failures, + }; + } + + emptySectionCorrectness = evaluateEmptySections( + evidenceSnapshot, + report, + failures + ); + + const reportClaims = new Set(); + + const citedEvidenceIds = new Set(); + + for (const [sectionName, section] of Object.entries(report.sections)) { + for (const item of getSectionItems(section)) { + const text = typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" + ? normalizeClaim(item.text) + : null; + + if (text) { + reportClaims.add(text); + if (claims.has(text)) { + failures.push({ + category: "duplicate_claim", + section: sectionName, + claim: text, + explanation: "Same factual claim appears multiple times.", + }); + } + claims.add(text); + } + + if ( + typeof item === "object" && + item !== null && + "kind" in item + ) { + const kind = getItemKind(item); + if ( + kind === "observed_fact" || + kind === "contradictory_evidence" || + kind === "recommendation" + ) { + totalClaimsRequiringCitation++; + if ( + "sourceIds" in item && + Array.isArray(item.sourceIds) && + item.sourceIds.length > 0 + ) { + totalClaimsWithCitationIds++; + } + } + } + + if ( + typeof item === "object" && + item !== null && + "sourceIds" in item && + Array.isArray(item.sourceIds) + ) { + + for (const sourceId of item.sourceIds) { + citedEvidenceIds.add(sourceId); + const evidence = evidenceById.get(sourceId); + + if (evidence) { + totalSourceTypeChecks++; + + if ( + sectionName === "whatChanged" && + typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" && + isTemporalChangeClaim(item.text) && + evidence.sourceType === "workspace_document" + ) { + sourceTypeViolations++; + + failures.push({ + category: "invalid_source_type", + section: sectionName, + claim: item.text, + explanation: + "A workspace document can establish current document context, but cannot by itself establish that a change occurred during the reporting period.", + }); + } else if (!isValidSourceForSection(sectionName, evidence.sourceType)) { + sourceTypeViolations++; + + failures.push({ + category: "invalid_source_type", + section: sectionName, + explanation: `Source type "${evidence.sourceType}" is not valid for section "${sectionName}"`, + }); + } + + if ( + sectionName === "whatShipped" && + typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" + ) { + shippedClaimChecks++; + + if ( + evidence.sourceType === "document_change" && + futureReleaseMentioned( + `${item.text} ${evidence.title} ${evidence.excerpt}`, + evidenceSnapshot.reportingPeriod.end + ) + ) { + failures.push({ + category: "future_release_claim", + section: sectionName, + claim: item.text, + explanation: + "A planned or launched release is dated after the reporting period and cannot establish shipped-this-period.", + }); + } + + if ( + ["github_activity", "document_change", "workspace_document"].includes(evidence.sourceType) && + !evidenceIndicatesShipped(evidence) + ) { + unsupportedShippedClaims++; + + failures.push({ + category: "unsupported_shipped_claim", + section: sectionName, + claim: item.text, + explanation: + "Evidence does not contain a shipping signal.", + }); + } + } + } + + totalCitations++; + if (evidenceById.has(sourceId)) { + validCitations++; + } else { + failures.push({ + category: "invalid_citation", + section: sectionName, + explanation: `Unknown sourceId: ${sourceId}`, + }); + } + } + if ( + typeof item === "object" && + item !== null && + "text" in item && + typeof item.text === "string" + ) { + + const kind = getItemKind(item); + + if (kind !== "recommendation" && + kind !== "contradictory_evidence" && + sectionName !== "whatShipped") { + unsupportedClaimChecks++; + + const citedEvidence = item.sourceIds + .map((id:string) => evidenceById.get(id)) + .filter( + (e): e is FounderWeeklyReviewEvidenceSnapshot["items"][number] => + Boolean(e) + ); + + const combinedEvidenceText = citedEvidence + .map(e => `${e.title} ${e.excerpt}`) + .join(" "); + + if ( + citedEvidence.length > 1 && + evidenceConflicts(citedEvidence) + ) { + failures.push({ + category: "conflicting_evidence", + section: sectionName, + claim: item.text, + explanation: + "Claim ignores conflicting cited evidence.", + }); + } else if ( + citedEvidence.length > 0 && + !claimSupportedByEvidence( + item.text, + combinedEvidenceText + ) + ) { + unsupportedClaims++; + + failures.push({ + category: "unsupported_claim", + section: sectionName, + claim: item.text, + explanation: + "Claim is not directly supported by cited evidence.", + }); + } + } + } + } + } + } + + const citationValidity = + totalCitations === 0 + ? 1 + : validCitations / totalCitations; + + const citationCoverage = + totalClaimsRequiringCitation === 0 + ? 1 + : totalClaimsWithCitationIds / totalClaimsRequiringCitation; + + const sourceTypeViolationRate = + totalSourceTypeChecks === 0 + ? 0 + : sourceTypeViolations / totalSourceTypeChecks; + + const hasHardFailure = failures.some((f) => + [ // duplicate_claim not included: more of a quality issue + "malformed_payload", + "invalid_citation", + "invalid_source_type", + "unsupported_shipped_claim", + "future_release_claim", + "invalid_empty_section", + "conflicting_evidence" + ].includes(f.category) + ); + + const coveredEvidence = + evidenceSnapshot.items.filter((evidence) => + citedEvidenceIds.has(evidence.sourceId) + ).length; + + const evidenceCoverage = + evidenceSnapshot.items.length === 0 + ? 1 + : coveredEvidence / evidenceSnapshot.items.length; + + const materialEvidence = evidenceSnapshot.items.filter( + (evidence) => + [ + "github_activity", + "document_change", + "founder_context", + "workspace_document", + ].includes(evidence.sourceType) + ); + + + const materialEvidenceCovered = + materialEvidence.filter((evidence) => + citedEvidenceIds.has(evidence.sourceId) + ).length; + + + const materialityScore = + materialEvidence.length === 0 + ? 1 + : materialEvidenceCovered / materialEvidence.length; + + + const overallScore = hasHardFailure + ? 0 + : citationValidity * 0.25 + + citationCoverage * 0.15 + + evidenceCoverage * 0.25 + + materialityScore * 0.15 + + (1 - sourceTypeViolationRate) * 0.20; + + return { + passed: failures.length === 0, + + hasHardFailure, + + deterministic: { + canonicalSchemaValid, + citationValidity, + citationCoverage, + unsupportedClaimRate: + unsupportedClaimChecks === 0 + ? 0 + : unsupportedClaims / unsupportedClaimChecks, + + unsupportedShippedClaimRate: + shippedClaimChecks === 0 + ? 0 + : unsupportedShippedClaims / shippedClaimChecks, + sourceTypeViolationRate, + evidenceCoverage, + materialityScore, + duplicateClaimRate: + claims.size === 0 + ? 0 + : Math.min( + 1, + failures.filter( + (f) => f.category === "duplicate_claim" + ).length / claims.size + ), + emptySectionCorrectness, + }, + + overallScore, + + failures, + }; +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/evidence-digest.ts b/packages/features/src/founder-weekly-review/evidence-digest.ts new file mode 100644 index 000000000..c615ea834 --- /dev/null +++ b/packages/features/src/founder-weekly-review/evidence-digest.ts @@ -0,0 +1,18 @@ +import { createHash } from "node:crypto"; + +import type { FounderWeeklyReviewEvidenceSnapshot } from "./contracts"; + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value as Record) + .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([key, entry]) => [key, canonicalize(entry)])); + } + return value; +} + +/** Digest of the complete frozen evidence snapshot, including v2 raw/group audit data. */ +export function buildFounderWeeklyReviewEvidenceDigest(snapshot: FounderWeeklyReviewEvidenceSnapshot): string { + return createHash("sha256").update(JSON.stringify(canonicalize(snapshot)), "utf8").digest("hex"); +} diff --git a/packages/features/src/founder-weekly-review/evidence-service.ts b/packages/features/src/founder-weekly-review/evidence-service.ts new file mode 100644 index 000000000..bc4b7f55d --- /dev/null +++ b/packages/features/src/founder-weekly-review/evidence-service.ts @@ -0,0 +1,356 @@ +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + FounderWeeklyReviewEvidenceSnapshotSchema, + type FounderWeeklyReviewEvidenceItem, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewEvidenceWarning, + type DocumentChangeAuditSnapshot, + type ReportingPeriod, +} from "./contracts"; +import { resolveReportingPeriodBounds } from "./reporting-period"; +import { and, asc, eq, gte, lt } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { + document, + documentContextChunks, + documentVersions, +} from "@launchstack/core/db/schema"; +import { + alignVersionChunks, + selectVersionPairsForReportingPeriod, + type DocumentChangePairInput, + type DocumentVersionForComparison, + type VersionChunk, +} from "./document-change"; +import { + materializeDocumentChangesWithAnalyzer, + type DocumentChangeMaterialityAnalyzer, +} from "./document-change-materiality-analyzer"; +import type { DeterministicMaterialChangeDiagnostics } from "./document-change-materiality"; +import { + buildWorkspaceDocumentEvidence, + normalizeFounderContextRetrievalQuery, + type WorkspaceDocumentRetrievalInput, + type WorkspaceDocumentRetrievalResult, +} from "./workspace-document"; + +/** Approved, stored category value. Do not fuzzy-match or seed this category. */ +export const CUSTOMER_FEEDBACK_CATEGORY = "Customer Feedback"; +const MAX_SNAPSHOT_ITEMS = 500; +const MAX_ITEMS_PER_SOURCE = 250; +const MAX_EXCERPT_LENGTH = 4000; +const MAX_WARNINGS = 100; + +function normalizeText(value: string | null | undefined): string | null { + const trimmed = value?.replace(/\s+/g, " ").trim(); + return trimmed ? trimmed : null; +} + +function boundExcerpt(value: string): { excerpt: string; truncated: boolean } { + return value.length <= MAX_EXCERPT_LENGTH + ? { excerpt: value, truncated: false } + : { excerpt: value.slice(0, MAX_EXCERPT_LENGTH), truncated: true }; +} + +function warning(code: string, message: string, sourceType?: FounderWeeklyReviewEvidenceItem["sourceType"]): FounderWeeklyReviewEvidenceWarning { + return { code, message: message.slice(0, 512), ...(sourceType ? { sourceType } : {}) }; +} + +export class FounderWeeklyReviewEvidenceConflictError extends Error { + readonly code = "duplicate_evidence_source_id_conflict"; + constructor(readonly sourceId: string) { + super(`Conflicting evidence items share sourceId: ${sourceId}`); + } +} + +export interface DocumentVersionRow { + documentId: bigint; + documentTitle: string; + documentCategory: string | null; + versionId: number; + versionNumber: number; + uploadedBy: string | null; + changelog: string | null; + createdAt: Date; +} + +export interface CustomerFeedbackChunkRow { + documentId: bigint; + versionId: number; + versionNumber: number; + documentTitle: string; + documentCategory: string; + createdAt: Date; + chunkId: number | null; + chunkContent: string | null; + pageNumber: number | null; +} + +export function mapCustomerFeedbackChunkToEvidenceItem( + row: CustomerFeedbackChunkRow +): FounderWeeklyReviewEvidenceItem | null { + const content = row.chunkContent?.trim(); + if (!content || row.chunkId === null) return null; + const bounded = boundExcerpt(content); + return { + sourceType: "customer_feedback", + sourceId: `customer_feedback:doc:${row.documentId}:version:${row.versionId}:section:${row.chunkId}`, + title: row.documentTitle, + sourceTimestamp: row.createdAt.toISOString(), + excerpt: bounded.excerpt, + workspaceDeepLink: `/employer/documents/viewer?docId=${row.documentId}`, + metadata: { + documentId: row.documentId.toString(), + documentVersionId: row.versionId, + sectionId: row.chunkId, + versionNumber: row.versionNumber, + documentCategory: row.documentCategory, + pageNumber: row.pageNumber, + excerptTruncated: bounded.truncated, + }, + }; +} + +export function mapDocumentVersionToEvidenceItem(row: DocumentVersionRow): FounderWeeklyReviewEvidenceItem { + const changelog = normalizeText(row.changelog); + const bounded = changelog ? boundExcerpt(changelog) : null; + return { + sourceType: "document_change", + sourceId: `document_change:doc:${row.documentId}:version:${row.versionId}`, + title: row.documentTitle, + sourceTimestamp: row.createdAt.toISOString(), + excerpt: bounded?.excerpt ?? `Version ${row.versionNumber} uploaded`, + workspaceDeepLink: `/employer/documents/viewer?docId=${row.documentId}`, + metadata: { + documentId: row.documentId.toString(), documentVersionId: row.versionId, + versionNumber: row.versionNumber, documentCategory: row.documentCategory, + uploadedBy: row.uploadedBy, hasChangelog: changelog !== null, + changelogTruncated: bounded?.truncated ?? false, + }, + }; +} + +export interface FounderWeeklyReviewEvidenceActor { externalUserId: string } +export interface BuildFounderWeeklyReviewEvidenceSnapshotInput { + companyId: bigint; + reportingPeriod: ReportingPeriod; + workspaceTimezone: string; + founderContext?: string; + actor?: FounderWeeklyReviewEvidenceActor; + /** Stable idempotency identity supplied by the request layer. */ + contextEntryId?: string; + requestKey?: string; + capturedAt?: Date; + maxItems?: number; +} + +export interface FounderWeeklyReviewEvidenceSourceResult { + items: FounderWeeklyReviewEvidenceItem[]; + warnings: FounderWeeklyReviewEvidenceWarning[]; + documentChangeAudit?: DocumentChangeAuditSnapshot; +} + +/** Implemented in apps/web so pure feature logic never depends on Drizzle. */ +export interface FounderWeeklyReviewDocumentChangeStore { + listVersionsBeforePeriodEnd(companyId: bigint, endExclusive: Date): Promise; + getDocumentChunksForVersion(input: { companyId: bigint; documentId: bigint; versionId: number }): Promise< + { state: "complete" | "partial" | "missing"; chunks: VersionChunk[]; warnings: string[] } + >; +} + +export type FounderWeeklyReviewDocumentChangeSource = + | { kind: "computed"; store: FounderWeeklyReviewDocumentChangeStore } + | { kind: "legacy" } + | { kind: "unconfigured" }; + +export interface FounderWeeklyReviewWorkspaceDocumentStore { + retrieveRelevantCurrentDocumentChunks(input: WorkspaceDocumentRetrievalInput): Promise; +} + +export class FounderWeeklyReviewEvidenceService { + constructor( + private readonly db: DbClient = getDb(), + private readonly now: () => Date = () => new Date(), + private readonly documentChangeSource: FounderWeeklyReviewDocumentChangeSource = { kind: "unconfigured" }, + private readonly workspaceDocumentStore?: FounderWeeklyReviewWorkspaceDocumentStore, + private readonly documentChangeMaterialityAnalyzer?: DocumentChangeMaterialityAnalyzer, + private readonly onDocumentChangeDiagnostics?: (diagnostics: DeterministicMaterialChangeDiagnostics) => void, + ) {} + + async collectDocumentChangeEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + return (await this.collectDocumentChangeEvidenceResult(companyId, startInclusive, endExclusive)).items; + } + + private async collectDocumentChangeEvidenceResult(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + if (this.documentChangeSource.kind === "computed") { + const store = this.documentChangeSource.store; + const versions = await store.listVersionsBeforePeriodEnd(companyId, endExclusive); + const pairs = selectVersionPairsForReportingPeriod(versions, startInclusive, endExclusive); + const pairInputs: DocumentChangePairInput[] = []; + const warnings: FounderWeeklyReviewEvidenceWarning[] = []; + for (const pair of pairs) { + const [previous, current] = await Promise.all([ + store.getDocumentChunksForVersion({ companyId, documentId: pair.documentId, versionId: pair.previousVersionId }), + store.getDocumentChunksForVersion({ companyId, documentId: pair.documentId, versionId: pair.currentVersionId }), + ]); + if (previous.state === "missing" || current.state === "missing") { + warnings.push(warning("document_change_chunks_missing", "A version pair could not be compared because processed historical chunks are missing.", "document_change")); + continue; + } + if (previous.state === "partial" || current.state === "partial") { + warnings.push(warning("document_change_chunks_partial", "A version pair was compared with partial chunk provenance.", "document_change")); + } + pairInputs.push({ pair, alignments: alignVersionChunks(previous.chunks, current.chunks) }); + } + const materialized = await materializeDocumentChangesWithAnalyzer( + pairInputs, + this.documentChangeMaterialityAnalyzer, + ); + this.onDocumentChangeDiagnostics?.(materialized.diagnostics); + const items = [...materialized.items]; + warnings.push(...materialized.warnings.map((item) => warning(item.code, item.message, "document_change"))); + const pairedCurrentVersionIds = new Set(pairs.map((pair) => pair.currentVersionId)); + const inPeriodWithNoPair = versions.some((version) => + version.createdAt >= startInclusive + && version.createdAt < endExclusive + && !pairedCurrentVersionIds.has(version.versionId) + ); + if (inPeriodWithNoPair) warnings.push(warning("document_change_baseline_missing", "An in-period document version has no predecessor, so no content diff was generated.", "document_change")); + const ordered = orderEvidenceItems(dedupeEvidenceItems(items)); + if (ordered.length > MAX_ITEMS_PER_SOURCE) warnings.push(warning("document_change_truncated", "Document change evidence was truncated to the per-source limit.", "document_change")); + return { items: ordered.slice(0, MAX_ITEMS_PER_SOURCE), warnings, documentChangeAudit: materialized.audit }; + } + if (this.documentChangeSource.kind !== "legacy") { + return { items: [], warnings: [warning("document_change_source_unconfigured", "Document-change collection requires the explicit computed-diff source.", "document_change")] }; + } + const rows = await this.db.select({ + documentId: documentVersions.documentId, documentTitle: document.title, + documentCategory: document.category, versionId: documentVersions.id, + versionNumber: documentVersions.versionNumber, uploadedBy: documentVersions.uploadedBy, + changelog: documentVersions.changelog, createdAt: documentVersions.createdAt, + }).from(documentVersions).innerJoin(document, eq(document.id, documentVersions.documentId)) + .where(and(gte(documentVersions.createdAt, startInclusive), lt(documentVersions.createdAt, endExclusive), eq(document.companyId, companyId))) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id)).limit(MAX_ITEMS_PER_SOURCE + 1); + return { + items: rows.slice(0, MAX_ITEMS_PER_SOURCE).map(mapDocumentVersionToEvidenceItem), + warnings: rows.length > MAX_ITEMS_PER_SOURCE + ? [warning("document_change_truncated", "Document change evidence was truncated to the per-source limit.", "document_change")] + : [], + }; + } + + async collectCustomerFeedbackEvidence(companyId: bigint, startInclusive: Date, endExclusive: Date): Promise { + const rows = await this.db.select({ + documentId: documentVersions.documentId, versionId: documentVersions.id, + versionNumber: documentVersions.versionNumber, documentTitle: document.title, + documentCategory: document.category, createdAt: documentVersions.createdAt, + chunkId: documentContextChunks.id, + chunkContent: documentContextChunks.content, + pageNumber: documentContextChunks.pageNumber, + }).from(documentVersions).innerJoin(document, eq(document.id, documentVersions.documentId)) + .leftJoin(documentContextChunks, and(eq(documentContextChunks.documentId, documentVersions.documentId), eq(documentContextChunks.versionId, documentVersions.id))) + .where(and(gte(documentVersions.createdAt, startInclusive), lt(documentVersions.createdAt, endExclusive), eq(document.companyId, companyId), eq(document.category, CUSTOMER_FEEDBACK_CATEGORY))) + .orderBy(asc(documentVersions.createdAt), asc(documentVersions.id), asc(documentContextChunks.id)).limit(MAX_ITEMS_PER_SOURCE + 1); + if (rows.length === 0) return { items: [], warnings: [warning("customer_feedback_unavailable", "No Customer Feedback documents were available for this reporting period.", "customer_feedback")] }; + const mapped = rows.map(mapCustomerFeedbackChunkToEvidenceItem); + const items = mapped.filter((item): item is FounderWeeklyReviewEvidenceItem => item !== null); + const warnings: FounderWeeklyReviewEvidenceWarning[] = []; + if (items.length === 0) warnings.push(warning("customer_feedback_missing_sections", "Customer Feedback document versions had no processed, citeable sections.", "customer_feedback")); + else if (items.length < rows.length) warnings.push(warning("customer_feedback_missing_sections", "Some Customer Feedback sections had no citeable content.", "customer_feedback")); + if (rows.length > MAX_ITEMS_PER_SOURCE) warnings.push(warning("customer_feedback_truncated", "Customer Feedback evidence was truncated to the per-source limit.", "customer_feedback")); + return { items: items.slice(0, MAX_ITEMS_PER_SOURCE), warnings }; + } + + collectFounderContextEvidence(input: Pick): FounderWeeklyReviewEvidenceSourceResult { + const context = normalizeText(input.founderContext); + if (!context) return { items: [], warnings: [] }; + const entryId = input.contextEntryId ?? input.requestKey; + if (!entryId) throw new Error("Founder context requires a stable contextEntryId or requestKey"); + if (entryId.length > 230) throw new Error("Founder context entry identity is too long"); + if (!input.actor) throw new Error("Founder context requires an authenticated actor"); + const bounded = boundExcerpt(context); + return { items: [{ sourceType: "founder_context", sourceId: `founder_context:entry:${entryId}`, + title: "Founder context", excerpt: bounded.excerpt, + metadata: { enteredBy: input.actor.externalUserId, provenance: "request_time_founder_input", excerptTruncated: bounded.truncated } }], warnings: [] }; + } + + async collectWorkspaceDocumentEvidence(companyId: bigint, founderContext: string | undefined): Promise { + const query = normalizeFounderContextRetrievalQuery(founderContext); + if (!query || !this.workspaceDocumentStore) return { items: [], warnings: [] }; + const result = await this.workspaceDocumentStore.retrieveRelevantCurrentDocumentChunks({ companyId, founderContext: query }); + if (result.state === "success") return { items: buildWorkspaceDocumentEvidence(result.hits), warnings: [] }; + if (result.state === "empty") return { items: [], warnings: [] }; + return { items: [], warnings: result.warnings.slice(0, MAX_WARNINGS).map((code) => warning(code, "Founder Context workspace retrieval was unavailable.", "workspace_document")) }; + } + + async collectFounderWeeklyReviewEvidence(input: BuildFounderWeeklyReviewEvidenceSnapshotInput): Promise { + return this.buildEvidenceSnapshot(input); + } + + async buildEvidenceSnapshot(input: BuildFounderWeeklyReviewEvidenceSnapshotInput): Promise { + const { startInclusive, endExclusive } = resolveReportingPeriodBounds(input.reportingPeriod, input.workspaceTimezone); + const documentChanges = await this.collectDocumentChangeEvidenceResult(input.companyId, startInclusive, endExclusive); + const feedback = await this.collectCustomerFeedbackEvidence(input.companyId, startInclusive, endExclusive); + const founderContext = this.collectFounderContextEvidence(input); + const workspaceDocuments = await this.collectWorkspaceDocumentEvidence(input.companyId, input.founderContext); + const sourceResults: FounderWeeklyReviewEvidenceSourceResult[] = [documentChanges, feedback, founderContext, workspaceDocuments]; + const items = orderEvidenceItems(dedupeEvidenceItems(sourceResults.flatMap((result) => result.items))); + const requestedMax = input.maxItems ?? MAX_SNAPSHOT_ITEMS; + const maxItems = Math.max(0, Math.min(MAX_SNAPSHOT_ITEMS, requestedMax)); + const warnings = dedupeWarnings(sourceResults.flatMap((result) => result.warnings)); + if (items.length > maxItems) warnings.push(warning("evidence_snapshot_truncated", "Evidence snapshot was truncated to its configured maximum.")); + const selectedItems = items.slice(0, maxItems); + const selectedSourceIds = new Set(selectedItems.map(item => item.sourceId)); + const collectedAudit = documentChanges.documentChangeAudit ?? { schemaVersion: "document-change-audit/v1" as const, rawChanges: [], groups: [] }; + const documentChangeAudit: DocumentChangeAuditSnapshot = { + ...collectedAudit, + groups: collectedAudit.groups.map(group => ({ + ...group, + evidenceSourceId: group.evidenceSourceId && selectedSourceIds.has(group.evidenceSourceId) ? group.evidenceSourceId : null, + })), + }; + return FounderWeeklyReviewEvidenceSnapshotSchema.parse({ schemaVersion: FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + capturedAt: (input.capturedAt ?? this.now()).toISOString(), reportingPeriod: input.reportingPeriod, + workspaceTimezone: input.workspaceTimezone, items: selectedItems, sourceWarnings: dedupeWarnings(warnings).slice(0, MAX_WARNINGS), + documentChangeAudit }); + } +} + +function canonicalItem(item: FounderWeeklyReviewEvidenceItem): string { return JSON.stringify(item); } +export function dedupeEvidenceItems(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceItem[] { + const bySourceId = new Map(); + for (const item of items) { + const existing = bySourceId.get(item.sourceId); + if (!existing) { bySourceId.set(item.sourceId, item); continue; } + if (canonicalItem(existing) !== canonicalItem(item)) throw new FounderWeeklyReviewEvidenceConflictError(item.sourceId); + } + return [...bySourceId.values()]; +} + +function compareOrdinal(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } +function evidenceMetadataText(item: FounderWeeklyReviewEvidenceItem, key: string): string { + const value = item.metadata[key]; + return typeof value === "string" || typeof value === "number" ? String(value) : ""; +} +function compareDocumentChangeEvidence(a: FounderWeeklyReviewEvidenceItem, b: FounderWeeklyReviewEvidenceItem): number { + if (a.sourceType !== "document_change" || b.sourceType !== "document_change") return 0; + const aDocument = evidenceMetadataText(a, "documentId"); const bDocument = evidenceMetadataText(b, "documentId"); + const document = /^\d+$/.test(aDocument) && /^\d+$/.test(bDocument) + ? (BigInt(aDocument) < BigInt(bDocument) ? -1 : BigInt(aDocument) > BigInt(bDocument) ? 1 : 0) + : compareOrdinal(aDocument, bDocument); + if (document !== 0) return document; + for (const key of ["previousVersionId", "currentVersionId", "structureOrdering", "pageNumber", "lineStart"] as const) { + const left = Number(evidenceMetadataText(a, key)); const right = Number(evidenceMetadataText(b, key)); + if (Number.isFinite(left) && Number.isFinite(right) && left !== right) return left - right; + } + return compareOrdinal(evidenceMetadataText(a, "structurePath"), evidenceMetadataText(b, "structurePath")); +} +export function orderEvidenceItems(items: FounderWeeklyReviewEvidenceItem[]): FounderWeeklyReviewEvidenceItem[] { + return [...items].sort((a, b) => compareOrdinal(a.sourceTimestamp ?? "", b.sourceTimestamp ?? "") || compareOrdinal(a.sourceType, b.sourceType) || compareDocumentChangeEvidence(a, b) || compareOrdinal(a.sourceId, b.sourceId)); +} + +export function dedupeWarnings(warnings: FounderWeeklyReviewEvidenceWarning[]): FounderWeeklyReviewEvidenceWarning[] { + const unique = new Map(); + for (const item of warnings) unique.set(`${item.code}\u0000${item.sourceType ?? ""}\u0000${item.message}`, item); + return [...unique.values()].sort((a, b) => compareOrdinal(a.code, b.code) || compareOrdinal(a.sourceType ?? "", b.sourceType ?? "") || compareOrdinal(a.message, b.message)); +} diff --git a/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts b/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts new file mode 100644 index 000000000..bddf5d61b --- /dev/null +++ b/packages/features/src/founder-weekly-review/generation-evidence-envelope.ts @@ -0,0 +1,375 @@ +import type { + FounderWeeklyReviewEvidenceItem, + FounderWeeklyReviewEvidenceSnapshot, +} from "./contracts"; + +export const FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION = + "founder-weekly-review-evidence-envelope/v1" as const; + +export const FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET = Object.freeze({ + totalSerializedCharacters: 72_000, + founderContextReservedCharacters: 4_000, + workspaceDocumentReservedCharacters: 24_000, + customerFeedbackReservedCharacters: 24_000, + documentChangeSerializedCharacters: 14_000, + documentChangeItems: 24, + documentChangeItemsPerDocument: 8, + estimatedCharactersPerToken: 4, +}); + +const SOURCE_TYPES = [ + "workspace_document", + "document_change", + "customer_feedback", + "github_activity", + "manual_note", + "founder_context", + "other", +] as const satisfies readonly FounderWeeklyReviewEvidenceItem["sourceType"][]; + +type SourceType = FounderWeeklyReviewEvidenceItem["sourceType"]; +type MetadataValue = FounderWeeklyReviewEvidenceItem["metadata"][string]; + +export type FounderWeeklyReviewPromptEvidenceItem = Pick< + FounderWeeklyReviewEvidenceItem, + "sourceId" | "sourceType" | "title" | "excerpt" +> & { + sourceTimestamp: string | null; + metadata: Record; +}; + +export type GenerationEvidenceEnvelopeDiagnostics = { + originalItemCount: number; + selectedItemCount: number; + excludedItemCount: number; + selectedBySourceType: Record; + excludedBySourceType: Record; + serializedCharacterCount: number; + estimatedTokenCount: number; + truncated: boolean; +}; + +export type GenerationEvidenceEnvelope = { + version: typeof FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION; + items: FounderWeeklyReviewPromptEvidenceItem[]; + diagnostics: GenerationEvidenceEnvelopeDiagnostics; +}; + +export class FounderWeeklyReviewGenerationEvidenceBudgetError extends Error { + readonly code = "generation_evidence_budget_exceeded"; + + constructor( + message = "Founder Weekly Review generation evidence exceeded its deterministic context budget." + ) { + super(message); + this.name = "FounderWeeklyReviewGenerationEvidenceBudgetError"; + } +} + +const PROMPT_METADATA_ALLOWLIST: Record = { + document_change: [ + "changeType", + "previousVersionNumber", + "currentVersionNumber", + "structurePath", + "alignmentMethod", + "userChangelog", + "category", + "materialityMethod", + "materialityConfidence", + "materialityVersion", + "structureTitle", + "rawChangeCount", + ], + customer_feedback: ["versionNumber", "documentCategory", "pageNumber"], + workspace_document: ["retrievalReason", "similarityScore"], + founder_context: ["provenance"], + github_activity: [], + manual_note: [], + other: [], +}; + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} +function metadataText(item: FounderWeeklyReviewEvidenceItem, key: string): string { + const value = item.metadata[key]; + return typeof value === "string" || typeof value === "number" ? String(value) : ""; +} +function similarityScore(item: FounderWeeklyReviewEvidenceItem): number { + const value = item.metadata.similarityScore; + return typeof value === "number" && Number.isFinite(value) ? value : Number.NEGATIVE_INFINITY; +} +function compareIntegerMetadata(a: FounderWeeklyReviewEvidenceItem, b: FounderWeeklyReviewEvidenceItem, key: string): number { + const left = metadataText(a, key); const right = metadataText(b, key); + if (/^-?\d+$/.test(left) && /^-?\d+$/.test(right)) { + const leftInteger = BigInt(left); const rightInteger = BigInt(right); + return leftInteger < rightInteger ? -1 : leftInteger > rightInteger ? 1 : 0; + } + return compareOrdinal(left, right); +} + +/** Stable source-specific ordering that does not depend on snapshot input order. */ +function compareEvidenceItems( + a: FounderWeeklyReviewEvidenceItem, + b: FounderWeeklyReviewEvidenceItem +): number { + const timestamp = compareOrdinal(a.sourceTimestamp ?? "", b.sourceTimestamp ?? ""); + if (timestamp !== 0) return timestamp; + const sourceType = compareOrdinal(a.sourceType, b.sourceType); + if (sourceType !== 0) return sourceType; + if (a.sourceType === "workspace_document" && b.sourceType === "workspace_document") { + const similarity = similarityScore(b) - similarityScore(a); + if (Number.isFinite(similarity) && similarity !== 0) return similarity; + } + if (a.sourceType === "document_change" && b.sourceType === "document_change") { + for (const key of ["documentId", "previousVersionId", "currentVersionId", "structureOrdering", "pageNumber", "lineStart"] as const) { + const compared = compareIntegerMetadata(a, b, key); + if (compared !== 0) return compared; + } + const structurePath = compareOrdinal(metadataText(a, "structurePath"), metadataText(b, "structurePath")); + if (structurePath !== 0) return structurePath; + } + return compareOrdinal(a.sourceId, b.sourceId); +} + +function allowlistedMetadata(item: FounderWeeklyReviewEvidenceItem): Record { + const metadata: Record = {}; + for (const key of PROMPT_METADATA_ALLOWLIST[item.sourceType]) { + const value = item.metadata[key]; + if (value !== undefined) metadata[key] = value; + } + return metadata; +} + +export function buildFounderWeeklyReviewPromptEvidenceItem( + item: FounderWeeklyReviewEvidenceItem +): FounderWeeklyReviewPromptEvidenceItem { + return { + sourceId: item.sourceId, + sourceType: item.sourceType, + title: item.title, + sourceTimestamp: item.sourceTimestamp ?? null, + excerpt: item.excerpt, + metadata: allowlistedMetadata(item), + }; +} + +function serializedItemsLength(items: readonly FounderWeeklyReviewPromptEvidenceItem[]): number { + return JSON.stringify(items).length; +} +function canAppendWithin( + selected: readonly FounderWeeklyReviewPromptEvidenceItem[], + candidate: FounderWeeklyReviewPromptEvidenceItem, + limit: number +): boolean { + return serializedItemsLength([...selected, candidate]) <= limit; +} +function sourceCounts(items: readonly { sourceType: SourceType }[]): Record { + const counts = Object.fromEntries(SOURCE_TYPES.map(sourceType => [sourceType, 0])) as Record< + SourceType, + number + >; + for (const item of items) counts[item.sourceType]++; + return counts; +} +function subtractCounts( + total: Record, + selected: Record +): Record { + return Object.fromEntries( + SOURCE_TYPES.map(sourceType => [sourceType, total[sourceType] - selected[sourceType]]) + ) as Record; +} + +function dedupeAndOrder( + items: readonly FounderWeeklyReviewEvidenceItem[] +): FounderWeeklyReviewEvidenceItem[] { + const ordered = [...items].sort( + (a, b) => + compareEvidenceItems(a, b) || + compareOrdinal( + JSON.stringify(buildFounderWeeklyReviewPromptEvidenceItem(a)), + JSON.stringify(buildFounderWeeklyReviewPromptEvidenceItem(b)) + ) + ); + const bySourceId = new Map(); + for (const item of ordered) + if (!bySourceId.has(item.sourceId)) bySourceId.set(item.sourceId, item); + return [...bySourceId.values()]; +} + +function selectReserved( + candidates: readonly FounderWeeklyReviewEvidenceItem[], + limit: number +): { selected: FounderWeeklyReviewEvidenceItem[]; excluded: FounderWeeklyReviewEvidenceItem[] } { + const selected: FounderWeeklyReviewEvidenceItem[] = []; + const projected: FounderWeeklyReviewPromptEvidenceItem[] = []; + const excluded: FounderWeeklyReviewEvidenceItem[] = []; + for (const item of candidates) { + const promptItem = buildFounderWeeklyReviewPromptEvidenceItem(item); + if (canAppendWithin(projected, promptItem, limit)) { + selected.push(item); + projected.push(promptItem); + } else excluded.push(item); + } + return { selected, excluded }; +} + +function documentKey(item: FounderWeeklyReviewEvidenceItem): string { + return ( + metadataText(item, "documentId") || + /^document_change:doc:([^:]+)/.exec(item.sourceId)?.[1] || + item.sourceId + ); +} + +function selectDocumentChanges( + candidates: readonly FounderWeeklyReviewEvidenceItem[] +): FounderWeeklyReviewEvidenceItem[] { + const byDocument = new Map(); + for (const item of candidates) { + const key = documentKey(item); + const group = byDocument.get(key) ?? []; + group.push(item); + byDocument.set(key, group); + } + const documents = [...byDocument.entries()].sort(([a], [b]) => compareOrdinal(a, b)); + const selected: FounderWeeklyReviewEvidenceItem[] = []; + const projected: FounderWeeklyReviewPromptEvidenceItem[] = []; + for ( + let round = 0; + round < FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItemsPerDocument; + round++ + ) { + for (const [, items] of documents) { + if ( + selected.length >= + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItems + ) + return selected; + const item = items[round]; + if (!item) continue; + const promptItem = buildFounderWeeklyReviewPromptEvidenceItem(item); + if ( + !canAppendWithin( + projected, + promptItem, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters + ) + ) + continue; + selected.push(item); + projected.push(promptItem); + } + } + return selected; +} + +/** Builds a bounded prompt projection without mutating the immutable snapshot. */ +export function buildGenerationEvidenceEnvelope( + snapshot: FounderWeeklyReviewEvidenceSnapshot +): GenerationEvidenceEnvelope { + const ordered = dedupeAndOrder(snapshot.items); + const founder = ordered.filter(item => item.sourceType === "founder_context"); + const workspace = ordered.filter(item => item.sourceType === "workspace_document"); + const feedback = ordered.filter(item => item.sourceType === "customer_feedback"); + const documentChanges = ordered.filter(item => item.sourceType === "document_change"); + const selectedOriginal: FounderWeeklyReviewEvidenceItem[] = founder.slice(0, 1); + const reservedWorkspace = selectReserved( + workspace, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.workspaceDocumentReservedCharacters + ); + const reservedFeedback = selectReserved( + feedback, + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.customerFeedbackReservedCharacters + ); + selectedOriginal.push( + ...reservedWorkspace.selected, + ...reservedFeedback.selected, + ...selectDocumentChanges(documentChanges) + ); + const selectedIds = new Set(selectedOriginal.map(item => item.sourceId)); + const sharedCandidates = ordered.filter( + item => + item.sourceType !== "document_change" && + item.sourceType !== "founder_context" && + !selectedIds.has(item.sourceId) + ); + for (const item of sharedCandidates) { + const candidateItems = [...selectedOriginal, item] + .sort(compareEvidenceItems) + .map(buildFounderWeeklyReviewPromptEvidenceItem); + if ( + serializedItemsLength(candidateItems) > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.totalSerializedCharacters + ) + continue; + selectedOriginal.push(item); + selectedIds.add(item.sourceId); + } + const items = selectedOriginal + .sort(compareEvidenceItems) + .map(buildFounderWeeklyReviewPromptEvidenceItem); + const serializedCharacterCount = serializedItemsLength(items); + const selectedBySourceType = sourceCounts(items); + const diagnostics: GenerationEvidenceEnvelopeDiagnostics = { + originalItemCount: snapshot.items.length, + selectedItemCount: items.length, + excludedItemCount: snapshot.items.length - items.length, + selectedBySourceType, + excludedBySourceType: subtractCounts(sourceCounts(snapshot.items), selectedBySourceType), + serializedCharacterCount, + estimatedTokenCount: Math.ceil( + serializedCharacterCount / + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.estimatedCharactersPerToken + ), + truncated: items.length !== snapshot.items.length, + }; + const envelope: GenerationEvidenceEnvelope = { + version: FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION, + items, + diagnostics, + }; + assertGenerationEvidenceEnvelopeWithinBudget(envelope); + return envelope; +} + +/** Local invariant check used immediately before prompt serialization/provider invocation. */ +export function assertGenerationEvidenceEnvelopeWithinBudget( + envelope: GenerationEvidenceEnvelope +): void { + const actualCharacters = serializedItemsLength(envelope.items); + const documentChanges = envelope.items.filter(item => item.sourceType === "document_change"); + const founderContexts = envelope.items.filter(item => item.sourceType === "founder_context"); + const perDocument = new Map(); + for (const item of documentChanges) { + const key = + typeof item.metadata.documentId === "string" || + typeof item.metadata.documentId === "number" + ? String(item.metadata.documentId) + : (/^document_change:doc:([^:]+)/.exec(item.sourceId)?.[1] ?? item.sourceId); + perDocument.set(key, (perDocument.get(key) ?? 0) + 1); + } + const invalid = + envelope.version !== FOUNDER_WEEKLY_REVIEW_EVIDENCE_ENVELOPE_VERSION || + actualCharacters !== envelope.diagnostics.serializedCharacterCount || + actualCharacters > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.totalSerializedCharacters || + founderContexts.length > 1 || + founderContexts.some( + item => + item.excerpt.length > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.founderContextReservedCharacters + ) || + documentChanges.length > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItems || + serializedItemsLength(documentChanges) > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeSerializedCharacters || + [...perDocument.values()].some( + count => + count > + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET.documentChangeItemsPerDocument + ); + if (invalid) throw new FounderWeeklyReviewGenerationEvidenceBudgetError(); +} diff --git a/packages/features/src/founder-weekly-review/generation-validation.ts b/packages/features/src/founder-weekly-review/generation-validation.ts new file mode 100644 index 000000000..9533ce61c --- /dev/null +++ b/packages/features/src/founder-weekly-review/generation-validation.ts @@ -0,0 +1,197 @@ +import type { + FounderWeeklyReviewEvidenceSnapshot, + FounderWeeklyReviewV2Payload, +} from "./contracts"; + +export class FounderWeeklyReviewGenerationValidationError extends Error { + constructor( + message: string, + public readonly details: ReadonlyArray = [] + ) { + super(message); + this.name = "FounderWeeklyReviewGenerationValidationError"; + } +} + +export interface FounderWeeklyReviewValidationDetail { + code: string; + section?: string; + itemIndex?: number; + sourceId?: string; +} + +/** Sources in the current model that can establish a reporting-period event. */ +const TEMPORAL_EVIDENCE_SOURCE_TYPES = new Set(["document_change"]); +const isTemporalEvidenceSource = (source: { sourceType?: string } | undefined) => + source !== undefined && TEMPORAL_EVIDENCE_SOURCE_TYPES.has(source.sourceType ?? ""); + +type EvidenceStatus = + | "planned" + | "changed" + | "shipped"; + +function getEvidenceStatus( + source: { metadata?: Record } | undefined +): EvidenceStatus | null { + const status = source?.metadata?.evidenceStatus; + + if ( + status === "planned" || + status === "changed" || + status === "shipped" + ) { + return status; + } + + return null; +} + +function assertClaimDoesNotOverstateEvidence( + text: string, + sources: Array<{ metadata?: Record } | undefined>, + section: string, + itemIndex: number, + sourceIds: readonly string[] +): void { + const claim = text.toLowerCase(); + + for (const source of sources) { + const status = getEvidenceStatus(source); + + if (!status) continue; + + if ( + status === "planned" && + /\b(shipped|released|launched|completed|delivered)\b/i.test(claim) + ) { + throw new FounderWeeklyReviewGenerationValidationError( + "Claim describes planned work as completed.", + [{ + code: "claim_overstates_planned_evidence", + section, + itemIndex, + sourceId: sourceIds[0], + }] + ); + } + } +} + +export function assertUniqueSnapshotSourceIds( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): void { + const sourceIds = evidenceSnapshot.items.map((item) => item.sourceId); + if (new Set(sourceIds).size !== sourceIds.length) { + throw new FounderWeeklyReviewGenerationValidationError( + "Evidence snapshot contains duplicate source IDs." + ); + } +} + +export function validateFounderWeeklyReviewV2Citations( + payload: FounderWeeklyReviewV2Payload, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): FounderWeeklyReviewV2Payload { + const evidenceBySourceId = new Map( + evidenceSnapshot.items.map((item) => [item.sourceId, item]) + ); + const factualSections = [ + "whatChanged", + "whatShipped", + "whatCustomersSaid", + "currentBlockers", + ] as const; + + for (const sectionName of factualSections) { + const section = payload.sections[sectionName]; + if (section.state === "no_evidence") continue; + for (const [itemIndex, item] of section.items.entries()) { + assertCitations(item.sourceIds, evidenceBySourceId, item.kind); + + const citedSources = item.sourceIds.map((sourceId) => evidenceBySourceId.get(sourceId)); + + assertClaimDoesNotOverstateEvidence( + item.text, + citedSources, + sectionName, + itemIndex, + item.sourceIds + ); + + if (item.kind === "contradictory_evidence" && item.sourceIds.length < 2) { + throw new FounderWeeklyReviewGenerationValidationError( + `${sectionName} contradictory_evidence must cite at least two sources.` + ); + } + if (sectionName === "whatCustomersSaid") { + for (const sourceId of item.sourceIds) { + const source = evidenceBySourceId.get(sourceId); + if (source?.sourceType === "founder_context") { + throw new FounderWeeklyReviewGenerationValidationError( + "founder_context must never be presented as customer feedback.", + [{ code: "founder_context_used_as_customer_feedback", section: sectionName, itemIndex, sourceId }] + ); + } + if (source?.sourceType !== "customer_feedback") { + throw new FounderWeeklyReviewGenerationValidationError( + `whatCustomersSaid may cite only customer_feedback evidence; received "${sourceId}".`, + [{ code: "customer_signals_requires_customer_feedback", section: sectionName, itemIndex, sourceId }] + ); + } + } + } + if (sectionName === "whatChanged" || sectionName === "whatShipped") { + const sources = item.sourceIds.map((sourceId) => evidenceBySourceId.get(sourceId) as { sourceType?: string } | undefined); + const workspaceOnly = sources.length > 0 && sources.every((source) => source?.sourceType === "workspace_document"); + const workspaceWithoutTemporalEvidence = sources.some((source) => source?.sourceType === "workspace_document") + && !sources.some(isTemporalEvidenceSource); + if (workspaceOnly || workspaceWithoutTemporalEvidence) { + throw new FounderWeeklyReviewGenerationValidationError( + `${sectionName} requires temporal evidence in addition to workspace_document context.`, + item.sourceIds.map((sourceId) => ({ + code: workspaceOnly ? "workspace_document_only_temporal_claim" : `${sectionName === "whatChanged" ? "what_changed" : "what_shipped"}_requires_temporal_evidence`, section: sectionName, itemIndex, sourceId, + })) + ); + } + } + } + } + + const priorities = payload.sections.nextPriorities; + if (priorities.state === "evidence") { + for (const item of priorities.items) { + assertCitations(item.sourceIds, evidenceBySourceId, item.kind); + if (item.kind !== "recommendation") { + throw new FounderWeeklyReviewGenerationValidationError( + "nextPriorities may contain recommendations only." + ); + } + } + } + + return payload; +} + +function assertCitations( + sourceIds: readonly string[], + evidenceBySourceId: ReadonlyMap, + itemKind: string +): void { + if (sourceIds.length === 0) { + throw new FounderWeeklyReviewGenerationValidationError( + `${itemKind} must cite at least one evidence source.` + ); + } + if (new Set(sourceIds).size !== sourceIds.length) { + throw new FounderWeeklyReviewGenerationValidationError( + `${itemKind} contains duplicate source citations.` + ); + } + for (const sourceId of sourceIds) { + if (!evidenceBySourceId.has(sourceId)) { + throw new FounderWeeklyReviewGenerationValidationError( + `${itemKind} cites source ID "${sourceId}" that is absent from the evidence snapshot.` + ); + } + } +} diff --git a/packages/features/src/founder-weekly-review/generator.ts b/packages/features/src/founder-weekly-review/generator.ts new file mode 100644 index 000000000..006ccb201 --- /dev/null +++ b/packages/features/src/founder-weekly-review/generator.ts @@ -0,0 +1,205 @@ +import { createHash } from "node:crypto"; +import type { ZodType } from "zod"; + +import { + FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + FounderWeeklyReviewV2PayloadSchema, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewModelMetadata, + type FounderWeeklyReviewV2Payload, +} from "./contracts"; +import { + assertUniqueSnapshotSourceIds, + FounderWeeklyReviewGenerationValidationError, + validateFounderWeeklyReviewV2Citations, +} from "./generation-validation"; +import { + buildFounderWeeklyReviewPrompt, + FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, +} from "./prompts"; +import { + buildGenerationEvidenceEnvelope, + type FounderWeeklyReviewPromptEvidenceItem, + type GenerationEvidenceEnvelopeDiagnostics, +} from "./generation-evidence-envelope"; +import { buildFounderWeeklyReviewEvidenceDigest } from "./evidence-digest"; + +export interface FounderWeeklyReviewResolvedGenerationMetadata { + provider: string; + model: string; + capability: string; + temperature?: number; + finishReason?: string; + usage?: Record; + providerRequestId?: string; +} + +export type FounderWeeklyReviewStructuredGenerator = (input: { + system?: string; + prompt: string; + schema: TSchema; + schemaName?: string; + generationPhase?: "initial" | "semantic-repair"; +}) => Promise<{ + object: ReturnType; + metadata: FounderWeeklyReviewResolvedGenerationMetadata; +}>; + +export interface GenerateFounderWeeklyReviewInput { + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + generate: FounderWeeklyReviewStructuredGenerator; +} + +export interface GenerateFounderWeeklyReviewResult { + reviewPayload: FounderWeeklyReviewV2Payload; + modelMetadata: FounderWeeklyReviewModelMetadata; +} + +export async function generateFounderWeeklyReview( + input: GenerateFounderWeeklyReviewInput +): Promise { + const { evidenceSnapshot, generate } = input; + assertUniqueSnapshotSourceIds(evidenceSnapshot); + + const evidenceEnvelope = buildGenerationEvidenceEnvelope(evidenceSnapshot); + const prompt = buildFounderWeeklyReviewPrompt(evidenceSnapshot, evidenceEnvelope); + const promptHash = createHash("sha256") + .update(FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT) + .update(prompt) + .digest("hex"); + + if (evidenceEnvelope.items.length === 0) { + return { + reviewPayload: buildEmptyReview(), + modelMetadata: buildMetadata( + { provider: "skipped", model: "none", capability: "founderWeeklyReview", temperature: 0 }, + promptHash, + true, + evidenceEnvelope.diagnostics, + evidenceSnapshot, + ), + }; + } + + const initial = await generate({ + system: FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, + prompt, + schema: FounderWeeklyReviewV2PayloadSchema, + schemaName: "founder_weekly_review_v2", + generationPhase: "initial", + }); + let result = initial; + let reviewPayload: FounderWeeklyReviewV2Payload; + try { + reviewPayload = validateFounderWeeklyReviewV2Citations( + FounderWeeklyReviewV2PayloadSchema.parse(initial.object), + evidenceSnapshot + ); + logGenerationValidation("initial", initial.metadata, "passed"); + } catch (error) { + if (!(error instanceof FounderWeeklyReviewGenerationValidationError)) throw error; + logGenerationValidation("initial", initial.metadata, "failed"); + const repaired = await generate({ + system: FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT, + prompt: buildSemanticRepairPrompt(initial.object, evidenceEnvelope.items, error), + schema: FounderWeeklyReviewV2PayloadSchema, + schemaName: "founder_weekly_review_v2", + generationPhase: "semantic-repair", + }); + result = repaired; + try { + reviewPayload = validateFounderWeeklyReviewV2Citations( + FounderWeeklyReviewV2PayloadSchema.parse(repaired.object), + evidenceSnapshot + ); + logGenerationValidation("semantic-repair", repaired.metadata, "passed"); + } catch (repairError) { + logGenerationValidation("semantic-repair", repaired.metadata, "failed"); + throw repairError; + } + } + + return { reviewPayload, modelMetadata: buildMetadata(result.metadata, promptHash, false, evidenceEnvelope.diagnostics, evidenceSnapshot) }; +} + +function buildSemanticRepairPrompt( + candidate: FounderWeeklyReviewV2Payload, + evidenceItems: readonly FounderWeeklyReviewPromptEvidenceItem[], + error: FounderWeeklyReviewGenerationValidationError +): string { + const errors = error.details.length > 0 + ? error.details + : [{ code: "report_validation_failed" }]; + const sources = evidenceItems.map(({ sourceId, sourceType }) => ({ sourceId, sourceType })); + return [ + "Correct the complete canonical Founder Weekly Review JSON candidate below.", + "Customer Signals may cite only customer_feedback sources.", + "founder_context is founder-provided direction, not customer testimony.", + "Remove a customer claim if it lacks customer_feedback support.", + "Do not invent or substitute a source ID.", + "Return the complete corrected canonical JSON object only.", + `Validation errors: ${JSON.stringify(errors)}`, + `Source IDs and source types: ${JSON.stringify(sources)}`, + `Previous canonical candidate: ${JSON.stringify(candidate)}`, + ].join("\n"); +} + +function logGenerationValidation( + phase: "initial" | "semantic-repair", + metadata: FounderWeeklyReviewResolvedGenerationMetadata, + result: "passed" | "failed" +): void { + console.info(`[fwr] generation phase=${phase} provider=${metadata.provider} model=${metadata.model} validation=${result}`); +} + +function buildMetadata( + metadata: FounderWeeklyReviewResolvedGenerationMetadata, + promptHash: string, + skipped: boolean, + diagnostics: GenerationEvidenceEnvelopeDiagnostics, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, +): FounderWeeklyReviewModelMetadata { + return { + provider: metadata.provider, + model: metadata.model, + capability: metadata.capability, + ...(metadata.temperature === undefined ? {} : { temperature: metadata.temperature }), + promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + promptHash, + evidenceSchemaVersion: evidenceSnapshot.schemaVersion, + reviewPayloadSchemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + ...(metadata.providerRequestId ? { completionId: metadata.providerRequestId } : {}), + attributes: { + ...(skipped ? { generationSkipped: true } : {}), + ...(metadata.finishReason ? { finishReason: metadata.finishReason } : {}), + ...(metadata.usage ? { usage: JSON.stringify(metadata.usage) } : {}), + evidenceEnvelopeOriginalItems: diagnostics.originalItemCount, + evidenceEnvelopeSelectedItems: diagnostics.selectedItemCount, + evidenceEnvelopeExcludedItems: diagnostics.excludedItemCount, + evidenceEnvelopeCharacters: diagnostics.serializedCharacterCount, + evidenceEnvelopeEstimatedTokens: diagnostics.estimatedTokenCount, + evidenceEnvelopeTruncated: diagnostics.truncated, + evidenceEnvelopeSelectedByType: JSON.stringify(diagnostics.selectedBySourceType), + evidenceEnvelopeExcludedByType: JSON.stringify(diagnostics.excludedBySourceType), + evidenceDigest: buildFounderWeeklyReviewEvidenceDigest(evidenceSnapshot), + }, + }; +} + +function buildEmptyReview(): FounderWeeklyReviewV2Payload { + const noEvidence = (message: string, cta: string) => ({ + state: "no_evidence" as const, + noEvidence: { code: "no_relevant_evidence", message, cta }, + }); + return { + schemaVersion: FOUNDER_WEEKLY_REVIEW_V2_SCHEMA_VERSION, + sections: { + whatChanged: noEvidence("No change evidence was supplied for this period.", "Add document changes or founder notes for this reporting period."), + whatShipped: noEvidence("No shipment evidence was supplied for this period.", "Add release notes, deployment records, or GitHub activity."), + whatCustomersSaid: noEvidence("No customer feedback was supplied for this period.", "Add customer calls, support feedback, or survey evidence."), + currentBlockers: noEvidence("No blocker evidence was supplied for this period.", "Add founder context, project notes, or issue evidence describing blockers."), + nextPriorities: noEvidence("No evidence is available to ground priorities.", "Add evidence for this period before requesting recommended priorities."), + }, + }; +} diff --git a/packages/features/src/founder-weekly-review/grader.ts b/packages/features/src/founder-weekly-review/grader.ts new file mode 100644 index 000000000..0e6c4269e --- /dev/null +++ b/packages/features/src/founder-weekly-review/grader.ts @@ -0,0 +1,55 @@ +import { z, type ZodType } from "zod"; +import { founderWeeklyReviewGraderRubric } from "./evaluation-prompt"; +import { LLMGraderResultSchema, type LLMGraderResult } from "./llm-grader"; +import{ + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewV2Payload, +} from "./contracts"; + +export type FounderWeeklyReviewGenerateFn = ( + input: { + system?: string; + prompt: string; + schema: TSchema; + schemaName?: string; + temperature?: number; + }, +) => Promise<{ + object: z.infer; + metadata: { + provider: string; + model: string; + promptVersion?: string; + }; +}>; + +export async function gradeFounderWeeklyReview( + params: { + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot; + report: FounderWeeklyReviewV2Payload; + generate: FounderWeeklyReviewGenerateFn; + }, +): Promise { + const result = await params.generate({ + schema: LLMGraderResultSchema, + schemaName: "founder_weekly_review_grader", + temperature: 0, + system: founderWeeklyReviewGraderRubric, + prompt: ` +Evidence: +${JSON.stringify(params.evidenceSnapshot, null, 2)} + +Report: +${JSON.stringify(params.report, null, 2)} +`, + }); + + return LLMGraderResultSchema.parse({ + ...result.object, + metadata: { + provider: result.metadata.provider, + model: result.metadata.model, + promptVersion: "founder_weekly_review_grader_v1", + }, + }); +} \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/index.ts b/packages/features/src/founder-weekly-review/index.ts new file mode 100644 index 000000000..90ed78834 --- /dev/null +++ b/packages/features/src/founder-weekly-review/index.ts @@ -0,0 +1,19 @@ +export * from "./contracts"; +export * from "./evidence-service"; +export * from "./reporting-period"; +export * from "./errors"; +export * from "./repository"; +export * from "./user-service"; +export * from "./worker-service"; +export * from "./generator"; +export * from "./generation-validation"; +export * from "./generation-evidence-envelope"; +export * from "./evaluation"; +export * from "./evaluation-prompt"; +export * from "./prompts"; +export * from "./document-change"; +export * from "./document-change-materiality"; +export * from "./document-change-materiality-analyzer"; +export * from "./evidence-digest"; +export * from "./workspace-document"; +export * from "./grader"; diff --git a/packages/features/src/founder-weekly-review/llm-grader.ts b/packages/features/src/founder-weekly-review/llm-grader.ts new file mode 100644 index 000000000..4d5d128c4 --- /dev/null +++ b/packages/features/src/founder-weekly-review/llm-grader.ts @@ -0,0 +1,32 @@ +import { z } from "zod"; + +export const LLMGraderResultSchema = z.object({ + overallScore: z.number().min(0).max(1), + + dimensions: z.object({ + groundedness: z.number().min(0).max(1), + materiality: z.number().min(0).max(1), + temporalAccuracy: z.number().min(0).max(1), + synthesisQuality: z.number().min(0).max(1), + actionability: z.number().min(0).max(1), + }), + + findings: z.array( + z.object({ + section: z.string(), + severity: z.enum(["low", "medium", "high"]), + sourceIds: z.array(z.string()), + explanation: z.string(), + }) + ), + + summary: z.string(), + + metadata: z.object({ + provider: z.string(), + model: z.string(), + promptVersion: z.string(), + }), +}); + +export type LLMGraderResult = z.infer; \ No newline at end of file diff --git a/packages/features/src/founder-weekly-review/prompts.ts b/packages/features/src/founder-weekly-review/prompts.ts new file mode 100644 index 000000000..390a28f04 --- /dev/null +++ b/packages/features/src/founder-weekly-review/prompts.ts @@ -0,0 +1,124 @@ +import type { FounderWeeklyReviewEvidenceSnapshot } from "./contracts"; +import { + FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + assertGenerationEvidenceEnvelopeWithinBudget, + buildGenerationEvidenceEnvelope, + type GenerationEvidenceEnvelope, +} from "./generation-evidence-envelope"; + +export const FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION = + "founder-weekly-review-generation/v2" as const; + +export const FOUNDER_WEEKLY_REVIEW_SYSTEM_PROMPT = `You generate a structured Founder Weekly Review from supplied evidence only. + +Never invent, assume, infer, or embellish customers, dates, metrics, people, decisions, shipped work, blockers, outcomes, or source IDs. Every factual item must cite one or more supplied source IDs exactly as given. Do not create or modify source IDs. Confidence is how strongly the generated claim is supported by its cited supplied evidence; it is not a score for source reliability or truthfulness. + +Omit unsupported claims or use no_evidence rather than assigning them a low confidence. Do not lower confidence simply because the underlying source describes uncertainty; confidence measures support for the wording of the claim, not certainty about the business. + +Write a concise, natural, professional, decision-oriented founder review, not an evidence transcript. Prioritize the few most material founder-level conclusions. Synthesize related evidence into one focused claim where possible; do not create one output item per evidence source, and do not repeat the same fact across sections unless the section semantics genuinely require it. Prefer concise factual statements over explanatory prose. + +Use at most 3 items in each section: whatChanged, whatShipped, whatCustomersSaid, currentBlockers, and nextPriorities. Use fewer items when fewer material conclusions are justified; never add filler to reach a limit. For observed facts, use one concise sentence whenever possible and do not restate the entire evidence excerpt. + +Prefer a few substantive items over many one-sentence paraphrases. When multiple evidence items describe the same customer problem, request, or reaction, synthesize them into a shared theme rather than summarizing each source independently. Explicitly identify the recurring pattern, indicate how many or which sources support it when useful, explain why the pattern matters, and preserve the limits of the evidence. Do not generalize beyond the supplied customer evidence. + +Keep the distinctions below explicit. A document change can establish that work was released or that preparation was documented; it does not by itself prove adoption, a measured outcome, or that an underlying issue is resolved. Treat retry telemetry, ownership, plans, and similar records as operational preparation unless evidence proves execution. Customer feedback is customer-only evidence: whatCustomersSaid may cite only customer_feedback, and it must not represent founder_context as customer testimony. + +Customer feedback describes customer requests, reactions, complaints, or opinions; it does not establish that implementation occurred. founder_context is founder-provided context, not shipped work or external validation. Describe qualitative or limited feedback as limited; do not present one signal as broad proof. + +Use whatShipped only for work that the evidence explicitly establishes as released during this reporting period. Do not infer that nothing shipped merely because no shipped evidence is present. When no shipped work is supported, use qualified language such as "No shipped work was identified in the available evidence" rather than asserting that nothing shipped. + +Use whatChanged for meaningful developments supported by the evidence, including separate non-shipped operational developments. Do not infer that nothing changed merely because there are no document_change items. If the available evidence does not establish a meaningful change, say that no documented change was identified rather than claiming that nothing changed. Do not restate a shipped item in whatChanged. + +Use currentBlockers for evidence-backed execution blockers and for material product, customer, or operational risks. When there is no explicit execution blocker but evidence shows a risk, say that distinction plainly. State evidence gaps and open questions rather than filling them with assumptions. Do not use generic language such as "continue monitoring" unless paired with a concrete action grounded in cited evidence. + +For nextPriorities, create separate recommendation items for distinct priorities; do not combine unrelated work into one sentence. Each recommendation must be evidence-backed. The text should be one concise recommended action. The optional rationale should be brief (preferably one short sentence), should add decision context without restating the recommendation, and should not reproduce source evidence. Omit rationale when it adds no useful context. Avoid repetitive wording across sections. + +Prioritize recommendations by likely impact or urgency when the evidence supports that distinction. Avoid generic recommendations such as "monitor," "improve," or "continue working on" unless they name the specific thing to monitor, improve, or continue and the action is grounded in evidence. Avoid repetitive wording across sections. + +When evidence conflicts, return contradictory_evidence with the conflicting source IDs. Do not choose a winner or reconcile it unless supplied evidence explicitly resolves the conflict. + +nextPriorities contains recommendations only. Every recommendation must have label "Recommendation" and be grounded in supplied evidence. If a section lacks relevant evidence, return its typed no_evidence state with a concrete CTA. sourceWarnings may inform the CTA but are not factual evidence and cannot be cited. + +For nextPriorities, sourceIds have a strict citation allowlist. + +ONLY these source types may appear in nextPriorities.sourceIds: +- founder_context +- workspace_document + +NEVER put any other source type in nextPriorities.sourceIds, including: +- customer_feedback +- document_change +- github_activity + +Customer feedback and document changes may influence the recommendation's wording or rationale, but they are not valid citation sources for nextPriorities. Do not include their sourceIds under any circumstances. + +Before producing each nextPriorities recommendation, check every sourceId: +1. Look up the source's sourceType. +2. If sourceType is founder_context or workspace_document, it may be cited. +3. If sourceType is anything else, do not include that sourceId. +4. If no founder_context or workspace_document evidence independently supports the priority, do not create the recommendation. Return no_evidence instead. + +Important: Do not "fix" an invalid recommendation by returning an empty sourceIds array. Omit the recommendation and use no_evidence when the citation allowlist cannot be satisfied. + +Example: +If founder_context supports "follow up on saved filter adoption" and customer_feedback says customers want saved filters, the recommendation MAY mention the customer signal in its rationale, but sourceIds MUST contain only the founder_context sourceId. +Evidence status rules: + +- Only write shipped claims when the supplied evidence explicitly establishes that the work was released, shipped, launched, deployed, completed, or otherwise made available during the reporting period. +- Planned, scheduled, upcoming, or future-dated work cannot be described as completed or shipped. +- Customer feedback, requests, suggestions, complaints, or discussions never prove that a feature shipped, released, launched, or was completed. +- Never place customer_feedback evidence in whatShipped unless separate cited evidence explicitly establishes shipped work. +- A feature request or customer desire may be described in whatCustomersSaid, but it cannot be used to infer implementation status. +- Claim strength must not exceed evidence strength. + +Temporal interpretation rules: + +- Prefer the evidence's sourceTimestamp and explicit evidence content when determining whether something happened during the reporting period. +- Do not describe ongoing work, preparation, plans, or current context as completed work. +- Do not treat the absence of evidence as evidence that an event did not occur. +- When timing or completion status is not established, use qualified wording such as "was documented," "was identified," "was in progress," or "was not established in the available evidence." +- Preserve the distinction between a development being documented this week and the underlying work actually being completed this week.` + +/** Canonical, stable prompt serialization over the bounded evidence envelope. */ +export function buildFounderWeeklyReviewPrompt( + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot, + suppliedEnvelope?: GenerationEvidenceEnvelope, +): string { + const envelope = suppliedEnvelope ?? buildGenerationEvidenceEnvelope(evidenceSnapshot); + assertGenerationEvidenceEnvelopeWithinBudget(envelope); + return JSON.stringify(sortObjectKeysRecursively({ + promptVersion: FOUNDER_WEEKLY_REVIEW_PROMPT_VERSION, + evidenceEnvelopeVersion: envelope.version, + evidenceEnvelopeBudget: FOUNDER_WEEKLY_REVIEW_GENERATION_EVIDENCE_BUDGET, + evidenceEnvelopeDiagnostics: envelope.diagnostics, + reportingPeriod: evidenceSnapshot.reportingPeriod, + workspaceTimezone: evidenceSnapshot.workspaceTimezone, + evidence: envelope.items, + sourceWarnings: evidenceSnapshot.sourceWarnings, + requiredSections: [ + "whatChanged", + "whatShipped", + "whatCustomersSaid", + "currentBlockers", + "nextPriorities", + ], + })); +} + +/** Sort object keys recursively while retaining the exact supplied order of arrays. */ +function sortObjectKeysRecursively(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sortObjectKeysRecursively); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value as Record) + .sort() + .map((key) => [ + key, + sortObjectKeysRecursively((value as Record)[key]), + ]) + ); + } + return value; +} diff --git a/packages/features/src/founder-weekly-review/reporting-period.ts b/packages/features/src/founder-weekly-review/reporting-period.ts new file mode 100644 index 000000000..e26d113a9 --- /dev/null +++ b/packages/features/src/founder-weekly-review/reporting-period.ts @@ -0,0 +1,47 @@ +import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; +import timezone from "dayjs/plugin/timezone"; + +import type { ReportingPeriod } from "./contracts"; + +dayjs.extend(utc); +dayjs.extend(timezone); + +export interface ReportingPeriodBounds { + startInclusive: Date; + endExclusive: Date; +} + +// convert calendar dates in workspace timezone to UTC for querying +// end is the last included day +export function resolveReportingPeriodBounds( + period: ReportingPeriod, + workspaceTimezone: string +): ReportingPeriodBounds { + assertValidTimeZone(workspaceTimezone); + + const startInclusive = dayjs.tz( + `${period.start}T00:00:00`, + workspaceTimezone + ); + + const endExclusive = dayjs.tz( + `${period.end}T00:00:00`, + workspaceTimezone + ).add(1, "day"); + + return { + startInclusive: startInclusive.toDate(), + endExclusive: endExclusive.toDate(), + }; +} + +// dayjs.tz throws dayjs's own error for a bad zone; validate up front so callers +// get a clear, feature-specific message instead. +function assertValidTimeZone(timeZone: string): void { + try { + new Intl.DateTimeFormat("en-US", { timeZone }); + } catch { + throw new Error(`Invalid workspace timezone: ${timeZone}`); + } +} diff --git a/packages/features/src/founder-weekly-review/repository.ts b/packages/features/src/founder-weekly-review/repository.ts new file mode 100644 index 000000000..751639e26 --- /dev/null +++ b/packages/features/src/founder-weekly-review/repository.ts @@ -0,0 +1,666 @@ +import { and, desc, eq, sql } from "drizzle-orm"; +import { getDb, type DbClient } from "@launchstack/core/db"; +import { + founderWeeklyReviewOperations, + founderWeeklyReviewRuns, + type FounderWeeklyReviewOperationRow, + type FounderWeeklyReviewRunRow, +} from "@launchstack/core/db/schema"; + +import { + FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + type FounderWeeklyReviewPayloadSchemaVersion, + type FounderWeeklyReviewEvidenceSchemaVersion, + type CreateFounderWeeklyReviewRunInput, + type FounderWeeklyReviewClaimInput, + type FounderWeeklyReviewCollectionClaimInput, + type FounderWeeklyReviewGenerationFailure, + type FounderWeeklyReviewModelMetadata, + type FounderWeeklyReviewOperationRecord, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewRetryInput, + type FounderWeeklyReviewRunRecord, + parseFounderWeeklyReviewEvidenceSnapshot, + parseFounderWeeklyReviewCollectionInput, + parseFounderWeeklyReviewModelMetadata, + parseFounderWeeklyReviewPayload, +} from "./contracts"; +import { FounderWeeklyReviewInvalidPayloadError } from "./errors"; + +export interface ConditionalRunMutationResult { + updated: boolean; + run: FounderWeeklyReviewRunRecord | null; +} +export interface CreateFounderWeeklyReviewResult { + run: FounderWeeklyReviewRunRecord; + created: boolean; +} + +export interface RetryFounderWeeklyReviewResult { + outcome: "updated" | "idempotent" | "conflict" | "not_found"; + run: FounderWeeklyReviewRunRecord | null; + operation: FounderWeeklyReviewOperationRecord | null; +} + +function mapRunRow(row: FounderWeeklyReviewRunRow): FounderWeeklyReviewRunRecord { + const reviewPayload = row.reviewPayload + ? parseFounderWeeklyReviewPayload(row.reviewPayload) + : null; + if (reviewPayload && reviewPayload.schemaVersion !== row.reviewSchemaVersion) { + throw new FounderWeeklyReviewInvalidPayloadError( + `Founder weekly review run "${row.id}" has mismatched review payload and review schema versions.` + ); + } + const evidenceSnapshot = row.evidenceSnapshot + ? parseFounderWeeklyReviewEvidenceSnapshot(row.evidenceSnapshot) + : null; + if (evidenceSnapshot && evidenceSnapshot.schemaVersion !== row.evidenceSchemaVersion) { + throw new FounderWeeklyReviewInvalidPayloadError( + `Founder weekly review run "${row.id}" has mismatched evidence snapshot and evidence schema versions.` + ); + } + return { + id: row.id, + companyId: row.companyId, + requestKey: row.requestKey, + reportingPeriod: { + start: row.reportingPeriodStart, + end: row.reportingPeriodEnd, + }, + status: row.status, + reviewPayload, + reviewSchemaVersion: row.reviewSchemaVersion as FounderWeeklyReviewPayloadSchemaVersion, + evidenceSnapshot, + evidenceSchemaVersion: + row.evidenceSchemaVersion as FounderWeeklyReviewEvidenceSchemaVersion, + collectionInput: parseFounderWeeklyReviewCollectionInput(row.collectionInput), + collectionClaimId: row.collectionClaimId ?? null, + collectionStartedAt: row.collectionStartedAt ?? null, + evidenceCollectedAt: row.evidenceCollectedAt ?? null, + modelMetadata: row.modelMetadata + ? parseFounderWeeklyReviewModelMetadata(row.modelMetadata) + : null, + createdByActorId: row.createdByActorId, + retryCount: row.retryCount, + failureSequence: row.failureSequence, + generationAttempt: row.generationAttempt, + generationClaimId: row.generationClaimId ?? null, + generationJobId: row.generationJobId ?? null, + queuedAt: row.queuedAt, + claimedAt: row.claimedAt ?? null, + generationStartedAt: row.generationStartedAt ?? null, + generatedAt: row.generatedAt ?? null, + publishedAt: row.publishedAt ?? null, + errorCode: row.errorCode ?? null, + errorMessage: row.errorMessage ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt ?? null, + }; +} + +function mapOperationRow(row: FounderWeeklyReviewOperationRow): FounderWeeklyReviewOperationRecord { + return { + id: row.id, + runId: row.runId, + companyId: row.companyId, + operationType: row.operationType, + requestKey: row.requestKey, + sourceFailureSequence: row.sourceFailureSequence, + actorId: row.actorId, + createdAt: row.createdAt, + }; +} + +function truncateErrorMessage(value?: string | null): string | null { + if (!value) { + return null; + } + return value.length <= 1024 ? value : value.slice(0, 1024); +} + +export class FounderWeeklyReviewRepository { + constructor(private readonly db: DbClient = getDb()) {} + + async createOrGetByRequestKey( + input: CreateFounderWeeklyReviewRunInput + ): Promise { + return (await this.createOrGetByRequestKeyWithResult(input)).run; + } + + async createOrGetByRequestKeyWithResult( + input: CreateFounderWeeklyReviewRunInput + ): Promise { + const [inserted] = await this.db + .insert(founderWeeklyReviewRuns) + .values({ + id: input.id, + companyId: input.companyId, + requestKey: input.requestKey, + reportingPeriodStart: input.reportingPeriod.start, + reportingPeriodEnd: input.reportingPeriod.end, + status: "queued", + reviewPayload: null, + reviewSchemaVersion: FOUNDER_WEEKLY_REVIEW_SCHEMA_VERSION, + evidenceSnapshot: input.evidenceSnapshot ?? null, + evidenceSchemaVersion: input.evidenceSnapshot?.schemaVersion ?? FOUNDER_WEEKLY_REVIEW_EVIDENCE_SCHEMA_VERSION, + collectionInput: input.collectionInput ?? { + workspaceTimezone: input.evidenceSnapshot?.workspaceTimezone ?? "UTC", + actorExternalUserId: input.createdByActorId.replace(/^user:/, ""), + }, + modelMetadata: null, + createdByActorId: input.createdByActorId, + queuedAt: new Date(), + updatedAt: new Date(), + }) + .onConflictDoNothing({ + target: [ + founderWeeklyReviewRuns.companyId, + founderWeeklyReviewRuns.requestKey, + ], + }) + .returning(); + + if (inserted) { + return { run: mapRunRow(inserted), created: true }; + } + + const existing = await this.getByCompanyAndRequestKey( + input.companyId, + input.requestKey + ); + if (!existing) { + throw new Error("Failed to create or retrieve founder weekly review run"); + } + return { run: existing, created: false }; + } + + async getByCompanyAndRunId( + companyId: bigint, + runId: string + ): Promise { + const [row] = await this.db + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId) + ) + ) + .limit(1); + + return row ? mapRunRow(row) : null; + } + + async getByCompanyAndRequestKey( + companyId: bigint, + requestKey: string + ): Promise { + const [row] = await this.db + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.requestKey, requestKey) + ) + ) + .limit(1); + + return row ? mapRunRow(row) : null; + } + + async listByCompany(companyId: bigint): Promise { + const rows = await this.db + .select() + .from(founderWeeklyReviewRuns) + .where(eq(founderWeeklyReviewRuns.companyId, companyId)) + .orderBy(desc(founderWeeklyReviewRuns.createdAt), desc(founderWeeklyReviewRuns.id)); + + return rows.map(mapRunRow); + } + + async updateDraftConditionally( + companyId: bigint, + runId: string, + reviewPayload: FounderWeeklyReviewPayload + ): Promise { + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + reviewPayload, + reviewSchemaVersion: reviewPayload.schemaVersion, + updatedAt: new Date(), + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId), + eq(founderWeeklyReviewRuns.status, "draft") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(companyId, runId), + }; + } + + async publishConditionally( + companyId: bigint, + runId: string + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "published", + publishedAt: now, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId), + eq(founderWeeklyReviewRuns.status, "draft") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(companyId, runId), + }; + } + + async claimQueuedRun( + input: FounderWeeklyReviewClaimInput + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "generating", + generationAttempt: sql`${founderWeeklyReviewRuns.generationAttempt} + 1`, + generationClaimId: input.generationClaimId, + generationJobId: input.generationJobId ?? null, + claimedAt: now, + generationStartedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "queued"), + sql`${founderWeeklyReviewRuns.evidenceSnapshot} IS NOT NULL` + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async claimEvidenceCollection(input: FounderWeeklyReviewCollectionClaimInput): Promise { + const now = new Date(); + const [row] = await this.db.update(founderWeeklyReviewRuns).set({ + status: "collecting", + collectionClaimId: input.collectionClaimId, + collectionStartedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "queued"), + sql`${founderWeeklyReviewRuns.evidenceSnapshot} IS NULL`, + )).returning(); + return row ? { updated: true, run: mapRunRow(row) } : { + updated: false, run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async attachEvidenceSnapshotIfAbsent( + input: FounderWeeklyReviewCollectionClaimInput, + evidenceSnapshot: import("./contracts").FounderWeeklyReviewEvidenceSnapshot, + ): Promise { + const now = new Date(); + const [row] = await this.db.update(founderWeeklyReviewRuns).set({ + status: "queued", + evidenceSnapshot, + evidenceSchemaVersion: evidenceSnapshot.schemaVersion, + evidenceCollectedAt: now, + collectionClaimId: null, + queuedAt: now, + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "collecting"), + eq(founderWeeklyReviewRuns.collectionClaimId, input.collectionClaimId), + sql`${founderWeeklyReviewRuns.evidenceSnapshot} IS NULL`, + )).returning(); + return row ? { updated: true, run: mapRunRow(row) } : { + updated: false, run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async markCollectionFailed(input: FounderWeeklyReviewCollectionClaimInput, failure: FounderWeeklyReviewGenerationFailure): Promise { + const now = new Date(); + const [row] = await this.db.update(founderWeeklyReviewRuns).set({ + status: "failed", + failureSequence: sql`${founderWeeklyReviewRuns.failureSequence} + 1`, + errorCode: failure.errorCode, + errorMessage: truncateErrorMessage(failure.errorMessage), + updatedAt: now, + }).where(and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "collecting"), + eq(founderWeeklyReviewRuns.collectionClaimId, input.collectionClaimId), + )).returning(); + return row ? { updated: true, run: mapRunRow(row) } : { + updated: false, run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async saveGeneratedDraftWithClaim( + input: FounderWeeklyReviewClaimInput, + reviewPayload: FounderWeeklyReviewPayload, + modelMetadata: FounderWeeklyReviewModelMetadata | null + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "draft", + reviewPayload, + reviewSchemaVersion: reviewPayload.schemaVersion, + modelMetadata, + generatedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "generating"), + eq(founderWeeklyReviewRuns.generationClaimId, input.generationClaimId) + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async markGenerationFailedWithClaim( + input: FounderWeeklyReviewClaimInput, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "failed", + failureSequence: sql`${founderWeeklyReviewRuns.failureSequence} + 1`, + errorCode: failure.errorCode, + errorMessage: truncateErrorMessage(failure.errorMessage), + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "generating"), + eq(founderWeeklyReviewRuns.generationClaimId, input.generationClaimId) + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(input.companyId, input.runId), + }; + } + + async markQueuedRunFailed( + companyId: bigint, + runId: string, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const now = new Date(); + const [row] = await this.db + .update(founderWeeklyReviewRuns) + .set({ + status: "failed", + failureSequence: sql`${founderWeeklyReviewRuns.failureSequence} + 1`, + errorCode: failure.errorCode, + errorMessage: truncateErrorMessage(failure.errorMessage), + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId), + eq(founderWeeklyReviewRuns.status, "queued") + ) + ) + .returning(); + + if (row) { + return { updated: true, run: mapRunRow(row) }; + } + + return { + updated: false, + run: await this.getByCompanyAndRunId(companyId, runId), + }; + } + + async retryFailedRun( + input: FounderWeeklyReviewRetryInput + ): Promise { + return this.db.transaction(async (tx) => { + const [currentRow] = await tx + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId) + ) + ) + .limit(1); + + if (!currentRow) { + return { + outcome: "not_found", + run: null, + operation: null, + }; + } + + const [existingOperationRow] = await tx + .select() + .from(founderWeeklyReviewOperations) + .where( + and( + eq(founderWeeklyReviewOperations.companyId, input.companyId), + eq(founderWeeklyReviewOperations.runId, input.runId), + eq(founderWeeklyReviewOperations.operationType, "retry"), + eq(founderWeeklyReviewOperations.requestKey, input.requestKey) + ) + ) + .limit(1); + + const currentRun = mapRunRow(currentRow); + + if (existingOperationRow) { + const operation = mapOperationRow(existingOperationRow); + if (operation.sourceFailureSequence !== currentRun.failureSequence) { + return { + outcome: "conflict", + run: currentRun, + operation, + }; + } + + return { + outcome: "idempotent", + run: currentRun, + operation, + }; + } + + if (currentRun.status !== "failed") { + return { + outcome: "conflict", + run: currentRun, + operation: null, + }; + } + + const [insertedOperationRow] = await tx + .insert(founderWeeklyReviewOperations) + .values({ + id: input.operationId, + runId: input.runId, + companyId: input.companyId, + operationType: "retry", + requestKey: input.requestKey, + sourceFailureSequence: currentRun.failureSequence, + actorId: input.actorId, + }) + .onConflictDoNothing({ + target: [ + founderWeeklyReviewOperations.runId, + founderWeeklyReviewOperations.operationType, + founderWeeklyReviewOperations.requestKey, + ], + }) + .returning(); + + if (!insertedOperationRow) { + const [conflictingOperationRow] = await tx + .select() + .from(founderWeeklyReviewOperations) + .where( + and( + eq(founderWeeklyReviewOperations.companyId, input.companyId), + eq(founderWeeklyReviewOperations.runId, input.runId), + eq(founderWeeklyReviewOperations.operationType, "retry"), + eq(founderWeeklyReviewOperations.requestKey, input.requestKey) + ) + ) + .limit(1); + + return { + outcome: + conflictingOperationRow && + conflictingOperationRow.sourceFailureSequence === + currentRun.failureSequence + ? "idempotent" + : "conflict", + run: await this.getRunInsideTransaction(tx, input.companyId, input.runId), + operation: conflictingOperationRow + ? mapOperationRow(conflictingOperationRow) + : null, + }; + } + + const now = new Date(); + const [updatedRow] = await tx + .update(founderWeeklyReviewRuns) + .set({ + status: "queued", + retryCount: sql`${founderWeeklyReviewRuns.retryCount} + 1`, + generationClaimId: null, + generationJobId: null, + claimedAt: null, + generationStartedAt: null, + generatedAt: null, + queuedAt: now, + errorCode: null, + errorMessage: null, + updatedAt: now, + }) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, input.companyId), + eq(founderWeeklyReviewRuns.id, input.runId), + eq(founderWeeklyReviewRuns.status, "failed"), + eq( + founderWeeklyReviewRuns.failureSequence, + currentRun.failureSequence + ) + ) + ) + .returning(); + + if (!updatedRow) { + return { + outcome: "conflict", + run: await this.getRunInsideTransaction(tx, input.companyId, input.runId), + operation: mapOperationRow(insertedOperationRow), + }; + } + + return { + outcome: "updated", + run: mapRunRow(updatedRow), + operation: mapOperationRow(insertedOperationRow), + }; + }); + } + + private async getRunInsideTransaction( + tx: Pick, + companyId: bigint, + runId: string + ): Promise { + const [row] = await tx + .select() + .from(founderWeeklyReviewRuns) + .where( + and( + eq(founderWeeklyReviewRuns.companyId, companyId), + eq(founderWeeklyReviewRuns.id, runId) + ) + ) + .limit(1); + return row ? mapRunRow(row) : null; + } +} diff --git a/packages/features/src/founder-weekly-review/user-service.ts b/packages/features/src/founder-weekly-review/user-service.ts new file mode 100644 index 000000000..0ebc5d7cc --- /dev/null +++ b/packages/features/src/founder-weekly-review/user-service.ts @@ -0,0 +1,207 @@ +import { randomUUID } from "node:crypto"; +import { ZodError } from "zod"; + +import { + buildFounderWeeklyReviewActorId, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewCollectionInput, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewRunRecord, + type FounderWeeklyReviewUserActor, + parseFounderWeeklyReviewEvidenceSnapshot, + parseFounderWeeklyReviewPayload, +} from "./contracts"; +import { + FounderWeeklyReviewConflictError, + FounderWeeklyReviewForbiddenError, + FounderWeeklyReviewInvalidPayloadError, + FounderWeeklyReviewInvalidTransitionError, + FounderWeeklyReviewNotFoundError, +} from "./errors"; +import { FounderWeeklyReviewRepository } from "./repository"; + +const ALLOWED_WORKSPACE_ROLES = new Set(["owner", "admin", "editor"]); + +export interface CreateFounderWeeklyReviewRunRequest { + requestKey: string; + reportingPeriod: { + start: string; + end: string; + }; + evidenceSnapshot?: FounderWeeklyReviewEvidenceSnapshot; + collectionInput?: FounderWeeklyReviewCollectionInput; +} +export interface CreateFounderWeeklyReviewRunResult { + run: FounderWeeklyReviewRunRecord; + created: boolean; +} +export interface RetryFounderWeeklyReviewRunResult { + run: FounderWeeklyReviewRunRecord; + transitionApplied: boolean; +} + +function assertWorkspaceMutationRole(role: string): void { + if (!ALLOWED_WORKSPACE_ROLES.has(role)) { + throw new FounderWeeklyReviewForbiddenError(); + } +} + +function assertReportingPeriodMatchesSnapshot( + reportingPeriod: { start: string; end: string }, + evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot +): void { + if ( + reportingPeriod.start !== evidenceSnapshot.reportingPeriod.start || + reportingPeriod.end !== evidenceSnapshot.reportingPeriod.end + ) { + throw new FounderWeeklyReviewInvalidPayloadError( + "Evidence snapshot reporting period must match the run reporting period." + ); + } +} + +export class FounderWeeklyReviewUserService { + constructor( + private readonly repository = new FounderWeeklyReviewRepository() + ) {} + + async createOrGetRun( + actor: FounderWeeklyReviewUserActor, + input: CreateFounderWeeklyReviewRunRequest + ): Promise { + assertWorkspaceMutationRole(actor.role); + let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | undefined; + try { evidenceSnapshot = input.evidenceSnapshot ? parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot) : undefined; } + catch (error) { if (error instanceof ZodError) throw new FounderWeeklyReviewInvalidPayloadError(error.message); throw error; } + if (evidenceSnapshot) assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); + + return (await this.repository.createOrGetByRequestKeyWithResult({ + id: `fwr_${randomUUID()}`, + companyId: actor.companyId, + requestKey: input.requestKey, + reportingPeriod: input.reportingPeriod, + evidenceSnapshot, + collectionInput: input.collectionInput ?? { workspaceTimezone: evidenceSnapshot?.workspaceTimezone ?? "UTC", actorExternalUserId: actor.externalUserId }, + createdByActorId: buildFounderWeeklyReviewActorId(actor), + })).run; + } + + async createOrGetRunWithMetadata(actor: FounderWeeklyReviewUserActor, input: CreateFounderWeeklyReviewRunRequest): Promise { + assertWorkspaceMutationRole(actor.role); + let evidenceSnapshot: FounderWeeklyReviewEvidenceSnapshot | undefined; + try { evidenceSnapshot = input.evidenceSnapshot ? parseFounderWeeklyReviewEvidenceSnapshot(input.evidenceSnapshot) : undefined; } + catch (error) { if (error instanceof ZodError) throw new FounderWeeklyReviewInvalidPayloadError(error.message); throw error; } + if (evidenceSnapshot) assertReportingPeriodMatchesSnapshot(input.reportingPeriod, evidenceSnapshot); + return this.repository.createOrGetByRequestKeyWithResult({ id: `fwr_${randomUUID()}`, companyId: actor.companyId, requestKey: input.requestKey, reportingPeriod: input.reportingPeriod, evidenceSnapshot, collectionInput: input.collectionInput ?? { workspaceTimezone: evidenceSnapshot?.workspaceTimezone ?? "UTC", actorExternalUserId: actor.externalUserId }, createdByActorId: buildFounderWeeklyReviewActorId(actor) }); + } + + async getRun( + actor: FounderWeeklyReviewUserActor, + runId: string + ): Promise { + const run = await this.repository.getByCompanyAndRunId(actor.companyId, runId); + if (!run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + return run; + } + + async listRuns(actor: FounderWeeklyReviewUserActor): Promise { + return this.repository.listByCompany(actor.companyId); + } + + async retryFailedRun( + actor: FounderWeeklyReviewUserActor, + runId: string, + requestKey: string + ): Promise { + assertWorkspaceMutationRole(actor.role); + const result = await this.repository.retryFailedRun({ + operationId: `fwrop_${randomUUID()}`, + companyId: actor.companyId, + runId, + requestKey, + actorId: buildFounderWeeklyReviewActorId(actor), + }); + + if (result.outcome === "not_found" || !result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + + if (result.outcome === "updated" || result.outcome === "idempotent") { + if (result.run.status === "failed" || result.run.status === "queued") { + return result.run; + } + } + + if (result.run.status !== "failed") { + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "retry"); + } + + throw new FounderWeeklyReviewConflictError( + `Retry request key "${requestKey}" belongs to a different failure cycle for run "${runId}".` + ); + } + + async retryFailedRunWithMetadata(actor: FounderWeeklyReviewUserActor, runId: string, requestKey: string): Promise { + assertWorkspaceMutationRole(actor.role); + const result = await this.repository.retryFailedRun({ operationId: `fwrop_${randomUUID()}`, companyId: actor.companyId, runId, requestKey, actorId: buildFounderWeeklyReviewActorId(actor) }); + if (result.outcome === "not_found" || !result.run) throw new FounderWeeklyReviewNotFoundError(runId); + if (result.outcome === "updated") return { run: result.run, transitionApplied: true }; + if (result.outcome === "idempotent" && (result.run.status === "failed" || result.run.status === "queued")) return { run: result.run, transitionApplied: false }; + if (result.run.status !== "failed") throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "retry"); + throw new FounderWeeklyReviewConflictError(`Retry request key "${requestKey}" belongs to a different failure cycle for run "${runId}".`); + } + + async updateDraft( + actor: FounderWeeklyReviewUserActor, + runId: string, + reviewPayload: FounderWeeklyReviewPayload + ): Promise { + assertWorkspaceMutationRole(actor.role); + let payload: FounderWeeklyReviewPayload; + try { + payload = parseFounderWeeklyReviewPayload(reviewPayload); + } catch (error) { + if (error instanceof ZodError) { + throw new FounderWeeklyReviewInvalidPayloadError(error.message); + } + throw error; + } + const result = await this.repository.updateDraftConditionally( + actor.companyId, + runId, + payload + ); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + if (result.updated) { + return result.run; + } + if (result.run.status === "published") { + throw new FounderWeeklyReviewInvalidTransitionError( + result.run.status, + "edit" + ); + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "edit"); + } + + async publishDraft( + actor: FounderWeeklyReviewUserActor, + runId: string + ): Promise { + assertWorkspaceMutationRole(actor.role); + const result = await this.repository.publishConditionally(actor.companyId, runId); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + if (result.updated || result.run.status === "published") { + return result.run; + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "publish"); + } +} diff --git a/packages/features/src/founder-weekly-review/worker-service.ts b/packages/features/src/founder-weekly-review/worker-service.ts new file mode 100644 index 000000000..a3d5544dc --- /dev/null +++ b/packages/features/src/founder-weekly-review/worker-service.ts @@ -0,0 +1,160 @@ +import { ZodError } from "zod"; + +import { + type FounderWeeklyReviewClaimInput, + type FounderWeeklyReviewCollectionClaimInput, + type FounderWeeklyReviewEvidenceSnapshot, + type FounderWeeklyReviewGenerationFailure, + type FounderWeeklyReviewModelMetadata, + type FounderWeeklyReviewPayload, + type FounderWeeklyReviewRunRecord, + parseFounderWeeklyReviewModelMetadata, + parseFounderWeeklyReviewPayload, +} from "./contracts"; +import { + FounderWeeklyReviewClaimOwnershipMismatchError, + FounderWeeklyReviewConflictError, + FounderWeeklyReviewInvalidPayloadError, + FounderWeeklyReviewInvalidTransitionError, + FounderWeeklyReviewNotFoundError, +} from "./errors"; +import { FounderWeeklyReviewRepository } from "./repository"; + +export interface FounderWeeklyReviewWorkerContext extends FounderWeeklyReviewClaimInput {} +export interface FounderWeeklyReviewCollectionContext extends FounderWeeklyReviewCollectionClaimInput {} + +export class FounderWeeklyReviewWorkerService { + constructor( + private readonly repository = new FounderWeeklyReviewRepository() + ) {} + + async getRun(companyId: bigint, runId: string): Promise { + const run = await this.repository.getByCompanyAndRunId(companyId, runId); + if (!run) throw new FounderWeeklyReviewNotFoundError(runId); + return run; + } + + async claimQueuedRun( + context: FounderWeeklyReviewWorkerContext + ): Promise { + const result = await this.repository.claimQueuedRun(context); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(context.runId); + } + if (result.updated) { + return result.run; + } + if ( + result.run.status === "generating" && + result.run.generationClaimId === context.generationClaimId + ) { + return result.run; + } + throw new FounderWeeklyReviewConflictError( + `Founder weekly review run "${context.runId}" is already owned by another generation claim or moved out of queue.` + ); + } + + async claimEvidenceCollection(context: FounderWeeklyReviewCollectionContext): Promise { + const result = await this.repository.claimEvidenceCollection(context); + if (!result.run) throw new FounderWeeklyReviewNotFoundError(context.runId); + if (result.updated) return result.run; + if (result.run.status === "collecting" && result.run.collectionClaimId === context.collectionClaimId) return result.run; + if (result.run.evidenceSnapshot) return result.run; + throw new FounderWeeklyReviewConflictError(`Founder weekly review run "${context.runId}" is already owned by another collection claim.`); + } + + async attachEvidenceSnapshotIfAbsent(context: FounderWeeklyReviewCollectionContext, snapshot: FounderWeeklyReviewEvidenceSnapshot): Promise { + const result = await this.repository.attachEvidenceSnapshotIfAbsent(context, snapshot); + if (!result.run) throw new FounderWeeklyReviewNotFoundError(context.runId); + if (result.updated || result.run.evidenceSnapshot) return result.run; + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + + async markCollectionFailed(context: FounderWeeklyReviewCollectionContext, failure: FounderWeeklyReviewGenerationFailure): Promise { + const result = await this.repository.markCollectionFailed(context, failure); + if (!result.run) throw new FounderWeeklyReviewNotFoundError(context.runId); + if (result.updated || (result.run.status === "failed" && result.run.collectionClaimId === context.collectionClaimId)) return result.run; + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + + async saveGeneratedDraft( + context: FounderWeeklyReviewWorkerContext, + reviewPayload: FounderWeeklyReviewPayload, + modelMetadata: FounderWeeklyReviewModelMetadata | null + ): Promise { + let payload: FounderWeeklyReviewPayload; + let metadata: FounderWeeklyReviewModelMetadata | null; + try { + payload = parseFounderWeeklyReviewPayload(reviewPayload); + metadata = modelMetadata + ? parseFounderWeeklyReviewModelMetadata(modelMetadata) + : null; + } catch (error) { + if (error instanceof ZodError) { + throw new FounderWeeklyReviewInvalidPayloadError(error.message); + } + throw error; + } + const result = await this.repository.saveGeneratedDraftWithClaim( + context, + payload, + metadata + ); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(context.runId); + } + if (result.updated) { + return result.run; + } + if (result.run.generationClaimId !== context.generationClaimId) { + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "save draft"); + } + + async markGenerationFailed( + context: FounderWeeklyReviewWorkerContext, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const result = await this.repository.markGenerationFailedWithClaim( + context, + failure + ); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(context.runId); + } + if (result.updated) { + return result.run; + } + if ( + result.run.status === "failed" && + result.run.generationClaimId === context.generationClaimId + ) { + return result.run; + } + if (result.run.generationClaimId !== context.generationClaimId) { + throw new FounderWeeklyReviewClaimOwnershipMismatchError(context.runId); + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "mark failed"); + } + + async markQueuedRunFailed( + companyId: bigint, + runId: string, + failure: FounderWeeklyReviewGenerationFailure + ): Promise { + const result = await this.repository.markQueuedRunFailed(companyId, runId, failure); + + if (!result.run) { + throw new FounderWeeklyReviewNotFoundError(runId); + } + if (result.updated) { + return result.run; + } + throw new FounderWeeklyReviewInvalidTransitionError(result.run.status, "mark failed"); + } +} diff --git a/packages/features/src/founder-weekly-review/workspace-document.ts b/packages/features/src/founder-weekly-review/workspace-document.ts new file mode 100644 index 000000000..96507965b --- /dev/null +++ b/packages/features/src/founder-weekly-review/workspace-document.ts @@ -0,0 +1,58 @@ +import type { FounderWeeklyReviewEvidenceItem } from "./contracts"; + +export const MAX_WORKSPACE_RETRIEVAL_CANDIDATES = 12; +export const MAX_WORKSPACE_EVIDENCE_ITEMS = 8; +export const MAX_WORKSPACE_EVIDENCE_PER_DOCUMENT = 2; +export const MAX_WORKSPACE_EXCERPT_LENGTH = 4000; + +export type WorkspaceDocumentRetrievalInput = { companyId: bigint; founderContext: string; topK?: number }; +export type WorkspaceDocumentHit = { + documentId: bigint; documentTitle: string; versionId: bigint; contextChunkId: number; retrievalChunkId?: number; + content: string; similarityScore: number; structureId?: bigint | null; structurePath?: string | null; structureTitle?: string | null; + pageNumber?: number | null; lineStart?: number | null; lineEnd?: number | null; +}; +export type WorkspaceDocumentRetrievalResult = + | { state: "success"; hits: WorkspaceDocumentHit[] } + | { state: "empty"; hits: [] } + | { state: "unavailable"; hits: []; warnings: string[] }; + +const normalize = (value: string | null | undefined) => value?.replace(/\s+/g, " ").trim() || null; +const bound = (value: string, max: number) => value.length <= max ? value : `${value.slice(0, max - 1)}…`; +const compareBigInt = (left: bigint, right: bigint) => left < right ? -1 : left > right ? 1 : 0; +const compareHits = (a: WorkspaceDocumentHit, b: WorkspaceDocumentHit) => b.similarityScore - a.similarityScore || compareBigInt(a.documentId, b.documentId) || compareBigInt(a.versionId, b.versionId) || a.contextChunkId - b.contextChunkId; + +export function normalizeFounderContextRetrievalQuery(founderContext: string | null | undefined): string | null { + const normalized = normalize(founderContext); + return normalized ? bound(normalized, 4000) : null; +} + +/** Deterministically deduplicate, diversify, and cap provider/database hits. */ +export function selectWorkspaceDocumentHits(hits: readonly WorkspaceDocumentHit[]): WorkspaceDocumentHit[] { + const selected: WorkspaceDocumentHit[] = []; const contexts = new Set(); const perDocument = new Map(); + for (const hit of [...hits].filter((hit) => normalize(hit.content) && Number.isFinite(hit.similarityScore)).sort(compareHits)) { + const contextKey = `${hit.documentId}:${hit.versionId}:${hit.contextChunkId}`; + if (contexts.has(contextKey)) continue; + const count = perDocument.get(hit.documentId.toString()) ?? 0; + if (count >= MAX_WORKSPACE_EVIDENCE_PER_DOCUMENT) continue; + contexts.add(contextKey); perDocument.set(hit.documentId.toString(), count + 1); selected.push(hit); + if (selected.length >= MAX_WORKSPACE_EVIDENCE_ITEMS) break; + } + return selected; +} + +export function buildWorkspaceDocumentEvidence(hits: readonly WorkspaceDocumentHit[]): FounderWeeklyReviewEvidenceItem[] { + return selectWorkspaceDocumentHits(hits).map((hit) => ({ + sourceType: "workspace_document" as const, + sourceId: `workspace_document:doc:${hit.documentId}:version:${hit.versionId}:chunk:${hit.contextChunkId}`, + title: hit.documentTitle, + excerpt: bound(normalize(hit.content)!, MAX_WORKSPACE_EXCERPT_LENGTH), + workspaceDeepLink: `/employer/documents/viewer?docId=${hit.documentId}`, + metadata: { + documentId: hit.documentId.toString(), documentVersionId: hit.versionId.toString(), chunkId: hit.contextChunkId, + retrievalChunkId: hit.retrievalChunkId ?? null, retrievalReason: "founder_context_relevance", similarityScore: hit.similarityScore, + structureId: hit.structureId?.toString() ?? null, structurePath: hit.structurePath ? bound(hit.structurePath, 512) : null, + structureTitle: hit.structureTitle ? bound(hit.structureTitle, 512) : null, pageNumber: hit.pageNumber ?? null, + lineStart: hit.lineStart ?? null, lineEnd: hit.lineEnd ?? null, + }, + })); +} diff --git a/packages/features/src/index.ts b/packages/features/src/index.ts index cb0ff5c3b..d9a0fe4d5 100644 --- a/packages/features/src/index.ts +++ b/packages/features/src/index.ts @@ -1 +1 @@ -export {}; +export * from "./founder-weekly-review"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 452c39241..62d6c6213 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -608,6 +608,9 @@ importers: '@launchstack/core': specifier: workspace:* version: link:../core + dayjs: + specifier: ^1.11.18 + version: 1.11.19 docxtemplater: specifier: ^3.68.3 version: 3.68.3 diff --git a/scripts/backfill-embedding-credentials.ts b/scripts/backfill-embedding-credentials.ts index f658cc456..bf8a3deda 100644 --- a/scripts/backfill-embedding-credentials.ts +++ b/scripts/backfill-embedding-credentials.ts @@ -1,113 +1,12 @@ -import "dotenv/config"; - -import { eq, isNotNull, or, sql } from "drizzle-orm"; - -import { db } from "../src/server/db"; -import { company } from "../src/server/db/schema"; -import { upsertCompanyCredentials } from "../src/lib/ai/company-credentials"; - /** - * Backfill plaintext embedding credentials from the legacy columns on - * `pdr_ai_v2_company` into the encrypted `pdr_ai_v2_company_embedding_credentials` - * table, then NULL out the legacy columns. - * - * Safe to re-run: `upsertCompanyCredentials` upserts, and the legacy NULL - * step is idempotent. Requires `EMBEDDING_SECRETS_KEY` to be set. - * - * Run with: pnpm tsx scripts/backfill-embedding-credentials.ts + * Moved to apps/web/scripts/backfill-embedding-credentials.ts. * - * Flags: - * --dry-run Log what would change without writing anything. + * Run with: + * pnpm db:backfill:embedding-credentials */ -const DRY_RUN = process.argv.includes("--dry-run"); - -async function main() { - if (!process.env.EMBEDDING_SECRETS_KEY) { - console.error( - "Refusing to run: EMBEDDING_SECRETS_KEY is not set. Generate one with:\n" + - " node -e \"console.log(require('crypto').randomBytes(32).toString('base64'))\"", - ); - process.exit(1); - } - - const rows = await db - .select({ - id: company.id, - name: company.name, - openAIApiKey: company.embeddingOpenAIApiKey, - huggingFaceApiKey: company.embeddingHuggingFaceApiKey, - ollamaBaseUrl: company.embeddingOllamaBaseUrl, - ollamaModel: company.embeddingOllamaModel, - }) - .from(company) - .where( - or( - isNotNull(company.embeddingOpenAIApiKey), - isNotNull(company.embeddingHuggingFaceApiKey), - isNotNull(company.embeddingOllamaBaseUrl), - isNotNull(company.embeddingOllamaModel), - ), - ); - - console.log( - `Found ${rows.length} company row(s) with legacy embedding credentials${DRY_RUN ? " (dry-run)" : ""}.`, - ); - - let migrated = 0; - for (const row of rows) { - const input: { - openAIApiKey?: string | null; - huggingFaceApiKey?: string | null; - ollamaBaseUrl?: string | null; - ollamaModel?: string | null; - } = {}; - if (row.openAIApiKey) input.openAIApiKey = row.openAIApiKey; - if (row.huggingFaceApiKey) input.huggingFaceApiKey = row.huggingFaceApiKey; - if (row.ollamaBaseUrl) input.ollamaBaseUrl = row.ollamaBaseUrl; - if (row.ollamaModel) input.ollamaModel = row.ollamaModel; - - if (Object.keys(input).length === 0) continue; - - console.log( - ` company #${row.id} (${row.name}): fields = [${Object.keys(input).join(", ")}]`, - ); - - if (DRY_RUN) continue; - - await upsertCompanyCredentials(row.id, input); - - // Null out the legacy columns so this row won't be picked up on re-run. - await db - .update(company) - .set({ - embeddingOpenAIApiKey: null, - embeddingHuggingFaceApiKey: null, - embeddingOllamaBaseUrl: null, - embeddingOllamaModel: null, - }) - .where(eq(company.id, row.id)); - - migrated += 1; - } - - if (DRY_RUN) { - console.log( - `Dry run complete. ${rows.length} row(s) would be migrated. No changes written.`, - ); - } else { - console.log( - `Migrated ${migrated} row(s). Once verified, apply drizzle/0011_drop_plaintext_embedding_credentials.sql to remove the legacy columns.`, - ); - } - - // Close the connection so the script exits cleanly. - await db.execute(sql`SELECT 1`); -} - -main() - .then(() => process.exit(0)) - .catch((err) => { - console.error("Backfill failed:", err); - process.exit(1); - }); +console.error( + "This script moved to apps/web/scripts/backfill-embedding-credentials.ts.\n" + + "Run: pnpm db:backfill:embedding-credentials", +); +process.exit(1);