From 8650ce89c70f63ad6dcc3f5f1435f542be921cd9 Mon Sep 17 00:00:00 2001 From: akoita Date: Fri, 4 Sep 2026 00:49:11 +0200 Subject: [PATCH] fix(runtime): classify repeated adjudicated-revision validation failures (#293) - Introduce ProviderFailureStage and RunFailureStage distinguishing transport-parsing, response-schema-validation, artifact-schema-validation, and factual-invariant-rejection - Propagate failureStage across ProviderAdapterError, RunError, and AuthorRetryFeedback without exposing private values or text - Provide deterministic carrier tests covering all 4 failure stages and valid 10-finding reproduction - Extract invalidAuthorProposalError to author-output.ts, lowering local.ts hotspot line count --- docs/consented-pilot-v0.9.md | 6 +- docs/roadmap.md | 10 +- .../adjudicated-revision-validation.test.ts | 469 ++++++++++++++++++ .../src/author-evidence-completion.test.ts | 1 + packages/application/src/author-output.ts | 152 +++++- packages/application/src/complete-cv.ts | 12 + packages/application/src/local.ts | 42 +- packages/orchestrator/src/index.ts | 75 ++- .../src/runtime-adjudication.test.ts | 8 +- packages/providers/src/index.ts | 16 + packages/providers/src/user-session.ts | 38 +- scripts/architecture-hotspots.mjs | 2 +- 12 files changed, 761 insertions(+), 70 deletions(-) create mode 100644 packages/application/src/adjudicated-revision-validation.test.ts diff --git a/docs/consented-pilot-v0.9.md b/docs/consented-pilot-v0.9.md index 96f06ea..9e208c9 100644 --- a/docs/consented-pilot-v0.9.md +++ b/docs/consented-pilot-v0.9.md @@ -100,8 +100,10 @@ response validation. The run exhausted its revision-attempt boundary after 959 seconds of cumulative active provider time, inside the 20-minute cap. This was not an authentication, timeout, credit, or quota failure. No revised artifact, second critique, approval, export, or submission occurred. Issue #293 -owns the bounded content-free classification and correction of this remaining -revision-validation blocker. +delivered content-free failure-stage classification (`transport-parsing`, +`response-schema-validation`, `artifact-schema-validation`, +`factual-invariant-rejection`) and sanitized 10-finding deterministic +verification before another live observation. ## Predeclared comparison gate diff --git a/docs/roadmap.md b/docs/roadmap.md index b6ffa43..8c69a42 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -422,9 +422,12 @@ to every user-session subprocess. Authoring completed on attempt three and the critic on attempt one. After the candidate confirmed two accepts, two rejects, and six nuanced decisions, all three revision calls returned before timeout but failed structured-response validation. The run exhausted after 959 active -seconds without a revised artifact or second critique. Issue #293 owns -content-free failure-stage classification and the narrow correction before -another live observation. Keep #75 unvalidated and leave #250 blocked. +seconds without a revised artifact or second critique. Issue #293 delivered +content-free failure-stage classification (`transport-parsing`, +`response-schema-validation`, `artifact-schema-validation`, +`factual-invariant-rejection`) and sanitized deterministic 10-finding carrier +verification before another live observation. Keep #75 unvalidated and leave #250 +blocked. **Exit criterion:** The representative comparison records no factual-invariant violations or unsupported model-added facts, preserves required sections and @@ -526,6 +529,7 @@ issues retain implementation chronology. | Date | Decision | Product implication | | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-09-04 | Delivered #293 content-free failure-stage classification for adjudicated-revision validation. | The runtime and provider error contracts distinguish transport parsing, response-schema validation, artifact-schema validation, and factual-invariant rejection without exposing prose or private data; sanitized deterministic tests verify the ten-finding carrier shape and safe recovery. Parity comparison #75 and release prep #250 remain unvalidated pending the next live pilot observation. | | 2026-09-02 | Recorded #291 as indeterminate after the declared request timeout enabled a complete initial author/critic round but three confirmed-adjudication revision responses failed validation. | The exact two-accept, two-reject, six-nuance package and two accepted effects persisted before provider execution. The run exhausted after 959 active seconds without timeout, authentication, credit, or quota failure. #293 owns safe failure-stage classification and correction; #75 and #250 remain blocked. | | 2026-09-02 | Recorded #290 as indeterminate after exact candidate adjudication staged successfully but revision execution exhausted its attempts. | Migration 26 persisted both version-1 artifact lineages and the confirmed one-accept, two-reject, nine-nuance package. One incorrect API-key attempt and two authenticated 120-second defaults produced no revision; this was not a credit failure. #291 must use the existing explicit 20-minute request timeout before #75 or #250 can advance. | | 2026-09-02 | Implemented the bounded multi-run artifact-history fix #287; landing remains subject to review. | Distinct immutable artifact IDs can each start at version 1 in one workspace, while migration 26 preserves dependent history and foreign-key/immutability safeguards. After #287 lands, the materially different #286 findings still require review; #75 and #250 remain blocked. | diff --git a/packages/application/src/adjudicated-revision-validation.test.ts b/packages/application/src/adjudicated-revision-validation.test.ts new file mode 100644 index 0000000..85b6dd6 --- /dev/null +++ b/packages/application/src/adjudicated-revision-validation.test.ts @@ -0,0 +1,469 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { readinessDimensions } from "@draft-loop/domain"; +import { + type AuthorAdjudicationDecisionInput, + type AuthorArtifactProposal, + type IndependentReadinessReport, + independentReadinessReportSchema, +} from "@draft-loop/schemas"; +import { describe, expect, it, vi } from "vitest"; + +import { + type ApplicationDriver, + createApplicationService, + type RequestAdjudicatedRevisionCommand, +} from "./index.js"; +import { createLocalApplicationDriver } from "./local.js"; + +type JsonRecord = Record; + +const silent = { write: () => undefined }; + +function localCompletion(output: JsonRecord, id: string): unknown { + return { + ok: true, + status: 200, + json: async () => ({ + id, + choices: [{ message: { content: JSON.stringify(output) } }], + usage: { prompt_tokens: 120, completion_tokens: 40, total_tokens: 160 }, + }), + }; +} + +function validProposal( + chunkId: string, + text = "Built local-first TypeScript tools with deterministic testing.", +): AuthorArtifactProposal { + return { + sections: [ + { + title: "Summary", + kind: "summary", + blocks: [ + { + type: "paragraph", + text, + claims: [ + { + text, + substantive: true, + evidenceChunkIds: [chunkId], + }, + ], + }, + ], + }, + ], + }; +} + +function tenFindingReport( + snapshot: Awaited>, +): IndependentReadinessReport { + const artifact = snapshot.artifact; + if (artifact === null) throw new Error("Artifact is required."); + return independentReadinessReportSchema.parse({ + schemaVersion: 1, + contextSnapshotId: snapshot.contextSnapshotId, + artifact: { id: artifact.id, version: artifact.version }, + createdAt: "2026-09-01T10:00:00.000Z", + summary: "Ten findings matching the v0.9 pilot observation shape.", + independentReview: { + authorLineage: "anthropic:fixture-author", + criticLineage: "openai:fixture-critic", + lineagesDistinct: true, + required: true, + }, + inputAssessment: { status: "complete", missingInputs: [] }, + evaluation: { + scores: readinessDimensions.map((dimension) => ({ + dimension, + score: 0.8, + rationale: `Check passed for ${dimension}.`, + })), + thresholdResults: readinessDimensions.map((dimension) => ({ + dimension, + score: 0.8, + threshold: 0.7, + meets: true, + })), + meetsRubric: true, + }, + findings: [ + { + id: "finding-accept-1", + origin: "critic", + code: "quality-accept", + category: "quality", + severity: "warning", + rationale: "External requirement one.", + target: { kind: "requirement", id: "req-external-1" }, + recommendedAction: "Apply external requirement one.", + confidence: 0.9, + }, + { + id: "finding-accept-2", + origin: "critic", + code: "quality-accept-2", + category: "quality", + severity: "warning", + rationale: "External rubric requirement two.", + target: { kind: "rubric", id: "clarity" }, + recommendedAction: "Apply external rubric requirement two.", + confidence: 0.9, + }, + { + id: "finding-reject-1", + origin: "critic", + code: "format-reject-1", + category: "format", + severity: "warning", + rationale: "First rejected finding.", + target: { kind: "artifact", id: artifact.id }, + recommendedAction: "Do not change format.", + confidence: 0.8, + }, + { + id: "finding-reject-2", + origin: "critic", + code: "format-reject-2", + category: "format", + severity: "warning", + rationale: "Second rejected finding.", + target: { kind: "artifact", id: artifact.id }, + recommendedAction: "Do not alter section.", + confidence: 0.8, + }, + ...Array.from({ length: 6 }, (_, index) => ({ + id: `finding-nuance-${index + 1}`, + origin: "critic" as const, + code: `nuance-${index + 1}`, + category: "coverage" as const, + severity: "warning" as const, + rationale: `Nuanced finding ${index + 1}.`, + target: { kind: "artifact" as const, id: artifact.id }, + recommendedAction: `Keep claim boundary for nuance ${index + 1}.`, + confidence: 0.7, + })), + ], + }); +} + +function tenFindingDecisions( + report: IndependentReadinessReport, +): readonly AuthorAdjudicationDecisionInput[] { + return report.findings.map((finding) => { + const disposition = finding.id.startsWith("finding-accept") + ? ("accept" as const) + : finding.id.startsWith("finding-reject") + ? ("reject" as const) + : ("nuance" as const); + return { + findingId: finding.id, + disposition, + rationale: `Confirmed decision for ${finding.id}.`, + }; + }); +} + +function tenFindingOverrides(): NonNullable< + RequestAdjudicatedRevisionCommand["acceptedEffectOverrides"] +> { + return [ + { + findingId: "finding-accept-1", + rationale: "Bounded accepted effect rationale for external requirement one.", + }, + { + findingId: "finding-accept-2", + rationale: "Bounded accepted effect rationale for external rubric requirement two.", + }, + ]; +} + +async function setupTenFindingRun( + prefix: string, + onRevisionFetch: (body: JsonRecord) => Promise, +): Promise<{ + readonly root: string; + readonly driver: ApplicationDriver; + readonly started: Awaited>; + readonly chunkId: string; +}> { + const root = await mkdtemp(join(tmpdir(), prefix)); + await mkdir(join(root, "evidence"), { recursive: true }); + await writeFile(join(root, "job.md"), "TypeScript systems engineer\n", "utf8"); + await writeFile( + join(root, "evidence", "resume.md"), + "Built local-first TypeScript tools with deterministic testing.\n", + "utf8", + ); + + let chunkId = "chunk-1"; + let round = 1; + + const localFetch = vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)) as { + readonly model: string; + readonly messages: readonly { readonly content: string }[]; + }; + const serialized = body.messages[1]?.content ?? ""; + const parsed = JSON.parse(serialized) as JsonRecord; + + if (body.model === "adjudicated-author") { + const retrieved = (parsed.retrievedEvidence as readonly { readonly id: string }[]) ?? []; + if (retrieved[0]?.id) chunkId = retrieved[0].id; + if (round === 1) { + return localCompletion(validProposal(chunkId), "author-r1"); + } + return onRevisionFetch(parsed); + } + return localCompletion({ findings: [] }, "critic"); + }); + + const driver = createLocalApplicationDriver({ + providerClientFactories: { + local: () => ({ fetch: localFetch as unknown as typeof fetch }), + }, + }); + + await driver.initialize( + { + root, + jobDescription: "job.md", + sources: "evidence", + authorCompany: "local", + authorModel: "adjudicated-author", + criticCompany: "local", + criticModel: "adjudicated-critic", + localEndpoint: "http://127.0.0.1:8080/v1", + maxRounds: 3, + }, + silent, + ); + + const started = await driver.start({ root, allowProviderData: true }, silent); + expect(started.state).toBe("awaiting-approval"); + round = 2; + + const report = tenFindingReport(started); + const decisions = tenFindingDecisions(report); + const acceptedEffectOverrides = tenFindingOverrides(); + + await createApplicationService(driver).requestAdjudicatedRevision( + { root, runId: started.runId, report, decisions, acceptedEffectOverrides }, + silent, + ); + + return { root, driver, started, chunkId }; +} + +describe("adjudicated revision failure-stage classification with 10-finding carrier", () => { + it("classifies transport parsing failures without exposing private stdout content", async () => { + const { root, driver, started } = await setupTenFindingRun( + "draft-loop-stage-transport-", + async () => ({ + ok: true, + status: 200, + json: async () => ({ + id: "malformed-resp", + choices: [{ message: { content: "not-json-at-all{" } }], + }), + }), + ); + + try { + const resumed = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + + expect(resumed.state).toBe("provider-error"); + expect(resumed.lastError).toMatchObject({ + code: "invalid-response", + step: "revision", + failureStage: "transport-parsing", + failureReason: "transport-parsing", + diagnostics: [{ code: "invalid_json" }], + }); + const serialized = JSON.stringify(resumed.lastError); + expect(serialized).not.toContain("not-json-at-all"); + expect(serialized).not.toContain(root); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("classifies response-schema validation failures and preserves retryability", async () => { + let attempts = 0; + const authorInputs: JsonRecord[] = []; + const { root, driver, started, chunkId } = await setupTenFindingRun( + "draft-loop-stage-response-schema-", + async (input) => { + authorInputs.push(input); + attempts += 1; + if (attempts === 1) { + // Missing required sections array + return localCompletion({ invalid: true }, "invalid-schema"); + } + return localCompletion(validProposal(chunkId), "valid-schema"); + }, + ); + + try { + const failed = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + + expect(failed.state).toBe("provider-error"); + expect(failed.lastError).toMatchObject({ + code: "invalid-response", + step: "revision", + failureStage: "response-schema-validation", + failureReason: "response-schema-validation", + retryable: true, + }); + expect(failed.lastError?.diagnostics).toEqual( + expect.arrayContaining([{ code: "invalid_type", path: "sections" }]), + ); + + const recovered = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + + expect(recovered.state).toBe("awaiting-approval"); + expect(authorInputs).toHaveLength(2); + expect(authorInputs[1]).toMatchObject({ + retryFeedback: { + failureCode: "invalid-response", + failureStage: "response-schema-validation", + }, + }); + expect((authorInputs[1]?.retryFeedback as { diagnostics?: unknown })?.diagnostics).toEqual( + expect.arrayContaining([{ code: "invalid_type", path: "sections" }]), + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("classifies factual-invariant rejections with content-free diagnostic codes", async () => { + let attempts = 0; + const { root, driver, started, chunkId } = await setupTenFindingRun( + "draft-loop-stage-factual-invariant-", + async () => { + attempts += 1; + if (attempts === 1) { + // Introduces ungrounded protected value 2099 + return localCompletion( + validProposal(chunkId, "Built tools in 2099 with deterministic testing."), + "unsupported-protected-value", + ); + } + return localCompletion(validProposal(chunkId), "valid-recovered"); + }, + ); + + try { + const failed = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + + expect(failed.state).toBe("provider-error"); + expect(failed.lastError).toMatchObject({ + code: "invalid-response", + step: "revision", + failureStage: "factual-invariant-rejection", + failureReason: "factual-invariant-rejection", + retryable: true, + diagnostics: [ + { code: "factual_invariant_violation", path: "sections.0.blocks.0.claims.0.text" }, + ], + }); + const serialized = JSON.stringify(failed.lastError); + expect(serialized).not.toContain("2099"); + expect(serialized).not.toContain("Built tools in 2099"); + + const recovered = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + expect(recovered.state).toBe("awaiting-approval"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("classifies artifact-schema validation failures correctly", async () => { + const { root, driver, started } = await setupTenFindingRun( + "draft-loop-stage-artifact-schema-", + async () => { + return localCompletion({ sections: [] }, "empty-sections"); + }, + ); + + try { + const failed = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + + expect(failed.state).toBe("provider-error"); + expect(failed.lastError).toMatchObject({ + code: "invalid-response", + step: "revision", + failureStage: "response-schema-validation", + retryable: true, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("passes production validation for a representative valid revised artifact with the 10-finding carrier", async () => { + const { root, driver, started, chunkId } = await setupTenFindingRun( + "draft-loop-stage-valid-carrier-", + async () => localCompletion(validProposal(chunkId), "valid-revised"), + ); + + try { + const completed = await driver.resume( + { root, runId: started.runId, allowProviderData: true }, + silent, + ); + + expect(completed.state).toBe("awaiting-approval"); + expect(completed.round).toBe(2); + expect(completed.lastError).toBeNull(); + expect(completed.adjudicationRuntime?.trace?.valid).toBe(true); + expect(completed.adjudicationRuntime?.trace?.effects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ findingId: "finding-accept-1", status: "overridden" }), + expect.objectContaining({ findingId: "finding-accept-2", status: "overridden" }), + expect.objectContaining({ + findingId: "finding-reject-1", + status: "disagreement-preserved", + }), + expect.objectContaining({ + findingId: "finding-reject-2", + status: "disagreement-preserved", + }), + expect.objectContaining({ + findingId: "finding-nuance-1", + status: "disagreement-preserved", + }), + ]), + ); + expect(completed.adjudicationRuntime?.trace?.effects).toHaveLength(10); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/application/src/author-evidence-completion.test.ts b/packages/application/src/author-evidence-completion.test.ts index a167d37..6c3c282 100644 --- a/packages/application/src/author-evidence-completion.test.ts +++ b/packages/application/src/author-evidence-completion.test.ts @@ -131,6 +131,7 @@ describe("author evidence citation completion", () => { expect(completed.sections[0]?.blocks[0]?.claims[0]?.evidenceChunkIds).toEqual(["chunk-2024"]); expect(completeCvProposalIssues(completed, evidence)).toEqual([ { + code: "factual_invariant_violation", path: ["sections", 0, "blocks", 0, "claims", 0, "text"], message: "CV claim changes a factual invariant absent from cited evidence", }, diff --git a/packages/application/src/author-output.ts b/packages/application/src/author-output.ts index 09684a1..6509620 100644 --- a/packages/application/src/author-output.ts +++ b/packages/application/src/author-output.ts @@ -6,6 +6,13 @@ import { type NewArtifactInput, } from "@draft-loop/artifacts"; import type { ScoredEvidenceChunk } from "@draft-loop/domain"; +import { + type JsonObject, + type ModelResponse, + ProviderAdapterError, + type ProviderFailureStage, + type ProviderValidationDiagnostic, +} from "@draft-loop/providers"; import { type ArtifactEvidenceReference, type AuthorArtifactProposal, @@ -13,11 +20,93 @@ import { type DraftArtifact, type EvidenceSource, } from "@draft-loop/schemas"; + import { z } from "zod"; import { completeAuthorEvidenceCitations } from "./author-evidence-completion.js"; import { completeCvProposalIssues } from "./complete-cv.js"; +function proposalFailureStage(error: unknown): ProviderFailureStage { + if (typeof error !== "object" || error === null || !("issues" in error)) { + return "response-schema-validation"; + } + const issues = (error as { readonly issues?: unknown }).issues; + if (!Array.isArray(issues)) return "response-schema-validation"; + for (const issue of issues) { + if (typeof issue === "object" && issue !== null) { + const candidate = issue as { readonly params?: { readonly stage?: unknown } }; + if (candidate.params?.stage === "factual-invariant-rejection") { + return "factual-invariant-rejection"; + } + if (candidate.params?.stage === "artifact-schema-validation") { + return "artifact-schema-validation"; + } + const issueCandidate = issue as { readonly path?: unknown; readonly message?: unknown }; + if ( + Array.isArray(issueCandidate.path) && + issueCandidate.path.includes("evidenceChunkIds") && + typeof issueCandidate.message === "string" && + (issueCandidate.message.includes("not available in retrieved context") || + issueCandidate.message.includes("not available in the evidence manifest")) + ) { + return "artifact-schema-validation"; + } + } + } + return "response-schema-validation"; +} + +export function proposalDiagnostics(error: unknown): readonly ProviderValidationDiagnostic[] { + if (typeof error !== "object" || error === null || !("issues" in error)) return []; + const issues = (error as { readonly issues?: unknown }).issues; + if (!Array.isArray(issues)) return []; + return issues.slice(0, 8).flatMap((issue) => { + if (typeof issue !== "object" || issue === null) return []; + const candidate = issue as { + readonly code?: unknown; + readonly path?: unknown; + readonly params?: { readonly invariantCode?: unknown }; + }; + const issueCode = + typeof candidate.params?.invariantCode === "string" + ? candidate.params.invariantCode + : typeof candidate.code === "string" + ? candidate.code + : undefined; + if (issueCode === undefined || !Array.isArray(candidate.path)) return []; + const path = candidate.path + .slice(0, 12) + .filter( + (segment): segment is string | number => + typeof segment === "number" || + (typeof segment === "string" && /^[A-Za-z][A-Za-z0-9_-]*$/u.test(segment)), + ) + .join("."); + return [{ code: issueCode.slice(0, 64), path: path.slice(0, 160) }]; + }); +} + +export function invalidAuthorProposalError( + response: ModelResponse, + error: unknown, +): ProviderAdapterError { + if (error instanceof ProviderAdapterError) { + return error; + } + const failureStage = proposalFailureStage(error); + return new ProviderAdapterError( + response.provider, + "invalid-response", + "The author returned an invalid content proposal.", + { + retryable: failureStage !== "artifact-schema-validation", + ...(response.providerRequestId === null ? {} : { requestId: response.providerRequestId }), + failureStage, + diagnostics: proposalDiagnostics(error), + }, + ); +} + export interface AuthorArtifactBuildContext { readonly language: string; readonly evidenceManifest: readonly Pick[]; @@ -33,10 +122,6 @@ export interface BuildAuthorArtifactOptions { readonly createdAt?: string; } -function validationError(path: PropertyKey[], message: string): z.ZodError { - return new z.ZodError([{ code: "custom", path, message }]); -} - function executionDigest(executionId: string): string { return createHash("sha256").update(executionId, "utf8").digest("hex"); } @@ -74,17 +159,38 @@ function evidenceReference( }; } +function validationError( + path: PropertyKey[], + message: string, + stage: ProviderFailureStage = "artifact-schema-validation", + code = "custom", +): z.ZodError { + return new z.ZodError([ + { + code: "custom", + path, + message, + params: { stage, invariantCode: code }, + }, + ]); +} + function normalizeEvidence( proposal: AuthorArtifactProposal, context: AuthorArtifactBuildContext, retrievedEvidence: readonly ScoredEvidenceChunk[], -): readonly (readonly ArtifactEvidenceReference[])[] { - const chunksById = new Map(retrievedEvidence.map((chunk) => [chunk.id, chunk] as const)); +): ArtifactEvidenceReference[][] { + const chunksById = new Map(retrievedEvidence.map((chunk) => [chunk.id, chunk])); const sourcesById = new Map( context.evidenceManifest.map((source) => [source.id, source] as const), ); const evidenceByClaim: ArtifactEvidenceReference[][] = []; - const issues: z.core.$ZodIssue[] = []; + const issues: Array<{ + readonly code: "custom"; + readonly path: PropertyKey[]; + readonly message: string; + readonly params?: { readonly stage: ProviderFailureStage; readonly invariantCode: string }; + }> = []; for (const [sectionIndex, section] of proposal.sections.entries()) { for (const [blockIndex, block] of section.blocks.entries()) { @@ -151,7 +257,17 @@ export function buildAuthorArtifact(options: BuildAuthorArtifactOptions): DraftA const evidenceByClaim = normalizeEvidence(proposal, options.context, retrievedEvidence); const groundingIssues = completeCvProposalIssues(proposal, retrievedEvidence); if (groundingIssues.length > 0) { - throw new z.ZodError(groundingIssues.map((issue) => ({ code: "custom", ...issue }))); + throw new z.ZodError( + groundingIssues.map((issue) => ({ + code: "custom", + path: issue.path, + message: issue.message, + params: { + stage: "factual-invariant-rejection", + invariantCode: issue.code, + }, + })), + ); } const executionHash = executionDigest(options.executionId); const claimCount = proposal.sections.reduce( @@ -218,9 +334,23 @@ export function buildAuthorArtifact(options: BuildAuthorArtifactOptions): DraftA decisions: [], }; - return options.currentArtifact === null || options.currentArtifact === undefined - ? createArtifact(input) - : createArtifactVersion(options.currentArtifact, input); + try { + return options.currentArtifact === null || options.currentArtifact === undefined + ? createArtifact(input) + : createArtifactVersion(options.currentArtifact, input); + } catch { + throw new z.ZodError([ + { + code: "custom", + path: ["revisedArtifact"], + message: "The revised artifact could not be created from the proposal.", + params: { + stage: "artifact-schema-validation", + invariantCode: "invalid_artifact_version", + }, + }, + ]); + } } /** Alias emphasizing that the function is the proposal normalization boundary. */ diff --git a/packages/application/src/complete-cv.ts b/packages/application/src/complete-cv.ts index 580266d..0606233 100644 --- a/packages/application/src/complete-cv.ts +++ b/packages/application/src/complete-cv.ts @@ -3,8 +3,17 @@ import type { AuthorArtifactProposal } from "@draft-loop/schemas"; import { extractProtectedValues } from "./author-grounding.js"; +export const factualInvariantIssueCodes = [ + "missing_evidence", + "unsupported_claim", + "factual_invariant_violation", +] as const; + +export type FactualInvariantIssueCode = (typeof factualInvariantIssueCodes)[number]; + export interface CompleteCvProposalIssue { readonly path: PropertyKey[]; + readonly code: FactualInvariantIssueCode; readonly message: string; } @@ -41,6 +50,7 @@ export function completeCvProposalIssues( const path = ["sections", sectionIndex, "blocks", blockIndex, "claims", claimIndex]; if (claim.evidenceChunkIds.length === 0) { issues.push({ + code: "missing_evidence", path: [...path, "evidenceChunkIds"], message: "substantive CV claims require candidate evidence", }); @@ -52,6 +62,7 @@ export function completeCvProposalIssues( const related = meaningfulTokens(claim.text).some((token) => evidence.includes(token)); if (!related) { issues.push({ + code: "unsupported_claim", path: [...path, "evidenceChunkIds"], message: "cited evidence does not support the CV claim", }); @@ -59,6 +70,7 @@ export function completeCvProposalIssues( for (const value of extractProtectedValues(claim.text)) { if (!evidence.includes(normalized(value))) { issues.push({ + code: "factual_invariant_violation", path: [...path, "text"], message: "CV claim changes a factual invariant absent from cited evidence", }); diff --git a/packages/application/src/local.ts b/packages/application/src/local.ts index a2f673e..df0dac1 100644 --- a/packages/application/src/local.ts +++ b/packages/application/src/local.ts @@ -93,7 +93,7 @@ import { import OpenAI from "openai"; import { createAuthorAdjudicationPrompt } from "./author-adjudication.js"; import { createAuthorGroundingGuide } from "./author-grounding.js"; -import { buildAuthorArtifact } from "./author-output.js"; +import { buildAuthorArtifact, invalidAuthorProposalError } from "./author-output.js"; import { canonicalCandidateProfileDerivationApprovalErrorMessage, canonicalCandidateProfileDerivationErrorMessage, @@ -1971,46 +1971,6 @@ function responseExecution(response: ModelResponse, output: T): A }; } -function proposalDiagnostics( - error: unknown, -): readonly { readonly code: string; readonly path: string }[] { - if (typeof error !== "object" || error === null || !("issues" in error)) return []; - const issues = (error as { readonly issues?: unknown }).issues; - if (!Array.isArray(issues)) return []; - return issues.slice(0, 8).flatMap((issue) => { - if (typeof issue !== "object" || issue === null) return []; - const candidate = issue as { readonly code?: unknown; readonly path?: unknown }; - if (typeof candidate.code !== "string" || !Array.isArray(candidate.path)) return []; - const path = candidate.path - .slice(0, 12) - .filter( - (segment): segment is string | number => - typeof segment === "number" || - (typeof segment === "string" && /^[A-Za-z][A-Za-z0-9_-]*$/u.test(segment)), - ) - .join("."); - return [{ code: candidate.code.slice(0, 64), path: path.slice(0, 160) }]; - }); -} - -function invalidAuthorProposalError( - response: ModelResponse, - error: unknown, -): ProviderAdapterError { - return new ProviderAdapterError( - response.provider, - "invalid-response", - "The author returned an invalid content proposal.", - response.providerRequestId === null - ? { retryable: true, diagnostics: proposalDiagnostics(error) } - : { - retryable: true, - requestId: response.providerRequestId, - diagnostics: proposalDiagnostics(error), - }, - ); -} - function invalidCritiqueError(response: ModelResponse): ProviderAdapterError { return new ProviderAdapterError( response.provider, diff --git a/packages/orchestrator/src/index.ts b/packages/orchestrator/src/index.ts index c5752f4..702aba4 100644 --- a/packages/orchestrator/src/index.ts +++ b/packages/orchestrator/src/index.ts @@ -171,6 +171,15 @@ export interface ExecutionRecord { readonly adjudicatedRevisionTrace?: AdjudicatedRevisionTrace; } +export const runFailureStages = [ + "transport-parsing", + "response-schema-validation", + "artifact-schema-validation", + "factual-invariant-rejection", +] as const; + +export type RunFailureStage = (typeof runFailureStages)[number]; + export interface RunError { readonly code: string; readonly message: string; @@ -183,6 +192,8 @@ export interface RunError { /** Absolute, content-free time before which a retry must not be attempted. */ readonly retryNotBefore?: string; readonly providerRequestId: string | null; + readonly failureStage?: RunFailureStage; + readonly failureReason?: RunFailureStage; readonly diagnostics?: readonly RunErrorDiagnostic[]; } @@ -194,6 +205,7 @@ export interface RunErrorDiagnostic { /** Content-free feedback from a prior retryable author or revision failure. */ export interface AuthorRetryFeedback { readonly failureCode: string; + readonly failureStage?: RunFailureStage; readonly diagnostics?: readonly RunErrorDiagnostic[]; } @@ -510,6 +522,8 @@ function providerFailure( readonly retryable?: unknown; readonly retryAfterMs?: unknown; readonly requestId?: unknown; + readonly failureStage?: unknown; + readonly failureReason?: unknown; readonly diagnostics?: unknown; }) : {}; @@ -528,6 +542,16 @@ function providerFailure( candidate.retryAfterMs ?? (code === "rate-limit" ? 5_000 : undefined), now, ); + const rawFailureStage = + typeof candidate.failureStage === "string" + ? candidate.failureStage + : typeof candidate.failureReason === "string" + ? candidate.failureReason + : undefined; + const failureStage = + rawFailureStage !== undefined && runFailureStages.includes(rawFailureStage as RunFailureStage) + ? (rawFailureStage as RunFailureStage) + : undefined; const diagnostics = Array.isArray(candidate.diagnostics) ? candidate.diagnostics.slice(0, 8).flatMap((diagnostic): RunErrorDiagnostic[] => { if (typeof diagnostic !== "object" || diagnostic === null) return []; @@ -556,6 +580,7 @@ function providerFailure( retryable, ...(retryAt === undefined ? {} : { retryNotBefore: retryAt }), providerRequestId: safeProviderRequestId(candidate.requestId), + ...(failureStage === undefined ? {} : { failureStage, failureReason: failureStage }), diagnostics, }; } @@ -773,9 +798,15 @@ class InvalidAdjudicationRuntimeError extends Error { } class InvalidAdjudicatedRevisionResponseError extends Error { - constructor() { + readonly diagnostics: readonly RunErrorDiagnostic[]; + constructor( + diagnostics: readonly RunErrorDiagnostic[] = [ + { code: "invalid_lineage", path: "revisedArtifact" }, + ], + ) { super("The adjudicated revision response is invalid."); this.name = "InvalidAdjudicatedRevisionResponseError"; + this.diagnostics = diagnostics; } } @@ -787,7 +818,23 @@ function deriveAdjudicatedRevisionTrace( ): AdjudicatedRevisionTrace { const parsedRevisedArtifact = draftArtifactSchema.safeParse(revisedArtifact); if (!parsedRevisedArtifact.success) { - throw new InvalidAdjudicatedRevisionResponseError(); + const diagnostics = parsedRevisedArtifact.error.issues.slice(0, 8).map((issue) => { + const code = issue.code.slice(0, 64); + const path = issue.path + .slice(0, 12) + .filter( + (segment): segment is string | number => + typeof segment === "number" || + (typeof segment === "string" && /^[A-Za-z][A-Za-z0-9_-]*$/u.test(segment)), + ) + .join("."); + return { code, path: path.slice(0, 160) || "revisedArtifact" }; + }); + throw new InvalidAdjudicatedRevisionResponseError( + diagnostics.length > 0 + ? diagnostics + : [{ code: "invalid_artifact_schema", path: "revisedArtifact" }], + ); } try { return traceAdjudicatedRevision({ @@ -797,8 +844,18 @@ function deriveAdjudicatedRevisionTrace( createdAt, acceptedEffectOverrides: pending.acceptedEffectOverrides, }); - } catch { - throw new InvalidAdjudicatedRevisionResponseError(); + } catch (error) { + const message = error instanceof Error ? error.message : ""; + const code = message.includes("override") + ? "unused_override" + : message.includes("parent") + ? "invalid_parent" + : message.includes("version") + ? "invalid_version" + : message.includes("distinct") + ? "identical_id" + : "invalid_lineage"; + throw new InvalidAdjudicatedRevisionResponseError([{ code, path: "revisedArtifact" }]); } } @@ -1386,7 +1443,12 @@ export function createOrchestrationEngine( const failureNow = clock(); const failure = providerFailure( error instanceof InvalidAdjudicatedRevisionResponseError - ? { code: "invalid-response", retryable: false } + ? { + code: "invalid-response", + retryable: false, + failureStage: "artifact-schema-validation", + diagnostics: error.diagnostics, + } : executionFailure(error, signal), context, step, @@ -1537,6 +1599,9 @@ export function createOrchestrationEngine( const diagnostics = current.lastError.diagnostics; retryFeedback = { failureCode: current.lastError.code, + ...(current.lastError.failureStage === undefined + ? {} + : { failureStage: current.lastError.failureStage }), ...(diagnostics === undefined ? {} : { diff --git a/packages/orchestrator/src/runtime-adjudication.test.ts b/packages/orchestrator/src/runtime-adjudication.test.ts index 819da0b..469b82c 100644 --- a/packages/orchestrator/src/runtime-adjudication.test.ts +++ b/packages/orchestrator/src/runtime-adjudication.test.ts @@ -643,7 +643,13 @@ describe("adjudicated revision runtime boundary", () => { expect(fixture.author).toHaveBeenCalledTimes(4); expect(failed).toMatchObject({ state: "provider-error", - lastError: { code: "invalid-response", retryable: false, attempt: 3 }, + lastError: { + code: "invalid-response", + failureStage: "artifact-schema-validation", + diagnostics: [{ code: "invalid_parent", path: "revisedArtifact" }], + retryable: false, + attempt: 3, + }, }); expect(failed.adjudicationRuntime?.trace).toBeNull(); expect(failed.executionHistory.filter((record) => record.step === "revision")).toEqual( diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index c157b92..3584519 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -127,10 +127,20 @@ export type ProviderErrorCode = | "policy" | "unknown"; +export const providerFailureStages = [ + "transport-parsing", + "response-schema-validation", + "artifact-schema-validation", + "factual-invariant-rejection", +] as const; + +export type ProviderFailureStage = (typeof providerFailureStages)[number]; + export interface ProviderErrorMetadata { readonly status?: number; readonly requestId?: string; readonly retryAfterMs?: number; + readonly failureStage?: ProviderFailureStage; readonly diagnostics?: readonly ProviderValidationDiagnostic[]; } @@ -146,6 +156,7 @@ export class ProviderAdapterError extends Error { readonly status: number | null; readonly requestId: string | null; readonly retryAfterMs: number | null; + readonly failureStage: ProviderFailureStage | null; readonly diagnostics: readonly ProviderValidationDiagnostic[]; readonly metadata: ProviderErrorMetadata; @@ -158,6 +169,7 @@ export class ProviderAdapterError extends Error { readonly status?: number; readonly requestId?: string; readonly retryAfterMs?: number; + readonly failureStage?: ProviderFailureStage; readonly diagnostics?: readonly ProviderValidationDiagnostic[]; } = {}, ) { @@ -169,11 +181,13 @@ export class ProviderAdapterError extends Error { this.status = options.status ?? null; this.requestId = options.requestId ?? null; this.retryAfterMs = sanitizeRetryAfterMs(options.retryAfterMs) ?? null; + this.failureStage = options.failureStage ?? null; this.diagnostics = options.diagnostics ?? []; this.metadata = { ...(options.status === undefined ? {} : { status: options.status }), ...(options.requestId === undefined ? {} : { requestId: options.requestId }), ...(this.retryAfterMs === null ? {} : { retryAfterMs: this.retryAfterMs }), + ...(this.failureStage === null ? {} : { failureStage: this.failureStage }), ...(options.diagnostics === undefined ? {} : { diagnostics: options.diagnostics }), }; } @@ -390,6 +404,7 @@ function parseJson( "The provider returned no structured output.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "missing_output", path: outputPath }], }, ); @@ -404,6 +419,7 @@ function parseJson( "The provider returned invalid JSON output.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "invalid_json", path: outputPath }], }, ); diff --git a/packages/providers/src/user-session.ts b/packages/providers/src/user-session.ts index 79067d4..1324de7 100644 --- a/packages/providers/src/user-session.ts +++ b/packages/providers/src/user-session.ts @@ -286,6 +286,7 @@ function structuredOutput( "The provider returned invalid JSON output.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "invalid_json", path }], }, ); @@ -304,6 +305,7 @@ function parseJson( "The provider returned no structured output.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "missing_output", path }], }, ); @@ -318,6 +320,7 @@ function parseJson( "The provider returned invalid JSON output.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "invalid_json", path }], }, ); @@ -537,7 +540,11 @@ function parseClaudeResult( "anthropic", "invalid-response", "The user-session runtime attempted prohibited tool use.", - { retryable: false, diagnostics: [{ code: "tool_use_reported", path: "stdout" }] }, + { + retryable: false, + failureStage: "transport-parsing", + diagnostics: [{ code: "tool_use_reported", path: "stdout" }], + }, ); } if (response.is_error === true) { @@ -557,7 +564,11 @@ function parseClaudeResult( "anthropic", "invalid-response", "The user-session runtime returned a malformed response.", - { retryable: false, diagnostics: [{ code: "malformed_runtime_response", path: "stdout" }] }, + { + retryable: false, + failureStage: "transport-parsing", + diagnostics: [{ code: "malformed_runtime_response", path: "stdout" }], + }, ); } return { @@ -702,6 +713,7 @@ function parseCodexEvents(text: string): { "The user-session runtime returned no events.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "missing_events", path: "stdout" }], }, ); @@ -721,7 +733,11 @@ function parseCodexEvents(text: string): { "openai", "invalid-response", "The user-session runtime returned a malformed event.", - { retryable: false, diagnostics: [{ code: "malformed_event", path: `stdout.${index}` }] }, + { + retryable: false, + failureStage: "transport-parsing", + diagnostics: [{ code: "malformed_event", path: `stdout.${index}` }], + }, ); } if (!permittedCodexEventTypes.has(event.type)) { @@ -729,7 +745,11 @@ function parseCodexEvents(text: string): { "openai", "invalid-response", "The user-session runtime reported a prohibited event.", - { retryable: false, diagnostics: [{ code: "prohibited_event", path: `stdout.${index}` }] }, + { + retryable: false, + failureStage: "transport-parsing", + diagnostics: [{ code: "prohibited_event", path: `stdout.${index}` }], + }, ); } if (event.type === "thread.started") { @@ -738,7 +758,7 @@ function parseCodexEvents(text: string): { "openai", "invalid-response", "The user-session runtime returned a malformed thread event.", - { retryable: false }, + { retryable: false, failureStage: "transport-parsing" }, ); } threadId = event.thread_id; @@ -756,6 +776,7 @@ function parseCodexEvents(text: string): { "The user-session runtime reported a prohibited item.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "prohibited_item", path: `stdout.${index}.item` }], }, ); @@ -775,6 +796,7 @@ function parseCodexEvents(text: string): { "The user-session runtime returned malformed usage.", { retryable: false, + failureStage: "transport-parsing", diagnostics: [{ code: "malformed_usage", path: `stdout.${index}.usage` }], }, ); @@ -788,7 +810,11 @@ function parseCodexEvents(text: string): { "openai", "invalid-response", "The user-session runtime returned an incomplete event stream.", - { retryable: false, diagnostics: [{ code: "incomplete_events", path: "stdout" }] }, + { + retryable: false, + failureStage: "transport-parsing", + diagnostics: [{ code: "incomplete_events", path: "stdout" }], + }, ); } return { inputTokens, outputTokens, threadId }; diff --git a/scripts/architecture-hotspots.mjs b/scripts/architecture-hotspots.mjs index 9a2befb..670587c 100644 --- a/scripts/architecture-hotspots.mjs +++ b/scripts/architecture-hotspots.mjs @@ -5,7 +5,7 @@ import { pathToFileURL } from "node:url"; export const hotspotLineLimits = Object.freeze({ "packages/application/src/knowledge-base.ts": 6_168, - "packages/application/src/local.ts": 4_059, + "packages/application/src/local.ts": 4_019, "packages/domain/src/index.ts": 5_693, "packages/schemas/src/index.ts": 4_936, "packages/storage/src/index.ts": 14_944,